From 9e04e57949674123b9b35b4070b387cec7353092 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 9 Feb 2023 15:52:11 +0100 Subject: [PATCH 01/69] Allow to give access only to certain named collections --- src/Access/AccessRights.cpp | 38 +++++++--- src/Access/Common/AccessFlags.cpp | 10 ++- src/Access/Common/AccessFlags.h | 5 ++ src/Access/Common/AccessRightsElement.cpp | 23 +++--- src/Access/Common/AccessRightsElement.h | 2 + src/Access/Common/AccessType.h | 10 +-- src/Access/ContextAccess.cpp | 4 +- src/Parsers/Access/ASTGrantQuery.cpp | 21 +++--- src/Parsers/Access/ParserGrantQuery.cpp | 34 ++++++++- .../System/StorageSystemNamedCollections.cpp | 6 +- .../System/StorageSystemPrivileges.cpp | 3 + .../test_named_collections/test.py | 70 +++++++++++++++++++ 12 files changed, 189 insertions(+), 37 deletions(-) diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 7e21e3c2c4d..595afadfe1c 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -61,12 +61,22 @@ namespace res.any_database = true; res.any_table = true; res.any_column = true; + res.any_named_collection = !access_flags.isNamedCollectionAccessOnly(); break; } case 1: { - res.any_database = false; - res.database = full_name[0]; + res.any_named_collection = !access_flags.isNamedCollectionAccessOnly(); + if (!res.any_named_collection) + { + res.any_database = true; + res.named_collection = full_name[0]; + } + else + { + res.any_database = false; + res.database = full_name[0]; + } res.any_table = true; res.any_column = true; break; @@ -317,8 +327,8 @@ public: const Node * child = tryGetChild(name); if (child) return child->isGranted(flags_to_check, subnames...); - else - return flags.contains(flags_to_check); + + return flags.contains(flags_to_check); } template @@ -783,7 +793,9 @@ void AccessRights::grantImplHelper(const AccessRightsElement & element) { assert(!element.is_partial_revoke); assert(!element.grant_option || with_grant_option); - if (element.any_database) + if (!element.any_named_collection) + grantImpl(element.access_flags, element.named_collection); + else if (element.any_database) grantImpl(element.access_flags); else if (element.any_table) grantImpl(element.access_flags, element.database); @@ -825,7 +837,10 @@ void AccessRights::grant(const AccessFlags & flags, std::string_view database, s void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) { grantImpl(flags, database, table, columns); } void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl(flags, database, table, columns); } void AccessRights::grant(const AccessRightsElement & element) { grantImpl(element); } -void AccessRights::grant(const AccessRightsElements & elements) { grantImpl(elements); } +void AccessRights::grant(const AccessRightsElements & elements) +{ + grantImpl(elements); +} void AccessRights::grantWithGrantOption(const AccessFlags & flags) { grantImpl(flags); } void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database) { grantImpl(flags, database); } @@ -858,7 +873,9 @@ template void AccessRights::revokeImplHelper(const AccessRightsElement & element) { assert(!element.grant_option || grant_option); - if (element.any_database) + if (!element.any_named_collection) + revokeImpl(element.access_flags, element.named_collection); + else if (element.any_database) revokeImpl(element.access_flags); else if (element.any_table) revokeImpl(element.access_flags, element.database); @@ -912,7 +929,7 @@ void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { re AccessRightsElements AccessRights::getElements() const { -#if 0 +#if 1 logTree(); #endif if (!root) @@ -934,6 +951,7 @@ bool AccessRights::isGrantedImpl(const AccessFlags & flags, const Args &... args { auto helper = [&](const std::unique_ptr & root_node) -> bool { + logTree(); if (!root_node) return flags.isEmpty(); return root_node->isGranted(flags, args...); @@ -948,7 +966,9 @@ template bool AccessRights::isGrantedImplHelper(const AccessRightsElement & element) const { assert(!element.grant_option || grant_option); - if (element.any_database) + if (!element.any_named_collection) + return isGrantedImpl(element.access_flags, element.named_collection); + else if (element.any_database) return isGrantedImpl(element.access_flags); else if (element.any_table) return isGrantedImpl(element.access_flags, element.database); diff --git a/src/Access/Common/AccessFlags.cpp b/src/Access/Common/AccessFlags.cpp index bef165ba4e6..533fbb0e524 100644 --- a/src/Access/Common/AccessFlags.cpp +++ b/src/Access/Common/AccessFlags.cpp @@ -96,11 +96,13 @@ namespace const Flags & getAllFlags() const { return all_flags; } const Flags & getGlobalFlags() const { return all_flags_for_target[GLOBAL]; } + const Flags & getGlobalWithParameterFlags() const { return all_flags_for_target[GLOBAL_WITH_PARAMETER]; } const Flags & getDatabaseFlags() const { return all_flags_for_target[DATABASE]; } const Flags & getTableFlags() const { return all_flags_for_target[TABLE]; } const Flags & getColumnFlags() const { return all_flags_for_target[COLUMN]; } const Flags & getDictionaryFlags() const { return all_flags_for_target[DICTIONARY]; } const Flags & getAllFlagsGrantableOnGlobalLevel() const { return getAllFlags(); } + const Flags & getAllFlagsGrantableOnNamedCollectionLevel() const { return all_flags_for_target[NAMED_COLLECTION]; } const Flags & getAllFlagsGrantableOnDatabaseLevel() const { return all_flags_grantable_on_database_level; } const Flags & getAllFlagsGrantableOnTableLevel() const { return all_flags_grantable_on_table_level; } const Flags & getAllFlagsGrantableOnColumnLevel() const { return getColumnFlags(); } @@ -116,6 +118,8 @@ namespace VIEW = TABLE, COLUMN, DICTIONARY, + GLOBAL_WITH_PARAMETER, + NAMED_COLLECTION, }; struct Node; @@ -295,7 +299,7 @@ namespace collectAllFlags(child.get()); all_flags_grantable_on_table_level = all_flags_for_target[TABLE] | all_flags_for_target[DICTIONARY] | all_flags_for_target[COLUMN]; - all_flags_grantable_on_database_level = all_flags_for_target[DATABASE] | all_flags_grantable_on_table_level; + all_flags_grantable_on_database_level = all_flags_for_target[DATABASE] | all_flags_for_target[NAMED_COLLECTION] | all_flags_grantable_on_table_level; } Helper() @@ -345,7 +349,7 @@ namespace std::unordered_map keyword_to_flags_map; std::vector access_type_to_flags_mapping; Flags all_flags; - Flags all_flags_for_target[static_cast(DICTIONARY) + 1]; + Flags all_flags_for_target[static_cast(NAMED_COLLECTION) + 1]; Flags all_flags_grantable_on_database_level; Flags all_flags_grantable_on_table_level; }; @@ -361,11 +365,13 @@ std::vector AccessFlags::toAccessTypes() const { return Helper::inst std::vector AccessFlags::toKeywords() const { return Helper::instance().flagsToKeywords(flags); } AccessFlags AccessFlags::allFlags() { return Helper::instance().getAllFlags(); } AccessFlags AccessFlags::allGlobalFlags() { return Helper::instance().getGlobalFlags(); } +AccessFlags AccessFlags::allGlobalWithParameterFlags() { return Helper::instance().getGlobalWithParameterFlags(); } AccessFlags AccessFlags::allDatabaseFlags() { return Helper::instance().getDatabaseFlags(); } AccessFlags AccessFlags::allTableFlags() { return Helper::instance().getTableFlags(); } AccessFlags AccessFlags::allColumnFlags() { return Helper::instance().getColumnFlags(); } AccessFlags AccessFlags::allDictionaryFlags() { return Helper::instance().getDictionaryFlags(); } AccessFlags AccessFlags::allFlagsGrantableOnGlobalLevel() { return Helper::instance().getAllFlagsGrantableOnGlobalLevel(); } +AccessFlags AccessFlags::allFlagsGrantableOnNamedCollectionLevel() { return Helper::instance().getAllFlagsGrantableOnNamedCollectionLevel(); } AccessFlags AccessFlags::allFlagsGrantableOnDatabaseLevel() { return Helper::instance().getAllFlagsGrantableOnDatabaseLevel(); } AccessFlags AccessFlags::allFlagsGrantableOnTableLevel() { return Helper::instance().getAllFlagsGrantableOnTableLevel(); } AccessFlags AccessFlags::allFlagsGrantableOnColumnLevel() { return Helper::instance().getAllFlagsGrantableOnColumnLevel(); } diff --git a/src/Access/Common/AccessFlags.h b/src/Access/Common/AccessFlags.h index c4e0b7ac281..5443c505245 100644 --- a/src/Access/Common/AccessFlags.h +++ b/src/Access/Common/AccessFlags.h @@ -50,6 +50,7 @@ public: bool isEmpty() const { return flags.none(); } explicit operator bool() const { return !isEmpty(); } bool contains(const AccessFlags & other) const { return (flags & other.flags) == other.flags; } + bool isNamedCollectionAccessOnly() const { return (flags & ~allFlagsGrantableOnNamedCollectionLevel()).isEmpty(); } friend bool operator ==(const AccessFlags & left, const AccessFlags & right) { return left.flags == right.flags; } friend bool operator !=(const AccessFlags & left, const AccessFlags & right) { return !(left == right); } @@ -76,6 +77,8 @@ public: /// Returns all the global flags. static AccessFlags allGlobalFlags(); + static AccessFlags allGlobalWithParameterFlags(); + /// Returns all the flags related to a database. static AccessFlags allDatabaseFlags(); @@ -104,6 +107,8 @@ public: /// The same as allColumnFlags(). static AccessFlags allFlagsGrantableOnColumnLevel(); + static AccessFlags allFlagsGrantableOnNamedCollectionLevel(); + static constexpr size_t SIZE = 256; private: using Flags = std::bitset; diff --git a/src/Access/Common/AccessRightsElement.cpp b/src/Access/Common/AccessRightsElement.cpp index 69a2354f25d..e2eb14ad9cb 100644 --- a/src/Access/Common/AccessRightsElement.cpp +++ b/src/Access/Common/AccessRightsElement.cpp @@ -21,24 +21,31 @@ namespace result += ")"; } - void formatONClause(const String & database, bool any_database, const String & table, bool any_table, String & result) + void formatONClause(const AccessRightsElement & element, String & result) { result += "ON "; - if (any_database) + if (!element.any_named_collection) + { + if (element.named_collection.empty()) + result += "*"; + else + result += backQuoteIfNeed(element.named_collection); + } + else if (element.any_database) { result += "*.*"; } else { - if (!database.empty()) + if (!element.database.empty()) { - result += backQuoteIfNeed(database); + result += backQuoteIfNeed(element.database); result += "."; } - if (any_table) + if (element.any_table) result += "*"; else - result += backQuoteIfNeed(table); + result += backQuoteIfNeed(element.table); } } @@ -96,7 +103,7 @@ namespace String result; formatAccessFlagsWithColumns(element.access_flags, element.columns, element.any_column, result); result += " "; - formatONClause(element.database, element.any_database, element.table, element.any_table, result); + formatONClause(element, result); if (with_options) formatOptions(element.grant_option, element.is_partial_revoke, result); return result; @@ -129,7 +136,7 @@ namespace if (!next_element_uses_same_table_and_options) { part += " "; - formatONClause(element.database, element.any_database, element.table, element.any_table, part); + formatONClause(element, part); if (with_options) formatOptions(element.grant_option, element.is_partial_revoke, part); if (result.empty()) diff --git a/src/Access/Common/AccessRightsElement.h b/src/Access/Common/AccessRightsElement.h index 5f65b6bcd12..27657ea3960 100644 --- a/src/Access/Common/AccessRightsElement.h +++ b/src/Access/Common/AccessRightsElement.h @@ -14,9 +14,11 @@ struct AccessRightsElement String database; String table; Strings columns; + String named_collection; bool any_database = true; bool any_table = true; bool any_column = true; + bool any_named_collection = true; bool grant_option = false; bool is_partial_revoke = false; diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 497327c1bad..703b3106a9a 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -12,7 +12,7 @@ enum class AccessType /// Macro M should be defined as M(name, aliases, node_type, parent_group_name) /// where name is identifier with underscores (instead of spaces); /// aliases is a string containing comma-separated list; -/// node_type either specifies access type's level (GLOBAL/DATABASE/TABLE/DICTIONARY/VIEW/COLUMNS), +/// node_type either specifies access type's level (GLOBAL/NAMED_COLLECTION/DATABASE/TABLE/DICTIONARY/VIEW/COLUMNS), /// or specifies that the access type is a GROUP of other access types; /// parent_group_name is the name of the group containing this access type (or NONE if there is no such group). #define APPLY_FOR_ACCESS_TYPES(M) \ @@ -69,7 +69,7 @@ enum class AccessType M(ALTER_FREEZE_PARTITION, "FREEZE PARTITION, UNFREEZE", TABLE, ALTER_TABLE) \ \ M(ALTER_DATABASE_SETTINGS, "ALTER DATABASE SETTING, ALTER MODIFY DATABASE SETTING, MODIFY DATABASE SETTING", DATABASE, ALTER_DATABASE) /* allows to execute ALTER MODIFY SETTING */\ - M(ALTER_NAMED_COLLECTION, "", GROUP, ALTER) /* allows to execute ALTER NAMED COLLECTION */\ + M(ALTER_NAMED_COLLECTION, "", NAMED_COLLECTION, ALTER) /* allows to execute ALTER NAMED COLLECTION */\ \ M(ALTER_TABLE, "", GROUP, ALTER) \ M(ALTER_DATABASE, "", GROUP, ALTER) \ @@ -89,7 +89,7 @@ enum class AccessType M(CREATE_TEMPORARY_TABLE, "", GLOBAL, CREATE) /* allows to create and manipulate temporary tables; implicitly enabled by the grant CREATE_TABLE on any table */ \ M(CREATE_FUNCTION, "", GLOBAL, CREATE) /* allows to execute CREATE FUNCTION */ \ - M(CREATE_NAMED_COLLECTION, "", GLOBAL, CREATE) /* allows to execute CREATE NAMED COLLECTION */ \ + M(CREATE_NAMED_COLLECTION, "", NAMED_COLLECTION, CREATE) /* allows to execute CREATE NAMED COLLECTION */ \ M(CREATE, "", GROUP, ALL) /* allows to execute {CREATE|ATTACH} */ \ \ M(DROP_DATABASE, "", DATABASE, DROP) /* allows to execute {DROP|DETACH} DATABASE */\ @@ -98,7 +98,7 @@ enum class AccessType implicitly enabled by the grant DROP_TABLE */\ M(DROP_DICTIONARY, "", DICTIONARY, DROP) /* allows to execute {DROP|DETACH} DICTIONARY */\ M(DROP_FUNCTION, "", GLOBAL, DROP) /* allows to execute DROP FUNCTION */\ - M(DROP_NAMED_COLLECTION, "", GLOBAL, DROP) /* allows to execute DROP NAMED COLLECTION */\ + M(DROP_NAMED_COLLECTION, "", NAMED_COLLECTION, DROP) /* allows to execute DROP NAMED COLLECTION */\ M(DROP, "", GROUP, ALL) /* allows to execute {DROP|DETACH} */\ \ M(TRUNCATE, "TRUNCATE TABLE", TABLE, ALL) \ @@ -134,7 +134,7 @@ enum class AccessType M(SHOW_QUOTAS, "SHOW CREATE QUOTA", GLOBAL, SHOW_ACCESS) \ M(SHOW_SETTINGS_PROFILES, "SHOW PROFILES, SHOW CREATE SETTINGS PROFILE, SHOW CREATE PROFILE", GLOBAL, SHOW_ACCESS) \ M(SHOW_ACCESS, "", GROUP, ACCESS_MANAGEMENT) \ - M(SHOW_NAMED_COLLECTIONS, "SHOW NAMED COLLECTIONS", GLOBAL, ACCESS_MANAGEMENT) \ + M(SHOW_NAMED_COLLECTIONS, "SHOW NAMED COLLECTIONS", NAMED_COLLECTION, ACCESS_MANAGEMENT) \ M(ACCESS_MANAGEMENT, "", GROUP, ALL) \ \ M(SYSTEM_SHUTDOWN, "SYSTEM KILL, SHUTDOWN", GLOBAL, SYSTEM) \ diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index fbaacb2263b..6d6362a98b2 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -606,7 +606,9 @@ template bool ContextAccess::checkAccessImplHelper(const AccessRightsElement & element) const { assert(!element.grant_option || grant_option); - if (element.any_database) + if (!element.any_named_collection) + return checkAccessImpl(element.access_flags, element.named_collection); + else if (element.any_database) return checkAccessImpl(element.access_flags); else if (element.any_table) return checkAccessImpl(element.access_flags, element.database); diff --git a/src/Parsers/Access/ASTGrantQuery.cpp b/src/Parsers/Access/ASTGrantQuery.cpp index f92541ec672..e4aa11967c6 100644 --- a/src/Parsers/Access/ASTGrantQuery.cpp +++ b/src/Parsers/Access/ASTGrantQuery.cpp @@ -27,21 +27,25 @@ namespace } - void formatONClause(const String & database, bool any_database, const String & table, bool any_table, const IAST::FormatSettings & settings) + void formatONClause(const AccessRightsElement & element, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ON " << (settings.hilite ? IAST::hilite_none : ""); - if (any_database) + if (!element.any_named_collection) + { + settings.ostr << backQuoteIfNeed(element.named_collection); + } + else if (element.any_database) { settings.ostr << "*.*"; } else { - if (!database.empty()) - settings.ostr << backQuoteIfNeed(database) << "."; - if (any_table) + if (!element.database.empty()) + settings.ostr << backQuoteIfNeed(element.database) << "."; + if (element.any_table) settings.ostr << "*"; else - settings.ostr << backQuoteIfNeed(table); + settings.ostr << backQuoteIfNeed(element.table); } } @@ -71,14 +75,15 @@ namespace { const auto & next_element = elements[i + 1]; if ((element.database == next_element.database) && (element.any_database == next_element.any_database) - && (element.table == next_element.table) && (element.any_table == next_element.any_table)) + && (element.table == next_element.table) && (element.any_table == next_element.any_table) + && (element.named_collection == next_element.named_collection)) next_element_on_same_db_and_table = true; } if (!next_element_on_same_db_and_table) { settings.ostr << " "; - formatONClause(element.database, element.any_database, element.table, element.any_table, settings); + formatONClause(element, settings); } } diff --git a/src/Parsers/Access/ParserGrantQuery.cpp b/src/Parsers/Access/ParserGrantQuery.cpp index 2211969c61e..a2f4e2a4921 100644 --- a/src/Parsers/Access/ParserGrantQuery.cpp +++ b/src/Parsers/Access/ParserGrantQuery.cpp @@ -123,12 +123,38 @@ namespace if (!parseAccessFlagsWithColumns(pos, expected, access_and_columns)) return false; + String database_name, table_name, collection_name; + bool any_database = false, any_table = false, any_named_collection = true; + + size_t named_collection_access = 0; + for (const auto & elem : access_and_columns) + { + if (elem.first.isNamedCollectionAccessOnly()) + ++named_collection_access; + } + const bool grant_named_collection_access = named_collection_access == access_and_columns.size(); + if (!ParserKeyword{"ON"}.ignore(pos, expected)) return false; - String database_name, table_name; - bool any_database = false, any_table = false; - if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, database_name, any_database, table_name, any_table)) + if (grant_named_collection_access) + { + ASTPtr collection; + if (ParserToken{TokenType::Asterisk}.ignore(pos, expected)) + { + any_named_collection = true; + } + else if (ParserIdentifier{}.parse(pos, collection, expected)) + { + any_named_collection = false; + collection_name = getIdentifierName(collection); + } + else + return false; + + any_database = any_table = true; + } + else if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, database_name, any_database, table_name, any_table)) return false; for (auto & [access_flags, columns] : access_and_columns) @@ -140,6 +166,8 @@ namespace element.any_database = any_database; element.database = database_name; element.any_table = any_table; + element.any_named_collection = any_named_collection; + element.named_collection = collection_name; element.table = table_name; res_elements.emplace_back(std::move(element)); } diff --git a/src/Storages/System/StorageSystemNamedCollections.cpp b/src/Storages/System/StorageSystemNamedCollections.cpp index bc1e3a45e6b..0b7522d3845 100644 --- a/src/Storages/System/StorageSystemNamedCollections.cpp +++ b/src/Storages/System/StorageSystemNamedCollections.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -29,11 +30,14 @@ StorageSystemNamedCollections::StorageSystemNamedCollections(const StorageID & t void StorageSystemNamedCollections::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const { - context->checkAccess(AccessType::SHOW_NAMED_COLLECTIONS); + const auto & access = context->getAccess(); auto collections = NamedCollectionFactory::instance().getAll(); for (const auto & [name, collection] : collections) { + if (!access->isGranted(AccessType::SHOW_NAMED_COLLECTIONS, name)) + continue; + res_columns[0]->insert(name); auto * column_map = typeid_cast(res_columns[1].get()); diff --git a/src/Storages/System/StorageSystemPrivileges.cpp b/src/Storages/System/StorageSystemPrivileges.cpp index 70163979f72..14ca76df7d8 100644 --- a/src/Storages/System/StorageSystemPrivileges.cpp +++ b/src/Storages/System/StorageSystemPrivileges.cpp @@ -28,6 +28,8 @@ namespace DICTIONARY, VIEW, COLUMN, + GLOBAL_WITH_PARAMETER, + NAMED_COLLECTION, }; DataTypeEnum8::Values getLevelEnumValues() @@ -39,6 +41,7 @@ namespace enum_values.emplace_back("DICTIONARY", static_cast(DICTIONARY)); enum_values.emplace_back("VIEW", static_cast(VIEW)); enum_values.emplace_back("COLUMN", static_cast(COLUMN)); + enum_values.emplace_back("NAMED_COLLECTION", static_cast(NAMED_COLLECTION)); return enum_values; } } diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 3b102f1aa70..0f8999f43dd 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -105,6 +105,76 @@ def test_access(cluster): assert int(node.query("select count() from system.named_collections")) > 0 +def test_granular_access(cluster): + node = cluster.instances["node"] + assert 1 == int(node.query("SELECT count() FROM system.named_collections")) + assert ( + "collection1" == node.query("SELECT name FROM system.named_collections").strip() + ) + + node.query("CREATE USER kek") + node.query("GRANT select ON *.* TO kek") + assert 0 == int( + node.query("SELECT count() FROM system.named_collections", user="kek") + ) + + node.query("GRANT show named collections ON collection1 TO kek") + assert 1 == int( + node.query("SELECT count() FROM system.named_collections", user="kek") + ) + assert ( + "collection1" + == node.query("SELECT name FROM system.named_collections", user="kek").strip() + ) + + node.query("CREATE NAMED COLLECTION collection2 AS key1=1, key2='value2'") + assert 2 == int(node.query("SELECT count() FROM system.named_collections")) + assert ( + "collection1\ncollection2" + == node.query("select name from system.named_collections").strip() + ) + + assert 1 == int( + node.query("SELECT count() FROM system.named_collections", user="kek") + ) + assert ( + "collection1" + == node.query("select name from system.named_collections", user="kek").strip() + ) + + node.query("GRANT show named collections ON collection2 TO kek") + assert 2 == int( + node.query("SELECT count() FROM system.named_collections", user="kek") + ) + assert ( + "collection1\ncollection2" + == node.query("select name from system.named_collections", user="kek").strip() + ) + node.restart_clickhouse() + assert ( + "collection1\ncollection2" + == node.query("select name from system.named_collections", user="kek").strip() + ) + + node.query("CREATE USER koko") + node.query("GRANT select ON *.* TO koko") + assert 0 == int( + node.query("SELECT count() FROM system.named_collections", user="koko") + ) + node.query("GRANT show named collections ON * TO koko") + assert ( + "collection1\ncollection2" + == node.query("select name from system.named_collections", user="koko").strip() + ) + node.restart_clickhouse() + assert ( + "collection1\ncollection2" + == node.query("select name from system.named_collections", user="koko").strip() + ) + + node.query("DROP NAMED COLLECTION collection2") + + def test_config_reload(cluster): node = cluster.instances["node"] assert ( From 214ffe0bb0f7d19b70649fac60886c900fb97925 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 9 Feb 2023 17:44:26 +0100 Subject: [PATCH 02/69] Remove unused code --- src/Access/Common/AccessFlags.cpp | 3 --- src/Access/Common/AccessFlags.h | 2 -- src/Access/Common/AccessRightsElement.cpp | 8 ++++---- src/Access/Common/AccessRightsElement.h | 2 ++ src/Storages/System/StorageSystemPrivileges.cpp | 1 - 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/Access/Common/AccessFlags.cpp b/src/Access/Common/AccessFlags.cpp index 533fbb0e524..c8d2b1ca4d9 100644 --- a/src/Access/Common/AccessFlags.cpp +++ b/src/Access/Common/AccessFlags.cpp @@ -96,7 +96,6 @@ namespace const Flags & getAllFlags() const { return all_flags; } const Flags & getGlobalFlags() const { return all_flags_for_target[GLOBAL]; } - const Flags & getGlobalWithParameterFlags() const { return all_flags_for_target[GLOBAL_WITH_PARAMETER]; } const Flags & getDatabaseFlags() const { return all_flags_for_target[DATABASE]; } const Flags & getTableFlags() const { return all_flags_for_target[TABLE]; } const Flags & getColumnFlags() const { return all_flags_for_target[COLUMN]; } @@ -118,7 +117,6 @@ namespace VIEW = TABLE, COLUMN, DICTIONARY, - GLOBAL_WITH_PARAMETER, NAMED_COLLECTION, }; @@ -365,7 +363,6 @@ std::vector AccessFlags::toAccessTypes() const { return Helper::inst std::vector AccessFlags::toKeywords() const { return Helper::instance().flagsToKeywords(flags); } AccessFlags AccessFlags::allFlags() { return Helper::instance().getAllFlags(); } AccessFlags AccessFlags::allGlobalFlags() { return Helper::instance().getGlobalFlags(); } -AccessFlags AccessFlags::allGlobalWithParameterFlags() { return Helper::instance().getGlobalWithParameterFlags(); } AccessFlags AccessFlags::allDatabaseFlags() { return Helper::instance().getDatabaseFlags(); } AccessFlags AccessFlags::allTableFlags() { return Helper::instance().getTableFlags(); } AccessFlags AccessFlags::allColumnFlags() { return Helper::instance().getColumnFlags(); } diff --git a/src/Access/Common/AccessFlags.h b/src/Access/Common/AccessFlags.h index 5443c505245..b81b73b8350 100644 --- a/src/Access/Common/AccessFlags.h +++ b/src/Access/Common/AccessFlags.h @@ -77,8 +77,6 @@ public: /// Returns all the global flags. static AccessFlags allGlobalFlags(); - static AccessFlags allGlobalWithParameterFlags(); - /// Returns all the flags related to a database. static AccessFlags allDatabaseFlags(); diff --git a/src/Access/Common/AccessRightsElement.cpp b/src/Access/Common/AccessRightsElement.cpp index e2eb14ad9cb..011db851bc4 100644 --- a/src/Access/Common/AccessRightsElement.cpp +++ b/src/Access/Common/AccessRightsElement.cpp @@ -24,12 +24,12 @@ namespace void formatONClause(const AccessRightsElement & element, String & result) { result += "ON "; - if (!element.any_named_collection) + if (element.isNamedCollectionAccess()) { - if (element.named_collection.empty()) - result += "*"; - else + if (!element.any_named_collection) result += backQuoteIfNeed(element.named_collection); + else + result += "*"; } else if (element.any_database) { diff --git a/src/Access/Common/AccessRightsElement.h b/src/Access/Common/AccessRightsElement.h index 27657ea3960..653f813ff35 100644 --- a/src/Access/Common/AccessRightsElement.h +++ b/src/Access/Common/AccessRightsElement.h @@ -69,6 +69,8 @@ struct AccessRightsElement /// If the database is empty, replaces it with `current_database`. Otherwise does nothing. void replaceEmptyDatabase(const String & current_database); + bool isNamedCollectionAccess() const { return access_flags.isNamedCollectionAccessOnly(); } + /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table". String toString() const; String toStringWithoutOptions() const; diff --git a/src/Storages/System/StorageSystemPrivileges.cpp b/src/Storages/System/StorageSystemPrivileges.cpp index 14ca76df7d8..ee412d0e648 100644 --- a/src/Storages/System/StorageSystemPrivileges.cpp +++ b/src/Storages/System/StorageSystemPrivileges.cpp @@ -28,7 +28,6 @@ namespace DICTIONARY, VIEW, COLUMN, - GLOBAL_WITH_PARAMETER, NAMED_COLLECTION, }; From c13b0b8a065ec5fa2275fa7ed27705c1f481eebc Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 10 Feb 2023 13:14:11 +0100 Subject: [PATCH 03/69] Fix tests --- src/Access/AccessRights.cpp | 7 +++---- src/Access/Common/AccessFlags.h | 3 ++- src/Access/Common/AccessRightsElement.h | 2 +- src/Parsers/Access/ParserGrantQuery.cpp | 4 ++-- tests/queries/0_stateless/01271_show_privileges.reference | 8 ++++---- .../0_stateless/02117_show_create_table_system.reference | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 595afadfe1c..ca118203541 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -61,15 +61,14 @@ namespace res.any_database = true; res.any_table = true; res.any_column = true; - res.any_named_collection = !access_flags.isNamedCollectionAccessOnly(); + res.any_named_collection = true; break; } case 1: { - res.any_named_collection = !access_flags.isNamedCollectionAccessOnly(); - if (!res.any_named_collection) + if (access_flags.isNamedCollectionAccess()) { - res.any_database = true; + res.any_named_collection = false; res.named_collection = full_name[0]; } else diff --git a/src/Access/Common/AccessFlags.h b/src/Access/Common/AccessFlags.h index b81b73b8350..f475896b6e5 100644 --- a/src/Access/Common/AccessFlags.h +++ b/src/Access/Common/AccessFlags.h @@ -50,7 +50,8 @@ public: bool isEmpty() const { return flags.none(); } explicit operator bool() const { return !isEmpty(); } bool contains(const AccessFlags & other) const { return (flags & other.flags) == other.flags; } - bool isNamedCollectionAccessOnly() const { return (flags & ~allFlagsGrantableOnNamedCollectionLevel()).isEmpty(); } + bool containsOnly(const AccessFlags & other) const { return flags == other.flags; } + bool isNamedCollectionAccess() const { return containsOnly(AccessFlags::allFlagsGrantableOnNamedCollectionLevel()); } friend bool operator ==(const AccessFlags & left, const AccessFlags & right) { return left.flags == right.flags; } friend bool operator !=(const AccessFlags & left, const AccessFlags & right) { return !(left == right); } diff --git a/src/Access/Common/AccessRightsElement.h b/src/Access/Common/AccessRightsElement.h index 653f813ff35..db6a1f6872f 100644 --- a/src/Access/Common/AccessRightsElement.h +++ b/src/Access/Common/AccessRightsElement.h @@ -69,7 +69,7 @@ struct AccessRightsElement /// If the database is empty, replaces it with `current_database`. Otherwise does nothing. void replaceEmptyDatabase(const String & current_database); - bool isNamedCollectionAccess() const { return access_flags.isNamedCollectionAccessOnly(); } + bool isNamedCollectionAccess() const { return access_flags.isNamedCollectionAccess(); } /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table". String toString() const; diff --git a/src/Parsers/Access/ParserGrantQuery.cpp b/src/Parsers/Access/ParserGrantQuery.cpp index a2f4e2a4921..06660642fbf 100644 --- a/src/Parsers/Access/ParserGrantQuery.cpp +++ b/src/Parsers/Access/ParserGrantQuery.cpp @@ -129,14 +129,14 @@ namespace size_t named_collection_access = 0; for (const auto & elem : access_and_columns) { - if (elem.first.isNamedCollectionAccessOnly()) + if (elem.first.isNamedCollectionAccess()) ++named_collection_access; } - const bool grant_named_collection_access = named_collection_access == access_and_columns.size(); if (!ParserKeyword{"ON"}.ignore(pos, expected)) return false; + const bool grant_named_collection_access = named_collection_access && named_collection_access == access_and_columns.size(); if (grant_named_collection_access) { ASTPtr collection; diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 58b1cab6e20..518b1a84abb 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -39,7 +39,7 @@ ALTER MOVE PARTITION ['ALTER MOVE PART','MOVE PARTITION','MOVE PART'] TABLE ALTE ALTER FETCH PARTITION ['ALTER FETCH PART','FETCH PARTITION'] TABLE ALTER TABLE ALTER FREEZE PARTITION ['FREEZE PARTITION','UNFREEZE'] TABLE ALTER TABLE ALTER DATABASE SETTINGS ['ALTER DATABASE SETTING','ALTER MODIFY DATABASE SETTING','MODIFY DATABASE SETTING'] DATABASE ALTER DATABASE -ALTER NAMED COLLECTION [] \N ALTER +ALTER NAMED COLLECTION [] NAMED_COLLECTION ALTER ALTER TABLE [] \N ALTER ALTER DATABASE [] \N ALTER ALTER VIEW REFRESH ['ALTER LIVE VIEW REFRESH','REFRESH VIEW'] VIEW ALTER VIEW @@ -52,14 +52,14 @@ CREATE VIEW [] VIEW CREATE CREATE DICTIONARY [] DICTIONARY CREATE CREATE TEMPORARY TABLE [] GLOBAL CREATE CREATE FUNCTION [] GLOBAL CREATE -CREATE NAMED COLLECTION [] GLOBAL CREATE +CREATE NAMED COLLECTION [] NAMED_COLLECTION CREATE CREATE [] \N ALL DROP DATABASE [] DATABASE DROP DROP TABLE [] TABLE DROP DROP VIEW [] VIEW DROP DROP DICTIONARY [] DICTIONARY DROP DROP FUNCTION [] GLOBAL DROP -DROP NAMED COLLECTION [] GLOBAL DROP +DROP NAMED COLLECTION [] NAMED_COLLECTION DROP DROP [] \N ALL TRUNCATE ['TRUNCATE TABLE'] TABLE ALL OPTIMIZE ['OPTIMIZE TABLE'] TABLE ALL @@ -89,7 +89,7 @@ SHOW ROW POLICIES ['SHOW POLICIES','SHOW CREATE ROW POLICY','SHOW CREATE POLICY' SHOW QUOTAS ['SHOW CREATE QUOTA'] GLOBAL SHOW ACCESS SHOW SETTINGS PROFILES ['SHOW PROFILES','SHOW CREATE SETTINGS PROFILE','SHOW CREATE PROFILE'] GLOBAL SHOW ACCESS SHOW ACCESS [] \N ACCESS MANAGEMENT -SHOW NAMED COLLECTIONS ['SHOW NAMED COLLECTIONS'] GLOBAL ACCESS MANAGEMENT +SHOW NAMED COLLECTIONS ['SHOW NAMED COLLECTIONS'] NAMED_COLLECTION ACCESS MANAGEMENT ACCESS MANAGEMENT [] \N ALL SYSTEM SHUTDOWN ['SYSTEM KILL','SHUTDOWN'] GLOBAL SYSTEM SYSTEM DROP DNS CACHE ['SYSTEM DROP DNS','DROP DNS CACHE','DROP DNS'] GLOBAL SYSTEM DROP CACHE diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index dc7cdddf8ec..484243bd523 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -571,7 +571,7 @@ CREATE TABLE system.privileges ( `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'addressToLine' = 136, 'addressToLineWithInlines' = 137, 'addressToSymbol' = 138, 'demangle' = 139, 'INTROSPECTION' = 140, 'FILE' = 141, 'URL' = 142, 'REMOTE' = 143, 'MONGO' = 144, 'MEILISEARCH' = 145, 'MYSQL' = 146, 'POSTGRES' = 147, 'SQLITE' = 148, 'ODBC' = 149, 'JDBC' = 150, 'HDFS' = 151, 'S3' = 152, 'HIVE' = 153, 'SOURCES' = 154, 'CLUSTER' = 155, 'ALL' = 156, 'NONE' = 157), `aliases` Array(String), - `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)), + `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5, 'NAMED_COLLECTION' = 6)), `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'addressToLine' = 136, 'addressToLineWithInlines' = 137, 'addressToSymbol' = 138, 'demangle' = 139, 'INTROSPECTION' = 140, 'FILE' = 141, 'URL' = 142, 'REMOTE' = 143, 'MONGO' = 144, 'MEILISEARCH' = 145, 'MYSQL' = 146, 'POSTGRES' = 147, 'SQLITE' = 148, 'ODBC' = 149, 'JDBC' = 150, 'HDFS' = 151, 'S3' = 152, 'HIVE' = 153, 'SOURCES' = 154, 'CLUSTER' = 155, 'ALL' = 156, 'NONE' = 157)) ) ENGINE = SystemPrivileges From 6f985b8ae0b8546dc0022862ae5ad913b83d6615 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 10 Feb 2023 18:49:18 +0100 Subject: [PATCH 04/69] Fix unit test, cleanup code --- src/Access/AccessRights.cpp | 12 ++++-------- src/Access/Common/AccessRightsElement.cpp | 6 +++--- src/Access/tests/gtest_access_rights_ops.cpp | 4 ++-- src/Parsers/Access/ASTGrantQuery.cpp | 7 +++++-- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index ca118203541..65363babb4f 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -326,8 +326,8 @@ public: const Node * child = tryGetChild(name); if (child) return child->isGranted(flags_to_check, subnames...); - - return flags.contains(flags_to_check); + else + return flags.contains(flags_to_check); } template @@ -836,10 +836,7 @@ void AccessRights::grant(const AccessFlags & flags, std::string_view database, s void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) { grantImpl(flags, database, table, columns); } void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl(flags, database, table, columns); } void AccessRights::grant(const AccessRightsElement & element) { grantImpl(element); } -void AccessRights::grant(const AccessRightsElements & elements) -{ - grantImpl(elements); -} +void AccessRights::grant(const AccessRightsElements & elements) { grantImpl(elements); } void AccessRights::grantWithGrantOption(const AccessFlags & flags) { grantImpl(flags); } void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database) { grantImpl(flags, database); } @@ -928,7 +925,7 @@ void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { re AccessRightsElements AccessRights::getElements() const { -#if 1 +#if 0 logTree(); #endif if (!root) @@ -950,7 +947,6 @@ bool AccessRights::isGrantedImpl(const AccessFlags & flags, const Args &... args { auto helper = [&](const std::unique_ptr & root_node) -> bool { - logTree(); if (!root_node) return flags.isEmpty(); return root_node->isGranted(flags, args...); diff --git a/src/Access/Common/AccessRightsElement.cpp b/src/Access/Common/AccessRightsElement.cpp index 011db851bc4..70a6b3bea57 100644 --- a/src/Access/Common/AccessRightsElement.cpp +++ b/src/Access/Common/AccessRightsElement.cpp @@ -26,10 +26,10 @@ namespace result += "ON "; if (element.isNamedCollectionAccess()) { - if (!element.any_named_collection) - result += backQuoteIfNeed(element.named_collection); - else + if (element.any_named_collection) result += "*"; + else + result += backQuoteIfNeed(element.named_collection); } else if (element.any_database) { diff --git a/src/Access/tests/gtest_access_rights_ops.cpp b/src/Access/tests/gtest_access_rights_ops.cpp index e21ebda2a31..d6f827a02c5 100644 --- a/src/Access/tests/gtest_access_rights_ops.cpp +++ b/src/Access/tests/gtest_access_rights_ops.cpp @@ -48,9 +48,9 @@ TEST(AccessRights, Union) ASSERT_EQ(lhs.toString(), "GRANT INSERT ON *.*, " "GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, " - "CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, " + "CREATE DICTIONARY, CREATE NAMED COLLECTION, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, DROP NAMED COLLECTION, " "TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, " - "SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, " + "SHOW ROW POLICIES, SHOW NAMED COLLECTIONS, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, " "SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, " "SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, " "SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*"); diff --git a/src/Parsers/Access/ASTGrantQuery.cpp b/src/Parsers/Access/ASTGrantQuery.cpp index e4aa11967c6..8d66ac991be 100644 --- a/src/Parsers/Access/ASTGrantQuery.cpp +++ b/src/Parsers/Access/ASTGrantQuery.cpp @@ -30,9 +30,12 @@ namespace void formatONClause(const AccessRightsElement & element, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ON " << (settings.hilite ? IAST::hilite_none : ""); - if (!element.any_named_collection) + if (element.isNamedCollectionAccess()) { - settings.ostr << backQuoteIfNeed(element.named_collection); + if (element.any_named_collection) + settings.ostr << "*"; + else + settings.ostr << backQuoteIfNeed(element.named_collection); } else if (element.any_database) { From 78c809608cb7b0b8b81bc339062d6535af192880 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 10 Feb 2023 23:37:34 +0100 Subject: [PATCH 05/69] Fix --- src/Access/Common/AccessFlags.h | 3 +-- src/Parsers/Access/ParserGrantQuery.cpp | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Access/Common/AccessFlags.h b/src/Access/Common/AccessFlags.h index f475896b6e5..68ff28516e5 100644 --- a/src/Access/Common/AccessFlags.h +++ b/src/Access/Common/AccessFlags.h @@ -50,8 +50,7 @@ public: bool isEmpty() const { return flags.none(); } explicit operator bool() const { return !isEmpty(); } bool contains(const AccessFlags & other) const { return (flags & other.flags) == other.flags; } - bool containsOnly(const AccessFlags & other) const { return flags == other.flags; } - bool isNamedCollectionAccess() const { return containsOnly(AccessFlags::allFlagsGrantableOnNamedCollectionLevel()); } + bool isNamedCollectionAccess() const { return AccessFlags::allFlagsGrantableOnNamedCollectionLevel().contains(*this); } friend bool operator ==(const AccessFlags & left, const AccessFlags & right) { return left.flags == right.flags; } friend bool operator !=(const AccessFlags & left, const AccessFlags & right) { return !(left == right); } diff --git a/src/Parsers/Access/ParserGrantQuery.cpp b/src/Parsers/Access/ParserGrantQuery.cpp index 06660642fbf..efc2e1c3ea5 100644 --- a/src/Parsers/Access/ParserGrantQuery.cpp +++ b/src/Parsers/Access/ParserGrantQuery.cpp @@ -155,7 +155,9 @@ namespace any_database = any_table = true; } else if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, database_name, any_database, table_name, any_table)) + { return false; + } for (auto & [access_flags, columns] : access_and_columns) { From c79f252a568b12989eb0ab5f17caee949324ba5e Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 11 Feb 2023 12:14:37 +0100 Subject: [PATCH 06/69] Fix test --- src/Access/Common/AccessFlags.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Access/Common/AccessFlags.h b/src/Access/Common/AccessFlags.h index 68ff28516e5..f43e54f3f33 100644 --- a/src/Access/Common/AccessFlags.h +++ b/src/Access/Common/AccessFlags.h @@ -50,7 +50,7 @@ public: bool isEmpty() const { return flags.none(); } explicit operator bool() const { return !isEmpty(); } bool contains(const AccessFlags & other) const { return (flags & other.flags) == other.flags; } - bool isNamedCollectionAccess() const { return AccessFlags::allFlagsGrantableOnNamedCollectionLevel().contains(*this); } + bool isNamedCollectionAccess() const { return !isEmpty() && AccessFlags::allFlagsGrantableOnNamedCollectionLevel().contains(*this); } friend bool operator ==(const AccessFlags & left, const AccessFlags & right) { return left.flags == right.flags; } friend bool operator !=(const AccessFlags & left, const AccessFlags & right) { return !(left == right); } From 7e0a98cbf190e4bfd0d695285ae14db509e9e5d4 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 11 Feb 2023 12:36:25 +0100 Subject: [PATCH 07/69] Add test --- .../test_named_collections/test.py | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 0f8999f43dd..d2d6455caec 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -105,7 +105,7 @@ def test_access(cluster): assert int(node.query("select count() from system.named_collections")) > 0 -def test_granular_access(cluster): +def test_granular_access_show_query(cluster): node = cluster.instances["node"] assert 1 == int(node.query("SELECT count() FROM system.named_collections")) assert ( @@ -175,6 +175,56 @@ def test_granular_access(cluster): node.query("DROP NAMED COLLECTION collection2") +def test_granular_access_create_alter_drop_query(cluster): + node = cluster.instances["node"] + node.query("CREATE USER kek") + node.query("GRANT select ON *.* TO kek") + assert 0 == int( + node.query("SELECT count() FROM system.named_collections", user="kek") + ) + + assert ( + "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant CREATE NAMED COLLECTION" + in node.query_and_get_error( + "CREATE NAMED COLLECTION collection2 AS key1=1, key2='value2'", user="kek" + ) + ) + node.query("GRANT create named collection ON collection2 TO kek") + node.query_and_get_error( + "CREATE NAMED COLLECTION collection2 AS key1=1, key2='value2'", user="kek" + ) + assert 0 == int( + node.query("select count() from system.named_collections", user="kek") + ) + + node.query("GRANT show named collections ON collection2 TO kek") + # assert ( + # "collection2" + # == node.query("select name from system.named_collections", user="kek").strip() + # ) + # assert ( + # "1" + # == node.query( + # "select collection['key1'] from system.named_collections where name = 'collection2'" + # ).strip() + # ) + + # assert ( + # "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant ALTER NAMED COLLECTION" + # in node.query_and_get_error( + # "ALTER NAMED COLLECTION collection2 SET key1=2", user="kek" + # ) + # ) + # node.query("GRANT alter named collection ON collection2 TO kek") + # node.query("ALTER NAMED COLLECTION collection2 SET key1=2", user="kek") + # assert ( + # "2" + # == node.query( + # "select collection['key1'] from system.named_collections where name = 'collection2'" + # ).strip() + # ) + + def test_config_reload(cluster): node = cluster.instances["node"] assert ( From 7beb84365019617bfeeab5487a3e8dadfa647ca7 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 11 Feb 2023 19:57:41 +0100 Subject: [PATCH 08/69] Fix --- .../InterpreterAlterNamedCollectionQuery.cpp | 5 +- .../InterpreterCreateNamedCollectionQuery.cpp | 4 +- .../InterpreterDropNamedCollectionQuery.cpp | 5 +- .../test_named_collections/test.py | 82 ++++++++++--------- 4 files changed, 52 insertions(+), 44 deletions(-) diff --git a/src/Interpreters/InterpreterAlterNamedCollectionQuery.cpp b/src/Interpreters/InterpreterAlterNamedCollectionQuery.cpp index 040a8714983..478735c432b 100644 --- a/src/Interpreters/InterpreterAlterNamedCollectionQuery.cpp +++ b/src/Interpreters/InterpreterAlterNamedCollectionQuery.cpp @@ -12,9 +12,10 @@ namespace DB BlockIO InterpreterAlterNamedCollectionQuery::execute() { auto current_context = getContext(); - current_context->checkAccess(AccessType::ALTER_NAMED_COLLECTION); - const auto & query = query_ptr->as(); + + current_context->checkAccess(AccessType::ALTER_NAMED_COLLECTION, query.collection_name); + if (!query.cluster.empty()) { DDLQueryOnClusterParams params; diff --git a/src/Interpreters/InterpreterCreateNamedCollectionQuery.cpp b/src/Interpreters/InterpreterCreateNamedCollectionQuery.cpp index 8a1a8d9dde6..bac59998062 100644 --- a/src/Interpreters/InterpreterCreateNamedCollectionQuery.cpp +++ b/src/Interpreters/InterpreterCreateNamedCollectionQuery.cpp @@ -13,10 +13,10 @@ namespace DB BlockIO InterpreterCreateNamedCollectionQuery::execute() { auto current_context = getContext(); - current_context->checkAccess(AccessType::CREATE_NAMED_COLLECTION); - const auto & query = query_ptr->as(); + current_context->checkAccess(AccessType::CREATE_NAMED_COLLECTION, query.collection_name); + if (!query.cluster.empty()) { DDLQueryOnClusterParams params; diff --git a/src/Interpreters/InterpreterDropNamedCollectionQuery.cpp b/src/Interpreters/InterpreterDropNamedCollectionQuery.cpp index 064a13012a6..cc3444bb4df 100644 --- a/src/Interpreters/InterpreterDropNamedCollectionQuery.cpp +++ b/src/Interpreters/InterpreterDropNamedCollectionQuery.cpp @@ -12,9 +12,10 @@ namespace DB BlockIO InterpreterDropNamedCollectionQuery::execute() { auto current_context = getContext(); - current_context->checkAccess(AccessType::DROP_NAMED_COLLECTION); - const auto & query = query_ptr->as(); + + current_context->checkAccess(AccessType::DROP_NAMED_COLLECTION, query.collection_name); + if (!query.cluster.empty()) { DDLQueryOnClusterParams params; diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index d2d6455caec..ba403d3f48b 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -70,20 +70,15 @@ def replace_in_users_config(node, old, new): ) -def test_access(cluster): +def test_default_access(cluster): node = cluster.instances["node_no_default_access"] - assert ( - "DB::Exception: default: Not enough privileges. To execute this query it's necessary to have grant SHOW NAMED COLLECTIONS ON *.*" - in node.query_and_get_error("select count() from system.named_collections") - ) + assert 0 == int(node.query("select count() from system.named_collections")) node = cluster.instances["node_no_default_access_but_with_access_management"] - assert ( - "DB::Exception: default: Not enough privileges. To execute this query it's necessary to have grant SHOW NAMED COLLECTIONS ON *.*" - in node.query_and_get_error("select count() from system.named_collections") - ) + assert 0 == int(node.query("select count() from system.named_collections")) node = cluster.instances["node"] assert int(node.query("select count() from system.named_collections")) > 0 + replace_in_users_config( node, "show_named_collections>1", "show_named_collections>0" ) @@ -91,10 +86,8 @@ def test_access(cluster): ["bash", "-c", f"cat /etc/clickhouse-server/users.d/users.xml"] ) node.restart_clickhouse() - assert ( - "DB::Exception: default: Not enough privileges. To execute this query it's necessary to have grant SHOW NAMED COLLECTIONS ON *.*" - in node.query_and_get_error("select count() from system.named_collections") - ) + assert 0 == int(node.query("select count() from system.named_collections")) + replace_in_users_config( node, "show_named_collections>0", "show_named_collections>1" ) @@ -112,6 +105,7 @@ def test_granular_access_show_query(cluster): "collection1" == node.query("SELECT name FROM system.named_collections").strip() ) + node.query("DROP USER IF EXISTS kek") node.query("CREATE USER kek") node.query("GRANT select ON *.* TO kek") assert 0 == int( @@ -156,6 +150,7 @@ def test_granular_access_show_query(cluster): == node.query("select name from system.named_collections", user="kek").strip() ) + node.query("DROP USER IF EXISTS koko") node.query("CREATE USER koko") node.query("GRANT select ON *.* TO koko") assert 0 == int( @@ -177,6 +172,7 @@ def test_granular_access_show_query(cluster): def test_granular_access_create_alter_drop_query(cluster): node = cluster.instances["node"] + node.query("DROP USER IF EXISTS kek") node.query("CREATE USER kek") node.query("GRANT select ON *.* TO kek") assert 0 == int( @@ -190,7 +186,7 @@ def test_granular_access_create_alter_drop_query(cluster): ) ) node.query("GRANT create named collection ON collection2 TO kek") - node.query_and_get_error( + node.query( "CREATE NAMED COLLECTION collection2 AS key1=1, key2='value2'", user="kek" ) assert 0 == int( @@ -198,31 +194,41 @@ def test_granular_access_create_alter_drop_query(cluster): ) node.query("GRANT show named collections ON collection2 TO kek") - # assert ( - # "collection2" - # == node.query("select name from system.named_collections", user="kek").strip() - # ) - # assert ( - # "1" - # == node.query( - # "select collection['key1'] from system.named_collections where name = 'collection2'" - # ).strip() - # ) + assert ( + "collection2" + == node.query("select name from system.named_collections", user="kek").strip() + ) + assert ( + "1" + == node.query( + "select collection['key1'] from system.named_collections where name = 'collection2'" + ).strip() + ) - # assert ( - # "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant ALTER NAMED COLLECTION" - # in node.query_and_get_error( - # "ALTER NAMED COLLECTION collection2 SET key1=2", user="kek" - # ) - # ) - # node.query("GRANT alter named collection ON collection2 TO kek") - # node.query("ALTER NAMED COLLECTION collection2 SET key1=2", user="kek") - # assert ( - # "2" - # == node.query( - # "select collection['key1'] from system.named_collections where name = 'collection2'" - # ).strip() - # ) + assert ( + "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant ALTER NAMED COLLECTION" + in node.query_and_get_error( + "ALTER NAMED COLLECTION collection2 SET key1=2", user="kek" + ) + ) + node.query("GRANT alter named collection ON collection2 TO kek") + node.query("ALTER NAMED COLLECTION collection2 SET key1=2", user="kek") + assert ( + "2" + == node.query( + "select collection['key1'] from system.named_collections where name = 'collection2'" + ).strip() + ) + + assert ( + "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant DROP NAMED COLLECTION" + in node.query_and_get_error("DROP NAMED COLLECTION collection2", user="kek") + ) + node.query("GRANT drop named collection ON collection2 TO kek") + node.query("DROP NAMED COLLECTION collection2", user="kek") + assert 0 == int( + node.query("select count() from system.named_collections", user="kek") + ) def test_config_reload(cluster): From 33572f9acd62e2d94109ae1c309e3aa8796d6fdc Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 14 Feb 2023 19:39:21 +0100 Subject: [PATCH 09/69] Better --- src/Access/Common/AccessRightsElement.h | 6 ++++-- src/Parsers/Access/ASTGrantQuery.cpp | 4 +--- src/Parsers/Access/ParserGrantQuery.cpp | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Access/Common/AccessRightsElement.h b/src/Access/Common/AccessRightsElement.h index db6a1f6872f..9e972fbc43c 100644 --- a/src/Access/Common/AccessRightsElement.h +++ b/src/Access/Common/AccessRightsElement.h @@ -52,8 +52,10 @@ struct AccessRightsElement bool sameDatabaseAndTable(const AccessRightsElement & other) const { - return (database == other.database) && (any_database == other.any_database) && (table == other.table) - && (any_table == other.any_table); + return (database == other.database) && (any_database == other.any_database) + && (table == other.table) && (any_table == other.any_table) + && (named_collection == other.named_collection) && (any_named_collection == other.any_named_collection) + && (isNamedCollectionAccess() == other.isNamedCollectionAccess()); } bool sameOptions(const AccessRightsElement & other) const diff --git a/src/Parsers/Access/ASTGrantQuery.cpp b/src/Parsers/Access/ASTGrantQuery.cpp index 8d66ac991be..d29ae3b8d3f 100644 --- a/src/Parsers/Access/ASTGrantQuery.cpp +++ b/src/Parsers/Access/ASTGrantQuery.cpp @@ -77,9 +77,7 @@ namespace if (i != elements.size() - 1) { const auto & next_element = elements[i + 1]; - if ((element.database == next_element.database) && (element.any_database == next_element.any_database) - && (element.table == next_element.table) && (element.any_table == next_element.any_table) - && (element.named_collection == next_element.named_collection)) + if (element.sameDatabaseAndTable(next_element)) next_element_on_same_db_and_table = true; } diff --git a/src/Parsers/Access/ParserGrantQuery.cpp b/src/Parsers/Access/ParserGrantQuery.cpp index efc2e1c3ea5..5d688c82187 100644 --- a/src/Parsers/Access/ParserGrantQuery.cpp +++ b/src/Parsers/Access/ParserGrantQuery.cpp @@ -136,8 +136,7 @@ namespace if (!ParserKeyword{"ON"}.ignore(pos, expected)) return false; - const bool grant_named_collection_access = named_collection_access && named_collection_access == access_and_columns.size(); - if (grant_named_collection_access) + if (named_collection_access && named_collection_access == access_and_columns.size()) { ASTPtr collection; if (ParserToken{TokenType::Asterisk}.ignore(pos, expected)) From 0c53f33e6f1deb1c708dffe9364d8bba0acf3774 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 15 Feb 2023 16:59:43 +0100 Subject: [PATCH 10/69] Make code more common: add GlobalWithParameter level --- src/Access/AccessRights.cpp | 20 +++++++-------- src/Access/Common/AccessFlags.cpp | 30 +++++++++++++++++++++-- src/Access/Common/AccessFlags.h | 19 +++++++++++--- src/Access/Common/AccessRightsElement.cpp | 8 +++--- src/Access/Common/AccessRightsElement.h | 14 +++++++---- src/Access/ContextAccess.cpp | 22 ++++++++++------- src/Common/ErrorCodes.cpp | 1 + src/Parsers/Access/ASTGrantQuery.cpp | 6 ++--- src/Parsers/Access/ParserGrantQuery.cpp | 28 +++++++++++---------- 9 files changed, 100 insertions(+), 48 deletions(-) diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 65363babb4f..a200111a311 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -61,15 +61,15 @@ namespace res.any_database = true; res.any_table = true; res.any_column = true; - res.any_named_collection = true; + res.any_global_with_parameter = true; break; } case 1: { - if (access_flags.isNamedCollectionAccess()) + if (access_flags.isGlobalWithParameter()) { - res.any_named_collection = false; - res.named_collection = full_name[0]; + res.any_global_with_parameter = false; + res.parameter = full_name[0]; } else { @@ -792,8 +792,8 @@ void AccessRights::grantImplHelper(const AccessRightsElement & element) { assert(!element.is_partial_revoke); assert(!element.grant_option || with_grant_option); - if (!element.any_named_collection) - grantImpl(element.access_flags, element.named_collection); + if (!element.any_global_with_parameter) + grantImpl(element.access_flags, element.parameter); else if (element.any_database) grantImpl(element.access_flags); else if (element.any_table) @@ -869,8 +869,8 @@ template void AccessRights::revokeImplHelper(const AccessRightsElement & element) { assert(!element.grant_option || grant_option); - if (!element.any_named_collection) - revokeImpl(element.access_flags, element.named_collection); + if (!element.any_global_with_parameter) + revokeImpl(element.access_flags, element.parameter); else if (element.any_database) revokeImpl(element.access_flags); else if (element.any_table) @@ -961,8 +961,8 @@ template bool AccessRights::isGrantedImplHelper(const AccessRightsElement & element) const { assert(!element.grant_option || grant_option); - if (!element.any_named_collection) - return isGrantedImpl(element.access_flags, element.named_collection); + if (!element.any_global_with_parameter) + return isGrantedImpl(element.access_flags, element.parameter); else if (element.any_database) return isGrantedImpl(element.access_flags); else if (element.any_table) diff --git a/src/Access/Common/AccessFlags.cpp b/src/Access/Common/AccessFlags.cpp index c8d2b1ca4d9..a018e7d3acb 100644 --- a/src/Access/Common/AccessFlags.cpp +++ b/src/Access/Common/AccessFlags.cpp @@ -15,6 +15,7 @@ namespace ErrorCodes { extern const int UNKNOWN_ACCESS_TYPE; extern const int LOGICAL_ERROR; + extern const int MIXED_ACCESS_PARAMETER_TYPES; } namespace @@ -96,12 +97,14 @@ namespace const Flags & getAllFlags() const { return all_flags; } const Flags & getGlobalFlags() const { return all_flags_for_target[GLOBAL]; } + const Flags & getGlobalWithParameterFlags() const { return all_flags_grantable_on_global_with_parameter_level; } const Flags & getDatabaseFlags() const { return all_flags_for_target[DATABASE]; } const Flags & getTableFlags() const { return all_flags_for_target[TABLE]; } const Flags & getColumnFlags() const { return all_flags_for_target[COLUMN]; } const Flags & getDictionaryFlags() const { return all_flags_for_target[DICTIONARY]; } + const Flags & getNamedCollectionFlags() const { return all_flags_for_target[NAMED_COLLECTION]; } const Flags & getAllFlagsGrantableOnGlobalLevel() const { return getAllFlags(); } - const Flags & getAllFlagsGrantableOnNamedCollectionLevel() const { return all_flags_for_target[NAMED_COLLECTION]; } + const Flags & getAllFlagsGrantableOnGlobalWithParameterLevel() const { return getGlobalWithParameterFlags(); } const Flags & getAllFlagsGrantableOnDatabaseLevel() const { return all_flags_grantable_on_database_level; } const Flags & getAllFlagsGrantableOnTableLevel() const { return all_flags_grantable_on_table_level; } const Flags & getAllFlagsGrantableOnColumnLevel() const { return getColumnFlags(); } @@ -297,6 +300,7 @@ namespace collectAllFlags(child.get()); all_flags_grantable_on_table_level = all_flags_for_target[TABLE] | all_flags_for_target[DICTIONARY] | all_flags_for_target[COLUMN]; + all_flags_grantable_on_global_with_parameter_level = all_flags_for_target[NAMED_COLLECTION]; all_flags_grantable_on_database_level = all_flags_for_target[DATABASE] | all_flags_for_target[NAMED_COLLECTION] | all_flags_grantable_on_table_level; } @@ -350,9 +354,29 @@ namespace Flags all_flags_for_target[static_cast(NAMED_COLLECTION) + 1]; Flags all_flags_grantable_on_database_level; Flags all_flags_grantable_on_table_level; + Flags all_flags_grantable_on_global_with_parameter_level; }; } +bool AccessFlags::isGlobalWithParameter() const +{ + return getParameterType() != AccessFlags::NONE; +} + +AccessFlags::ParameterType AccessFlags::getParameterType() const +{ + if (isEmpty() || contains(AccessFlags::allGlobalFlags())) + return AccessFlags::NONE; + + /// All flags refer to NAMED COLLECTION access type. + if (AccessFlags::allNamedCollectionFlags().contains(*this)) + return AccessFlags::NAMED_COLLECTION; + + if (!contains(AccessFlags::allGlobalWithParameterFlags())) + return AccessFlags::NONE; + + throw Exception(ErrorCodes::MIXED_ACCESS_PARAMETER_TYPES, "Having mixed parameter types: {}", toString()); +} AccessFlags::AccessFlags(AccessType type) : flags(Helper::instance().accessTypeToFlags(type)) {} AccessFlags::AccessFlags(std::string_view keyword) : flags(Helper::instance().keywordToFlags(keyword)) {} @@ -363,12 +387,14 @@ std::vector AccessFlags::toAccessTypes() const { return Helper::inst std::vector AccessFlags::toKeywords() const { return Helper::instance().flagsToKeywords(flags); } AccessFlags AccessFlags::allFlags() { return Helper::instance().getAllFlags(); } AccessFlags AccessFlags::allGlobalFlags() { return Helper::instance().getGlobalFlags(); } +AccessFlags AccessFlags::allGlobalWithParameterFlags() { return Helper::instance().getGlobalWithParameterFlags(); } AccessFlags AccessFlags::allDatabaseFlags() { return Helper::instance().getDatabaseFlags(); } AccessFlags AccessFlags::allTableFlags() { return Helper::instance().getTableFlags(); } AccessFlags AccessFlags::allColumnFlags() { return Helper::instance().getColumnFlags(); } AccessFlags AccessFlags::allDictionaryFlags() { return Helper::instance().getDictionaryFlags(); } +AccessFlags AccessFlags::allNamedCollectionFlags() { return Helper::instance().getNamedCollectionFlags(); } AccessFlags AccessFlags::allFlagsGrantableOnGlobalLevel() { return Helper::instance().getAllFlagsGrantableOnGlobalLevel(); } -AccessFlags AccessFlags::allFlagsGrantableOnNamedCollectionLevel() { return Helper::instance().getAllFlagsGrantableOnNamedCollectionLevel(); } +AccessFlags AccessFlags::allFlagsGrantableOnGlobalWithParameterLevel() { return Helper::instance().getAllFlagsGrantableOnGlobalWithParameterLevel(); } AccessFlags AccessFlags::allFlagsGrantableOnDatabaseLevel() { return Helper::instance().getAllFlagsGrantableOnDatabaseLevel(); } AccessFlags AccessFlags::allFlagsGrantableOnTableLevel() { return Helper::instance().getAllFlagsGrantableOnTableLevel(); } AccessFlags AccessFlags::allFlagsGrantableOnColumnLevel() { return Helper::instance().getAllFlagsGrantableOnColumnLevel(); } diff --git a/src/Access/Common/AccessFlags.h b/src/Access/Common/AccessFlags.h index f43e54f3f33..b923b24be47 100644 --- a/src/Access/Common/AccessFlags.h +++ b/src/Access/Common/AccessFlags.h @@ -48,9 +48,16 @@ public: AccessFlags operator ~() const { AccessFlags res; res.flags = ~flags; return res; } bool isEmpty() const { return flags.none(); } + bool isAll() const { return flags.all(); } explicit operator bool() const { return !isEmpty(); } bool contains(const AccessFlags & other) const { return (flags & other.flags) == other.flags; } - bool isNamedCollectionAccess() const { return !isEmpty() && AccessFlags::allFlagsGrantableOnNamedCollectionLevel().contains(*this); } + bool isGlobalWithParameter() const; + enum ParameterType + { + NONE, + NAMED_COLLECTION, + }; + ParameterType getParameterType() const; friend bool operator ==(const AccessFlags & left, const AccessFlags & right) { return left.flags == right.flags; } friend bool operator !=(const AccessFlags & left, const AccessFlags & right) { return !(left == right); } @@ -77,6 +84,8 @@ public: /// Returns all the global flags. static AccessFlags allGlobalFlags(); + static AccessFlags allGlobalWithParameterFlags(); + /// Returns all the flags related to a database. static AccessFlags allDatabaseFlags(); @@ -89,10 +98,16 @@ public: /// Returns all the flags related to a dictionary. static AccessFlags allDictionaryFlags(); + /// Returns all the flags related to a named collection. + static AccessFlags allNamedCollectionFlags(); + /// Returns all the flags which could be granted on the global level. /// The same as allFlags(). static AccessFlags allFlagsGrantableOnGlobalLevel(); + /// Returns all the flags which could be granted on the global with parameter level. + static AccessFlags allFlagsGrantableOnGlobalWithParameterLevel(); + /// Returns all the flags which could be granted on the database level. /// Returns allDatabaseFlags() | allTableFlags() | allDictionaryFlags() | allColumnFlags(). static AccessFlags allFlagsGrantableOnDatabaseLevel(); @@ -105,8 +120,6 @@ public: /// The same as allColumnFlags(). static AccessFlags allFlagsGrantableOnColumnLevel(); - static AccessFlags allFlagsGrantableOnNamedCollectionLevel(); - static constexpr size_t SIZE = 256; private: using Flags = std::bitset; diff --git a/src/Access/Common/AccessRightsElement.cpp b/src/Access/Common/AccessRightsElement.cpp index 70a6b3bea57..2f6f1264a65 100644 --- a/src/Access/Common/AccessRightsElement.cpp +++ b/src/Access/Common/AccessRightsElement.cpp @@ -24,12 +24,12 @@ namespace void formatONClause(const AccessRightsElement & element, String & result) { result += "ON "; - if (element.isNamedCollectionAccess()) + if (element.isGlobalWithParameter()) { - if (element.any_named_collection) + if (element.any_global_with_parameter) result += "*"; else - result += backQuoteIfNeed(element.named_collection); + result += backQuoteIfNeed(element.parameter); } else if (element.any_database) { @@ -206,6 +206,8 @@ void AccessRightsElement::eraseNonGrantable() access_flags &= AccessFlags::allFlagsGrantableOnTableLevel(); else if (!any_database) access_flags &= AccessFlags::allFlagsGrantableOnDatabaseLevel(); + else if (!any_global_with_parameter) + access_flags &= AccessFlags::allFlagsGrantableOnGlobalWithParameterLevel(); else access_flags &= AccessFlags::allFlagsGrantableOnGlobalLevel(); } diff --git a/src/Access/Common/AccessRightsElement.h b/src/Access/Common/AccessRightsElement.h index 9e972fbc43c..e881767b185 100644 --- a/src/Access/Common/AccessRightsElement.h +++ b/src/Access/Common/AccessRightsElement.h @@ -11,14 +11,17 @@ namespace DB struct AccessRightsElement { AccessFlags access_flags; + String database; String table; Strings columns; - String named_collection; + String parameter; + bool any_database = true; bool any_table = true; bool any_column = true; - bool any_named_collection = true; + bool any_global_with_parameter = true; + bool grant_option = false; bool is_partial_revoke = false; @@ -54,8 +57,9 @@ struct AccessRightsElement { return (database == other.database) && (any_database == other.any_database) && (table == other.table) && (any_table == other.any_table) - && (named_collection == other.named_collection) && (any_named_collection == other.any_named_collection) - && (isNamedCollectionAccess() == other.isNamedCollectionAccess()); + && (parameter == other.parameter) && (any_global_with_parameter == other.any_global_with_parameter) + && (access_flags.getParameterType() == other.access_flags.getParameterType()) + && (isGlobalWithParameter() == other.isGlobalWithParameter()); } bool sameOptions(const AccessRightsElement & other) const @@ -71,7 +75,7 @@ struct AccessRightsElement /// If the database is empty, replaces it with `current_database`. Otherwise does nothing. void replaceEmptyDatabase(const String & current_database); - bool isNamedCollectionAccess() const { return access_flags.isNamedCollectionAccess(); } + bool isGlobalWithParameter() const { return access_flags.isGlobalWithParameter(); } /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table". String toString() const; diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 6d6362a98b2..4abb161fd80 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -502,13 +502,17 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg if (!flags) return true; - /// Access to temporary tables is controlled in an unusual way, not like normal tables. - /// Creating of temporary tables is controlled by AccessType::CREATE_TEMPORARY_TABLES grant, - /// and other grants are considered as always given. - /// The DatabaseCatalog class won't resolve StorageID for temporary tables - /// which shouldn't be accessed. - if (getDatabase(args...) == DatabaseCatalog::TEMPORARY_DATABASE) - return access_granted(); + const auto parameter_type = flags.getParameterType(); + if (parameter_type == AccessFlags::NONE) + { + /// Access to temporary tables is controlled in an unusual way, not like normal tables. + /// Creating of temporary tables is controlled by AccessType::CREATE_TEMPORARY_TABLES grant, + /// and other grants are considered as always given. + /// The DatabaseCatalog class won't resolve StorageID for temporary tables + /// which shouldn't be accessed. + if (getDatabase(args...) == DatabaseCatalog::TEMPORARY_DATABASE) + return access_granted(); + } auto acs = getAccessRightsWithImplicit(); bool granted; @@ -606,8 +610,8 @@ template bool ContextAccess::checkAccessImplHelper(const AccessRightsElement & element) const { assert(!element.grant_option || grant_option); - if (!element.any_named_collection) - return checkAccessImpl(element.access_flags, element.named_collection); + if (!element.any_global_with_parameter) + return checkAccessImpl(element.access_flags, element.parameter); else if (element.any_database) return checkAccessImpl(element.access_flags); else if (element.any_table) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 028663a2176..cffe5fd357c 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -648,6 +648,7 @@ M(677, THREAD_WAS_CANCELED) \ M(678, IO_URING_INIT_FAILED) \ M(679, IO_URING_SUBMIT_ERROR) \ + M(690, MIXED_ACCESS_PARAMETER_TYPES) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Parsers/Access/ASTGrantQuery.cpp b/src/Parsers/Access/ASTGrantQuery.cpp index d29ae3b8d3f..f1a1f9184a5 100644 --- a/src/Parsers/Access/ASTGrantQuery.cpp +++ b/src/Parsers/Access/ASTGrantQuery.cpp @@ -30,12 +30,12 @@ namespace void formatONClause(const AccessRightsElement & element, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ON " << (settings.hilite ? IAST::hilite_none : ""); - if (element.isNamedCollectionAccess()) + if (element.isGlobalWithParameter()) { - if (element.any_named_collection) + if (element.any_global_with_parameter) settings.ostr << "*"; else - settings.ostr << backQuoteIfNeed(element.named_collection); + settings.ostr << backQuoteIfNeed(element.parameter); } else if (element.any_database) { diff --git a/src/Parsers/Access/ParserGrantQuery.cpp b/src/Parsers/Access/ParserGrantQuery.cpp index 5d688c82187..d58599ead56 100644 --- a/src/Parsers/Access/ParserGrantQuery.cpp +++ b/src/Parsers/Access/ParserGrantQuery.cpp @@ -123,30 +123,30 @@ namespace if (!parseAccessFlagsWithColumns(pos, expected, access_and_columns)) return false; - String database_name, table_name, collection_name; - bool any_database = false, any_table = false, any_named_collection = true; + String database_name, table_name, parameter; + bool any_database = false, any_table = false, any_global_with_parameter = true; - size_t named_collection_access = 0; + size_t is_global_with_parameter = 0; for (const auto & elem : access_and_columns) { - if (elem.first.isNamedCollectionAccess()) - ++named_collection_access; + if (elem.first.isGlobalWithParameter()) + ++is_global_with_parameter; } if (!ParserKeyword{"ON"}.ignore(pos, expected)) return false; - if (named_collection_access && named_collection_access == access_and_columns.size()) + if (is_global_with_parameter && is_global_with_parameter == access_and_columns.size()) { - ASTPtr collection; + ASTPtr parameter_ast; if (ParserToken{TokenType::Asterisk}.ignore(pos, expected)) { - any_named_collection = true; + any_global_with_parameter = true; } - else if (ParserIdentifier{}.parse(pos, collection, expected)) + else if (ParserIdentifier{}.parse(pos, parameter_ast, expected)) { - any_named_collection = false; - collection_name = getIdentifierName(collection); + any_global_with_parameter = false; + parameter = getIdentifierName(parameter_ast); } else return false; @@ -167,9 +167,9 @@ namespace element.any_database = any_database; element.database = database_name; element.any_table = any_table; - element.any_named_collection = any_named_collection; - element.named_collection = collection_name; + element.any_global_with_parameter = any_global_with_parameter; element.table = table_name; + element.parameter = parameter; res_elements.emplace_back(std::move(element)); } @@ -202,6 +202,8 @@ namespace throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted on the table level", old_flags.toString()); else if (!element.any_database) throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted on the database level", old_flags.toString()); + else if (!element.any_global_with_parameter) + throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted on the global with parameter level", old_flags.toString()); else throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted", old_flags.toString()); }); From c6b371ddeaa7ff69d1372d838d46be5f126a3632 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 15 Feb 2023 18:32:00 +0100 Subject: [PATCH 11/69] Better --- src/Access/AccessRights.cpp | 2 +- src/Access/Common/AccessFlags.cpp | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index a200111a311..6096612059a 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -214,7 +214,7 @@ namespace switch (level) { case GLOBAL_LEVEL: return AccessFlags::allFlagsGrantableOnGlobalLevel(); - case DATABASE_LEVEL: return AccessFlags::allFlagsGrantableOnDatabaseLevel(); + case DATABASE_LEVEL: return AccessFlags::allFlagsGrantableOnDatabaseLevel() | AccessFlags::allFlagsGrantableOnGlobalWithParameterLevel(); case TABLE_LEVEL: return AccessFlags::allFlagsGrantableOnTableLevel(); case COLUMN_LEVEL: return AccessFlags::allFlagsGrantableOnColumnLevel(); } diff --git a/src/Access/Common/AccessFlags.cpp b/src/Access/Common/AccessFlags.cpp index a018e7d3acb..4f8a9798ec4 100644 --- a/src/Access/Common/AccessFlags.cpp +++ b/src/Access/Common/AccessFlags.cpp @@ -301,7 +301,7 @@ namespace all_flags_grantable_on_table_level = all_flags_for_target[TABLE] | all_flags_for_target[DICTIONARY] | all_flags_for_target[COLUMN]; all_flags_grantable_on_global_with_parameter_level = all_flags_for_target[NAMED_COLLECTION]; - all_flags_grantable_on_database_level = all_flags_for_target[DATABASE] | all_flags_for_target[NAMED_COLLECTION] | all_flags_grantable_on_table_level; + all_flags_grantable_on_database_level = all_flags_for_target[DATABASE] | all_flags_grantable_on_table_level; } Helper() @@ -365,16 +365,13 @@ bool AccessFlags::isGlobalWithParameter() const AccessFlags::ParameterType AccessFlags::getParameterType() const { - if (isEmpty() || contains(AccessFlags::allGlobalFlags())) + if (isEmpty() || !AccessFlags::allGlobalWithParameterFlags().contains(*this)) return AccessFlags::NONE; /// All flags refer to NAMED COLLECTION access type. if (AccessFlags::allNamedCollectionFlags().contains(*this)) return AccessFlags::NAMED_COLLECTION; - if (!contains(AccessFlags::allGlobalWithParameterFlags())) - return AccessFlags::NONE; - throw Exception(ErrorCodes::MIXED_ACCESS_PARAMETER_TYPES, "Having mixed parameter types: {}", toString()); } From a54b0116704ca7a1c60e82d7944532680a1ddb6b Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 20 Feb 2023 21:37:38 +0100 Subject: [PATCH 12/69] Finish for mysql --- .../NamedCollections/NamedCollections.cpp | 11 ++++ .../NamedCollections/NamedCollections.h | 2 + src/Databases/DatabaseFactory.cpp | 46 ++----------- .../MySQL/ConnectionMySQLSettings.cpp | 65 ------------------- src/Databases/MySQL/ConnectionMySQLSettings.h | 38 ----------- src/Databases/MySQL/DatabaseMySQL.cpp | 6 +- src/Databases/MySQL/DatabaseMySQL.h | 6 +- src/Dictionaries/MySQLDictionarySource.cpp | 22 +++---- .../ExternalDataSourceConfiguration.cpp | 23 ------- .../ExternalDataSourceConfiguration.h | 6 -- src/Storages/MySQL/MySQLHelpers.cpp | 10 +-- src/Storages/MySQL/MySQLHelpers.h | 5 +- src/Storages/MySQL/MySQLSettings.cpp | 12 ++++ src/Storages/MySQL/MySQLSettings.h | 4 ++ src/Storages/NamedCollectionsHelpers.cpp | 25 +++++-- src/Storages/NamedCollectionsHelpers.h | 7 +- src/Storages/StorageMySQL.cpp | 63 ++++++++++++------ src/Storages/StorageMySQL.h | 24 ++++++- src/Storages/StoragePostgreSQL.cpp | 48 ++++++++------ src/Storages/StoragePostgreSQL.h | 4 +- src/Storages/StorageURL.h | 1 - src/TableFunctions/TableFunctionMySQL.cpp | 1 - src/TableFunctions/TableFunctionMySQL.h | 4 +- 23 files changed, 179 insertions(+), 254 deletions(-) delete mode 100644 src/Databases/MySQL/ConnectionMySQLSettings.cpp delete mode 100644 src/Databases/MySQL/ConnectionMySQLSettings.h diff --git a/src/Common/NamedCollections/NamedCollections.cpp b/src/Common/NamedCollections/NamedCollections.cpp index 5db46826b19..2f80392c9ab 100644 --- a/src/Common/NamedCollections/NamedCollections.cpp +++ b/src/Common/NamedCollections/NamedCollections.cpp @@ -200,6 +200,11 @@ public: return std::unique_ptr(new Impl(collection_config, keys)); } + bool has(const Key & key) const + { + return Configuration::hasConfigValue(*config, key); + } + template T get(const Key & key) const { return Configuration::getConfigValue(*config, key); @@ -341,6 +346,12 @@ MutableNamedCollectionPtr NamedCollection::create( new NamedCollection(std::move(impl), collection_name, source_id, is_mutable)); } +bool NamedCollection::has(const Key & key) const +{ + std::lock_guard lock(mutex); + return pimpl->has(key); +} + template T NamedCollection::get(const Key & key) const { std::lock_guard lock(mutex); diff --git a/src/Common/NamedCollections/NamedCollections.h b/src/Common/NamedCollections/NamedCollections.h index 5ff9404ed69..a5b4349aaa3 100644 --- a/src/Common/NamedCollections/NamedCollections.h +++ b/src/Common/NamedCollections/NamedCollections.h @@ -33,6 +33,8 @@ public: SourceId source_id_, bool is_mutable_); + bool has(const Key & key) const; + template T get(const Key & key) const; template T getOrDefault(const Key & key, const T & default_value) const; diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 51378b66b08..97ec0de9552 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -24,11 +23,11 @@ #if USE_MYSQL # include -# include # include # include # include # include +# include # include # include #endif @@ -183,21 +182,13 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String if (!engine->arguments) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine `{}` must have arguments", engine_name); - StorageMySQLConfiguration configuration; + StorageMySQL::Configuration configuration; ASTs & arguments = engine->arguments->children; - auto mysql_settings = std::make_unique(); + auto mysql_settings = std::make_unique(); - if (auto named_collection = getExternalDataSourceConfiguration(arguments, context, true, true, *mysql_settings)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(arguments)) { - auto [common_configuration, storage_specific_args, settings_changes] = named_collection.value(); - - configuration.set(common_configuration); - configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; - mysql_settings->applyChanges(settings_changes); - - if (!storage_specific_args.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "MySQL database require mysql_hostname, mysql_database_name, mysql_username, mysql_password arguments."); + configuration = StorageMySQL::processNamedCollectionResult(*named_collection, *mysql_settings, false); } else { @@ -326,19 +317,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { - validateNamedCollection( - *named_collection, - {"host", "port", "user", "password", "database"}, - {"schema", "on_conflict", "use_table_cache"}); - - configuration.host = named_collection->get("host"); - configuration.port = static_cast(named_collection->get("port")); - configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; - configuration.username = named_collection->get("user"); - configuration.password = named_collection->get("password"); - configuration.database = named_collection->get("database"); - configuration.schema = named_collection->getOrDefault("schema", ""); - configuration.on_conflict = named_collection->getOrDefault("on_conflict", ""); + configuration = StoragePostgreSQL::processNamedCollectionResult(*named_collection); use_table_cache = named_collection->getOrDefault("use_tables_cache", 0); } else @@ -401,18 +380,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { - validateNamedCollection( - *named_collection, - {"host", "port", "user", "password", "database"}, - {"schema"}); - - configuration.host = named_collection->get("host"); - configuration.port = static_cast(named_collection->get("port")); - configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; - configuration.username = named_collection->get("user"); - configuration.password = named_collection->get("password"); - configuration.database = named_collection->get("database"); - configuration.schema = named_collection->getOrDefault("schema", ""); + configuration = StoragePostgreSQL::processNamedCollectionResult(*named_collection); } else { diff --git a/src/Databases/MySQL/ConnectionMySQLSettings.cpp b/src/Databases/MySQL/ConnectionMySQLSettings.cpp deleted file mode 100644 index 50c4c7e0613..00000000000 --- a/src/Databases/MySQL/ConnectionMySQLSettings.cpp +++ /dev/null @@ -1,65 +0,0 @@ -#include - -#include -#include -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int UNKNOWN_SETTING; - extern const int BAD_ARGUMENTS; -} - -IMPLEMENT_SETTINGS_TRAITS(ConnectionMySQLSettingsTraits, LIST_OF_MYSQL_DATABASE_SETTINGS) - -void ConnectionMySQLSettings::loadFromQuery(ASTStorage & storage_def) -{ - if (storage_def.settings) - { - try - { - applyChanges(storage_def.settings->changes); - } - catch (Exception & e) - { - if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "{} for database {}", e.message(), storage_def.engine->name); - else - e.rethrow(); - } - } - else - { - auto settings_ast = std::make_shared(); - settings_ast->is_standalone = false; - storage_def.set(storage_def.settings, settings_ast); - } - - SettingsChanges & changes = storage_def.settings->changes; -#define ADD_IF_ABSENT(NAME) \ - if (std::find_if(changes.begin(), changes.end(), \ - [](const SettingChange & c) { return c.name == #NAME; }) \ - == changes.end()) \ - changes.push_back(SettingChange{#NAME, static_cast(NAME)}); - - APPLY_FOR_IMMUTABLE_CONNECTION_MYSQL_SETTINGS(ADD_IF_ABSENT) -#undef ADD_IF_ABSENT -} - -void ConnectionMySQLSettings::loadFromQueryContext(ContextPtr context) -{ - if (!context->hasQueryContext()) - return; - - const Settings & settings = context->getQueryContext()->getSettingsRef(); - - if (settings.mysql_datatypes_support_level.value != mysql_datatypes_support_level.value) - set("mysql_datatypes_support_level", settings.mysql_datatypes_support_level.toString()); -} - - -} diff --git a/src/Databases/MySQL/ConnectionMySQLSettings.h b/src/Databases/MySQL/ConnectionMySQLSettings.h deleted file mode 100644 index 34902cbe9be..00000000000 --- a/src/Databases/MySQL/ConnectionMySQLSettings.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace DB -{ - -class ASTStorage; - -#define LIST_OF_CONNECTION_MYSQL_SETTINGS(M, ALIAS) \ - M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \ - -/// Settings that should not change after the creation of a database. -#define APPLY_FOR_IMMUTABLE_CONNECTION_MYSQL_SETTINGS(M) \ - M(mysql_datatypes_support_level) - -#define LIST_OF_MYSQL_DATABASE_SETTINGS(M, ALIAS) \ - LIST_OF_CONNECTION_MYSQL_SETTINGS(M, ALIAS) \ - LIST_OF_MYSQL_SETTINGS(M, ALIAS) - -DECLARE_SETTINGS_TRAITS(ConnectionMySQLSettingsTraits, LIST_OF_MYSQL_DATABASE_SETTINGS) - - -/** Settings for the MySQL database engine. - * Could be loaded from a CREATE DATABASE query (SETTINGS clause) and Query settings. - */ -struct ConnectionMySQLSettings : public BaseSettings -{ - void loadFromQuery(ASTStorage & storage_def); - - void loadFromQueryContext(ContextPtr context); -}; - -} diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 85944319999..df1726611b4 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -53,7 +53,7 @@ DatabaseMySQL::DatabaseMySQL( const String & metadata_path_, const ASTStorage * database_engine_define_, const String & database_name_in_mysql_, - std::unique_ptr settings_, + std::unique_ptr settings_, mysqlxx::PoolWithFailover && pool, bool attach) : IDatabase(database_name_) @@ -61,7 +61,7 @@ DatabaseMySQL::DatabaseMySQL( , metadata_path(metadata_path_) , database_engine_define(database_engine_define_->clone()) , database_name_in_mysql(database_name_in_mysql_) - , database_settings(std::move(settings_)) + , mysql_settings(std::move(settings_)) , mysql_pool(std::move(pool)) /// NOLINT { try @@ -309,7 +309,7 @@ DatabaseMySQL::fetchTablesColumnsList(const std::vector & tables_name, C database_name_in_mysql, tables_name, settings, - database_settings->mysql_datatypes_support_level); + mysql_settings->mysql_datatypes_support_level); } void DatabaseMySQL::shutdown() diff --git a/src/Databases/MySQL/DatabaseMySQL.h b/src/Databases/MySQL/DatabaseMySQL.h index a9c06074237..f34a2fff4f7 100644 --- a/src/Databases/MySQL/DatabaseMySQL.h +++ b/src/Databases/MySQL/DatabaseMySQL.h @@ -9,8 +9,8 @@ #include #include #include +#include #include -#include #include #include @@ -44,7 +44,7 @@ public: const String & metadata_path, const ASTStorage * database_engine_define, const String & database_name_in_mysql, - std::unique_ptr settings_, + std::unique_ptr settings_, mysqlxx::PoolWithFailover && pool, bool attach); @@ -93,7 +93,7 @@ private: String metadata_path; ASTPtr database_engine_define; String database_name_in_mysql; - std::unique_ptr database_settings; + std::unique_ptr mysql_settings; std::atomic quit{false}; std::condition_variable cond; diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index c8491d99255..82a2762e61e 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -13,9 +13,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -68,27 +68,21 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) auto settings_config_prefix = config_prefix + ".mysql"; std::shared_ptr pool; MySQLSettings mysql_settings; - auto has_config_key = [&](const String & key) - { - return dictionary_allowed_keys.contains(key) || key.starts_with("replica") || mysql_settings.has(key); - }; - StorageMySQLConfiguration configuration; - auto named_collection = created_from_ddl - ? getExternalDataSourceConfiguration(config, settings_config_prefix, global_context, has_config_key, mysql_settings) - : std::nullopt; + + StorageMySQL::Configuration configuration; + auto named_collection = created_from_ddl ? tryGetNamedCollectionWithOverrides(config, settings_config_prefix) : nullptr; if (named_collection) { - if (created_from_ddl) - global_context->getRemoteHostFilter().checkHostAndPort(configuration.host, toString(configuration.port)); + named_collection->remove("name"); + configuration = StorageMySQL::processNamedCollectionResult(*named_collection, mysql_settings); + global_context->getRemoteHostFilter().checkHostAndPort(configuration.host, toString(configuration.port)); - mysql_settings.applyChanges(named_collection->settings_changes); - configuration.set(named_collection->configuration); - configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; const auto & settings = global_context->getSettingsRef(); if (!mysql_settings.isChanged("connect_timeout")) mysql_settings.connect_timeout = settings.external_storage_connect_timeout_sec; if (!mysql_settings.isChanged("read_write_timeout")) mysql_settings.read_write_timeout = settings.external_storage_rw_timeout_sec; + pool = std::make_shared(createMySQLPoolWithFailover(configuration, mysql_settings)); } else diff --git a/src/Storages/ExternalDataSourceConfiguration.cpp b/src/Storages/ExternalDataSourceConfiguration.cpp index d7c3fe44f38..4882c644f74 100644 --- a/src/Storages/ExternalDataSourceConfiguration.cpp +++ b/src/Storages/ExternalDataSourceConfiguration.cpp @@ -15,10 +15,6 @@ #if USE_RDKAFKA #include #endif -#if USE_MYSQL -#include -#include -#endif #if USE_NATSIO #include #endif @@ -473,23 +469,4 @@ template SettingsChanges getSettingsChangesFromConfig( const BaseSettings & settings, const Poco::Util::AbstractConfiguration & config, const String & config_prefix); -#if USE_MYSQL -template -std::optional getExternalDataSourceConfiguration( - const ASTs & args, ContextPtr context, bool is_database_engine, bool throw_on_no_collection, const BaseSettings & storage_settings); - -template -std::optional getExternalDataSourceConfiguration( - const ASTs & args, ContextPtr context, bool is_database_engine, bool throw_on_no_collection, const BaseSettings & storage_settings); - -template -std::optional getExternalDataSourceConfiguration( - const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, - ContextPtr context, HasConfigKeyFunc has_config_key, const BaseSettings & settings); - -template -SettingsChanges getSettingsChangesFromConfig( - const BaseSettings & settings, const Poco::Util::AbstractConfiguration & config, const String & config_prefix); - -#endif } diff --git a/src/Storages/ExternalDataSourceConfiguration.h b/src/Storages/ExternalDataSourceConfiguration.h index d042f763b11..7095f6b1b04 100644 --- a/src/Storages/ExternalDataSourceConfiguration.h +++ b/src/Storages/ExternalDataSourceConfiguration.h @@ -40,12 +40,6 @@ struct StoragePostgreSQLConfiguration : ExternalDataSourceConfiguration }; -struct StorageMySQLConfiguration : ExternalDataSourceConfiguration -{ - bool replace_query = false; - String on_duplicate_clause; -}; - using StorageSpecificArgs = std::vector>; struct ExternalDataSourceInfo diff --git a/src/Storages/MySQL/MySQLHelpers.cpp b/src/Storages/MySQL/MySQLHelpers.cpp index 127bdb96eaf..63a3436ea4a 100644 --- a/src/Storages/MySQL/MySQLHelpers.cpp +++ b/src/Storages/MySQL/MySQLHelpers.cpp @@ -2,9 +2,7 @@ #if USE_MYSQL #include -#include #include -#include namespace DB { @@ -14,8 +12,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -template mysqlxx::PoolWithFailover -createMySQLPoolWithFailover(const StorageMySQLConfiguration & configuration, const T & mysql_settings) +mysqlxx::PoolWithFailover createMySQLPoolWithFailover(const StorageMySQL::Configuration & configuration, const MySQLSettings & mysql_settings) { if (!mysql_settings.connection_pool_size) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Connection pool cannot have zero size"); @@ -30,11 +27,6 @@ createMySQLPoolWithFailover(const StorageMySQLConfiguration & configuration, con mysql_settings.read_write_timeout); } -template -mysqlxx::PoolWithFailover createMySQLPoolWithFailover(const StorageMySQLConfiguration & configuration, const MySQLSettings & mysql_settings); -template -mysqlxx::PoolWithFailover createMySQLPoolWithFailover(const StorageMySQLConfiguration & configuration, const ConnectionMySQLSettings & mysql_settings); - } #endif diff --git a/src/Storages/MySQL/MySQLHelpers.h b/src/Storages/MySQL/MySQLHelpers.h index 57b564c360c..71c331da16f 100644 --- a/src/Storages/MySQL/MySQLHelpers.h +++ b/src/Storages/MySQL/MySQLHelpers.h @@ -3,15 +3,14 @@ #if USE_MYSQL #include +#include namespace mysqlxx { class PoolWithFailover; } namespace DB { -struct StorageMySQLConfiguration; -template mysqlxx::PoolWithFailover -createMySQLPoolWithFailover(const StorageMySQLConfiguration & configuration, const T & mysql_settings); +mysqlxx::PoolWithFailover createMySQLPoolWithFailover(const StorageMySQL::Configuration & configuration, const MySQLSettings & mysql_settings); } diff --git a/src/Storages/MySQL/MySQLSettings.cpp b/src/Storages/MySQL/MySQLSettings.cpp index 5c1a2246ae9..b3bc11482f4 100644 --- a/src/Storages/MySQL/MySQLSettings.cpp +++ b/src/Storages/MySQL/MySQLSettings.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -43,4 +44,15 @@ void MySQLSettings::loadFromQuery(ASTStorage & storage_def) } } +void MySQLSettings::loadFromQueryContext(ContextPtr context) +{ + if (!context->hasQueryContext()) + return; + + const Settings & settings = context->getQueryContext()->getSettingsRef(); + + if (settings.mysql_datatypes_support_level.value != mysql_datatypes_support_level.value) + set("mysql_datatypes_support_level", settings.mysql_datatypes_support_level.toString()); +} + } diff --git a/src/Storages/MySQL/MySQLSettings.h b/src/Storages/MySQL/MySQLSettings.h index 603b477b856..40771d0aacb 100644 --- a/src/Storages/MySQL/MySQLSettings.h +++ b/src/Storages/MySQL/MySQLSettings.h @@ -2,6 +2,8 @@ #include #include +#include +#include namespace Poco::Util @@ -22,6 +24,7 @@ class ASTSetQuery; M(Bool, connection_auto_close, true, "Auto-close connection after query execution, i.e. disable connection reuse.", 0) \ M(UInt64, connect_timeout, DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, "Connect timeout (in seconds)", 0) \ M(UInt64, read_write_timeout, DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, "Read/write timeout (in seconds)", 0) \ + M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \ DECLARE_SETTINGS_TRAITS(MySQLSettingsTraits, LIST_OF_MYSQL_SETTINGS) @@ -34,6 +37,7 @@ struct MySQLSettings : public MySQLBaseSettings { void loadFromQuery(ASTStorage & storage_def); void loadFromQuery(const ASTSetQuery & settings_def); + void loadFromQueryContext(ContextPtr context); }; diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index cefed555781..0c7426bb682 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -53,7 +53,7 @@ namespace } -NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts) +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts) { if (asts.empty()) return nullptr; @@ -62,11 +62,11 @@ NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts) if (!collection) return nullptr; - if (asts.size() == 1) - return collection; - auto collection_copy = collection->duplicate(); + if (asts.size() == 1) + return collection_copy; + for (auto * it = std::next(asts.begin()); it != asts.end(); ++it) { auto value_override = getKeyValueFromAST(*it); @@ -82,6 +82,23 @@ NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts) return collection_copy; } +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) +{ + auto collection_name = config.getString(config_prefix + ".name", ""); + if (collection_name.empty()) + return nullptr; + + const auto & collection = NamedCollectionFactory::instance().get(collection_name); + auto collection_copy = collection->duplicate(); + + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(config_prefix, keys); + for (const auto & key : keys) + collection_copy->setOrUpdate(key, config.getString(config_prefix + '.' + key)); + + return collection_copy; +} + HTTPHeaderEntries getHeadersFromNamedCollection(const NamedCollection & collection) { HTTPHeaderEntries headers; diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 9c70a46d755..5f8f316d023 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -16,7 +16,12 @@ namespace ErrorCodes namespace DB { -NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts); +/// Helper function to get named collection for table engine. +/// Table engines have collection name as first argument of ast and other arguments are key-value overrides. +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts); +/// Helper function to get named collection for dictionary source. +/// Dictionaries have collection name as name argument of dict configuration and other arguments are overrides. +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); HTTPHeaderEntries getHeadersFromNamedCollection(const NamedCollection & collection); diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index ee647043407..bc39e76be29 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -20,6 +20,7 @@ #include #include #include +#include namespace DB @@ -235,31 +236,53 @@ SinkToStoragePtr StorageMySQL::write(const ASTPtr & /*query*/, const StorageMeta local_context->getSettingsRef().mysql_max_rows_to_insert); } - -StorageMySQLConfiguration StorageMySQL::getConfiguration(ASTs engine_args, ContextPtr context_, MySQLBaseSettings & storage_settings) +StorageMySQL::Configuration StorageMySQL::processNamedCollectionResult( + const NamedCollection & named_collection, MySQLSettings & storage_settings, bool require_table) { - StorageMySQLConfiguration configuration; + StorageMySQL::Configuration configuration; - if (auto named_collection = getExternalDataSourceConfiguration( - engine_args, context_, /* is_database_engine */false, /* throw_on_no_collection */true, storage_settings)) + std::unordered_set optional_arguments = {"replace_query", "on_duplicate_clause", "addresses_expr", "host", "port"}; + auto mysql_settings = storage_settings.all(); + for (const auto & setting : mysql_settings) + optional_arguments.insert(setting.getName()); + + std::unordered_set required_arguments = {"user", "password", "database", "table"}; + if (require_table) + required_arguments.insert("table"); + validateNamedCollection(named_collection, required_arguments, optional_arguments); + + configuration.addresses_expr = named_collection.getOrDefault("addresses_expr", ""); + if (configuration.addresses_expr.empty()) { - auto [common_configuration, storage_specific_args, settings_changes] = named_collection.value(); - configuration.set(common_configuration); + configuration.host = named_collection.get("host"); + configuration.port = static_cast(named_collection.get("port")); configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; - storage_settings.applyChanges(settings_changes); + } - for (const auto & [arg_name, arg_value] : storage_specific_args) - { - if (arg_name == "replace_query") - configuration.replace_query = checkAndGetLiteralArgument(arg_value, "replace_query"); - else if (arg_name == "on_duplicate_clause") - configuration.on_duplicate_clause = checkAndGetLiteralArgument(arg_value, "on_duplicate_clause"); - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Unexpected key-value argument." - "Got: {}, but expected one of:" - "host, port, username, password, database, table, replace_query, on_duplicate_clause.", arg_name); - } + configuration.username = named_collection.get("user"); + configuration.password = named_collection.get("password"); + configuration.database = named_collection.get("database"); + if (require_table) + configuration.table = named_collection.get("table"); + configuration.replace_query = named_collection.getOrDefault("replace_query", false); + configuration.on_duplicate_clause = named_collection.getOrDefault("on_duplicate_clause", ""); + + for (const auto & setting : mysql_settings) + { + const auto & setting_name = setting.getName(); + if (named_collection.has(setting_name)) + storage_settings.set(setting_name, named_collection.get(setting_name)); + } + + return configuration; +} + +StorageMySQL::Configuration StorageMySQL::getConfiguration(ASTs engine_args, ContextPtr context_, MySQLSettings & storage_settings) +{ + StorageMySQL::Configuration configuration; + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) + { + configuration = StorageMySQL::processNamedCollectionResult(*named_collection, storage_settings); } else { diff --git a/src/Storages/StorageMySQL.h b/src/Storages/StorageMySQL.h index bf9a24c9bfe..1c0e2639717 100644 --- a/src/Storages/StorageMySQL.h +++ b/src/Storages/StorageMySQL.h @@ -6,7 +6,6 @@ #include #include -#include #include namespace Poco @@ -17,6 +16,8 @@ class Logger; namespace DB { +class NamedCollection; + /** Implements storage in the MySQL database. * Use ENGINE = mysql(host_port, database_name, table_name, user_name, password) * Read only. @@ -50,7 +51,26 @@ public: SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override; - static StorageMySQLConfiguration getConfiguration(ASTs engine_args, ContextPtr context_, MySQLBaseSettings & storage_settings); + struct Configuration + { + String host; + UInt16 port = 0; + String username = "default"; + String password; + String database; + String table; + + bool replace_query = false; + String on_duplicate_clause; + + std::vector> addresses; /// Failover replicas. + String addresses_expr; + }; + + static Configuration getConfiguration(ASTs engine_args, ContextPtr context_, MySQLSettings & storage_settings); + + static Configuration processNamedCollectionResult( + const NamedCollection & named_collection, MySQLSettings & storage_settings, bool require_table = true); private: friend class StorageMySQLSink; diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 400430b9ea2..3fa0c137f7b 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -387,31 +387,41 @@ SinkToStoragePtr StoragePostgreSQL::write( return std::make_shared(metadata_snapshot, pool->get(), remote_table_name, remote_table_schema, on_conflict); } +StoragePostgreSQL::Configuration StoragePostgreSQL::processNamedCollectionResult(const NamedCollection & named_collection, bool require_table) +{ + StoragePostgreSQL::Configuration configuration; + std::unordered_set required_arguments = {"user", "password", "database", "table"}; + if (require_table) + required_arguments.insert("table"); + validateNamedCollection( + named_collection, required_arguments, + {"schema", "on_conflict", "addresses_expr", "host", "port"}); + + configuration.addresses_expr = named_collection.getOrDefault("addresses_expr", ""); + if (configuration.addresses_expr.empty()) + { + configuration.host = named_collection.get("host"); + configuration.port = static_cast(named_collection.get("port")); + configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; + } + + configuration.username = named_collection.get("user"); + configuration.password = named_collection.get("password"); + configuration.database = named_collection.get("database"); + if (require_table) + configuration.table = named_collection.get("table"); + configuration.schema = named_collection.getOrDefault("schema", ""); + configuration.on_conflict = named_collection.getOrDefault("on_conflict", ""); + + return configuration; +} StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine_args, ContextPtr context) { StoragePostgreSQL::Configuration configuration; if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { - validateNamedCollection( - *named_collection, - {"user", "password", "database", "table"}, - {"schema", "on_conflict", "addresses_expr", "host", "port"}); - - configuration.addresses_expr = named_collection->getOrDefault("addresses_expr", ""); - if (configuration.addresses_expr.empty()) - { - configuration.host = named_collection->get("host"); - configuration.port = static_cast(named_collection->get("port")); - configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; - } - - configuration.username = named_collection->get("user"); - configuration.password = named_collection->get("password"); - configuration.database = named_collection->get("database"); - configuration.table = named_collection->get("table"); - configuration.schema = named_collection->getOrDefault("schema", ""); - configuration.on_conflict = named_collection->getOrDefault("on_conflict", ""); + configuration = StoragePostgreSQL::processNamedCollectionResult(*named_collection); } else { diff --git a/src/Storages/StoragePostgreSQL.h b/src/Storages/StoragePostgreSQL.h index f3df24ebd35..b3ff342da10 100644 --- a/src/Storages/StoragePostgreSQL.h +++ b/src/Storages/StoragePostgreSQL.h @@ -5,7 +5,6 @@ #if USE_LIBPQXX #include #include -#include namespace Poco { @@ -20,6 +19,7 @@ using PoolWithFailoverPtr = std::shared_ptr; namespace DB { +class NamedCollection; class StoragePostgreSQL final : public IStorage { @@ -64,6 +64,8 @@ public: static Configuration getConfiguration(ASTs engine_args, ContextPtr context); + static Configuration processNamedCollectionResult(const NamedCollection & named_collection, bool require_table = true); + private: String remote_table_name; String remote_table_schema; diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index acf49f3cb71..d033747d0a2 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index 0cbad7bd9fd..1080f12021f 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include diff --git a/src/TableFunctions/TableFunctionMySQL.h b/src/TableFunctions/TableFunctionMySQL.h index 794e8632ae2..5a230530bc4 100644 --- a/src/TableFunctions/TableFunctionMySQL.h +++ b/src/TableFunctions/TableFunctionMySQL.h @@ -3,7 +3,7 @@ #if USE_MYSQL #include -#include +#include #include @@ -30,7 +30,7 @@ private: void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; mutable std::optional pool; - std::optional configuration; + std::optional configuration; }; } From 68e06ecb991a4e84492618c4592b47abf037dcaa Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 21 Feb 2023 14:33:37 +0100 Subject: [PATCH 13/69] Replace for table function remote, and external storage --- .../ExternalDataSourceConfiguration.cpp | 114 ------- .../ExternalDataSourceConfiguration.h | 20 -- src/Storages/MeiliSearch/StorageMeiliSearch.h | 1 - src/Storages/StorageExternalDistributed.cpp | 306 +++++------------- src/Storages/StorageExternalDistributed.h | 24 +- src/Storages/StorageMongoDB.h | 1 - src/Storages/StorageURL.h | 10 +- src/TableFunctions/TableFunctionRemote.cpp | 74 ++--- 8 files changed, 110 insertions(+), 440 deletions(-) diff --git a/src/Storages/ExternalDataSourceConfiguration.cpp b/src/Storages/ExternalDataSourceConfiguration.cpp index 4882c644f74..b742391bad5 100644 --- a/src/Storages/ExternalDataSourceConfiguration.cpp +++ b/src/Storages/ExternalDataSourceConfiguration.cpp @@ -90,116 +90,6 @@ void ExternalDataSourceConfiguration::set(const ExternalDataSourceConfiguration } -template -std::optional getExternalDataSourceConfiguration( - const ASTs & args, ContextPtr context, bool is_database_engine, bool throw_on_no_collection, const BaseSettings & storage_settings) -{ - if (args.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "External data source must have arguments"); - - ExternalDataSourceConfiguration configuration; - StorageSpecificArgs non_common_args; - - if (const auto * collection = typeid_cast(args[0].get())) - { - const auto & config = context->getConfigRef(); - const auto & collection_prefix = fmt::format("named_collections.{}", collection->name()); - - if (!config.has(collection_prefix)) - { - /// For table function remote we do not throw on no collection, because then we consider first arg - /// as cluster definition from config. - if (!throw_on_no_collection) - return std::nullopt; - - throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no collection named `{}` in config", collection->name()); - } - - SettingsChanges config_settings = getSettingsChangesFromConfig(storage_settings, config, collection_prefix); - - configuration.host = config.getString(collection_prefix + ".host", ""); - configuration.port = config.getInt(collection_prefix + ".port", 0); - configuration.username = config.getString(collection_prefix + ".user", ""); - configuration.password = config.getString(collection_prefix + ".password", ""); - configuration.quota_key = config.getString(collection_prefix + ".quota_key", ""); - configuration.database = config.getString(collection_prefix + ".database", ""); - configuration.table = config.getString(collection_prefix + ".table", config.getString(collection_prefix + ".collection", "")); - configuration.schema = config.getString(collection_prefix + ".schema", ""); - configuration.addresses_expr = config.getString(collection_prefix + ".addresses_expr", ""); - - if (!configuration.addresses_expr.empty() && !configuration.host.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot have `addresses_expr` and `host`, `port` in configuration at the same time"); - - if ((args.size() == 1) && ((configuration.addresses_expr.empty() && (configuration.host.empty() || configuration.port == 0)) - || configuration.database.empty() || (configuration.table.empty() && !is_database_engine))) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Named collection of connection parameters is missing some " - "of the parameters and no key-value arguments are added"); - } - - /// Check key-value arguments. - for (size_t i = 1; i < args.size(); ++i) - { - if (const auto * ast_function = typeid_cast(args[i].get())) - { - const auto * args_expr = assert_cast(ast_function->arguments.get()); - auto function_args = args_expr->children; - if (function_args.size() != 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument"); - - auto arg_name = function_args[0]->as()->name(); - if (function_args[1]->as()) - { - non_common_args.emplace_back(std::make_pair(arg_name, function_args[1])); - continue; - } - - auto arg_value_ast = evaluateConstantExpressionOrIdentifierAsLiteral(function_args[1], context); - auto * arg_value_literal = arg_value_ast->as(); - if (arg_value_literal) - { - auto arg_value = arg_value_literal->value; - - if (arg_name == "host") - configuration.host = arg_value.safeGet(); - else if (arg_name == "port") - configuration.port = arg_value.safeGet(); - else if (arg_name == "user") - configuration.username = arg_value.safeGet(); - else if (arg_name == "password") - configuration.password = arg_value.safeGet(); - else if (arg_name == "quota_key") - configuration.quota_key = arg_value.safeGet(); - else if (arg_name == "database") - configuration.database = arg_value.safeGet(); - else if (arg_name == "table") - configuration.table = arg_value.safeGet(); - else if (arg_name == "schema") - configuration.schema = arg_value.safeGet(); - else if (arg_name == "addresses_expr") - configuration.addresses_expr = arg_value.safeGet(); - else if (storage_settings.has(arg_name)) - config_settings.emplace_back(arg_name, arg_value); - else - non_common_args.emplace_back(std::make_pair(arg_name, arg_value_ast)); - } - else - { - non_common_args.emplace_back(std::make_pair(arg_name, arg_value_ast)); - } - } - else - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument"); - } - } - - return ExternalDataSourceInfo{ .configuration = configuration, .specific_args = non_common_args, .settings_changes = config_settings }; - } - return std::nullopt; -} - static void validateConfigKeys( const Poco::Util::AbstractConfiguration & dict_config, const String & config_prefix, HasConfigKeyFunc has_config_key_func) { @@ -456,10 +346,6 @@ template bool getExternalDataSourceConfiguration(const ASTs & args, BaseSettings & settings, ContextPtr context); #endif -template -std::optional getExternalDataSourceConfiguration( - const ASTs & args, ContextPtr context, bool is_database_engine, bool throw_on_no_collection, const BaseSettings & storage_settings); - template std::optional getExternalDataSourceConfiguration( const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, diff --git a/src/Storages/ExternalDataSourceConfiguration.h b/src/Storages/ExternalDataSourceConfiguration.h index 7095f6b1b04..6eab7c2e072 100644 --- a/src/Storages/ExternalDataSourceConfiguration.h +++ b/src/Storages/ExternalDataSourceConfiguration.h @@ -34,12 +34,6 @@ struct ExternalDataSourceConfiguration }; -struct StoragePostgreSQLConfiguration : ExternalDataSourceConfiguration -{ - String on_conflict; -}; - - using StorageSpecificArgs = std::vector>; struct ExternalDataSourceInfo @@ -49,20 +43,6 @@ struct ExternalDataSourceInfo SettingsChanges settings_changes; }; -/* If there is a storage engine's configuration specified in the named_collections, - * this function returns valid for usage ExternalDataSourceConfiguration struct - * otherwise std::nullopt is returned. - * - * If any configuration options are provided as key-value engine arguments, they will override - * configuration values, i.e. ENGINE = PostgreSQL(postgresql_configuration, database = 'postgres_database'); - * - * Any key-value engine argument except common (`host`, `port`, `username`, `password`, `database`) - * is returned in EngineArgs struct. - */ -template -std::optional getExternalDataSourceConfiguration( - const ASTs & args, ContextPtr context, bool is_database_engine = false, bool throw_on_no_collection = true, const BaseSettings & storage_settings = {}); - using HasConfigKeyFunc = std::function; template diff --git a/src/Storages/MeiliSearch/StorageMeiliSearch.h b/src/Storages/MeiliSearch/StorageMeiliSearch.h index 5fa7ac2c0e3..30ff2f9b9fa 100644 --- a/src/Storages/MeiliSearch/StorageMeiliSearch.h +++ b/src/Storages/MeiliSearch/StorageMeiliSearch.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index 9eeb619b899..e07a4554a60 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -31,154 +32,20 @@ namespace ErrorCodes StorageExternalDistributed::StorageExternalDistributed( const StorageID & table_id_, - ExternalStorageEngine table_engine, - const String & cluster_description, - const ExternalDataSourceConfiguration & configuration, + std::unordered_set && shards_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, - const String & comment, - ContextPtr context) + const String & comment) : IStorage(table_id_) + , shards(shards_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); - - size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; - std::vector shards_descriptions = parseRemoteDescription(cluster_description, 0, cluster_description.size(), ',', max_addresses); - std::vector> addresses; - -#if USE_MYSQL || USE_LIBPQXX - - /// For each shard pass replicas description into storage, replicas are managed by storage's PoolWithFailover. - for (const auto & shard_description : shards_descriptions) - { - StoragePtr shard; - - switch (table_engine) - { -#if USE_MYSQL - case ExternalStorageEngine::MySQL: - { - addresses = parseRemoteDescriptionForExternalDatabase(shard_description, max_addresses, 3306); - - mysqlxx::PoolWithFailover pool( - configuration.database, - addresses, - configuration.username, - configuration.password); - - shard = std::make_shared( - table_id_, - std::move(pool), - configuration.database, - configuration.table, - /* replace_query = */ false, - /* on_duplicate_clause = */ "", - columns_, - constraints_, - String{}, - context, - MySQLSettings{}); - break; - } -#endif -#if USE_LIBPQXX - - case ExternalStorageEngine::PostgreSQL: - { - addresses = parseRemoteDescriptionForExternalDatabase(shard_description, max_addresses, 5432); - StoragePostgreSQL::Configuration postgres_conf; - postgres_conf.addresses = addresses; - postgres_conf.username = configuration.username; - postgres_conf.password = configuration.password; - postgres_conf.database = configuration.database; - postgres_conf.table = configuration.table; - postgres_conf.schema = configuration.schema; - - const auto & settings = context->getSettingsRef(); - auto pool = std::make_shared( - postgres_conf, - settings.postgresql_connection_pool_size, - settings.postgresql_connection_pool_wait_timeout, - POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, - settings.postgresql_connection_pool_auto_close_connection); - - shard = std::make_shared(table_id_, std::move(pool), configuration.table, columns_, constraints_, String{}); - break; - } -#endif - default: - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Unsupported table engine. Supported engines are: MySQL, PostgreSQL, URL"); - } - } - - shards.emplace(std::move(shard)); - } - -#else - (void)configuration; - (void)cluster_description; - (void)addresses; - (void)table_engine; -#endif } - -StorageExternalDistributed::StorageExternalDistributed( - const String & addresses_description, - const StorageID & table_id, - const String & format_name, - const std::optional & format_settings, - const String & compression_method, - const ColumnsDescription & columns, - const ConstraintsDescription & constraints, - ContextPtr context) - : IStorage(table_id) -{ - StorageInMemoryMetadata storage_metadata; - storage_metadata.setColumns(columns); - storage_metadata.setConstraints(constraints); - setInMemoryMetadata(storage_metadata); - - size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; - /// Generate addresses without splitting for failover options - std::vector url_descriptions = parseRemoteDescription(addresses_description, 0, addresses_description.size(), ',', max_addresses); - std::vector uri_options; - - for (const auto & url_description : url_descriptions) - { - /// For each uri (which acts like shard) check if it has failover options - uri_options = parseRemoteDescription(url_description, 0, url_description.size(), '|', max_addresses); - StoragePtr shard; - - if (uri_options.size() > 1) - { - shard = std::make_shared( - uri_options, - table_id, - format_name, - format_settings, - columns, constraints, context, - compression_method); - } - else - { - shard = std::make_shared( - url_description, table_id, format_name, format_settings, columns, constraints, String{}, context, compression_method); - - LOG_DEBUG(&Poco::Logger::get("StorageURLDistributed"), "Adding URL: {}", url_description); - } - - shards.emplace(std::move(shard)); - } -} - - void StorageExternalDistributed::read( QueryPlan & query_plan, const Names & column_names, @@ -226,7 +93,6 @@ void StorageExternalDistributed::read( query_plan.unitePlans(std::move(union_step), std::move(plans)); } - void registerStorageExternalDistributed(StorageFactory & factory) { factory.registerStorage("ExternalDistributed", [](const StorageFactory::Arguments & args) @@ -237,102 +103,94 @@ void registerStorageExternalDistributed(StorageFactory & factory) "Engine ExternalDistributed must have at least 2 arguments: " "engine_name, named_collection and/or description"); - auto engine_name = checkAndGetLiteralArgument(engine_args[0], "engine_name"); - StorageExternalDistributed::ExternalStorageEngine table_engine; - if (engine_name == "URL") - table_engine = StorageExternalDistributed::ExternalStorageEngine::URL; - else if (engine_name == "MySQL") - table_engine = StorageExternalDistributed::ExternalStorageEngine::MySQL; - else if (engine_name == "PostgreSQL") - table_engine = StorageExternalDistributed::ExternalStorageEngine::PostgreSQL; - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "External storage engine {} is not supported for StorageExternalDistributed. " - "Supported engines are: MySQL, PostgreSQL, URL", - engine_name); + auto context = args.getLocalContext(); + const auto & settings = context->getSettingsRef(); + size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; + auto get_addresses = [&](const std::string addresses_expr) + { + return parseRemoteDescription(addresses_expr, 0, addresses_expr.size(), ',', max_addresses); + }; + std::unordered_set shards; ASTs inner_engine_args(engine_args.begin() + 1, engine_args.end()); - String cluster_description; + auto engine_name = checkAndGetLiteralArgument(engine_args[0], "engine_name"); if (engine_name == "URL") { - StorageURL::Configuration configuration; - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) - { - StorageURL::processNamedCollectionResult(configuration, *named_collection); - StorageURL::collectHeaders(engine_args, configuration.headers, args.getLocalContext()); - } - else - { - for (auto & engine_arg : engine_args) - engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.getLocalContext()); - - cluster_description = checkAndGetLiteralArgument(engine_args[1], "cluster_description"); - configuration.format = checkAndGetLiteralArgument(engine_args[2], "format"); - configuration.compression_method = "auto"; - if (engine_args.size() == 4) - configuration.compression_method = checkAndGetLiteralArgument(engine_args[3], "compression_method"); - } - - + auto configuration = StorageURL::getConfiguration(inner_engine_args, context); + auto shards_addresses = get_addresses(configuration.addresses_expr); auto format_settings = StorageURL::getFormatSettingsFromArgs(args); - - return std::make_shared( - cluster_description, - args.table_id, - configuration.format, - format_settings, - configuration.compression_method, - args.columns, - args.constraints, - args.getContext()); + for (const auto & shard_address : shards_addresses) + { + auto uri_options = parseRemoteDescription(shard_address, 0, shard_address.size(), '|', max_addresses); + if (uri_options.size() > 1) + { + shards.insert( + std::make_shared( + uri_options, args.table_id, configuration.format, format_settings, + args.columns, args.constraints, context, configuration.compression_method)); + } + else + { + shards.insert(std::make_shared( + shard_address, args.table_id, configuration.format, format_settings, + args.columns, args.constraints, String{}, context, configuration.compression_method)); + } + } } +#if USE_MYSQL + else if (engine_name == "MySQL") + { + MySQLSettings mysql_settings; + auto configuration = StorageMySQL::getConfiguration(inner_engine_args, context, mysql_settings); + auto shards_addresses = get_addresses(configuration.addresses_expr); + for (const auto & shard_address : shards_addresses) + { + auto current_configuration{configuration}; + current_configuration.addresses = parseRemoteDescriptionForExternalDatabase(shard_address, max_addresses, 3306); + auto pool = createMySQLPoolWithFailover(configuration, mysql_settings); + shards.insert(std::make_shared( + args.table_id, std::move(pool), configuration.database, configuration.table, + /* replace_query = */ false, /* on_duplicate_clause = */ "", + args.columns, args.constraints, String{}, context, mysql_settings)); + } + } +#endif +#if USE_LIBPQXX + else if (engine_name == "PostgreSQL") + { + auto configuration = StoragePostgreSQL::getConfiguration(inner_engine_args, context); + auto shards_addresses = get_addresses(configuration.addresses_expr); + for (const auto & shard_address : shards_addresses) + { + auto current_configuration{configuration}; + current_configuration.addresses = parseRemoteDescriptionForExternalDatabase(shard_address, max_addresses, 5432); + auto pool = std::make_shared( + current_configuration, + settings.postgresql_connection_pool_size, + settings.postgresql_connection_pool_wait_timeout, + POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, + settings.postgresql_connection_pool_auto_close_connection); + shards.insert(std::make_shared( + args.table_id, std::move(pool), configuration.table, args.columns, args.constraints, String{})); + } + } +#endif else { - ExternalDataSourceConfiguration configuration; - if (auto named_collection = getExternalDataSourceConfiguration(inner_engine_args, args.getLocalContext())) - { - auto [common_configuration, storage_specific_args, _] = named_collection.value(); - configuration.set(common_configuration); - - for (const auto & [name, value] : storage_specific_args) - { - if (name == "description") - cluster_description = checkAndGetLiteralArgument(value, "cluster_description"); - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Unknown key-value argument {} for table function URL", name); - } - - if (cluster_description.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Engine ExternalDistribued must have `description` key-value argument or named collection parameter"); - } - else - { - if (engine_args.size() != 6) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage ExternalDistributed requires 5 parameters: " - "ExternalDistributed('engine_name', 'cluster_description', 'database', 'table', 'user', 'password')."); - - cluster_description = checkAndGetLiteralArgument(engine_args[1], "cluster_description"); - configuration.database = checkAndGetLiteralArgument(engine_args[2], "database"); - configuration.table = checkAndGetLiteralArgument(engine_args[3], "table"); - configuration.username = checkAndGetLiteralArgument(engine_args[4], "username"); - configuration.password = checkAndGetLiteralArgument(engine_args[5], "password"); - } - - - return std::make_shared( - args.table_id, - table_engine, - cluster_description, - configuration, - args.columns, - args.constraints, - args.comment, - args.getContext()); + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "External storage engine {} is not supported for StorageExternalDistributed. " + "Supported engines are: MySQL, PostgreSQL, URL", + engine_name); } + + return std::make_shared( + args.table_id, + std::move(shards), + args.columns, + args.constraints, + args.comment); }, { .source_access_type = AccessType::SOURCES, diff --git a/src/Storages/StorageExternalDistributed.h b/src/Storages/StorageExternalDistributed.h index a1bdb41dded..c4d37c3e5cc 100644 --- a/src/Storages/StorageExternalDistributed.h +++ b/src/Storages/StorageExternalDistributed.h @@ -18,32 +18,12 @@ struct ExternalDataSourceConfiguration; class StorageExternalDistributed final : public DB::IStorage { public: - enum class ExternalStorageEngine - { - MySQL, - PostgreSQL, - URL - }; - StorageExternalDistributed( const StorageID & table_id_, - ExternalStorageEngine table_engine, - const String & cluster_description, - const ExternalDataSourceConfiguration & configuration, + std::unordered_set && shards_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, - const String & comment, - ContextPtr context_); - - StorageExternalDistributed( - const String & addresses_description, - const StorageID & table_id, - const String & format_name, - const std::optional & format_settings, - const String & compression_method, - const ColumnsDescription & columns, - const ConstraintsDescription & constraints, - ContextPtr context); + const String & comment); std::string getName() const override { return "ExternalDistributed"; } diff --git a/src/Storages/StorageMongoDB.h b/src/Storages/StorageMongoDB.h index 941e9fcf4b0..682a027440d 100644 --- a/src/Storages/StorageMongoDB.h +++ b/src/Storages/StorageMongoDB.h @@ -3,7 +3,6 @@ #include #include -#include namespace DB { diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index d033747d0a2..24b1c7ee572 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -184,8 +184,9 @@ public: struct Configuration : public StatelessTableEngineConfiguration { std::string url; - std::string http_method; + std::string http_method = "auto"; HTTPHeaderEntries headers; + std::string addresses_expr; }; static Configuration getConfiguration(ASTs & args, ContextPtr context); @@ -219,13 +220,6 @@ public: size_t max_block_size, size_t num_streams) override; - struct Configuration - { - String url; - String compression_method = "auto"; - std::vector> headers; - }; - private: std::vector uri_options; }; diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 1877c9fe65b..dbc718536f2 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -2,8 +2,8 @@ #include #include -#include #include +#include #include #include #include @@ -34,10 +34,10 @@ namespace ErrorCodes void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr context) { ASTs & args_func = ast_function->children; - ExternalDataSourceConfiguration configuration; String cluster_name; String cluster_description; + String database, table, username, password; if (args_func.size() != 1) throw Exception(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); @@ -50,47 +50,21 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr * For now named collection can be used only for remote as cluster does not require credentials. */ size_t max_args = is_cluster_function ? 4 : 6; - auto named_collection = getExternalDataSourceConfiguration(args, context, false, false); - if (named_collection) + if (auto named_collection = tryGetNamedCollectionWithOverrides(args)) { if (is_cluster_function) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Named collection cannot be used for table function cluster"); - /** - * Common arguments: database, table, username, password, addresses_expr. - * Specific args (remote): sharding_key, or database (in case it is not ASTLiteral). - * None of the common arguments is empty at this point, it is checked in getExternalDataSourceConfiguration. - */ - auto [common_configuration, storage_specific_args, _] = named_collection.value(); - configuration.set(common_configuration); + validateNamedCollection( + *named_collection, + {"addresses_expr", "database", "table"}, + {"username", "password", "sharding_key"}); - for (const auto & [arg_name, arg_value] : storage_specific_args) - { - if (arg_name == "sharding_key") - { - sharding_key = arg_value; - } - else if (arg_name == "database") - { - const auto * function = arg_value->as(); - if (function && TableFunctionFactory::instance().isTableFunctionName(function->name)) - { - remote_table_function_ptr = arg_value; - } - else - { - auto database_literal = evaluateConstantExpressionOrIdentifierAsLiteral(arg_value, context); - configuration.database = checkAndGetLiteralArgument(database_literal, "database"); - } - } - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Unexpected key-value argument." - "Got: {}, but expected: sharding_key", arg_name); - } - cluster_description = configuration.addresses_expr; - if (cluster_description.empty()) - cluster_description = configuration.port ? configuration.host + ':' + toString(configuration.port) : configuration.host; + cluster_description = named_collection->getOrDefault("addresses_expr", ""); + database = named_collection->get("database"); + table = named_collection->get("table"); + username = named_collection->getOrDefault("username", ""); + password = named_collection->getOrDefault("password", ""); } else { @@ -159,11 +133,11 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr else { args[arg_num] = evaluateConstantExpressionForDatabaseName(args[arg_num], context); - configuration.database = checkAndGetLiteralArgument(args[arg_num], "database"); + database = checkAndGetLiteralArgument(args[arg_num], "database"); ++arg_num; - auto qualified_name = QualifiedTableName::parseFromString(configuration.database); + auto qualified_name = QualifiedTableName::parseFromString(database); if (qualified_name.database.empty()) { if (arg_num >= args.size()) @@ -179,8 +153,8 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr } } - configuration.database = std::move(qualified_name.database); - configuration.table = std::move(qualified_name.table); + database = std::move(qualified_name.database); + table = std::move(qualified_name.table); /// Cluster function may have sharding key for insert if (is_cluster_function && arg_num < args.size()) @@ -195,9 +169,9 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr { if (arg_num < args.size()) { - if (!get_string_literal(*args[arg_num], configuration.username)) + if (!get_string_literal(*args[arg_num], username)) { - configuration.username = "default"; + username = "default"; sharding_key = args[arg_num]; } ++arg_num; @@ -205,7 +179,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr if (arg_num < args.size() && !sharding_key) { - if (!get_string_literal(*args[arg_num], configuration.password)) + if (!get_string_literal(*args[arg_num], password)) { sharding_key = args[arg_num]; } @@ -267,19 +241,19 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr cluster = std::make_shared( context->getSettingsRef(), names, - configuration.username, - configuration.password, + username, + password, (secure ? (maybe_secure_port ? *maybe_secure_port : DBMS_DEFAULT_SECURE_PORT) : context->getTCPPort()), treat_local_as_remote, treat_local_port_as_remote, secure); } - if (!remote_table_function_ptr && configuration.table.empty()) + if (!remote_table_function_ptr && table.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The name of remote table cannot be empty"); - remote_table_id.database_name = configuration.database; - remote_table_id.table_name = configuration.table; + remote_table_id.database_name = database; + remote_table_id.table_name = table; } StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns) const From b3ca976f5f4b51001ed7e4035f4fc4f1db181903 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 21 Feb 2023 14:50:55 +0100 Subject: [PATCH 14/69] Finish for streaming engines --- .../ExternalDataSourceConfiguration.cpp | 58 ------------------- .../ExternalDataSourceConfiguration.h | 4 -- src/Storages/Kafka/StorageKafka.cpp | 26 ++++++--- src/Storages/NATS/StorageNATS.cpp | 14 ++++- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 15 ++++- 5 files changed, 40 insertions(+), 77 deletions(-) diff --git a/src/Storages/ExternalDataSourceConfiguration.cpp b/src/Storages/ExternalDataSourceConfiguration.cpp index b742391bad5..28bd058b802 100644 --- a/src/Storages/ExternalDataSourceConfiguration.cpp +++ b/src/Storages/ExternalDataSourceConfiguration.cpp @@ -288,64 +288,6 @@ void URLBasedDataSourceConfiguration::set(const URLBasedDataSourceConfiguration headers = conf.headers; } -template -bool getExternalDataSourceConfiguration(const ASTs & args, BaseSettings & settings, ContextPtr context) -{ - if (args.empty()) - return false; - - if (const auto * collection = typeid_cast(args[0].get())) - { - const auto & config = context->getConfigRef(); - const auto & config_prefix = fmt::format("named_collections.{}", collection->name()); - - if (!config.has(config_prefix)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no collection named `{}` in config", collection->name()); - - auto config_settings = getSettingsChangesFromConfig(settings, config, config_prefix); - - /// Check key-value arguments. - for (size_t i = 1; i < args.size(); ++i) - { - if (const auto * ast_function = typeid_cast(args[i].get())) - { - const auto * args_expr = assert_cast(ast_function->arguments.get()); - auto function_args = args_expr->children; - if (function_args.size() != 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument"); - - auto arg_name = function_args[0]->as()->name(); - auto arg_value_ast = evaluateConstantExpressionOrIdentifierAsLiteral(function_args[1], context); - auto arg_value = arg_value_ast->as()->value; - config_settings.emplace_back(arg_name, arg_value); - } - else - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument"); - } - } - - settings.applyChanges(config_settings); - return true; - } - return false; -} - -#if USE_AMQPCPP -template -bool getExternalDataSourceConfiguration(const ASTs & args, BaseSettings & settings, ContextPtr context); -#endif - -#if USE_RDKAFKA -template -bool getExternalDataSourceConfiguration(const ASTs & args, BaseSettings & settings, ContextPtr context); -#endif - -#if USE_NATSIO -template -bool getExternalDataSourceConfiguration(const ASTs & args, BaseSettings & settings, ContextPtr context); -#endif - template std::optional getExternalDataSourceConfiguration( const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, diff --git a/src/Storages/ExternalDataSourceConfiguration.h b/src/Storages/ExternalDataSourceConfiguration.h index 6eab7c2e072..b825548debe 100644 --- a/src/Storages/ExternalDataSourceConfiguration.h +++ b/src/Storages/ExternalDataSourceConfiguration.h @@ -65,7 +65,6 @@ struct ExternalDataSourcesByPriority ExternalDataSourcesByPriority getExternalDataSourceConfigurationByPriority(const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, ContextPtr context, HasConfigKeyFunc has_config_key); - struct URLBasedDataSourceConfiguration { String url; @@ -92,7 +91,4 @@ struct URLBasedDataSourceConfig std::optional getURLBasedDataSourceConfiguration( const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, ContextPtr context); -template -bool getExternalDataSourceConfiguration(const ASTs & args, BaseSettings & settings, ContextPtr context); - } diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 50fb7dffa34..cfb066ee1b8 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -19,13 +19,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include #include #include @@ -757,10 +757,23 @@ void registerStorageKafka(StorageFactory & factory) { ASTs & engine_args = args.engine_args; size_t args_count = engine_args.size(); - bool has_settings = args.storage_def->settings; + const bool has_settings = args.storage_def->settings; auto kafka_settings = std::make_unique(); - auto named_collection = getExternalDataSourceConfiguration(args.engine_args, *kafka_settings, args.getLocalContext()); + String collection_name; + if (auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args)) + { + for (const auto & setting : kafka_settings->all()) + { + const auto & setting_name = setting.getName(); + if (named_collection->has(setting_name)) + kafka_settings->set(setting_name, named_collection->get(setting_name)); + } + collection_name = assert_cast(args.engine_args[0].get())->name(); + } + else if (!has_settings) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Kafka engine must have settings"); + if (has_settings) { kafka_settings->loadFromQuery(*args.storage_def); @@ -824,12 +837,7 @@ void registerStorageKafka(StorageFactory & factory) * - Do intermediate commits when the batch consumed and handled */ - String collection_name; - if (named_collection) - { - collection_name = assert_cast(args.engine_args[0].get())->name(); - } - else + if (has_settings) { /* 0 = raw, 1 = evaluateConstantExpressionAsLiteral, 2=evaluateConstantExpressionOrIdentifierAsLiteral */ CHECK_KAFKA_STORAGE_ARGUMENT(1, kafka_broker_list, 0) diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp index f1724b8c14c..feb44fe92e4 100644 --- a/src/Storages/NATS/StorageNATS.cpp +++ b/src/Storages/NATS/StorageNATS.cpp @@ -10,13 +10,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include #include #include @@ -711,8 +711,16 @@ void registerStorageNATS(StorageFactory & factory) auto creator_fn = [](const StorageFactory::Arguments & args) { auto nats_settings = std::make_unique(); - bool with_named_collection = getExternalDataSourceConfiguration(args.engine_args, *nats_settings, args.getLocalContext()); - if (!with_named_collection && !args.storage_def->settings) + if (auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args)) + { + for (const auto & setting : nats_settings->all()) + { + const auto & setting_name = setting.getName(); + if (named_collection->has(setting_name)) + nats_settings->set(setting_name, named_collection->get(setting_name)); + } + } + else if (!args.storage_def->settings) throw Exception(ErrorCodes::BAD_ARGUMENTS, "NATS engine must have settings"); nats_settings->loadFromQuery(*args.storage_def); diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index f10a60419d1..c5ea6f810ef 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include @@ -1194,8 +1194,17 @@ void registerStorageRabbitMQ(StorageFactory & factory) auto creator_fn = [](const StorageFactory::Arguments & args) { auto rabbitmq_settings = std::make_unique(); - bool with_named_collection = getExternalDataSourceConfiguration(args.engine_args, *rabbitmq_settings, args.getLocalContext()); - if (!with_named_collection && !args.storage_def->settings) + + if (auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args)) + { + for (const auto & setting : rabbitmq_settings->all()) + { + const auto & setting_name = setting.getName(); + if (named_collection->has(setting_name)) + rabbitmq_settings->set(setting_name, named_collection->get(setting_name)); + } + } + else if (!args.storage_def->settings) throw Exception(ErrorCodes::BAD_ARGUMENTS, "RabbitMQ engine must have settings"); if (args.storage_def->settings) From d84215d005195f787a493adf6c7093f062fe6b4a Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 21 Feb 2023 16:20:57 +0100 Subject: [PATCH 15/69] Fix style check --- src/Storages/StorageExternalDistributed.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index e07a4554a60..a5c22d0807d 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -26,7 +25,6 @@ namespace DB namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int BAD_ARGUMENTS; } @@ -104,7 +102,7 @@ void registerStorageExternalDistributed(StorageFactory & factory) "engine_name, named_collection and/or description"); auto context = args.getLocalContext(); - const auto & settings = context->getSettingsRef(); + [[maybe_unused]] const auto & settings = context->getSettingsRef(); size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; auto get_addresses = [&](const std::string addresses_expr) { From b15be14792437e6298124fc96e15ab4a40b5d5b4 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 21 Feb 2023 18:48:54 +0100 Subject: [PATCH 16/69] Fix table function remote --- src/Storages/NamedCollectionsHelpers.cpp | 10 ++++++---- src/Storages/NamedCollectionsHelpers.h | 2 +- src/TableFunctions/TableFunctionRemote.cpp | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index 0c7426bb682..648707da027 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -15,7 +15,7 @@ namespace ErrorCodes namespace { - NamedCollectionPtr tryGetNamedCollectionFromASTs(ASTs asts) + NamedCollectionPtr tryGetNamedCollectionFromASTs(ASTs asts, bool throw_unknown_collection) { if (asts.empty()) return nullptr; @@ -25,7 +25,9 @@ namespace return nullptr; const auto & collection_name = identifier->name(); - return NamedCollectionFactory::instance().get(collection_name); + if (throw_unknown_collection) + return NamedCollectionFactory::instance().get(collection_name); + return NamedCollectionFactory::instance().tryGet(collection_name); } std::optional> getKeyValueFromAST(ASTPtr ast) @@ -53,12 +55,12 @@ namespace } -MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts) +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool throw_unknown_collection) { if (asts.empty()) return nullptr; - auto collection = tryGetNamedCollectionFromASTs(asts); + auto collection = tryGetNamedCollectionFromASTs(asts, throw_unknown_collection); if (!collection) return nullptr; diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 5f8f316d023..6e80b360411 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -18,7 +18,7 @@ namespace DB /// Helper function to get named collection for table engine. /// Table engines have collection name as first argument of ast and other arguments are key-value overrides. -MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts); +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool throw_unknown_collection = true); /// Helper function to get named collection for dictionary source. /// Dictionaries have collection name as name argument of dict configuration and other arguments are overrides. MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index dbc718536f2..79f052854ec 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -50,7 +50,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr * For now named collection can be used only for remote as cluster does not require credentials. */ size_t max_args = is_cluster_function ? 4 : 6; - if (auto named_collection = tryGetNamedCollectionWithOverrides(args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(args, false)) { if (is_cluster_function) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Named collection cannot be used for table function cluster"); From d372bf9e128327faced5ec10aad7558c6bb9cdf4 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Feb 2023 11:47:22 +0100 Subject: [PATCH 17/69] Fix test --- src/TableFunctions/TableFunctionRemote.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 79f052854ec..66f04346f16 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -37,7 +37,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr String cluster_name; String cluster_description; - String database, table, username, password; + String database, table, username = "default", password; if (args_func.size() != 1) throw Exception(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); From 0555ca27240c6864a080b60ddaec154f57ebf3e3 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Feb 2023 14:44:47 +0100 Subject: [PATCH 18/69] Review fixes --- src/Access/AccessRights.cpp | 58 +++++++++++++++---- src/Access/Common/AccessFlags.cpp | 15 +++++ src/Access/Common/AccessFlags.h | 1 + src/Access/Common/AccessRightsElement.cpp | 19 +++--- src/Access/Common/AccessRightsElement.h | 8 +-- src/Access/ContextAccess.cpp | 9 ++- .../Access/InterpreterShowGrantsQuery.cpp | 2 +- src/Parsers/Access/ASTGrantQuery.cpp | 11 +++- src/Parsers/Access/ParserGrantQuery.cpp | 10 ++-- .../test_named_collections/test.py | 40 +++++++++++++ 10 files changed, 140 insertions(+), 33 deletions(-) diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 6096612059a..420b1e34db4 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -61,23 +61,24 @@ namespace res.any_database = true; res.any_table = true; res.any_column = true; - res.any_global_with_parameter = true; + res.any_parameter = true; break; } case 1: { if (access_flags.isGlobalWithParameter()) { - res.any_global_with_parameter = false; + res.any_parameter = false; res.parameter = full_name[0]; } else { res.any_database = false; res.database = full_name[0]; + + res.any_table = true; + res.any_column = true; } - res.any_table = true; - res.any_column = true; break; } case 2: @@ -119,10 +120,26 @@ namespace size_t count_elements_with_diff_columns = sorted.countElementsWithDifferenceInColumnOnly(i); if (count_elements_with_diff_columns == 1) { - /// Easy case: one Element is converted to one AccessRightsElement. const auto & element = sorted[i]; if (element.access_flags) - res.emplace_back(element.getResult()); + { + auto per_parameter = element.access_flags.splitIntoParameterTypes(); + if (per_parameter.size() == 1) + { + /// Easy case: one Element is converted to one AccessRightsElement. + res.emplace_back(element.getResult()); + } + else + { + /// Difficult case: one element is converted into multiple AccessRightsElements. + for (const auto & [_, parameter_flags] : per_parameter) + { + auto current_element{element}; + current_element.access_flags = parameter_flags; + res.emplace_back(current_element.getResult()); + } + } + } ++i; } else @@ -146,6 +163,8 @@ namespace { return (element.full_name.size() != 3) || (element.full_name[0] != start_element.full_name[0]) || (element.full_name[1] != start_element.full_name[1]) || (element.grant_option != start_element.grant_option) + || (element.access_flags.isGlobalWithParameter() != start_element.access_flags.isGlobalWithParameter()) + || (element.access_flags.getParameterType() != start_element.access_flags.getParameterType()) || (element.is_partial_revoke != start_element.is_partial_revoke); }); @@ -792,8 +811,13 @@ void AccessRights::grantImplHelper(const AccessRightsElement & element) { assert(!element.is_partial_revoke); assert(!element.grant_option || with_grant_option); - if (!element.any_global_with_parameter) - grantImpl(element.access_flags, element.parameter); + if (element.isGlobalWithParameter()) + { + if (element.any_parameter) + grantImpl(element.access_flags); + else + grantImpl(element.access_flags, element.parameter); + } else if (element.any_database) grantImpl(element.access_flags); else if (element.any_table) @@ -869,8 +893,13 @@ template void AccessRights::revokeImplHelper(const AccessRightsElement & element) { assert(!element.grant_option || grant_option); - if (!element.any_global_with_parameter) - revokeImpl(element.access_flags, element.parameter); + if (element.isGlobalWithParameter()) + { + if (element.any_parameter) + revokeImpl(element.access_flags); + else + revokeImpl(element.access_flags, element.parameter); + } else if (element.any_database) revokeImpl(element.access_flags); else if (element.any_table) @@ -961,8 +990,13 @@ template bool AccessRights::isGrantedImplHelper(const AccessRightsElement & element) const { assert(!element.grant_option || grant_option); - if (!element.any_global_with_parameter) - return isGrantedImpl(element.access_flags, element.parameter); + if (element.isGlobalWithParameter()) + { + if (element.any_parameter) + return isGrantedImpl(element.access_flags); + else + return isGrantedImpl(element.access_flags, element.parameter); + } else if (element.any_database) return isGrantedImpl(element.access_flags); else if (element.any_table) diff --git a/src/Access/Common/AccessFlags.cpp b/src/Access/Common/AccessFlags.cpp index 4f8a9798ec4..8612fc2309e 100644 --- a/src/Access/Common/AccessFlags.cpp +++ b/src/Access/Common/AccessFlags.cpp @@ -363,6 +363,21 @@ bool AccessFlags::isGlobalWithParameter() const return getParameterType() != AccessFlags::NONE; } +std::unordered_map AccessFlags::splitIntoParameterTypes() const +{ + std::unordered_map result; + + auto named_collection_flags = AccessFlags::allNamedCollectionFlags() & *this; + if (named_collection_flags) + result.emplace(ParameterType::NAMED_COLLECTION, named_collection_flags); + + auto other_flags = (~AccessFlags::allNamedCollectionFlags()) & *this; + if (other_flags) + result.emplace(ParameterType::NONE, other_flags); + + return result; +} + AccessFlags::ParameterType AccessFlags::getParameterType() const { if (isEmpty() || !AccessFlags::allGlobalWithParameterFlags().contains(*this)) diff --git a/src/Access/Common/AccessFlags.h b/src/Access/Common/AccessFlags.h index b923b24be47..270ee1c0045 100644 --- a/src/Access/Common/AccessFlags.h +++ b/src/Access/Common/AccessFlags.h @@ -58,6 +58,7 @@ public: NAMED_COLLECTION, }; ParameterType getParameterType() const; + std::unordered_map splitIntoParameterTypes() const; friend bool operator ==(const AccessFlags & left, const AccessFlags & right) { return left.flags == right.flags; } friend bool operator !=(const AccessFlags & left, const AccessFlags & right) { return !(left == right); } diff --git a/src/Access/Common/AccessRightsElement.cpp b/src/Access/Common/AccessRightsElement.cpp index 2f6f1264a65..8d849297246 100644 --- a/src/Access/Common/AccessRightsElement.cpp +++ b/src/Access/Common/AccessRightsElement.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB @@ -26,7 +27,7 @@ namespace result += "ON "; if (element.isGlobalWithParameter()) { - if (element.any_global_with_parameter) + if (element.any_parameter) result += "*"; else result += backQuoteIfNeed(element.parameter); @@ -129,8 +130,10 @@ namespace if (i != elements.size() - 1) { const auto & next_element = elements[i + 1]; - if (element.sameDatabaseAndTable(next_element) && element.sameOptions(next_element)) + if (element.sameDatabaseAndTableAndParameter(next_element) && element.sameOptions(next_element)) + { next_element_uses_same_table_and_options = true; + } } if (!next_element_uses_same_table_and_options) @@ -171,6 +174,7 @@ AccessRightsElement::AccessRightsElement( , any_database(false) , any_table(false) , any_column(false) + , any_parameter(false) { } @@ -195,19 +199,20 @@ AccessRightsElement::AccessRightsElement( , any_database(false) , any_table(false) , any_column(false) + , any_parameter(false) { } void AccessRightsElement::eraseNonGrantable() { - if (!any_column) + if (isGlobalWithParameter() && !any_parameter) + access_flags &= AccessFlags::allFlagsGrantableOnGlobalWithParameterLevel(); + else if (!any_column) access_flags &= AccessFlags::allFlagsGrantableOnColumnLevel(); else if (!any_table) access_flags &= AccessFlags::allFlagsGrantableOnTableLevel(); else if (!any_database) access_flags &= AccessFlags::allFlagsGrantableOnDatabaseLevel(); - else if (!any_global_with_parameter) - access_flags &= AccessFlags::allFlagsGrantableOnGlobalWithParameterLevel(); else access_flags &= AccessFlags::allFlagsGrantableOnGlobalLevel(); } @@ -224,9 +229,9 @@ String AccessRightsElement::toStringWithoutOptions() const { return toStringImpl bool AccessRightsElements::empty() const { return std::all_of(begin(), end(), [](const AccessRightsElement & e) { return e.empty(); }); } -bool AccessRightsElements::sameDatabaseAndTable() const +bool AccessRightsElements::sameDatabaseAndTableAndParameter() const { - return (size() < 2) || std::all_of(std::next(begin()), end(), [this](const AccessRightsElement & e) { return e.sameDatabaseAndTable(front()); }); + return (size() < 2) || std::all_of(std::next(begin()), end(), [this](const AccessRightsElement & e) { return e.sameDatabaseAndTableAndParameter(front()); }); } bool AccessRightsElements::sameOptions() const diff --git a/src/Access/Common/AccessRightsElement.h b/src/Access/Common/AccessRightsElement.h index e881767b185..247b1e4e455 100644 --- a/src/Access/Common/AccessRightsElement.h +++ b/src/Access/Common/AccessRightsElement.h @@ -20,7 +20,7 @@ struct AccessRightsElement bool any_database = true; bool any_table = true; bool any_column = true; - bool any_global_with_parameter = true; + bool any_parameter = false; bool grant_option = false; bool is_partial_revoke = false; @@ -53,11 +53,11 @@ struct AccessRightsElement friend bool operator==(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() == right.toTuple(); } friend bool operator!=(const AccessRightsElement & left, const AccessRightsElement & right) { return !(left == right); } - bool sameDatabaseAndTable(const AccessRightsElement & other) const + bool sameDatabaseAndTableAndParameter(const AccessRightsElement & other) const { return (database == other.database) && (any_database == other.any_database) && (table == other.table) && (any_table == other.any_table) - && (parameter == other.parameter) && (any_global_with_parameter == other.any_global_with_parameter) + && (parameter == other.parameter) && (any_parameter == other.any_parameter) && (access_flags.getParameterType() == other.access_flags.getParameterType()) && (isGlobalWithParameter() == other.isGlobalWithParameter()); } @@ -91,7 +91,7 @@ public: using Base::Base; bool empty() const; - bool sameDatabaseAndTable() const; + bool sameDatabaseAndTableAndParameter() const; bool sameOptions() const; /// Resets flags which cannot be granted. diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 4abb161fd80..5f84326d210 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -610,8 +610,13 @@ template bool ContextAccess::checkAccessImplHelper(const AccessRightsElement & element) const { assert(!element.grant_option || grant_option); - if (!element.any_global_with_parameter) - return checkAccessImpl(element.access_flags, element.parameter); + if (element.isGlobalWithParameter()) + { + if (element.any_parameter) + return checkAccessImpl(element.access_flags); + else + return checkAccessImpl(element.access_flags, element.parameter); + } else if (element.any_database) return checkAccessImpl(element.access_flags); else if (element.any_table) diff --git a/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp b/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp index 2ff6d44e041..56fbb34a577 100644 --- a/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp @@ -48,7 +48,7 @@ namespace if (current_query) { const auto & prev_element = current_query->access_rights_elements.back(); - bool continue_with_current_query = element.sameDatabaseAndTable(prev_element) && element.sameOptions(prev_element); + bool continue_with_current_query = element.sameDatabaseAndTableAndParameter(prev_element) && element.sameOptions(prev_element); if (!continue_with_current_query) current_query = nullptr; } diff --git a/src/Parsers/Access/ASTGrantQuery.cpp b/src/Parsers/Access/ASTGrantQuery.cpp index f1a1f9184a5..71eff476965 100644 --- a/src/Parsers/Access/ASTGrantQuery.cpp +++ b/src/Parsers/Access/ASTGrantQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB @@ -32,7 +33,7 @@ namespace settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ON " << (settings.hilite ? IAST::hilite_none : ""); if (element.isGlobalWithParameter()) { - if (element.any_global_with_parameter) + if (element.any_parameter) settings.ostr << "*"; else settings.ostr << backQuoteIfNeed(element.parameter); @@ -56,6 +57,8 @@ namespace void formatElementsWithoutOptions(const AccessRightsElements & elements, const IAST::FormatSettings & settings) { bool no_output = true; + auto * log = &Poco::Logger::get("kssenii"); + LOG_TEST(log, "kssenii 0 - {}", elements.size()); for (size_t i = 0; i != elements.size(); ++i) { const auto & element = elements[i]; @@ -77,12 +80,16 @@ namespace if (i != elements.size() - 1) { const auto & next_element = elements[i + 1]; - if (element.sameDatabaseAndTable(next_element)) + if (element.sameDatabaseAndTableAndParameter(next_element)) + { + LOG_TEST(log, "kssenii 1"); next_element_on_same_db_and_table = true; + } } if (!next_element_on_same_db_and_table) { + LOG_TEST(log, "kssenii 2"); settings.ostr << " "; formatONClause(element, settings); } diff --git a/src/Parsers/Access/ParserGrantQuery.cpp b/src/Parsers/Access/ParserGrantQuery.cpp index d58599ead56..28a1846df74 100644 --- a/src/Parsers/Access/ParserGrantQuery.cpp +++ b/src/Parsers/Access/ParserGrantQuery.cpp @@ -124,7 +124,7 @@ namespace return false; String database_name, table_name, parameter; - bool any_database = false, any_table = false, any_global_with_parameter = true; + bool any_database = false, any_table = false, any_parameter = false; size_t is_global_with_parameter = 0; for (const auto & elem : access_and_columns) @@ -141,11 +141,11 @@ namespace ASTPtr parameter_ast; if (ParserToken{TokenType::Asterisk}.ignore(pos, expected)) { - any_global_with_parameter = true; + any_parameter = true; } else if (ParserIdentifier{}.parse(pos, parameter_ast, expected)) { - any_global_with_parameter = false; + any_parameter = false; parameter = getIdentifierName(parameter_ast); } else @@ -167,7 +167,7 @@ namespace element.any_database = any_database; element.database = database_name; element.any_table = any_table; - element.any_global_with_parameter = any_global_with_parameter; + element.any_parameter = any_parameter; element.table = table_name; element.parameter = parameter; res_elements.emplace_back(std::move(element)); @@ -202,7 +202,7 @@ namespace throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted on the table level", old_flags.toString()); else if (!element.any_database) throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted on the database level", old_flags.toString()); - else if (!element.any_global_with_parameter) + else if (!element.any_parameter) throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted on the global with parameter level", old_flags.toString()); else throw Exception(ErrorCodes::INVALID_GRANT, "{} cannot be granted", old_flags.toString()); diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index ba403d3f48b..4e9b8324aea 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -150,6 +150,10 @@ def test_granular_access_show_query(cluster): == node.query("select name from system.named_collections", user="kek").strip() ) + # check: + # GRANT show named collections ON * + # REVOKE show named collections ON collection + node.query("DROP USER IF EXISTS koko") node.query("CREATE USER koko") node.query("GRANT select ON *.* TO koko") @@ -167,6 +171,35 @@ def test_granular_access_show_query(cluster): == node.query("select name from system.named_collections", user="koko").strip() ) + node.query("REVOKE show named collections ON collection1 FROM koko;") + assert ( + "collection2" + == node.query("select name from system.named_collections", user="koko").strip() + ) + node.restart_clickhouse() + assert ( + "collection2" + == node.query("select name from system.named_collections", user="koko").strip() + ) + node.query("REVOKE show named collections ON collection2 FROM koko;") + assert ( + "" == node.query("select * from system.named_collections", user="koko").strip() + ) + + # check: + # GRANT show named collections ON collection + # REVOKE show named collections ON * + + node.query("GRANT show named collections ON collection2 TO koko") + assert ( + "collection2" + == node.query("select name from system.named_collections", user="koko").strip() + ) + node.query("REVOKE show named collections ON * FROM koko;") + assert ( + "" == node.query("select * from system.named_collections", user="koko").strip() + ) + node.query("DROP NAMED COLLECTION collection2") @@ -219,6 +252,13 @@ def test_granular_access_create_alter_drop_query(cluster): "select collection['key1'] from system.named_collections where name = 'collection2'" ).strip() ) + node.query("REVOKE create named collection ON collection2 FROM kek") + assert ( + "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant ALTER NAMED COLLECTION" + in node.query_and_get_error( + "ALTER NAMED COLLECTION collection2 SET key1=3", user="kek" + ) + ) assert ( "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant DROP NAMED COLLECTION" From 8b40723fedf11b162864e56abf9c337e1b775a04 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Feb 2023 16:19:36 +0100 Subject: [PATCH 19/69] Better fix --- .../test_named_collections/test.py | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index f219fa7a632..899ae130404 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -228,6 +228,54 @@ def test_granular_access_show_query(cluster): node.query("DROP NAMED COLLECTION collection2") +def test_show_grants(cluster): + node = cluster.instances["node"] + node.query("DROP USER IF EXISTS koko") + node.query("CREATE USER koko") + node.query("GRANT CREATE NAMED COLLECTION ON name1 TO koko") + node.query("GRANT select ON name1.* TO koko") + assert ( + "GRANT SELECT ON name1.* TO koko\nGRANT CREATE NAMED COLLECTION ON name1 TO koko" + in node.query("SHOW GRANTS FOR koko;").strip() + ) + + node.query("DROP USER IF EXISTS koko") + node.query("CREATE USER koko") + node.query("GRANT CREATE NAMED COLLECTION ON name1 TO koko") + node.query("GRANT select ON name1 TO koko") + assert ( + "GRANT SELECT ON default.name1 TO koko\nGRANT CREATE NAMED COLLECTION ON name1 TO koko" + in node.query("SHOW GRANTS FOR koko;").strip() + ) + + node.query("DROP USER IF EXISTS koko") + node.query("CREATE USER koko") + node.query("GRANT select ON name1 TO koko") + node.query("GRANT CREATE NAMED COLLECTION ON name1 TO koko") + assert ( + "GRANT SELECT ON default.name1 TO koko\nGRANT CREATE NAMED COLLECTION ON name1 TO koko" + in node.query("SHOW GRANTS FOR koko;").strip() + ) + + node.query("DROP USER IF EXISTS koko") + node.query("CREATE USER koko") + node.query("GRANT select ON *.* TO koko") + node.query("GRANT CREATE NAMED COLLECTION ON * TO koko") + assert ( + "GRANT SELECT ON *.* TO koko\nGRANT CREATE NAMED COLLECTION ON * TO koko" + in node.query("SHOW GRANTS FOR koko;").strip() + ) + + node.query("DROP USER IF EXISTS koko") + node.query("CREATE USER koko") + node.query("GRANT CREATE NAMED COLLECTION ON * TO koko") + node.query("GRANT select ON *.* TO koko") + assert ( + "GRANT SELECT ON *.* TO koko\nGRANT CREATE NAMED COLLECTION ON * TO koko" + in node.query("SHOW GRANTS FOR koko;").strip() + ) + + def test_granular_access_create_alter_drop_query(cluster): node = cluster.instances["node"] node.query("DROP USER IF EXISTS kek") @@ -277,7 +325,7 @@ def test_granular_access_create_alter_drop_query(cluster): "select collection['key1'] from system.named_collections where name = 'collection2'" ).strip() ) - node.query("REVOKE create named collection ON collection2 FROM kek") + node.query("REVOKE alter named collection ON collection2 FROM kek") assert ( "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant ALTER NAMED COLLECTION" in node.query_and_get_error( From 357ffcb61f4e568642a4e8270d37c883e22c203f Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Feb 2023 16:49:25 +0100 Subject: [PATCH 20/69] More review fixes --- src/Access/Common/AccessType.h | 9 +++++---- .../integration/test_named_collections/test.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index ba39a984358..84ec93d58f6 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -89,7 +89,7 @@ enum class AccessType M(CREATE_TEMPORARY_TABLE, "", GLOBAL, CREATE) /* allows to create and manipulate temporary tables; implicitly enabled by the grant CREATE_TABLE on any table */ \ M(CREATE_FUNCTION, "", GLOBAL, CREATE) /* allows to execute CREATE FUNCTION */ \ - M(CREATE_NAMED_COLLECTION, "", NAMED_COLLECTION, CREATE) /* allows to execute CREATE NAMED COLLECTION */ \ + M(CREATE_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) /* allows to execute CREATE NAMED COLLECTION */ \ M(CREATE, "", GROUP, ALL) /* allows to execute {CREATE|ATTACH} */ \ \ M(DROP_DATABASE, "", DATABASE, DROP) /* allows to execute {DROP|DETACH} DATABASE */\ @@ -98,7 +98,7 @@ enum class AccessType implicitly enabled by the grant DROP_TABLE */\ M(DROP_DICTIONARY, "", DICTIONARY, DROP) /* allows to execute {DROP|DETACH} DICTIONARY */\ M(DROP_FUNCTION, "", GLOBAL, DROP) /* allows to execute DROP FUNCTION */\ - M(DROP_NAMED_COLLECTION, "", NAMED_COLLECTION, DROP) /* allows to execute DROP NAMED COLLECTION */\ + M(DROP_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) /* allows to execute DROP NAMED COLLECTION */\ M(DROP, "", GROUP, ALL) /* allows to execute {DROP|DETACH} */\ \ M(TRUNCATE, "TRUNCATE TABLE", TABLE, ALL) \ @@ -134,9 +134,10 @@ enum class AccessType M(SHOW_QUOTAS, "SHOW CREATE QUOTA", GLOBAL, SHOW_ACCESS) \ M(SHOW_SETTINGS_PROFILES, "SHOW PROFILES, SHOW CREATE SETTINGS PROFILE, SHOW CREATE PROFILE", GLOBAL, SHOW_ACCESS) \ M(SHOW_ACCESS, "", GROUP, ACCESS_MANAGEMENT) \ - M(SHOW_NAMED_COLLECTIONS, "SHOW NAMED COLLECTIONS", NAMED_COLLECTION, ACCESS_MANAGEMENT) \ - M(SHOW_NAMED_COLLECTIONS_SECRETS, "SHOW NAMED COLLECTIONS SECRETS", NAMED_COLLECTION, ACCESS_MANAGEMENT) \ M(ACCESS_MANAGEMENT, "", GROUP, ALL) \ + M(SHOW_NAMED_COLLECTIONS, "SHOW NAMED COLLECTIONS", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) \ + M(SHOW_NAMED_COLLECTIONS_SECRETS, "SHOW NAMED COLLECTIONS SECRETS", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) \ + M(NAMED_COLLECTION_CONTROL, "", NAMED_COLLECTION, ALL) \ \ M(SYSTEM_SHUTDOWN, "SYSTEM KILL, SHUTDOWN", GLOBAL, SYSTEM) \ M(SYSTEM_DROP_DNS_CACHE, "SYSTEM DROP DNS, DROP DNS CACHE, DROP DNS", GLOBAL, SYSTEM_DROP_CACHE) \ diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 899ae130404..1e9995a0603 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -275,6 +275,24 @@ def test_show_grants(cluster): in node.query("SHOW GRANTS FOR koko;").strip() ) + node.query("DROP USER IF EXISTS koko") + node.query("CREATE USER koko") + node.query("GRANT CREATE NAMED COLLECTION ON * TO koko") + node.query("GRANT select ON * TO koko") + assert ( + "GRANT CREATE NAMED COLLECTION ON * TO koko\nGRANT SELECT ON default.* TO koko" + in node.query("SHOW GRANTS FOR koko;").strip() + ) + + node.query("DROP USER IF EXISTS koko") + node.query("CREATE USER koko") + node.query("GRANT select ON * TO koko") + node.query("GRANT CREATE NAMED COLLECTION ON * TO koko") + assert ( + "GRANT SELECT ON default.* TO koko\nGRANT CREATE NAMED COLLECTION ON * TO koko" + in node.query("SHOW GRANTS FOR koko;").strip() + ) + def test_granular_access_create_alter_drop_query(cluster): node = cluster.instances["node"] From 6224ca9cd68c03a7b7f257baa235e7f0cf1d7311 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Feb 2023 17:08:05 +0100 Subject: [PATCH 21/69] Fix --- src/Access/AccessRights.cpp | 8 ++++++++ src/Storages/System/StorageSystemNamedCollections.cpp | 1 - tests/integration/test_named_collections/test.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 420b1e34db4..424135acb81 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -219,11 +219,19 @@ namespace } }; + /** + * Levels: + * 1. GLOBAL + * 2. DATABASE_LEVEL 2. GLOBAL_WITH_PARAMETER (parameter example: named collection) + * 3. TABLE_LEVEL + * 4. COLUMN_LEVEL + */ enum Level { GLOBAL_LEVEL, DATABASE_LEVEL, + GLOBAL_WITH_PARAMETER = DATABASE_LEVEL, TABLE_LEVEL, COLUMN_LEVEL, }; diff --git a/src/Storages/System/StorageSystemNamedCollections.cpp b/src/Storages/System/StorageSystemNamedCollections.cpp index baba93aa3d5..1d94b0afd1b 100644 --- a/src/Storages/System/StorageSystemNamedCollections.cpp +++ b/src/Storages/System/StorageSystemNamedCollections.cpp @@ -10,7 +10,6 @@ #include #include #include -#include namespace DB diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 1e9995a0603..6a53f7e0a58 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -289,7 +289,7 @@ def test_show_grants(cluster): node.query("GRANT select ON * TO koko") node.query("GRANT CREATE NAMED COLLECTION ON * TO koko") assert ( - "GRANT SELECT ON default.* TO koko\nGRANT CREATE NAMED COLLECTION ON * TO koko" + "GRANT CREATE NAMED COLLECTION ON * TO koko\nGRANT SELECT ON default.* TO koko" in node.query("SHOW GRANTS FOR koko;").strip() ) From d55fed77e365514ffb77a99ec626bd5cc3eefe34 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Feb 2023 18:02:31 +0100 Subject: [PATCH 22/69] Fix --- src/Access/AccessRights.cpp | 7 ++++--- src/Access/Common/AccessRightsElement.h | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 424135acb81..37597552a41 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -68,14 +68,15 @@ namespace { if (access_flags.isGlobalWithParameter()) { - res.any_parameter = false; res.parameter = full_name[0]; + res.any_parameter = false; + res.any_database = false; } else { - res.any_database = false; res.database = full_name[0]; - + res.any_database = false; + res.any_parameter = false; res.any_table = true; res.any_column = true; } diff --git a/src/Access/Common/AccessRightsElement.h b/src/Access/Common/AccessRightsElement.h index 247b1e4e455..96850f0880e 100644 --- a/src/Access/Common/AccessRightsElement.h +++ b/src/Access/Common/AccessRightsElement.h @@ -49,7 +49,7 @@ struct AccessRightsElement bool empty() const { return !access_flags || (!any_column && columns.empty()); } - auto toTuple() const { return std::tie(access_flags, any_database, database, any_table, table, any_column, columns, grant_option, is_partial_revoke); } + auto toTuple() const { return std::tie(access_flags, any_database, database, any_table, table, any_column, columns, any_parameter, parameter, grant_option, is_partial_revoke); } friend bool operator==(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() == right.toTuple(); } friend bool operator!=(const AccessRightsElement & left, const AccessRightsElement & right) { return !(left == right); } From 95f414200fd2a99ae5f9a9173784b70d488b403b Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Feb 2023 18:05:03 +0100 Subject: [PATCH 23/69] Update .reference --- .../queries/0_stateless/01271_show_privileges.reference | 9 +++++---- .../0_stateless/02117_show_create_table_system.reference | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index aa43c91ed72..03661d2469f 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -52,14 +52,14 @@ CREATE VIEW [] VIEW CREATE CREATE DICTIONARY [] DICTIONARY CREATE CREATE TEMPORARY TABLE [] GLOBAL CREATE CREATE FUNCTION [] GLOBAL CREATE -CREATE NAMED COLLECTION [] NAMED_COLLECTION CREATE +CREATE NAMED COLLECTION [] NAMED_COLLECTION NAMED COLLECTION CONTROL CREATE [] \N ALL DROP DATABASE [] DATABASE DROP DROP TABLE [] TABLE DROP DROP VIEW [] VIEW DROP DROP DICTIONARY [] DICTIONARY DROP DROP FUNCTION [] GLOBAL DROP -DROP NAMED COLLECTION [] NAMED_COLLECTION DROP +DROP NAMED COLLECTION [] NAMED_COLLECTION NAMED COLLECTION CONTROL DROP [] \N ALL TRUNCATE ['TRUNCATE TABLE'] TABLE ALL OPTIMIZE ['OPTIMIZE TABLE'] TABLE ALL @@ -89,9 +89,10 @@ SHOW ROW POLICIES ['SHOW POLICIES','SHOW CREATE ROW POLICY','SHOW CREATE POLICY' SHOW QUOTAS ['SHOW CREATE QUOTA'] GLOBAL SHOW ACCESS SHOW SETTINGS PROFILES ['SHOW PROFILES','SHOW CREATE SETTINGS PROFILE','SHOW CREATE PROFILE'] GLOBAL SHOW ACCESS SHOW ACCESS [] \N ACCESS MANAGEMENT -SHOW NAMED COLLECTIONS ['SHOW NAMED COLLECTIONS'] NAMED_COLLECTION ACCESS MANAGEMENT -SHOW NAMED COLLECTIONS SECRETS ['SHOW NAMED COLLECTIONS SECRETS'] NAMED_COLLECTION ACCESS MANAGEMENT ACCESS MANAGEMENT [] \N ALL +SHOW NAMED COLLECTIONS ['SHOW NAMED COLLECTIONS'] NAMED_COLLECTION NAMED COLLECTION CONTROL +SHOW NAMED COLLECTIONS SECRETS ['SHOW NAMED COLLECTIONS SECRETS'] NAMED_COLLECTION NAMED COLLECTION CONTROL +NAMED COLLECTION CONTROL [] NAMED_COLLECTION ALL SYSTEM SHUTDOWN ['SYSTEM KILL','SHUTDOWN'] GLOBAL SYSTEM SYSTEM DROP DNS CACHE ['SYSTEM DROP DNS','DROP DNS CACHE','DROP DNS'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP MARK CACHE ['SYSTEM DROP MARK','DROP MARK CACHE','DROP MARKS'] GLOBAL SYSTEM DROP CACHE diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 4c00b65d269..90e018703b1 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -289,7 +289,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'ACCESS MANAGEMENT' = 91, 'SHOW NAMED COLLECTIONS' = 92, 'SHOW NAMED COLLECTIONS SECRETS' = 93, 'NAMED COLLECTION CONTROL' = 94, 'SYSTEM SHUTDOWN' = 95, 'SYSTEM DROP DNS CACHE' = 96, 'SYSTEM DROP MARK CACHE' = 97, 'SYSTEM DROP UNCOMPRESSED CACHE' = 98, 'SYSTEM DROP MMAP CACHE' = 99, 'SYSTEM DROP QUERY CACHE' = 100, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 101, 'SYSTEM DROP FILESYSTEM CACHE' = 102, 'SYSTEM DROP SCHEMA CACHE' = 103, 'SYSTEM DROP S3 CLIENT CACHE' = 104, 'SYSTEM DROP CACHE' = 105, 'SYSTEM RELOAD CONFIG' = 106, 'SYSTEM RELOAD USERS' = 107, 'SYSTEM RELOAD SYMBOLS' = 108, 'SYSTEM RELOAD DICTIONARY' = 109, 'SYSTEM RELOAD MODEL' = 110, 'SYSTEM RELOAD FUNCTION' = 111, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 112, 'SYSTEM RELOAD' = 113, 'SYSTEM RESTART DISK' = 114, 'SYSTEM MERGES' = 115, 'SYSTEM TTL MERGES' = 116, 'SYSTEM FETCHES' = 117, 'SYSTEM MOVES' = 118, 'SYSTEM DISTRIBUTED SENDS' = 119, 'SYSTEM REPLICATED SENDS' = 120, 'SYSTEM SENDS' = 121, 'SYSTEM REPLICATION QUEUES' = 122, 'SYSTEM DROP REPLICA' = 123, 'SYSTEM SYNC REPLICA' = 124, 'SYSTEM RESTART REPLICA' = 125, 'SYSTEM RESTORE REPLICA' = 126, 'SYSTEM WAIT LOADING PARTS' = 127, 'SYSTEM SYNC DATABASE REPLICA' = 128, 'SYSTEM SYNC TRANSACTION LOG' = 129, 'SYSTEM SYNC FILE CACHE' = 130, 'SYSTEM FLUSH DISTRIBUTED' = 131, 'SYSTEM FLUSH LOGS' = 132, 'SYSTEM FLUSH' = 133, 'SYSTEM THREAD FUZZER' = 134, 'SYSTEM UNFREEZE' = 135, 'SYSTEM' = 136, 'dictGet' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -570,10 +570,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'ACCESS MANAGEMENT' = 91, 'SHOW NAMED COLLECTIONS' = 92, 'SHOW NAMED COLLECTIONS SECRETS' = 93, 'NAMED COLLECTION CONTROL' = 94, 'SYSTEM SHUTDOWN' = 95, 'SYSTEM DROP DNS CACHE' = 96, 'SYSTEM DROP MARK CACHE' = 97, 'SYSTEM DROP UNCOMPRESSED CACHE' = 98, 'SYSTEM DROP MMAP CACHE' = 99, 'SYSTEM DROP QUERY CACHE' = 100, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 101, 'SYSTEM DROP FILESYSTEM CACHE' = 102, 'SYSTEM DROP SCHEMA CACHE' = 103, 'SYSTEM DROP S3 CLIENT CACHE' = 104, 'SYSTEM DROP CACHE' = 105, 'SYSTEM RELOAD CONFIG' = 106, 'SYSTEM RELOAD USERS' = 107, 'SYSTEM RELOAD SYMBOLS' = 108, 'SYSTEM RELOAD DICTIONARY' = 109, 'SYSTEM RELOAD MODEL' = 110, 'SYSTEM RELOAD FUNCTION' = 111, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 112, 'SYSTEM RELOAD' = 113, 'SYSTEM RESTART DISK' = 114, 'SYSTEM MERGES' = 115, 'SYSTEM TTL MERGES' = 116, 'SYSTEM FETCHES' = 117, 'SYSTEM MOVES' = 118, 'SYSTEM DISTRIBUTED SENDS' = 119, 'SYSTEM REPLICATED SENDS' = 120, 'SYSTEM SENDS' = 121, 'SYSTEM REPLICATION QUEUES' = 122, 'SYSTEM DROP REPLICA' = 123, 'SYSTEM SYNC REPLICA' = 124, 'SYSTEM RESTART REPLICA' = 125, 'SYSTEM RESTORE REPLICA' = 126, 'SYSTEM WAIT LOADING PARTS' = 127, 'SYSTEM SYNC DATABASE REPLICA' = 128, 'SYSTEM SYNC TRANSACTION LOG' = 129, 'SYSTEM SYNC FILE CACHE' = 130, 'SYSTEM FLUSH DISTRIBUTED' = 131, 'SYSTEM FLUSH LOGS' = 132, 'SYSTEM FLUSH' = 133, 'SYSTEM THREAD FUZZER' = 134, 'SYSTEM UNFREEZE' = 135, 'SYSTEM' = 136, 'dictGet' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5, 'NAMED_COLLECTION' = 6)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'ACCESS MANAGEMENT' = 91, 'SHOW NAMED COLLECTIONS' = 92, 'SHOW NAMED COLLECTIONS SECRETS' = 93, 'NAMED COLLECTION CONTROL' = 94, 'SYSTEM SHUTDOWN' = 95, 'SYSTEM DROP DNS CACHE' = 96, 'SYSTEM DROP MARK CACHE' = 97, 'SYSTEM DROP UNCOMPRESSED CACHE' = 98, 'SYSTEM DROP MMAP CACHE' = 99, 'SYSTEM DROP QUERY CACHE' = 100, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 101, 'SYSTEM DROP FILESYSTEM CACHE' = 102, 'SYSTEM DROP SCHEMA CACHE' = 103, 'SYSTEM DROP S3 CLIENT CACHE' = 104, 'SYSTEM DROP CACHE' = 105, 'SYSTEM RELOAD CONFIG' = 106, 'SYSTEM RELOAD USERS' = 107, 'SYSTEM RELOAD SYMBOLS' = 108, 'SYSTEM RELOAD DICTIONARY' = 109, 'SYSTEM RELOAD MODEL' = 110, 'SYSTEM RELOAD FUNCTION' = 111, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 112, 'SYSTEM RELOAD' = 113, 'SYSTEM RESTART DISK' = 114, 'SYSTEM MERGES' = 115, 'SYSTEM TTL MERGES' = 116, 'SYSTEM FETCHES' = 117, 'SYSTEM MOVES' = 118, 'SYSTEM DISTRIBUTED SENDS' = 119, 'SYSTEM REPLICATED SENDS' = 120, 'SYSTEM SENDS' = 121, 'SYSTEM REPLICATION QUEUES' = 122, 'SYSTEM DROP REPLICA' = 123, 'SYSTEM SYNC REPLICA' = 124, 'SYSTEM RESTART REPLICA' = 125, 'SYSTEM RESTORE REPLICA' = 126, 'SYSTEM WAIT LOADING PARTS' = 127, 'SYSTEM SYNC DATABASE REPLICA' = 128, 'SYSTEM SYNC TRANSACTION LOG' = 129, 'SYSTEM SYNC FILE CACHE' = 130, 'SYSTEM FLUSH DISTRIBUTED' = 131, 'SYSTEM FLUSH LOGS' = 132, 'SYSTEM FLUSH' = 133, 'SYSTEM THREAD FUZZER' = 134, 'SYSTEM UNFREEZE' = 135, 'SYSTEM' = 136, 'dictGet' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' From 536446d526852fcf5dd5fdc8a9abcc4c95246078 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Feb 2023 18:09:00 +0100 Subject: [PATCH 24/69] Remove debug logs --- src/Access/Common/AccessRightsElement.cpp | 1 - src/Parsers/Access/ASTGrantQuery.cpp | 5 ----- 2 files changed, 6 deletions(-) diff --git a/src/Access/Common/AccessRightsElement.cpp b/src/Access/Common/AccessRightsElement.cpp index 8d849297246..81cebd68b4c 100644 --- a/src/Access/Common/AccessRightsElement.cpp +++ b/src/Access/Common/AccessRightsElement.cpp @@ -1,7 +1,6 @@ #include #include #include -#include namespace DB diff --git a/src/Parsers/Access/ASTGrantQuery.cpp b/src/Parsers/Access/ASTGrantQuery.cpp index 71eff476965..44d727c7177 100644 --- a/src/Parsers/Access/ASTGrantQuery.cpp +++ b/src/Parsers/Access/ASTGrantQuery.cpp @@ -2,7 +2,6 @@ #include #include #include -#include namespace DB @@ -57,8 +56,6 @@ namespace void formatElementsWithoutOptions(const AccessRightsElements & elements, const IAST::FormatSettings & settings) { bool no_output = true; - auto * log = &Poco::Logger::get("kssenii"); - LOG_TEST(log, "kssenii 0 - {}", elements.size()); for (size_t i = 0; i != elements.size(); ++i) { const auto & element = elements[i]; @@ -82,14 +79,12 @@ namespace const auto & next_element = elements[i + 1]; if (element.sameDatabaseAndTableAndParameter(next_element)) { - LOG_TEST(log, "kssenii 1"); next_element_on_same_db_and_table = true; } } if (!next_element_on_same_db_and_table) { - LOG_TEST(log, "kssenii 2"); settings.ostr << " "; formatONClause(element, settings); } From 96b57c8a6594026a32d5e94fea3a49e8b19a2dc0 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Feb 2023 20:38:40 +0100 Subject: [PATCH 25/69] Better support for equal keys --- src/Storages/NamedCollectionsHelpers.h | 38 +++++++++++++++++++--- src/Storages/StorageMongoDB.cpp | 16 +++------ src/Storages/StorageMySQL.cpp | 12 +++---- src/Storages/StoragePostgreSQL.cpp | 14 ++++---- src/TableFunctions/TableFunctionRemote.cpp | 10 +++--- 5 files changed, 57 insertions(+), 33 deletions(-) diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 6e80b360411..085e21937ee 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -25,12 +25,42 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(const Poco::Util::A HTTPHeaderEntries getHeadersFromNamedCollection(const NamedCollection & collection); -template , - typename OptionalKeys = std::unordered_set> +struct ExternalDatabaseEqualKeysSet +{ + static constexpr std::array, 3> equal_keys{ + std::pair{"username", "user"}, std::pair{"database", "db"}, std::pair{"hostname", "host"}}; +}; +struct MongoDBEqualKeysSet +{ + static constexpr std::array, 4> equal_keys{ + std::pair{"username", "user"}, std::pair{"database", "db"}, std::pair{"hostname", "host"}, std::pair{"table", "collection"}}; +}; + +template struct ValidateKeysCmp +{ + constexpr bool operator()(const auto & lhs, const auto & rhs) const + { + if (lhs == rhs) + return true; + + for (const auto & equal : EqualKeys::equal_keys) + { + if (((equal.first == lhs) && (equal.second == rhs)) || ((equal.first == rhs) && (equal.second == lhs))) + return true; + } + return false; + } +}; + +template using ValidateKeysMultiset = std::unordered_multiset, ValidateKeysCmp>; +using ValidateKeysSet = std::unordered_multiset>; + + +template void validateNamedCollection( const NamedCollection & collection, - const RequiredKeys & required_keys, - const OptionalKeys & optional_keys, + const Keys & required_keys, + const Keys & optional_keys, const std::vector & optional_regex_keys = {}) { NamedCollection::Keys keys = collection.getKeys(); diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index 2cb85878000..59ecab03bd8 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace DB { @@ -171,13 +172,6 @@ SinkToStoragePtr StorageMongoDB::write(const ASTPtr & /* query */, const Storage return std::make_shared(collection_name, database_name, metadata_snapshot, connection); } -struct KeysCmp -{ - constexpr bool operator()(const auto & lhs, const auto & rhs) const - { - return lhs == rhs || ((lhs == "table") && (rhs == "collection")) || ((rhs == "table") && (lhs == "collection")); - } -}; StorageMongoDB::Configuration StorageMongoDB::getConfiguration(ASTs engine_args, ContextPtr context) { Configuration configuration; @@ -186,14 +180,14 @@ StorageMongoDB::Configuration StorageMongoDB::getConfiguration(ASTs engine_args, { validateNamedCollection( *named_collection, - std::unordered_multiset, KeysCmp>{"host", "port", "user", "password", "database", "collection", "table"}, + ValidateKeysMultiset{"host", "port", "user", "username", "password", "database", "db", "collection", "table"}, {"options"}); - configuration.host = named_collection->get("host"); + configuration.host = named_collection->getOrDefault("host", named_collection->getOrDefault("hostname", "")); configuration.port = static_cast(named_collection->get("port")); - configuration.username = named_collection->get("user"); + configuration.username = named_collection->getOrDefault("user", named_collection->getOrDefault("username", "")); configuration.password = named_collection->get("password"); - configuration.database = named_collection->get("database"); + configuration.database = named_collection->getOrDefault("database", named_collection->getOrDefault("db", "")); configuration.table = named_collection->getOrDefault("collection", named_collection->getOrDefault("table", "")); configuration.options = named_collection->getOrDefault("options", ""); } diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index bc39e76be29..6bc9232a29a 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -241,27 +241,27 @@ StorageMySQL::Configuration StorageMySQL::processNamedCollectionResult( { StorageMySQL::Configuration configuration; - std::unordered_set optional_arguments = {"replace_query", "on_duplicate_clause", "addresses_expr", "host", "port"}; + ValidateKeysMultiset optional_arguments = {"replace_query", "on_duplicate_clause", "addresses_expr", "host", "hostname", "port"}; auto mysql_settings = storage_settings.all(); for (const auto & setting : mysql_settings) optional_arguments.insert(setting.getName()); - std::unordered_set required_arguments = {"user", "password", "database", "table"}; + ValidateKeysMultiset required_arguments = {"user", "username", "password", "database", "db", "table"}; if (require_table) required_arguments.insert("table"); - validateNamedCollection(named_collection, required_arguments, optional_arguments); + validateNamedCollection>(named_collection, required_arguments, optional_arguments); configuration.addresses_expr = named_collection.getOrDefault("addresses_expr", ""); if (configuration.addresses_expr.empty()) { - configuration.host = named_collection.get("host"); + configuration.host = named_collection.getOrDefault("host", named_collection.getOrDefault("hostname", "")); configuration.port = static_cast(named_collection.get("port")); configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; } - configuration.username = named_collection.get("user"); + configuration.username = named_collection.getOrDefault("username", named_collection.getOrDefault("user", "")); configuration.password = named_collection.get("password"); - configuration.database = named_collection.get("database"); + configuration.database = named_collection.getOrDefault("db", named_collection.getOrDefault("database", "")); if (require_table) configuration.table = named_collection.get("table"); configuration.replace_query = named_collection.getOrDefault("replace_query", false); diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 3fa0c137f7b..e736f9edc18 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -390,24 +390,24 @@ SinkToStoragePtr StoragePostgreSQL::write( StoragePostgreSQL::Configuration StoragePostgreSQL::processNamedCollectionResult(const NamedCollection & named_collection, bool require_table) { StoragePostgreSQL::Configuration configuration; - std::unordered_set required_arguments = {"user", "password", "database", "table"}; + ValidateKeysMultiset required_arguments = {"user", "username", "password", "database", "db", "table"}; if (require_table) required_arguments.insert("table"); - validateNamedCollection( - named_collection, required_arguments, - {"schema", "on_conflict", "addresses_expr", "host", "port"}); + + validateNamedCollection>( + named_collection, required_arguments, {"schema", "on_conflict", "addresses_expr", "host", "hostname", "port"}); configuration.addresses_expr = named_collection.getOrDefault("addresses_expr", ""); if (configuration.addresses_expr.empty()) { - configuration.host = named_collection.get("host"); + configuration.host = named_collection.getOrDefault("host", named_collection.getOrDefault("hostname", "")); configuration.port = static_cast(named_collection.get("port")); configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; } - configuration.username = named_collection.get("user"); + configuration.username = named_collection.getOrDefault("username", named_collection.getOrDefault("user", "")); configuration.password = named_collection.get("password"); - configuration.database = named_collection.get("database"); + configuration.database = named_collection.getOrDefault("db", named_collection.getOrDefault("database", "")); if (require_table) configuration.table = named_collection.get("table"); configuration.schema = named_collection.getOrDefault("schema", ""); diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 66f04346f16..ff1c714bbc9 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -55,15 +55,15 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr if (is_cluster_function) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Named collection cannot be used for table function cluster"); - validateNamedCollection( + validateNamedCollection>( *named_collection, - {"addresses_expr", "database", "table"}, - {"username", "password", "sharding_key"}); + {"addresses_expr", "database", "db", "table"}, + {"username", "user", "password", "sharding_key"}); cluster_description = named_collection->getOrDefault("addresses_expr", ""); - database = named_collection->get("database"); + database = named_collection->getOrDefault("db", named_collection->getOrDefault("database", "")); table = named_collection->get("table"); - username = named_collection->getOrDefault("username", ""); + username = named_collection->getOrDefault("username", named_collection->getOrDefault("user", "")); password = named_collection->getOrDefault("password", ""); } else From c2bcc4119fa706926e54ed99ddb108a601cb9853 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Feb 2023 20:57:37 +0100 Subject: [PATCH 26/69] Better --- .../NamedCollections/NamedCollections.cpp | 34 +++++++++++++++++++ .../NamedCollections/NamedCollections.h | 4 +++ src/Storages/StorageMongoDB.cpp | 8 ++--- src/Storages/StorageMySQL.cpp | 4 +-- src/Storages/StoragePostgreSQL.cpp | 6 ++-- src/TableFunctions/TableFunctionRemote.cpp | 4 +-- 6 files changed, 49 insertions(+), 11 deletions(-) diff --git a/src/Common/NamedCollections/NamedCollections.cpp b/src/Common/NamedCollections/NamedCollections.cpp index 2f80392c9ab..50f88adab36 100644 --- a/src/Common/NamedCollections/NamedCollections.cpp +++ b/src/Common/NamedCollections/NamedCollections.cpp @@ -364,6 +364,28 @@ template T NamedCollection::getOrDefault(const Key & key, const T & return pimpl->getOrDefault(key, default_value); } +template T NamedCollection::getAny(const std::initializer_list & keys) const +{ + std::lock_guard lock(mutex); + for (const auto & key : keys) + { + if (pimpl->has(key)) + return pimpl->get(key); + } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "No such keys: {}", fmt::join(keys, ", ")); +} + +template T NamedCollection::getAnyOrDefault(const std::initializer_list & keys, const T & default_value) const +{ + std::lock_guard lock(mutex); + for (const auto & key : keys) + { + if (pimpl->has(key)) + return pimpl->get(key); + } + return default_value; +} + template void NamedCollection::set(const Key & key, const T & value) { assertMutable(); @@ -455,6 +477,18 @@ template Int64 NamedCollection::getOrDefault(const NamedCollection::Key & template Float64 NamedCollection::getOrDefault(const NamedCollection::Key & key, const Float64 & default_value) const; template bool NamedCollection::getOrDefault(const NamedCollection::Key & key, const bool & default_value) const; +template String NamedCollection::getAny(const std::initializer_list & key) const; +template UInt64 NamedCollection::getAny(const std::initializer_list & key) const; +template Int64 NamedCollection::getAny(const std::initializer_list & key) const; +template Float64 NamedCollection::getAny(const std::initializer_list & key) const; +template bool NamedCollection::getAny(const std::initializer_list & key) const; + +template String NamedCollection::getAnyOrDefault(const std::initializer_list & key, const String & default_value) const; +template UInt64 NamedCollection::getAnyOrDefault(const std::initializer_list & key, const UInt64 & default_value) const; +template Int64 NamedCollection::getAnyOrDefault(const std::initializer_list & key, const Int64 & default_value) const; +template Float64 NamedCollection::getAnyOrDefault(const std::initializer_list & key, const Float64 & default_value) const; +template bool NamedCollection::getAnyOrDefault(const std::initializer_list & key, const bool & default_value) const; + template void NamedCollection::set(const NamedCollection::Key & key, const String & value); template void NamedCollection::set(const NamedCollection::Key & key, const String & value); template void NamedCollection::set(const NamedCollection::Key & key, const UInt64 & value); diff --git a/src/Common/NamedCollections/NamedCollections.h b/src/Common/NamedCollections/NamedCollections.h index a5b4349aaa3..b82d5eb3152 100644 --- a/src/Common/NamedCollections/NamedCollections.h +++ b/src/Common/NamedCollections/NamedCollections.h @@ -39,6 +39,10 @@ public: template T getOrDefault(const Key & key, const T & default_value) const; + template T getAny(const std::initializer_list & keys) const; + + template T getAnyOrDefault(const std::initializer_list & keys, const T & default_value) const; + std::unique_lock lock(); template void set(const Key & key, const T & value); diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index 59ecab03bd8..57aa81efe0a 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -183,12 +183,12 @@ StorageMongoDB::Configuration StorageMongoDB::getConfiguration(ASTs engine_args, ValidateKeysMultiset{"host", "port", "user", "username", "password", "database", "db", "collection", "table"}, {"options"}); - configuration.host = named_collection->getOrDefault("host", named_collection->getOrDefault("hostname", "")); + configuration.host = named_collection->getAny({"host", "hostname"}); configuration.port = static_cast(named_collection->get("port")); - configuration.username = named_collection->getOrDefault("user", named_collection->getOrDefault("username", "")); + configuration.username = named_collection->getAny({"user", "username"}); configuration.password = named_collection->get("password"); - configuration.database = named_collection->getOrDefault("database", named_collection->getOrDefault("db", "")); - configuration.table = named_collection->getOrDefault("collection", named_collection->getOrDefault("table", "")); + configuration.database = named_collection->getAny({"database", "db"}); + configuration.table = named_collection->getAny({"collection", "table"}); configuration.options = named_collection->getOrDefault("options", ""); } else diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index 6bc9232a29a..fdeea044dee 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -259,9 +259,9 @@ StorageMySQL::Configuration StorageMySQL::processNamedCollectionResult( configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; } - configuration.username = named_collection.getOrDefault("username", named_collection.getOrDefault("user", "")); + configuration.username = named_collection.getAny({"username", "user"}); configuration.password = named_collection.get("password"); - configuration.database = named_collection.getOrDefault("db", named_collection.getOrDefault("database", "")); + configuration.database = named_collection.getAny({"db", "database"}); if (require_table) configuration.table = named_collection.get("table"); configuration.replace_query = named_collection.getOrDefault("replace_query", false); diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index e736f9edc18..1bfc056f316 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -400,14 +400,14 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::processNamedCollectionResult configuration.addresses_expr = named_collection.getOrDefault("addresses_expr", ""); if (configuration.addresses_expr.empty()) { - configuration.host = named_collection.getOrDefault("host", named_collection.getOrDefault("hostname", "")); + configuration.host = named_collection.getAny({"host", "hostname"}); configuration.port = static_cast(named_collection.get("port")); configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; } - configuration.username = named_collection.getOrDefault("username", named_collection.getOrDefault("user", "")); + configuration.username = named_collection.getAny({"username", "user"}); configuration.password = named_collection.get("password"); - configuration.database = named_collection.getOrDefault("db", named_collection.getOrDefault("database", "")); + configuration.database = named_collection.getAny({"db", "database"}); if (require_table) configuration.table = named_collection.get("table"); configuration.schema = named_collection.getOrDefault("schema", ""); diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index ff1c714bbc9..f6c773b0b97 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -61,9 +61,9 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr {"username", "user", "password", "sharding_key"}); cluster_description = named_collection->getOrDefault("addresses_expr", ""); - database = named_collection->getOrDefault("db", named_collection->getOrDefault("database", "")); + database = named_collection->getAnyOrDefault({"db", "database"}, "default"); table = named_collection->get("table"); - username = named_collection->getOrDefault("username", named_collection->getOrDefault("user", "")); + username = named_collection->getAnyOrDefault({"username", "user"}, "default"); password = named_collection->getOrDefault("password", ""); } else From f398c5d4717102fdd5d6338f994673f34c9a3c31 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Feb 2023 12:22:27 +0100 Subject: [PATCH 27/69] Fix style --- src/Common/NamedCollections/NamedCollections.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/NamedCollections/NamedCollections.cpp b/src/Common/NamedCollections/NamedCollections.cpp index 50f88adab36..533481f792a 100644 --- a/src/Common/NamedCollections/NamedCollections.cpp +++ b/src/Common/NamedCollections/NamedCollections.cpp @@ -17,6 +17,7 @@ namespace ErrorCodes extern const int NAMED_COLLECTION_DOESNT_EXIST; extern const int NAMED_COLLECTION_ALREADY_EXISTS; extern const int NAMED_COLLECTION_IS_IMMUTABLE; + extern const int BAD_ARGUMENTS; } namespace Configuration = NamedCollectionConfiguration; From b19264cf9f059c4a6c1c0e40fd9f6a00cc9ba168 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Feb 2023 12:32:13 +0100 Subject: [PATCH 28/69] Remove redundant --- src/Databases/PostgreSQL/DatabasePostgreSQL.h | 1 - src/Storages/ExternalDataSourceConfiguration.cpp | 10 ---------- 2 files changed, 11 deletions(-) diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.h b/src/Databases/PostgreSQL/DatabasePostgreSQL.h index 074a8728d0a..08583f4b6d9 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.h @@ -8,7 +8,6 @@ #include #include #include -#include namespace DB { diff --git a/src/Storages/ExternalDataSourceConfiguration.cpp b/src/Storages/ExternalDataSourceConfiguration.cpp index 28bd058b802..e503c5edaab 100644 --- a/src/Storages/ExternalDataSourceConfiguration.cpp +++ b/src/Storages/ExternalDataSourceConfiguration.cpp @@ -9,16 +9,6 @@ #include #include -#if USE_AMQPCPP -#include -#endif -#if USE_RDKAFKA -#include -#endif -#if USE_NATSIO -#include -#endif - #include namespace DB From a2f9ac88d94df5ecf21af09918d78fcd7db3c069 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Feb 2023 13:00:54 +0100 Subject: [PATCH 29/69] Fix unit test --- src/Access/Common/AccessType.h | 2 +- src/Access/tests/gtest_access_rights_ops.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 84ec93d58f6..0a8ea908cff 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -69,7 +69,7 @@ enum class AccessType M(ALTER_FREEZE_PARTITION, "FREEZE PARTITION, UNFREEZE", TABLE, ALTER_TABLE) \ \ M(ALTER_DATABASE_SETTINGS, "ALTER DATABASE SETTING, ALTER MODIFY DATABASE SETTING, MODIFY DATABASE SETTING", DATABASE, ALTER_DATABASE) /* allows to execute ALTER MODIFY SETTING */\ - M(ALTER_NAMED_COLLECTION, "", NAMED_COLLECTION, ALTER) /* allows to execute ALTER NAMED COLLECTION */\ + M(ALTER_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) /* allows to execute ALTER NAMED COLLECTION */\ \ M(ALTER_TABLE, "", GROUP, ALTER) \ M(ALTER_DATABASE, "", GROUP, ALTER) \ diff --git a/src/Access/tests/gtest_access_rights_ops.cpp b/src/Access/tests/gtest_access_rights_ops.cpp index d6f827a02c5..025f70af587 100644 --- a/src/Access/tests/gtest_access_rights_ops.cpp +++ b/src/Access/tests/gtest_access_rights_ops.cpp @@ -48,12 +48,12 @@ TEST(AccessRights, Union) ASSERT_EQ(lhs.toString(), "GRANT INSERT ON *.*, " "GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, " - "CREATE DICTIONARY, CREATE NAMED COLLECTION, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, DROP NAMED COLLECTION, " + "CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, " "TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, " - "SHOW ROW POLICIES, SHOW NAMED COLLECTIONS, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, " + "SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, " "SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, " "SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, " - "SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*"); + "SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*, GRANT NAMED COLLECTION CONTROL ON db1"); } From 03c9eeb1064078ed5d39776e4b27020373c1d52d Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Feb 2023 14:24:22 +0100 Subject: [PATCH 30/69] Fix tests --- src/Access/UsersConfigAccessStorage.cpp | 6 ++-- .../configs/users.d/users.xml | 3 +- .../configs/users.xml | 3 +- .../configs/users.xml | 3 +- .../configs/users.d/users.xml | 3 +- .../configs/users.d/users.xml | 1 + ...> users_only_named_collection_control.xml} | 2 +- .../test_named_collections/test.py | 36 ++++++++++--------- .../configs/users.d/users.xml | 3 +- .../configs/users.d/users.xml | 3 +- .../01271_show_privileges.reference | 2 +- 11 files changed, 32 insertions(+), 33 deletions(-) rename tests/integration/test_named_collections/configs/users.d/{users_no_default_access_with_access_management.xml => users_only_named_collection_control.xml} (74%) diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index b893554cb8a..562df61e8aa 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -233,10 +233,10 @@ namespace user->access.revokeGrantOption(AccessType::ALL); } - bool show_named_collections = config.getBool(user_config + ".show_named_collections", false); - if (!show_named_collections) + bool named_collection_control = config.getBool(user_config + ".named_collection_control", false); + if (!named_collection_control) { - user->access.revoke(AccessType::SHOW_NAMED_COLLECTIONS); + user->access.revoke(AccessType::NAMED_COLLECTION_CONTROL); } bool show_named_collections_secrets = config.getBool(user_config + ".show_named_collections_secrets", false); diff --git a/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml b/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml index 8556e73c82f..775c63350b0 100644 --- a/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml +++ b/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml @@ -4,8 +4,7 @@ default default - 1 - 1 + 1 diff --git a/tests/integration/test_create_query_constraints/configs/users.xml b/tests/integration/test_create_query_constraints/configs/users.xml index 8556e73c82f..775c63350b0 100644 --- a/tests/integration/test_create_query_constraints/configs/users.xml +++ b/tests/integration/test_create_query_constraints/configs/users.xml @@ -4,8 +4,7 @@ default default - 1 - 1 + 1 diff --git a/tests/integration/test_global_overcommit_tracker/configs/users.xml b/tests/integration/test_global_overcommit_tracker/configs/users.xml index 8556e73c82f..775c63350b0 100644 --- a/tests/integration/test_global_overcommit_tracker/configs/users.xml +++ b/tests/integration/test_global_overcommit_tracker/configs/users.xml @@ -4,8 +4,7 @@ default default - 1 - 1 + 1 diff --git a/tests/integration/test_grant_and_revoke/configs/users.d/users.xml b/tests/integration/test_grant_and_revoke/configs/users.d/users.xml index 8556e73c82f..775c63350b0 100644 --- a/tests/integration/test_grant_and_revoke/configs/users.d/users.xml +++ b/tests/integration/test_grant_and_revoke/configs/users.d/users.xml @@ -4,8 +4,7 @@ default default - 1 - 1 + 1 diff --git a/tests/integration/test_named_collections/configs/users.d/users.xml b/tests/integration/test_named_collections/configs/users.d/users.xml index 8556e73c82f..15da914f666 100644 --- a/tests/integration/test_named_collections/configs/users.d/users.xml +++ b/tests/integration/test_named_collections/configs/users.d/users.xml @@ -4,6 +4,7 @@ default default + 1 1 1 diff --git a/tests/integration/test_named_collections/configs/users.d/users_no_default_access_with_access_management.xml b/tests/integration/test_named_collections/configs/users.d/users_only_named_collection_control.xml similarity index 74% rename from tests/integration/test_named_collections/configs/users.d/users_no_default_access_with_access_management.xml rename to tests/integration/test_named_collections/configs/users.d/users_only_named_collection_control.xml index 83dc04f03aa..775c63350b0 100644 --- a/tests/integration/test_named_collections/configs/users.d/users_no_default_access_with_access_management.xml +++ b/tests/integration/test_named_collections/configs/users.d/users_only_named_collection_control.xml @@ -4,7 +4,7 @@ default default - 1 + 1 diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 6a53f7e0a58..1f27826d213 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -24,6 +24,16 @@ def cluster(): ], stay_alive=True, ) + cluster.add_instance( + "node_only_named_collection_control", + main_configs=[ + "configs/config.d/named_collections.xml", + ], + user_configs=[ + "configs/users.d/users_only_named_collection_control.xml", + ], + stay_alive=True, + ) cluster.add_instance( "node_no_default_access", main_configs=[ @@ -34,16 +44,6 @@ def cluster(): ], stay_alive=True, ) - cluster.add_instance( - "node_no_default_access_but_with_access_management", - main_configs=[ - "configs/config.d/named_collections.xml", - ], - user_configs=[ - "configs/users.d/users_no_default_access_with_access_management.xml", - ], - stay_alive=True, - ) logging.info("Starting cluster...") cluster.start() @@ -73,25 +73,29 @@ def replace_in_users_config(node, old, new): def test_default_access(cluster): node = cluster.instances["node_no_default_access"] assert 0 == int(node.query("select count() from system.named_collections")) - node = cluster.instances["node_no_default_access_but_with_access_management"] - assert 0 == int(node.query("select count() from system.named_collections")) + node = cluster.instances["node_only_named_collection_control"] + assert 1 == int(node.query("select count() from system.named_collections")) + assert ( + node.query("select collection['key1'] from system.named_collections").strip() + == "[HIDDEN]" + ) node = cluster.instances["node"] assert int(node.query("select count() from system.named_collections")) > 0 replace_in_users_config( - node, "show_named_collections>1", "show_named_collections>0" + node, "named_collection_control>1", "named_collection_control>0" ) - assert "show_named_collections>0" in node.exec_in_container( + assert "named_collection_control>0" in node.exec_in_container( ["bash", "-c", f"cat /etc/clickhouse-server/users.d/users.xml"] ) node.restart_clickhouse() assert 0 == int(node.query("select count() from system.named_collections")) replace_in_users_config( - node, "show_named_collections>0", "show_named_collections>1" + node, "named_collection_control>0", "named_collection_control>1" ) - assert "show_named_collections>1" in node.exec_in_container( + assert "named_collection_control>1" in node.exec_in_container( ["bash", "-c", f"cat /etc/clickhouse-server/users.d/users.xml"] ) node.restart_clickhouse() diff --git a/tests/integration/test_overcommit_tracker/configs/users.d/users.xml b/tests/integration/test_overcommit_tracker/configs/users.d/users.xml index 8556e73c82f..775c63350b0 100644 --- a/tests/integration/test_overcommit_tracker/configs/users.d/users.xml +++ b/tests/integration/test_overcommit_tracker/configs/users.d/users.xml @@ -4,8 +4,7 @@ default default - 1 - 1 + 1 diff --git a/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml b/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml index 8556e73c82f..775c63350b0 100644 --- a/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml +++ b/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml @@ -4,8 +4,7 @@ default default - 1 - 1 + 1 diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 03661d2469f..2d54531aff4 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -39,7 +39,7 @@ ALTER MOVE PARTITION ['ALTER MOVE PART','MOVE PARTITION','MOVE PART'] TABLE ALTE ALTER FETCH PARTITION ['ALTER FETCH PART','FETCH PARTITION'] TABLE ALTER TABLE ALTER FREEZE PARTITION ['FREEZE PARTITION','UNFREEZE'] TABLE ALTER TABLE ALTER DATABASE SETTINGS ['ALTER DATABASE SETTING','ALTER MODIFY DATABASE SETTING','MODIFY DATABASE SETTING'] DATABASE ALTER DATABASE -ALTER NAMED COLLECTION [] NAMED_COLLECTION ALTER +ALTER NAMED COLLECTION [] NAMED_COLLECTION NAMED COLLECTION CONTROL ALTER TABLE [] \N ALTER ALTER DATABASE [] \N ALTER ALTER VIEW REFRESH ['ALTER LIVE VIEW REFRESH','REFRESH VIEW'] VIEW ALTER VIEW From ad88251ee785139660735884f56bd5573b013944 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Feb 2023 17:42:04 +0100 Subject: [PATCH 31/69] Fix tests --- src/Common/NamedCollections/NamedCollections.cpp | 9 +++++++++ src/Common/NamedCollections/NamedCollections.h | 2 ++ src/Storages/NamedCollectionsHelpers.h | 4 ++-- src/Storages/StorageURL.h | 2 +- src/TableFunctions/TableFunctionRemote.cpp | 6 ++++-- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/Common/NamedCollections/NamedCollections.cpp b/src/Common/NamedCollections/NamedCollections.cpp index 533481f792a..0a0f29a8a82 100644 --- a/src/Common/NamedCollections/NamedCollections.cpp +++ b/src/Common/NamedCollections/NamedCollections.cpp @@ -353,6 +353,15 @@ bool NamedCollection::has(const Key & key) const return pimpl->has(key); } +bool NamedCollection::hasAny(const std::initializer_list & keys) const +{ + std::lock_guard lock(mutex); + for (const auto & key : keys) + if (pimpl->has(key)) + return true; + return false; +} + template T NamedCollection::get(const Key & key) const { std::lock_guard lock(mutex); diff --git a/src/Common/NamedCollections/NamedCollections.h b/src/Common/NamedCollections/NamedCollections.h index b82d5eb3152..4a0f020db21 100644 --- a/src/Common/NamedCollections/NamedCollections.h +++ b/src/Common/NamedCollections/NamedCollections.h @@ -35,6 +35,8 @@ public: bool has(const Key & key) const; + bool hasAny(const std::initializer_list & keys) const; + template T get(const Key & key) const; template T getOrDefault(const Key & key, const T & default_value) const; diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 085e21937ee..a2aed38ed08 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -27,8 +27,8 @@ HTTPHeaderEntries getHeadersFromNamedCollection(const NamedCollection & collecti struct ExternalDatabaseEqualKeysSet { - static constexpr std::array, 3> equal_keys{ - std::pair{"username", "user"}, std::pair{"database", "db"}, std::pair{"hostname", "host"}}; + static constexpr std::array, 5> equal_keys{ + std::pair{"username", "user"}, std::pair{"database", "db"}, std::pair{"hostname", "host"}, std::pair{"addresses_expr", "host"}, std::pair{"addresses_expr", "hostname"}}; }; struct MongoDBEqualKeysSet { diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 24b1c7ee572..c95cfa69e54 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -184,7 +184,7 @@ public: struct Configuration : public StatelessTableEngineConfiguration { std::string url; - std::string http_method = "auto"; + std::string http_method; HTTPHeaderEntries headers; std::string addresses_expr; }; diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index f6c773b0b97..1e093e957a7 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -57,10 +57,12 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr validateNamedCollection>( *named_collection, - {"addresses_expr", "database", "db", "table"}, - {"username", "user", "password", "sharding_key"}); + {"addresses_expr", "host", "database", "db", "table"}, + {"username", "user", "password", "sharding_key", "port"}); cluster_description = named_collection->getOrDefault("addresses_expr", ""); + if (cluster_description.empty() && named_collection->hasAny({"host", "hostname"})) + cluster_description = named_collection->has("port") ? named_collection->getAny({"host", "hostname"}) + ':' + toString(named_collection->get("port")) : named_collection->getAny({"host", "hostname"}); database = named_collection->getAnyOrDefault({"db", "database"}, "default"); table = named_collection->get("table"); username = named_collection->getAnyOrDefault({"username", "user"}, "default"); From 353fca74f073edc51163ad09558c6131626dae23 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Feb 2023 16:33:59 +0100 Subject: [PATCH 32/69] Fix config --- tests/config/users.d/access_management.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/users.d/access_management.xml b/tests/config/users.d/access_management.xml index f7963cdb7f2..45e7c23227e 100644 --- a/tests/config/users.d/access_management.xml +++ b/tests/config/users.d/access_management.xml @@ -2,7 +2,7 @@ 1 - 1 + 1 1 From d4e6fc454631e85a43575c3fc36480f95c6ade39 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Feb 2023 21:43:45 +0100 Subject: [PATCH 33/69] Fix test --- src/Storages/NamedCollectionsHelpers.h | 51 +++++++++++++++++----- src/TableFunctions/TableFunctionRemote.cpp | 14 +++--- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index a2aed38ed08..619adfc54b6 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -36,24 +36,40 @@ struct MongoDBEqualKeysSet std::pair{"username", "user"}, std::pair{"database", "db"}, std::pair{"hostname", "host"}, std::pair{"table", "collection"}}; }; -template struct ValidateKeysCmp +template struct NamedCollectionValidateKey { - constexpr bool operator()(const auto & lhs, const auto & rhs) const + NamedCollectionValidateKey() = default; + NamedCollectionValidateKey(const char * value_) : value(value_) {} + NamedCollectionValidateKey(std::string_view value_) : value(value_) {} + NamedCollectionValidateKey(const String & value_) : value(value_) {} + + std::string_view value; + + bool operator==(const auto & other) const { - if (lhs == rhs) + if (value == other.value) return true; for (const auto & equal : EqualKeys::equal_keys) { - if (((equal.first == lhs) && (equal.second == rhs)) || ((equal.first == rhs) && (equal.second == lhs))) + if (((equal.first == value) && (equal.second == other.value)) || ((equal.first == other.value) && (equal.second == value))) + { return true; + } } return false; } + + bool operator<(const auto & other) const + { + if (*this == other) + return false; + return value < other.value; + } }; -template using ValidateKeysMultiset = std::unordered_multiset, ValidateKeysCmp>; -using ValidateKeysSet = std::unordered_multiset>; +template using ValidateKeysMultiset = std::multiset>; +using ValidateKeysSet = std::multiset; template @@ -84,10 +100,10 @@ void validateNamedCollection( if (!match) { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Unexpected key {} in named collection. Required keys: {}, optional keys: {}", - backQuoteIfNeed(key), fmt::join(required_keys, ", "), fmt::join(optional_keys, ", ")); + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Unexpected key {} in named collection. Required keys: {}, optional keys: {}", + backQuoteIfNeed(key), fmt::join(required_keys, ", "), fmt::join(optional_keys, ", ")); } } @@ -101,3 +117,18 @@ void validateNamedCollection( } } + +template +struct fmt::formatter> +{ + constexpr static auto parse(format_parse_context & context) + { + return context.begin(); + } + + template + auto format(const DB::NamedCollectionValidateKey & elem, FormatContext & context) + { + return fmt::format_to(context.out(), "{}", elem.value); + } +}; diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 1e093e957a7..4952aa16efa 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -50,19 +50,19 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr * For now named collection can be used only for remote as cluster does not require credentials. */ size_t max_args = is_cluster_function ? 4 : 6; - if (auto named_collection = tryGetNamedCollectionWithOverrides(args, false)) + NamedCollectionPtr named_collection; + if (!is_cluster_function && (named_collection = tryGetNamedCollectionWithOverrides(args))) { - if (is_cluster_function) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Named collection cannot be used for table function cluster"); - validateNamedCollection>( *named_collection, - {"addresses_expr", "host", "database", "db", "table"}, - {"username", "user", "password", "sharding_key", "port"}); + {"addresses_expr", "host", "hostname", "table"}, + {"username", "user", "password", "sharding_key", "port", "database", "db"}); cluster_description = named_collection->getOrDefault("addresses_expr", ""); if (cluster_description.empty() && named_collection->hasAny({"host", "hostname"})) - cluster_description = named_collection->has("port") ? named_collection->getAny({"host", "hostname"}) + ':' + toString(named_collection->get("port")) : named_collection->getAny({"host", "hostname"}); + cluster_description = named_collection->has("port") + ? named_collection->getAny({"host", "hostname"}) + ':' + toString(named_collection->get("port")) + : named_collection->getAny({"host", "hostname"}); database = named_collection->getAnyOrDefault({"db", "database"}, "default"); table = named_collection->get("table"); username = named_collection->getAnyOrDefault({"username", "user"}, "default"); From de81a5f92da88a6ff43d0d1ee155eec268638093 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 28 Feb 2023 13:36:52 +0100 Subject: [PATCH 34/69] Fix integration tests --- .../test_access_control_on_cluster/configs/users.d/users.xml | 1 + .../integration/test_create_query_constraints/configs/users.xml | 1 + .../integration/test_global_overcommit_tracker/configs/users.xml | 1 + .../integration/test_grant_and_revoke/configs/users.d/users.xml | 1 + .../test_overcommit_tracker/configs/users.d/users.xml | 1 + .../configs/users.d/users.xml | 1 + 6 files changed, 6 insertions(+) diff --git a/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml b/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml index 775c63350b0..1c5f4d5a21f 100644 --- a/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml +++ b/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/integration/test_create_query_constraints/configs/users.xml b/tests/integration/test_create_query_constraints/configs/users.xml index 775c63350b0..1c5f4d5a21f 100644 --- a/tests/integration/test_create_query_constraints/configs/users.xml +++ b/tests/integration/test_create_query_constraints/configs/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/integration/test_global_overcommit_tracker/configs/users.xml b/tests/integration/test_global_overcommit_tracker/configs/users.xml index 775c63350b0..1c5f4d5a21f 100644 --- a/tests/integration/test_global_overcommit_tracker/configs/users.xml +++ b/tests/integration/test_global_overcommit_tracker/configs/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/integration/test_grant_and_revoke/configs/users.d/users.xml b/tests/integration/test_grant_and_revoke/configs/users.d/users.xml index 775c63350b0..1c5f4d5a21f 100644 --- a/tests/integration/test_grant_and_revoke/configs/users.d/users.xml +++ b/tests/integration/test_grant_and_revoke/configs/users.d/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/integration/test_overcommit_tracker/configs/users.d/users.xml b/tests/integration/test_overcommit_tracker/configs/users.d/users.xml index 775c63350b0..1c5f4d5a21f 100644 --- a/tests/integration/test_overcommit_tracker/configs/users.d/users.xml +++ b/tests/integration/test_overcommit_tracker/configs/users.d/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml b/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml index 775c63350b0..1c5f4d5a21f 100644 --- a/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml +++ b/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 From 4716273349c03204d0de25031753cbae5d8838ee Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 1 Mar 2023 13:37:05 +0100 Subject: [PATCH 35/69] Fix test --- src/Storages/NamedCollectionsHelpers.cpp | 32 ++++++++++++++++------ src/Storages/NamedCollectionsHelpers.h | 2 +- src/TableFunctions/TableFunctionRemote.cpp | 12 ++++++-- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index 54d15e1e40c..81801c68344 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -30,7 +30,7 @@ namespace return NamedCollectionFactory::instance().tryGet(collection_name); } - std::optional> getKeyValueFromAST(ASTPtr ast) + std::optional>> getKeyValueFromAST(ASTPtr ast, bool) { const auto * function = ast->as(); if (!function || function->name != "equals") @@ -42,20 +42,27 @@ namespace if (function_args.size() != 2) return std::nullopt; - auto literal_key = evaluateConstantExpressionOrIdentifierAsLiteral( - function_args[0], Context::getGlobalContextInstance()); + auto context = Context::getGlobalContextInstance(); + auto literal_key = evaluateConstantExpressionOrIdentifierAsLiteral(function_args[0], context); auto key = checkAndGetLiteralArgument(literal_key, "key"); - auto literal_value = evaluateConstantExpressionOrIdentifierAsLiteral( - function_args[1], Context::getGlobalContextInstance()); - auto value = literal_value->as()->value; + ASTPtr literal_value; + try + { + literal_value = evaluateConstantExpressionOrIdentifierAsLiteral(function_args[1], context); + } + catch (...) + { + return std::pair{key, function_args[1]}; + } + auto value = literal_value->as()->value; return std::pair{key, value}; } } -MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool throw_unknown_collection) +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool throw_unknown_collection, std::vector> * non_convertible) { if (asts.empty()) return nullptr; @@ -73,14 +80,21 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool thr for (auto * it = std::next(asts.begin()); it != asts.end(); ++it) { - auto value_override = getKeyValueFromAST(*it); + auto value_override = getKeyValueFromAST(*it, non_convertible != nullptr); + if (!value_override && !(*it)->as()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value argument or function"); if (!value_override) continue; + if (const ASTPtr * value = std::get_if(&value_override->second)) + { + non_convertible->emplace_back(value_override->first, *value); + continue; + } + const auto & [key, value] = *value_override; - collection_copy->setOrUpdate(key, toString(value)); + collection_copy->setOrUpdate(key, toString(std::get(value_override->second))); } return collection_copy; diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 619adfc54b6..40d83ff9a12 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -18,7 +18,7 @@ namespace DB /// Helper function to get named collection for table engine. /// Table engines have collection name as first argument of ast and other arguments are key-value overrides. -MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool throw_unknown_collection = true); +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool throw_unknown_collection = true, std::vector> * non_convertible = nullptr); /// Helper function to get named collection for dictionary source. /// Dictionaries have collection name as name argument of dict configuration and other arguments are overrides. MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 4952aa16efa..55c61b8a82a 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -51,19 +51,27 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr */ size_t max_args = is_cluster_function ? 4 : 6; NamedCollectionPtr named_collection; - if (!is_cluster_function && (named_collection = tryGetNamedCollectionWithOverrides(args))) + std::vector> non_convertible; + if (!is_cluster_function && (named_collection = tryGetNamedCollectionWithOverrides(args, false, &non_convertible))) { validateNamedCollection>( *named_collection, {"addresses_expr", "host", "hostname", "table"}, {"username", "user", "password", "sharding_key", "port", "database", "db"}); + if (!non_convertible.empty()) + { + if (non_convertible.size() != 1 || (non_convertible[0].first != "database" && non_convertible[0].first != "db")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected argument representation for {}", non_convertible[0].first); + remote_table_function_ptr = non_convertible[0].second; + } + else + database = named_collection->getAnyOrDefault({"db", "database"}, "default"); cluster_description = named_collection->getOrDefault("addresses_expr", ""); if (cluster_description.empty() && named_collection->hasAny({"host", "hostname"})) cluster_description = named_collection->has("port") ? named_collection->getAny({"host", "hostname"}) + ':' + toString(named_collection->get("port")) : named_collection->getAny({"host", "hostname"}); - database = named_collection->getAnyOrDefault({"db", "database"}, "default"); table = named_collection->get("table"); username = named_collection->getAnyOrDefault({"username", "user"}, "default"); password = named_collection->getOrDefault("password", ""); From 80d017629a6a9f6cb8a0c76d00607ebf9b68ccc2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 1 Mar 2023 17:00:10 +0100 Subject: [PATCH 36/69] Fix test --- src/Databases/DatabaseFactory.cpp | 4 ++-- src/Storages/StorageMySQL.cpp | 2 +- src/Storages/StoragePostgreSQL.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 97ec0de9552..57281ca93d4 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -317,7 +317,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { - configuration = StoragePostgreSQL::processNamedCollectionResult(*named_collection); + configuration = StoragePostgreSQL::processNamedCollectionResult(*named_collection, false); use_table_cache = named_collection->getOrDefault("use_tables_cache", 0); } else @@ -380,7 +380,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { - configuration = StoragePostgreSQL::processNamedCollectionResult(*named_collection); + configuration = StoragePostgreSQL::processNamedCollectionResult(*named_collection, false); } else { diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index fdeea044dee..7d619c518cf 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -246,7 +246,7 @@ StorageMySQL::Configuration StorageMySQL::processNamedCollectionResult( for (const auto & setting : mysql_settings) optional_arguments.insert(setting.getName()); - ValidateKeysMultiset required_arguments = {"user", "username", "password", "database", "db", "table"}; + ValidateKeysMultiset required_arguments = {"user", "username", "password", "database", "db"}; if (require_table) required_arguments.insert("table"); validateNamedCollection>(named_collection, required_arguments, optional_arguments); diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 1bfc056f316..c208ad8ef5d 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -390,7 +390,7 @@ SinkToStoragePtr StoragePostgreSQL::write( StoragePostgreSQL::Configuration StoragePostgreSQL::processNamedCollectionResult(const NamedCollection & named_collection, bool require_table) { StoragePostgreSQL::Configuration configuration; - ValidateKeysMultiset required_arguments = {"user", "username", "password", "database", "db", "table"}; + ValidateKeysMultiset required_arguments = {"user", "username", "password", "database", "db"}; if (require_table) required_arguments.insert("table"); From cd7cd0526bf693c87765410facb04439e852fb68 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 2 Mar 2023 19:04:33 +0100 Subject: [PATCH 37/69] Fix tests --- src/Databases/DatabaseFactory.cpp | 4 +-- src/Storages/MySQL/MySQLSettings.cpp | 20 +++++++++++--- src/Storages/MySQL/MySQLSettings.h | 2 +- src/Storages/NamedCollectionsHelpers.cpp | 16 +++++++----- src/Storages/NamedCollectionsHelpers.h | 26 +++++++++++++++---- src/Storages/StorageExternalDistributed.cpp | 2 +- src/Storages/StorageMySQL.cpp | 1 + src/TableFunctions/TableFunctionRemote.cpp | 19 +++++++++----- .../configs/named_collections.xml | 13 +++++++--- .../test_mask_sensitive_info/test.py | 8 +++--- .../configs/named_collections.xml | 2 -- .../configs/named_collections.xml | 1 - tests/integration/test_storage_mysql/test.py | 2 +- 13 files changed, 81 insertions(+), 35 deletions(-) diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 57281ca93d4..cbd9f84df60 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -221,8 +221,8 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String { if (engine_name == "MySQL") { - mysql_settings->loadFromQueryContext(context); - mysql_settings->loadFromQuery(*engine_define); /// higher priority + mysql_settings->loadFromQuery(*engine_define); + mysql_settings->loadFromQueryContext(context, *engine_define); /// Will override only if not changed. auto mysql_pool = createMySQLPoolWithFailover(configuration, *mysql_settings); diff --git a/src/Storages/MySQL/MySQLSettings.cpp b/src/Storages/MySQL/MySQLSettings.cpp index b3bc11482f4..67942114182 100644 --- a/src/Storages/MySQL/MySQLSettings.cpp +++ b/src/Storages/MySQL/MySQLSettings.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include namespace DB @@ -44,15 +46,27 @@ void MySQLSettings::loadFromQuery(ASTStorage & storage_def) } } -void MySQLSettings::loadFromQueryContext(ContextPtr context) +void MySQLSettings::loadFromQueryContext(ContextPtr context, ASTStorage & storage_def) { if (!context->hasQueryContext()) return; const Settings & settings = context->getQueryContext()->getSettingsRef(); - if (settings.mysql_datatypes_support_level.value != mysql_datatypes_support_level.value) - set("mysql_datatypes_support_level", settings.mysql_datatypes_support_level.toString()); + /// Setting from SETTING clause have bigger priority. + if (!mysql_datatypes_support_level.changed + && settings.mysql_datatypes_support_level.value != mysql_datatypes_support_level.value) + { + static constexpr auto setting_name = "mysql_datatypes_support_level"; + set(setting_name, settings.mysql_datatypes_support_level.toString()); + auto & changes = storage_def.settings->changes; + if (changes.end() == std::find_if( + changes.begin(), changes.end(), + [](const SettingChange & c) { return c.name == setting_name; })) + { + changes.push_back(SettingChange{setting_name, settings.mysql_datatypes_support_level.toString()}); + } + } } } diff --git a/src/Storages/MySQL/MySQLSettings.h b/src/Storages/MySQL/MySQLSettings.h index 40771d0aacb..850ac432aa1 100644 --- a/src/Storages/MySQL/MySQLSettings.h +++ b/src/Storages/MySQL/MySQLSettings.h @@ -37,7 +37,7 @@ struct MySQLSettings : public MySQLBaseSettings { void loadFromQuery(ASTStorage & storage_def); void loadFromQuery(const ASTSetQuery & settings_def); - void loadFromQueryContext(ContextPtr context); + void loadFromQueryContext(ContextPtr context, ASTStorage & storage_def); }; diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index 81801c68344..0cca2e4b9df 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -30,7 +30,7 @@ namespace return NamedCollectionFactory::instance().tryGet(collection_name); } - std::optional>> getKeyValueFromAST(ASTPtr ast, bool) + std::optional>> getKeyValueFromAST(ASTPtr ast, bool fallback_to_ast_value) { const auto * function = ast->as(); if (!function || function->name != "equals") @@ -53,7 +53,9 @@ namespace } catch (...) { - return std::pair{key, function_args[1]}; + if (fallback_to_ast_value) + return std::pair{key, function_args[1]}; + throw; } auto value = literal_value->as()->value; @@ -62,7 +64,8 @@ namespace } -MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool throw_unknown_collection, std::vector> * non_convertible) +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( + ASTs asts, bool throw_unknown_collection, std::vector> * complex_args) { if (asts.empty()) return nullptr; @@ -80,7 +83,7 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool thr for (auto * it = std::next(asts.begin()); it != asts.end(); ++it) { - auto value_override = getKeyValueFromAST(*it, non_convertible != nullptr); + auto value_override = getKeyValueFromAST(*it, complex_args != nullptr); if (!value_override && !(*it)->as()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value argument or function"); @@ -89,7 +92,7 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool thr if (const ASTPtr * value = std::get_if(&value_override->second)) { - non_convertible->emplace_back(value_override->first, *value); + complex_args->emplace_back(value_override->first, *value); continue; } @@ -100,7 +103,8 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool thr return collection_copy; } -MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( + const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) { auto collection_name = config.getString(config_prefix + ".name", ""); if (collection_name.empty()) diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 40d83ff9a12..8c6c1fb4e24 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -62,15 +62,29 @@ template struct NamedCollectionValidateKey bool operator<(const auto & other) const { - if (*this == other) - return false; - return value < other.value; + std::string_view canonical_self = value; + std::string_view canonical_other = other.value; + for (const auto & equal : EqualKeys::equal_keys) + { + if ((equal.first == value) || (equal.second == value)) + canonical_self = std::max(equal.first, equal.second); + if ((equal.first == other.value) || (equal.second == other.value)) + canonical_other = std::max(equal.first, equal.second); + } + + return canonical_self < canonical_other; } }; -template using ValidateKeysMultiset = std::multiset>; -using ValidateKeysSet = std::multiset; +template +std::ostream & operator << (std::ostream & ostr, const NamedCollectionValidateKey & key) +{ + ostr << key.value; + return ostr; +} +template using ValidateKeysMultiset = std::multiset, std::less>>; +using ValidateKeysSet = std::multiset; template void validateNamedCollection( @@ -91,7 +105,9 @@ void validateNamedCollection( } if (optional_keys.contains(key)) + { continue; + } auto match = std::find_if( optional_regex_keys.begin(), optional_regex_keys.end(), diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index a5c22d0807d..45ca659a8fe 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -146,7 +146,7 @@ void registerStorageExternalDistributed(StorageFactory & factory) { auto current_configuration{configuration}; current_configuration.addresses = parseRemoteDescriptionForExternalDatabase(shard_address, max_addresses, 3306); - auto pool = createMySQLPoolWithFailover(configuration, mysql_settings); + auto pool = createMySQLPoolWithFailover(current_configuration, mysql_settings); shards.insert(std::make_shared( args.table_id, std::move(pool), configuration.database, configuration.table, /* replace_query = */ false, /* on_duplicate_clause = */ "", diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index 7d619c518cf..61c715bdaeb 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -297,6 +297,7 @@ StorageMySQL::Configuration StorageMySQL::getConfiguration(ASTs engine_args, Con const auto & host_port = checkAndGetLiteralArgument(engine_args[0], "host:port"); size_t max_addresses = context_->getSettingsRef().glob_expansion_max_elements; + configuration.addresses_expr = host_port; configuration.addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 3306); configuration.database = checkAndGetLiteralArgument(engine_args[1], "database"); configuration.table = checkAndGetLiteralArgument(engine_args[2], "table"); diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 55c61b8a82a..eed9e07b532 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -51,18 +51,25 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr */ size_t max_args = is_cluster_function ? 4 : 6; NamedCollectionPtr named_collection; - std::vector> non_convertible; - if (!is_cluster_function && (named_collection = tryGetNamedCollectionWithOverrides(args, false, &non_convertible))) + std::vector> complex_args; + if (!is_cluster_function && (named_collection = tryGetNamedCollectionWithOverrides(args, false, &complex_args))) { validateNamedCollection>( *named_collection, {"addresses_expr", "host", "hostname", "table"}, {"username", "user", "password", "sharding_key", "port", "database", "db"}); - if (!non_convertible.empty()) + + if (!complex_args.empty()) { - if (non_convertible.size() != 1 || (non_convertible[0].first != "database" && non_convertible[0].first != "db")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected argument representation for {}", non_convertible[0].first); - remote_table_function_ptr = non_convertible[0].second; + for (const auto & [arg_name, arg_ast] : complex_args) + { + if (arg_name == "database" || arg_name == "db") + remote_table_function_ptr = arg_ast; + else if (arg_name == "sharding_key") + sharding_key = arg_ast; + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected argument representation for {}", arg_name); + } } else database = named_collection->getAnyOrDefault({"db", "database"}, "default"); diff --git a/tests/integration/test_mask_sensitive_info/configs/named_collections.xml b/tests/integration/test_mask_sensitive_info/configs/named_collections.xml index ee923a90171..a4b58f6f812 100644 --- a/tests/integration/test_mask_sensitive_info/configs/named_collections.xml +++ b/tests/integration/test_mask_sensitive_info/configs/named_collections.xml @@ -2,9 +2,16 @@ - - - + + user + pass + + + + + + + diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 3f71b047213..5df40c38041 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -126,7 +126,7 @@ def test_create_table(): f"MySQL(named_collection_2, database = 'mysql_db', host = 'mysql57', port = 3306, password = '{password}', table = 'mysql_table', user = 'mysql_user')", f"MySQL(named_collection_3, database = 'mysql_db', host = 'mysql57', port = 3306, table = 'mysql_table')", f"PostgreSQL(named_collection_4, host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user', password = '{password}')", - f"MongoDB(named_collection_5, host = 'mongo1', port = 5432, database = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '{password}')", + f"MongoDB(named_collection_5, host = 'mongo1', port = 5432, db = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '{password}')", f"S3(named_collection_6, url = 'http://minio1:9001/root/data/test8.csv', access_key_id = 'minio', secret_access_key = '{password}', format = 'CSV')", ] @@ -163,7 +163,7 @@ def test_create_table(): "CREATE TABLE table9 (`x` int) ENGINE = MySQL(named_collection_2, database = 'mysql_db', host = 'mysql57', port = 3306, password = '[HIDDEN]', table = 'mysql_table', user = 'mysql_user')", "CREATE TABLE table10 (x int) ENGINE = MySQL(named_collection_3, database = 'mysql_db', host = 'mysql57', port = 3306, table = 'mysql_table')", "CREATE TABLE table11 (`x` int) ENGINE = PostgreSQL(named_collection_4, host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user', password = '[HIDDEN]')", - "CREATE TABLE table12 (`x` int) ENGINE = MongoDB(named_collection_5, host = 'mongo1', port = 5432, database = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '[HIDDEN]'", + "CREATE TABLE table12 (`x` int) ENGINE = MongoDB(named_collection_5, host = 'mongo1', port = 5432, db = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '[HIDDEN]'", "CREATE TABLE table13 (`x` int) ENGINE = S3(named_collection_6, url = 'http://minio1:9001/root/data/test8.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]', format = 'CSV')", ], must_not_contain=[password], @@ -233,7 +233,7 @@ def test_table_functions(): f"remoteSecure('127.{{2..11}}', 'default', 'remote_table', 'remote_user', rand())", f"mysql(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '{password}')", f"postgresql(named_collection_2, password = '{password}', host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user')", - f"s3(named_collection_3, url = 'http://minio1:9001/root/data/test4.csv', access_key_id = 'minio', secret_access_key = '{password}')", + f"s3(named_collection_2, url = 'http://minio1:9001/root/data/test4.csv', access_key_id = 'minio', secret_access_key = '{password}')", f"remote(named_collection_4, addresses_expr = '127.{{2..11}}', database = 'default', table = 'remote_table', user = 'remote_user', password = '{password}', sharding_key = rand())", f"remoteSecure(named_collection_5, addresses_expr = '127.{{2..11}}', database = 'default', table = 'remote_table', user = 'remote_user', password = '{password}')", ] @@ -286,7 +286,7 @@ def test_table_functions(): "CREATE TABLE tablefunc24 (x int) AS remoteSecure('127.{2..11}', 'default', 'remote_table', 'remote_user', rand())", "CREATE TABLE tablefunc25 (`x` int) AS mysql(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '[HIDDEN]')", "CREATE TABLE tablefunc26 (`x` int) AS postgresql(named_collection_2, password = '[HIDDEN]', host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user')", - "CREATE TABLE tablefunc27 (`x` int) AS s3(named_collection_3, url = 'http://minio1:9001/root/data/test4.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]')", + "CREATE TABLE tablefunc27 (`x` int) AS s3(named_collection_2, url = 'http://minio1:9001/root/data/test4.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]')", "CREATE TABLE tablefunc28 (`x` int) AS remote(named_collection_4, addresses_expr = '127.{2..11}', database = 'default', table = 'remote_table', user = 'remote_user', password = '[HIDDEN]', sharding_key = rand())", "CREATE TABLE tablefunc29 (`x` int) AS remoteSecure(named_collection_5, addresses_expr = '127.{2..11}', database = 'default', table = 'remote_table', user = 'remote_user', password = '[HIDDEN]')", ], diff --git a/tests/integration/test_mysql_database_engine/configs/named_collections.xml b/tests/integration/test_mysql_database_engine/configs/named_collections.xml index fd18dfa6202..3b65536f20f 100644 --- a/tests/integration/test_mysql_database_engine/configs/named_collections.xml +++ b/tests/integration/test_mysql_database_engine/configs/named_collections.xml @@ -6,7 +6,6 @@ mysql57 3306 test_database - test_table
postgres @@ -19,7 +18,6 @@ mysql57 1111 clickhouse - test_table
diff --git a/tests/integration/test_storage_mysql/configs/named_collections.xml b/tests/integration/test_storage_mysql/configs/named_collections.xml index 4d3fbf6085c..04117f32d4b 100644 --- a/tests/integration/test_storage_mysql/configs/named_collections.xml +++ b/tests/integration/test_storage_mysql/configs/named_collections.xml @@ -36,7 +36,6 @@ mysql57 3306 clickhouse - test_settings
1 20123001 20123002 diff --git a/tests/integration/test_storage_mysql/test.py b/tests/integration/test_storage_mysql/test.py index 50f0c5519b5..0c9369a8efa 100644 --- a/tests/integration/test_storage_mysql/test.py +++ b/tests/integration/test_storage_mysql/test.py @@ -765,7 +765,7 @@ def test_settings(started_cluster): rw_timeout = 20123001 connect_timeout = 20123002 - node1.query(f"SELECT * FROM mysql(mysql_with_settings)") + node1.query(f"SELECT * FROM mysql(mysql_with_settings, table='test_settings')") assert node1.contains_in_log( f"with settings: connect_timeout={connect_timeout}, read_write_timeout={rw_timeout}" ) From 0fa3fb359286054a0c2c97c7b68ffd1e0180f6fe Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 3 Mar 2023 14:04:32 +0100 Subject: [PATCH 38/69] Fix show grants for user which has all grants --- src/Access/AccessRights.cpp | 23 +++++++++++++------ src/Access/Common/AccessRightsElement.cpp | 5 ++++ src/Access/Common/AccessRightsElement.h | 16 ++++++++++--- .../integration/test_grant_and_revoke/test.py | 3 +++ .../test_named_collections/test.py | 16 ++++++++++--- 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 37597552a41..cfa14e6c88b 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -124,20 +124,29 @@ namespace const auto & element = sorted[i]; if (element.access_flags) { - auto per_parameter = element.access_flags.splitIntoParameterTypes(); - if (per_parameter.size() == 1) + const bool all_granted = sorted.size() == 1 && element.access_flags.contains(AccessFlags::allFlags()); + if (all_granted) { /// Easy case: one Element is converted to one AccessRightsElement. res.emplace_back(element.getResult()); } else { - /// Difficult case: one element is converted into multiple AccessRightsElements. - for (const auto & [_, parameter_flags] : per_parameter) + auto per_parameter = element.access_flags.splitIntoParameterTypes(); + if (per_parameter.size() == 1) { - auto current_element{element}; - current_element.access_flags = parameter_flags; - res.emplace_back(current_element.getResult()); + /// Easy case: one Element is converted to one AccessRightsElement. + res.emplace_back(element.getResult()); + } + else + { + /// Difficult case: one element is converted into multiple AccessRightsElements. + for (const auto & [_, parameter_flags] : per_parameter) + { + auto current_element{element}; + current_element.access_flags = parameter_flags; + res.emplace_back(current_element.getResult()); + } } } } diff --git a/src/Access/Common/AccessRightsElement.cpp b/src/Access/Common/AccessRightsElement.cpp index 81cebd68b4c..e11d43634ec 100644 --- a/src/Access/Common/AccessRightsElement.cpp +++ b/src/Access/Common/AccessRightsElement.cpp @@ -233,6 +233,11 @@ bool AccessRightsElements::sameDatabaseAndTableAndParameter() const return (size() < 2) || std::all_of(std::next(begin()), end(), [this](const AccessRightsElement & e) { return e.sameDatabaseAndTableAndParameter(front()); }); } +bool AccessRightsElements::sameDatabaseAndTable() const +{ + return (size() < 2) || std::all_of(std::next(begin()), end(), [this](const AccessRightsElement & e) { return e.sameDatabaseAndTable(front()); }); +} + bool AccessRightsElements::sameOptions() const { return (size() < 2) || std::all_of(std::next(begin()), end(), [this](const AccessRightsElement & e) { return e.sameOptions(front()); }); diff --git a/src/Access/Common/AccessRightsElement.h b/src/Access/Common/AccessRightsElement.h index 96850f0880e..ba625fc43df 100644 --- a/src/Access/Common/AccessRightsElement.h +++ b/src/Access/Common/AccessRightsElement.h @@ -55,13 +55,22 @@ struct AccessRightsElement bool sameDatabaseAndTableAndParameter(const AccessRightsElement & other) const { - return (database == other.database) && (any_database == other.any_database) - && (table == other.table) && (any_table == other.any_table) - && (parameter == other.parameter) && (any_parameter == other.any_parameter) + return sameDatabaseAndTable(other) && sameParameter(other); + } + + bool sameParameter(const AccessRightsElement & other) const + { + return (parameter == other.parameter) && (any_parameter == other.any_parameter) && (access_flags.getParameterType() == other.access_flags.getParameterType()) && (isGlobalWithParameter() == other.isGlobalWithParameter()); } + bool sameDatabaseAndTable(const AccessRightsElement & other) const + { + return (database == other.database) && (any_database == other.any_database) + && (table == other.table) && (any_table == other.any_table); + } + bool sameOptions(const AccessRightsElement & other) const { return (grant_option == other.grant_option) && (is_partial_revoke == other.is_partial_revoke); @@ -92,6 +101,7 @@ public: bool empty() const; bool sameDatabaseAndTableAndParameter() const; + bool sameDatabaseAndTable() const; bool sameOptions() const; /// Resets flags which cannot be granted. diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 8d48f7449e4..4d89e6255d3 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -402,6 +402,9 @@ def test_introspection(): assert instance.query("SHOW GRANTS FOR B") == TSV( ["GRANT CREATE ON *.* TO B WITH GRANT OPTION"] ) + assert instance.query("SHOW GRANTS FOR default") == TSV( + ["GRANT ALL ON *.* TO default WITH GRANT OPTION"] + ) assert instance.query("SHOW GRANTS FOR A,B") == TSV( [ "GRANT SELECT ON test.table TO A", diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 1f27826d213..5574c77b886 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -100,7 +100,9 @@ def test_default_access(cluster): ) node.restart_clickhouse() assert ( - node.query("select collection['key1'] from system.named_collections").strip() + node.query( + "select collection['key1'] from system.named_collections where name = 'collection1'" + ).strip() == "value1" ) replace_in_users_config( @@ -111,7 +113,9 @@ def test_default_access(cluster): ) node.restart_clickhouse() assert ( - node.query("select collection['key1'] from system.named_collections").strip() + node.query( + "select collection['key1'] from system.named_collections where name = 'collection1'" + ).strip() == "[HIDDEN]" ) replace_in_users_config( @@ -122,13 +126,19 @@ def test_default_access(cluster): ) node.restart_clickhouse() assert ( - node.query("select collection['key1'] from system.named_collections").strip() + node.query( + "select collection['key1'] from system.named_collections where name = 'collection1'" + ).strip() == "value1" ) def test_granular_access_show_query(cluster): node = cluster.instances["node"] + assert ( + "GRANT ALL ON *.* TO default WITH GRANT OPTION" + == node.query("SHOW GRANTS FOR default").strip() + ) # includes named collections control assert 1 == int(node.query("SELECT count() FROM system.named_collections")) assert ( "collection1" == node.query("SELECT name FROM system.named_collections").strip() From f78da967b3f1b39bb628b6777be3539d2047fc34 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 3 Mar 2023 16:32:23 +0100 Subject: [PATCH 39/69] Fix test --- src/Storages/NamedCollectionsHelpers.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 8c6c1fb4e24..86e215bccf5 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -67,9 +67,9 @@ template struct NamedCollectionValidateKey for (const auto & equal : EqualKeys::equal_keys) { if ((equal.first == value) || (equal.second == value)) - canonical_self = std::max(equal.first, equal.second); + canonical_self = std::max(canonical_self, std::max(equal.first, equal.second)); if ((equal.first == other.value) || (equal.second == other.value)) - canonical_other = std::max(equal.first, equal.second); + canonical_other = std::max(canonical_other, std::max(equal.first, equal.second)); } return canonical_self < canonical_other; From 8f2d75cef851072cc32d1955fef0a0e8f5ec9a78 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 5 Mar 2023 12:50:29 +0100 Subject: [PATCH 40/69] Fix tests --- src/Databases/DatabaseFactory.cpp | 9 +++++---- src/Storages/Kafka/StorageKafka.cpp | 2 +- src/Storages/MeiliSearch/StorageMeiliSearch.cpp | 2 +- src/Storages/NATS/StorageNATS.cpp | 2 +- src/Storages/NamedCollectionsHelpers.cpp | 12 +++++++----- src/Storages/NamedCollectionsHelpers.h | 5 ++++- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 2 +- src/Storages/StorageMongoDB.cpp | 2 +- src/Storages/StorageMySQL.cpp | 7 +++---- src/Storages/StoragePostgreSQL.cpp | 6 +++--- src/Storages/StorageS3.cpp | 2 +- src/Storages/StorageURL.cpp | 2 +- src/TableFunctions/TableFunctionRemote.cpp | 2 +- src/TableFunctions/TableFunctionS3.cpp | 2 +- src/TableFunctions/TableFunctionURL.cpp | 2 +- tests/integration/test_mask_sensitive_info/test.py | 8 ++++---- tests/integration/test_storage_postgresql/test.py | 2 +- 17 files changed, 37 insertions(+), 32 deletions(-) diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index cbd9f84df60..47cf0c6b6d0 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -186,7 +186,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String ASTs & arguments = engine->arguments->children; auto mysql_settings = std::make_unique(); - if (auto named_collection = tryGetNamedCollectionWithOverrides(arguments)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(arguments, context)) { configuration = StorageMySQL::processNamedCollectionResult(*named_collection, *mysql_settings, false); } @@ -222,7 +222,8 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String if (engine_name == "MySQL") { mysql_settings->loadFromQuery(*engine_define); - mysql_settings->loadFromQueryContext(context, *engine_define); /// Will override only if not changed. + if (engine_define->settings) + mysql_settings->loadFromQueryContext(context, *engine_define); /// Will override only if not changed. auto mysql_pool = createMySQLPoolWithFailover(configuration, *mysql_settings); @@ -315,7 +316,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String auto use_table_cache = false; StoragePostgreSQL::Configuration configuration; - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, context)) { configuration = StoragePostgreSQL::processNamedCollectionResult(*named_collection, false); use_table_cache = named_collection->getOrDefault("use_tables_cache", 0); @@ -378,7 +379,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String ASTs & engine_args = engine->arguments->children; StoragePostgreSQL::Configuration configuration; - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, context)) { configuration = StoragePostgreSQL::processNamedCollectionResult(*named_collection, false); } diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 86410447ee1..dedad6f44d7 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -838,7 +838,7 @@ void registerStorageKafka(StorageFactory & factory) auto kafka_settings = std::make_unique(); String collection_name; - if (auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args, args.getLocalContext())) { for (const auto & setting : kafka_settings->all()) { diff --git a/src/Storages/MeiliSearch/StorageMeiliSearch.cpp b/src/Storages/MeiliSearch/StorageMeiliSearch.cpp index 56dad2a0d13..62a6c471070 100644 --- a/src/Storages/MeiliSearch/StorageMeiliSearch.cpp +++ b/src/Storages/MeiliSearch/StorageMeiliSearch.cpp @@ -129,7 +129,7 @@ SinkToStoragePtr StorageMeiliSearch::write(const ASTPtr & /*query*/, const Stora MeiliSearchConfiguration StorageMeiliSearch::getConfiguration(ASTs engine_args, ContextPtr context) { - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, context)) { validateNamedCollection(*named_collection, {"url", "index"}, {"key"}); diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp index feb44fe92e4..aa4ec77b0d8 100644 --- a/src/Storages/NATS/StorageNATS.cpp +++ b/src/Storages/NATS/StorageNATS.cpp @@ -711,7 +711,7 @@ void registerStorageNATS(StorageFactory & factory) auto creator_fn = [](const StorageFactory::Arguments & args) { auto nats_settings = std::make_unique(); - if (auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args, args.getLocalContext())) { for (const auto & setting : nats_settings->all()) { diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index 0cca2e4b9df..c6e928d9412 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -30,7 +30,7 @@ namespace return NamedCollectionFactory::instance().tryGet(collection_name); } - std::optional>> getKeyValueFromAST(ASTPtr ast, bool fallback_to_ast_value) + std::optional>> getKeyValueFromAST(ASTPtr ast, bool fallback_to_ast_value, ContextPtr context) { const auto * function = ast->as(); if (!function || function->name != "equals") @@ -42,14 +42,16 @@ namespace if (function_args.size() != 2) return std::nullopt; - auto context = Context::getGlobalContextInstance(); auto literal_key = evaluateConstantExpressionOrIdentifierAsLiteral(function_args[0], context); auto key = checkAndGetLiteralArgument(literal_key, "key"); ASTPtr literal_value; try { - literal_value = evaluateConstantExpressionOrIdentifierAsLiteral(function_args[1], context); + if (key == "database" || key == "db") + literal_value = evaluateConstantExpressionForDatabaseName(function_args[1], context); + else + literal_value = evaluateConstantExpressionOrIdentifierAsLiteral(function_args[1], context); } catch (...) { @@ -65,7 +67,7 @@ namespace MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( - ASTs asts, bool throw_unknown_collection, std::vector> * complex_args) + ASTs asts, ContextPtr context, bool throw_unknown_collection, std::vector> * complex_args) { if (asts.empty()) return nullptr; @@ -83,7 +85,7 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( for (auto * it = std::next(asts.begin()); it != asts.end(); ++it) { - auto value_override = getKeyValueFromAST(*it, complex_args != nullptr); + auto value_override = getKeyValueFromAST(*it, complex_args != nullptr, context); if (!value_override && !(*it)->as()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value argument or function"); diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 86e215bccf5..0510ed7e298 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -18,7 +18,7 @@ namespace DB /// Helper function to get named collection for table engine. /// Table engines have collection name as first argument of ast and other arguments are key-value overrides. -MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, bool throw_unknown_collection = true, std::vector> * non_convertible = nullptr); +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, ContextPtr context, bool throw_unknown_collection = true, std::vector> * non_convertible = nullptr); /// Helper function to get named collection for dictionary source. /// Dictionaries have collection name as name argument of dict configuration and other arguments are overrides. MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); @@ -109,6 +109,9 @@ void validateNamedCollection( continue; } + if (required_keys.contains(key)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Duplicate key {} in named collection", key); + auto match = std::find_if( optional_regex_keys.begin(), optional_regex_keys.end(), [&](const std::regex & regex) { return std::regex_search(key, regex); }) diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index c5ea6f810ef..b7fb2c6df64 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -1195,7 +1195,7 @@ void registerStorageRabbitMQ(StorageFactory & factory) { auto rabbitmq_settings = std::make_unique(); - if (auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args, args.getLocalContext())) { for (const auto & setting : rabbitmq_settings->all()) { diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index 57aa81efe0a..0bad4ab8bf0 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -176,7 +176,7 @@ StorageMongoDB::Configuration StorageMongoDB::getConfiguration(ASTs engine_args, { Configuration configuration; - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, context)) { validateNamedCollection( *named_collection, diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index 61c715bdaeb..7e133538e41 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -280,7 +280,7 @@ StorageMySQL::Configuration StorageMySQL::processNamedCollectionResult( StorageMySQL::Configuration StorageMySQL::getConfiguration(ASTs engine_args, ContextPtr context_, MySQLSettings & storage_settings) { StorageMySQL::Configuration configuration; - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, context_)) { configuration = StorageMySQL::processNamedCollectionResult(*named_collection, storage_settings); } @@ -294,11 +294,10 @@ StorageMySQL::Configuration StorageMySQL::getConfiguration(ASTs engine_args, Con for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context_); - const auto & host_port = checkAndGetLiteralArgument(engine_args[0], "host:port"); + configuration.addresses_expr = checkAndGetLiteralArgument(engine_args[0], "host:port"); size_t max_addresses = context_->getSettingsRef().glob_expansion_max_elements; - configuration.addresses_expr = host_port; - configuration.addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 3306); + configuration.addresses = parseRemoteDescriptionForExternalDatabase(configuration.addresses_expr, max_addresses, 3306); configuration.database = checkAndGetLiteralArgument(engine_args[1], "database"); configuration.table = checkAndGetLiteralArgument(engine_args[2], "table"); configuration.username = checkAndGetLiteralArgument(engine_args[3], "username"); diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index c208ad8ef5d..cf87d23bf94 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -419,7 +419,7 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::processNamedCollectionResult StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine_args, ContextPtr context) { StoragePostgreSQL::Configuration configuration; - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, context)) { configuration = StoragePostgreSQL::processNamedCollectionResult(*named_collection); } @@ -438,10 +438,10 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context); - const auto & host_port = checkAndGetLiteralArgument(engine_args[0], "host:port"); + configuration.addresses_expr = checkAndGetLiteralArgument(engine_args[0], "host:port"); size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; - configuration.addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 5432); + configuration.addresses = parseRemoteDescriptionForExternalDatabase(configuration.addresses_expr, max_addresses, 5432); if (configuration.addresses.size() == 1) { configuration.host = configuration.addresses[0].first; diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index ed290c38c1f..d605eaf2d13 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -1294,7 +1294,7 @@ StorageS3::Configuration StorageS3::getConfiguration(ASTs & engine_args, Context { StorageS3::Configuration configuration; - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, local_context)) { processNamedCollectionResult(configuration, *named_collection); } diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 152dda8f360..691254867ac 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -1091,7 +1091,7 @@ StorageURL::Configuration StorageURL::getConfiguration(ASTs & args, ContextPtr l { StorageURL::Configuration configuration; - if (auto named_collection = tryGetNamedCollectionWithOverrides(args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(args, local_context)) { StorageURL::processNamedCollectionResult(configuration, *named_collection); collectHeaders(args, configuration.headers, local_context); diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index eed9e07b532..1ee51bcb040 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -52,7 +52,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr size_t max_args = is_cluster_function ? 4 : 6; NamedCollectionPtr named_collection; std::vector> complex_args; - if (!is_cluster_function && (named_collection = tryGetNamedCollectionWithOverrides(args, false, &complex_args))) + if (!is_cluster_function && (named_collection = tryGetNamedCollectionWithOverrides(args, context, false, &complex_args))) { validateNamedCollection>( *named_collection, diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index 1994787f831..f082b192ee0 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -32,7 +32,7 @@ namespace ErrorCodes void TableFunctionS3::parseArgumentsImpl( const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & s3_configuration, bool get_format_from_file) { - if (auto named_collection = tryGetNamedCollectionWithOverrides(args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(args, context)) { StorageS3::processNamedCollectionResult(s3_configuration, *named_collection); } diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 5de6c6b4ccc..468f949203d 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -36,7 +36,7 @@ void TableFunctionURL::parseArguments(const ASTPtr & ast, ContextPtr context) auto & url_function_args = assert_cast(args[0].get())->children; - if (auto named_collection = tryGetNamedCollectionWithOverrides(url_function_args)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(url_function_args, context)) { StorageURL::processNamedCollectionResult(configuration, *named_collection); diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 5df40c38041..92232f7e6a8 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -234,8 +234,8 @@ def test_table_functions(): f"mysql(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '{password}')", f"postgresql(named_collection_2, password = '{password}', host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user')", f"s3(named_collection_2, url = 'http://minio1:9001/root/data/test4.csv', access_key_id = 'minio', secret_access_key = '{password}')", - f"remote(named_collection_4, addresses_expr = '127.{{2..11}}', database = 'default', table = 'remote_table', user = 'remote_user', password = '{password}', sharding_key = rand())", - f"remoteSecure(named_collection_5, addresses_expr = '127.{{2..11}}', database = 'default', table = 'remote_table', user = 'remote_user', password = '{password}')", + f"remote(named_collection_6, addresses_expr = '127.{{2..11}}', database = 'default', table = 'remote_table', user = 'remote_user', password = '{password}', sharding_key = rand())", + f"remoteSecure(named_collection_6, addresses_expr = '127.{{2..11}}', database = 'default', table = 'remote_table', user = 'remote_user', password = '{password}')", ] for i, table_function in enumerate(table_functions): @@ -287,8 +287,8 @@ def test_table_functions(): "CREATE TABLE tablefunc25 (`x` int) AS mysql(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '[HIDDEN]')", "CREATE TABLE tablefunc26 (`x` int) AS postgresql(named_collection_2, password = '[HIDDEN]', host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user')", "CREATE TABLE tablefunc27 (`x` int) AS s3(named_collection_2, url = 'http://minio1:9001/root/data/test4.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]')", - "CREATE TABLE tablefunc28 (`x` int) AS remote(named_collection_4, addresses_expr = '127.{2..11}', database = 'default', table = 'remote_table', user = 'remote_user', password = '[HIDDEN]', sharding_key = rand())", - "CREATE TABLE tablefunc29 (`x` int) AS remoteSecure(named_collection_5, addresses_expr = '127.{2..11}', database = 'default', table = 'remote_table', user = 'remote_user', password = '[HIDDEN]')", + "CREATE TABLE tablefunc28 (`x` int) AS remote(named_collection_6, addresses_expr = '127.{2..11}', database = 'default', table = 'remote_table', user = 'remote_user', password = '[HIDDEN]', sharding_key = rand())", + "CREATE TABLE tablefunc29 (`x` int) AS remoteSecure(named_collection_6, addresses_expr = '127.{2..11}', database = 'default', table = 'remote_table', user = 'remote_user', password = '[HIDDEN]')", ], must_not_contain=[password], ) diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index a3ebbe97451..3b7aae1ccdc 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -382,7 +382,7 @@ def test_postgres_distributed(started_cluster): """ CREATE TABLE test_shards2 (id UInt32, name String, age UInt32, money UInt32) - ENGINE = ExternalDistributed('PostgreSQL', postgres4, description='postgres{1|2}:5432,postgres{3|4}:5432'); """ + ENGINE = ExternalDistributed('PostgreSQL', postgres4, addresses_expr='postgres{1|2}:5432,postgres{3|4}:5432'); """ ) result = node2.query("SELECT DISTINCT(name) FROM test_shards2 ORDER BY name") From d9e75e5c0a39ff7d9eac2f7e8f10b09c1720bb1b Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 5 Mar 2023 13:13:32 +0100 Subject: [PATCH 41/69] Fix test --- tests/integration/test_named_collections/test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 5574c77b886..af5aab38264 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -410,6 +410,16 @@ def test_config_reload(cluster): ).strip() ) + replace_in_server_config(node, "value2", "value1") + node.query("SYSTEM RELOAD CONFIG") + + assert ( + "value1" + == node.query( + "select collection['key1'] from system.named_collections where name = 'collection1'" + ).strip() + ) + def test_sql_commands(cluster): node = cluster.instances["node"] From c06af1f1e774e9e7b8f792e3df3abb5605392b39 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 5 Mar 2023 22:12:51 +0100 Subject: [PATCH 42/69] Fix clang-tidy --- src/Databases/DatabaseFactory.cpp | 4 +-- src/Storages/Kafka/StorageKafka.cpp | 39 ++++++++++++-------------- src/Storages/MySQL/MySQLSettings.cpp | 12 ++++++-- src/Storages/NamedCollectionsHelpers.h | 3 +- 4 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 47cf0c6b6d0..89a799349bf 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -221,9 +221,9 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String { if (engine_name == "MySQL") { - mysql_settings->loadFromQuery(*engine_define); + mysql_settings->loadFromQueryContext(context, *engine_define); if (engine_define->settings) - mysql_settings->loadFromQueryContext(context, *engine_define); /// Will override only if not changed. + mysql_settings->loadFromQuery(*engine_define); auto mysql_pool = createMySQLPoolWithFailover(configuration, *mysql_settings); diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index dedad6f44d7..6de87e18855 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -914,27 +914,24 @@ void registerStorageKafka(StorageFactory & factory) * - Do intermediate commits when the batch consumed and handled */ - if (has_settings) - { - /* 0 = raw, 1 = evaluateConstantExpressionAsLiteral, 2=evaluateConstantExpressionOrIdentifierAsLiteral */ - CHECK_KAFKA_STORAGE_ARGUMENT(1, kafka_broker_list, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(2, kafka_topic_list, 1) - CHECK_KAFKA_STORAGE_ARGUMENT(3, kafka_group_name, 2) - CHECK_KAFKA_STORAGE_ARGUMENT(4, kafka_format, 2) - CHECK_KAFKA_STORAGE_ARGUMENT(5, kafka_row_delimiter, 2) - CHECK_KAFKA_STORAGE_ARGUMENT(6, kafka_schema, 2) - CHECK_KAFKA_STORAGE_ARGUMENT(7, kafka_num_consumers, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(8, kafka_max_block_size, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(9, kafka_skip_broken_messages, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(10, kafka_commit_every_batch, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(11, kafka_client_id, 2) - CHECK_KAFKA_STORAGE_ARGUMENT(12, kafka_poll_timeout_ms, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(13, kafka_flush_interval_ms, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(14, kafka_thread_per_consumer, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(15, kafka_handle_error_mode, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(16, kafka_commit_on_select, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(17, kafka_max_rows_per_message, 0) - } + /* 0 = raw, 1 = evaluateConstantExpressionAsLiteral, 2=evaluateConstantExpressionOrIdentifierAsLiteral */ + CHECK_KAFKA_STORAGE_ARGUMENT(1, kafka_broker_list, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(2, kafka_topic_list, 1) + CHECK_KAFKA_STORAGE_ARGUMENT(3, kafka_group_name, 2) + CHECK_KAFKA_STORAGE_ARGUMENT(4, kafka_format, 2) + CHECK_KAFKA_STORAGE_ARGUMENT(5, kafka_row_delimiter, 2) + CHECK_KAFKA_STORAGE_ARGUMENT(6, kafka_schema, 2) + CHECK_KAFKA_STORAGE_ARGUMENT(7, kafka_num_consumers, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(8, kafka_max_block_size, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(9, kafka_skip_broken_messages, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(10, kafka_commit_every_batch, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(11, kafka_client_id, 2) + CHECK_KAFKA_STORAGE_ARGUMENT(12, kafka_poll_timeout_ms, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(13, kafka_flush_interval_ms, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(14, kafka_thread_per_consumer, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(15, kafka_handle_error_mode, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(16, kafka_commit_on_select, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(17, kafka_max_rows_per_message, 0) #undef CHECK_KAFKA_STORAGE_ARGUMENT diff --git a/src/Storages/MySQL/MySQLSettings.cpp b/src/Storages/MySQL/MySQLSettings.cpp index 67942114182..fd53174f4f6 100644 --- a/src/Storages/MySQL/MySQLSettings.cpp +++ b/src/Storages/MySQL/MySQLSettings.cpp @@ -53,12 +53,18 @@ void MySQLSettings::loadFromQueryContext(ContextPtr context, ASTStorage & storag const Settings & settings = context->getQueryContext()->getSettingsRef(); - /// Setting from SETTING clause have bigger priority. - if (!mysql_datatypes_support_level.changed - && settings.mysql_datatypes_support_level.value != mysql_datatypes_support_level.value) + if (settings.mysql_datatypes_support_level.value != mysql_datatypes_support_level.value) { static constexpr auto setting_name = "mysql_datatypes_support_level"; set(setting_name, settings.mysql_datatypes_support_level.toString()); + + if (!storage_def.settings) + { + auto settings_ast = std::make_shared(); + settings_ast->is_standalone = false; + storage_def.set(storage_def.settings, settings_ast); + } + auto & changes = storage_def.settings->changes; if (changes.end() == std::find_if( changes.begin(), changes.end(), diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 0510ed7e298..1473a3fbe48 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -18,7 +18,8 @@ namespace DB /// Helper function to get named collection for table engine. /// Table engines have collection name as first argument of ast and other arguments are key-value overrides. -MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts, ContextPtr context, bool throw_unknown_collection = true, std::vector> * non_convertible = nullptr); +MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( + ASTs asts, ContextPtr context, bool throw_unknown_collection = true, std::vector> * complex_args = nullptr); /// Helper function to get named collection for dictionary source. /// Dictionaries have collection name as name argument of dict configuration and other arguments are overrides. MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); From 8567e3976b28dc724773f8ddcd8ba2c80fca6ed7 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 6 Mar 2023 12:30:25 +0100 Subject: [PATCH 43/69] Fix kafka --- src/Storages/Kafka/StorageKafka.cpp | 38 ++++++++++++++++------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 6de87e18855..61495c966cb 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -915,23 +915,27 @@ void registerStorageKafka(StorageFactory & factory) */ /* 0 = raw, 1 = evaluateConstantExpressionAsLiteral, 2=evaluateConstantExpressionOrIdentifierAsLiteral */ - CHECK_KAFKA_STORAGE_ARGUMENT(1, kafka_broker_list, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(2, kafka_topic_list, 1) - CHECK_KAFKA_STORAGE_ARGUMENT(3, kafka_group_name, 2) - CHECK_KAFKA_STORAGE_ARGUMENT(4, kafka_format, 2) - CHECK_KAFKA_STORAGE_ARGUMENT(5, kafka_row_delimiter, 2) - CHECK_KAFKA_STORAGE_ARGUMENT(6, kafka_schema, 2) - CHECK_KAFKA_STORAGE_ARGUMENT(7, kafka_num_consumers, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(8, kafka_max_block_size, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(9, kafka_skip_broken_messages, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(10, kafka_commit_every_batch, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(11, kafka_client_id, 2) - CHECK_KAFKA_STORAGE_ARGUMENT(12, kafka_poll_timeout_ms, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(13, kafka_flush_interval_ms, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(14, kafka_thread_per_consumer, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(15, kafka_handle_error_mode, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(16, kafka_commit_on_select, 0) - CHECK_KAFKA_STORAGE_ARGUMENT(17, kafka_max_rows_per_message, 0) + /// In case of named collection we already validated the arguments. + if (collection_name.empty()) + { + CHECK_KAFKA_STORAGE_ARGUMENT(1, kafka_broker_list, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(2, kafka_topic_list, 1) + CHECK_KAFKA_STORAGE_ARGUMENT(3, kafka_group_name, 2) + CHECK_KAFKA_STORAGE_ARGUMENT(4, kafka_format, 2) + CHECK_KAFKA_STORAGE_ARGUMENT(5, kafka_row_delimiter, 2) + CHECK_KAFKA_STORAGE_ARGUMENT(6, kafka_schema, 2) + CHECK_KAFKA_STORAGE_ARGUMENT(7, kafka_num_consumers, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(8, kafka_max_block_size, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(9, kafka_skip_broken_messages, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(10, kafka_commit_every_batch, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(11, kafka_client_id, 2) + CHECK_KAFKA_STORAGE_ARGUMENT(12, kafka_poll_timeout_ms, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(13, kafka_flush_interval_ms, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(14, kafka_thread_per_consumer, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(15, kafka_handle_error_mode, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(16, kafka_commit_on_select, 0) + CHECK_KAFKA_STORAGE_ARGUMENT(17, kafka_max_rows_per_message, 0) + } #undef CHECK_KAFKA_STORAGE_ARGUMENT From aa5127c2fdd7e5a0dbae8003f8e57c166a8bc4db Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 15 Mar 2023 08:33:45 +0100 Subject: [PATCH 44/69] Add sanity checks for writing number in variable length format And just to double check: # var_uint 9223372036854775807 ffffffffffffffff7f ffffffffffffffff7f ffffffffffffffff7f x: 9223372036854775807, y: 9223372036854775807 # var_uint 9223372036854775808 808080808080808080 808080808080808080 808080808080808080 x: 9223372036854775808, y: 0 Signed-off-by: Azat Khuzhin --- src/IO/VarInt.h | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/IO/VarInt.h b/src/IO/VarInt.h index 0869051034a..d026192cb7d 100644 --- a/src/IO/VarInt.h +++ b/src/IO/VarInt.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -14,7 +15,19 @@ namespace ErrorCodes } -/** Write UInt64 in variable length format (base128) NOTE Only up to 2^63 - 1 are supported. */ +/** Variable-Length Quantity (VLQ) Base-128 compression + * + * NOTE: Due to historical reasons, only up to 1<<63-1 are supported, which + * cannot be changed without breaking the backward compatibility. + * Also some drivers may support full 1<<64 range (i.e. python - + * clickhouse-driver), while others has the same limitations as ClickHouse + * (i.e. Rust - clickhouse-rs). + * So implementing VLQ for the whole 1<<64 range will require different set of + * helpers. + */ +constexpr size_t VAR_UINT_MAX = (1ULL<<63) - 1; + +/** Write UInt64 in variable length format (base128) */ void writeVarUInt(UInt64 x, std::ostream & ostr); void writeVarUInt(UInt64 x, WriteBuffer & ostr); char * writeVarUInt(UInt64 x, char * ostr); @@ -186,6 +199,7 @@ inline const char * readVarUInt(UInt64 & x, const char * istr, size_t size) inline void writeVarUInt(UInt64 x, WriteBuffer & ostr) { + chassert(x <= VAR_UINT_MAX); for (size_t i = 0; i < 9; ++i) { uint8_t byte = x & 0x7F; @@ -205,6 +219,7 @@ inline void writeVarUInt(UInt64 x, WriteBuffer & ostr) inline void writeVarUInt(UInt64 x, std::ostream & ostr) { + chassert(x <= VAR_UINT_MAX); for (size_t i = 0; i < 9; ++i) { uint8_t byte = x & 0x7F; @@ -222,6 +237,7 @@ inline void writeVarUInt(UInt64 x, std::ostream & ostr) inline char * writeVarUInt(UInt64 x, char * ostr) { + chassert(x <= VAR_UINT_MAX); for (size_t i = 0; i < 9; ++i) { uint8_t byte = x & 0x7F; From 727ad9289bd4b5b03d127c8faa877d82b76438f4 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 15 Mar 2023 13:10:35 +0100 Subject: [PATCH 45/69] Use VAR_UINT_MAX for unknown_packet_in_send_data Signed-off-by: Azat Khuzhin --- src/Server/TCPHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index b240c99fc7f..2a5e2c302b3 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1824,7 +1824,7 @@ void TCPHandler::sendData(const Block & block) { --unknown_packet_in_send_data; if (unknown_packet_in_send_data == 0) - writeVarUInt(UInt64(-1), *out); + writeVarUInt(VAR_UINT_MAX, *out); } writeVarUInt(Protocol::Server::Data, *out); From 7d8b643f25cb0945cff8996b276e89a585d77208 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Thu, 16 Mar 2023 17:17:56 +0100 Subject: [PATCH 46/69] Documentation: Update debian setup This change is necessary. `apt-key` is deprecated and will be removed, see https://manpages.debian.org/bullseye/apt/apt-key.8.en.html#DESCRIPTION --- docs/en/getting-started/install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 0867f3a0795..f5ca44415f9 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -118,9 +118,9 @@ It is recommended to use official pre-compiled `deb` packages for Debian or Ubun #### Setup the Debian repository ``` bash sudo apt-get install -y apt-transport-https ca-certificates dirmngr -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 +sudo gpg --no-default-keyring --keyring /usr/share/keyrings/clickhouse-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 8919F6BD2B48D754 -echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \ +echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main" | sudo tee \ /etc/apt/sources.list.d/clickhouse.list sudo apt-get update ``` From d7b2ea60ebc88ac1a370164661479de0b08e655b Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 17 Mar 2023 13:56:02 +0100 Subject: [PATCH 47/69] Review fixes --- src/Storages/NamedCollectionsHelpers.cpp | 4 ++-- src/Storages/StorageExternalDistributed.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index c6e928d9412..83128ab025a 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -61,7 +61,7 @@ namespace } auto value = literal_value->as()->value; - return std::pair{key, value}; + return std::pair{key, Field(value)}; } } @@ -85,7 +85,7 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( for (auto * it = std::next(asts.begin()); it != asts.end(); ++it) { - auto value_override = getKeyValueFromAST(*it, complex_args != nullptr, context); + auto value_override = getKeyValueFromAST(*it, /* fallback_to_ast_value */complex_args != nullptr, context); if (!value_override && !(*it)->as()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value argument or function"); diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index 45ca659a8fe..db1f33193ac 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -102,8 +102,8 @@ void registerStorageExternalDistributed(StorageFactory & factory) "engine_name, named_collection and/or description"); auto context = args.getLocalContext(); - [[maybe_unused]] const auto & settings = context->getSettingsRef(); - size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; + const auto & settings = context->getSettingsRef(); + size_t max_addresses = settings.glob_expansion_max_elements; auto get_addresses = [&](const std::string addresses_expr) { return parseRemoteDescription(addresses_expr, 0, addresses_expr.size(), ',', max_addresses); From 2958c5f0f130c6308b00af7f3c41c91357dedb72 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 17 Mar 2023 15:08:16 +0000 Subject: [PATCH 48/69] Fix logical error in evaluate constant expression --- .../evaluateConstantExpression.cpp | 3 +++ src/Parsers/ASTAsterisk.h | 1 + src/Parsers/ASTColumnsMatcher.h | 6 ++++++ src/Parsers/ASTColumnsTransformers.h | 7 +++++++ src/Parsers/ASTFunction.cpp | 21 +++++++++++++++++++ src/Parsers/ASTFunction.h | 2 ++ src/Parsers/ASTIdentifier.h | 1 + src/Parsers/ASTLiteral.h | 1 + src/Parsers/ASTQualifiedAsterisk.h | 1 + src/Parsers/ASTWithAlias.h | 2 ++ src/Parsers/IAST.h | 3 +++ .../02680_mysql_ast_logical_err.reference | 0 .../02680_mysql_ast_logical_err.sql | 2 ++ 13 files changed, 50 insertions(+) create mode 100644 tests/queries/0_stateless/02680_mysql_ast_logical_err.reference create mode 100644 tests/queries/0_stateless/02680_mysql_ast_logical_err.sql diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index ebefa0d9ce7..3e5684946c2 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -44,6 +44,9 @@ static std::pair> getFieldAndDataTypeFro std::pair> evaluateConstantExpression(const ASTPtr & node, const ContextPtr & context) { + if (!node->hasColumnName()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expression '{}' is not a constant expression", node->formatForErrorMessage()); + if (ASTLiteral * literal = node->as()) return getFieldAndDataTypeFromLiteral(literal); diff --git a/src/Parsers/ASTAsterisk.h b/src/Parsers/ASTAsterisk.h index 840b7996536..72fd82fc9a7 100644 --- a/src/Parsers/ASTAsterisk.h +++ b/src/Parsers/ASTAsterisk.h @@ -15,6 +15,7 @@ public: String getID(char) const override { return "Asterisk"; } ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } ASTPtr expression; ASTPtr transformers; diff --git a/src/Parsers/ASTColumnsMatcher.h b/src/Parsers/ASTColumnsMatcher.h index f31a8bd9a22..5e0149756f1 100644 --- a/src/Parsers/ASTColumnsMatcher.h +++ b/src/Parsers/ASTColumnsMatcher.h @@ -23,6 +23,8 @@ public: ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } + void setPattern(String pattern); const String & getPattern() const; const std::shared_ptr & getMatcher() const; @@ -46,6 +48,7 @@ public: String getID(char) const override { return "ColumnsListMatcher"; } ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } ASTPtr expression; ASTPtr column_list; @@ -62,6 +65,8 @@ public: ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } + const std::shared_ptr & getMatcher() const; void setPattern(String pattern, bool set_matcher = true); void setMatcher(std::shared_ptr matcher); @@ -84,6 +89,7 @@ public: String getID(char) const override { return "QualifiedColumnsListMatcher"; } ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } ASTPtr qualifier; ASTPtr column_list; diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h index e42949ebfd8..ae84f424a8b 100644 --- a/src/Parsers/ASTColumnsTransformers.h +++ b/src/Parsers/ASTColumnsTransformers.h @@ -48,6 +48,8 @@ public: } void transform(ASTs & nodes) const override; void appendColumnName(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } + void updateTreeHashImpl(SipHash & hash_state) const override; // Case 1 APPLY (quantile(0.9)) @@ -80,6 +82,7 @@ public: const std::shared_ptr & getMatcher() const; bool isColumnMatching(const String & column_name) const; void appendColumnName(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } void updateTreeHashImpl(SipHash & hash_state) const override; protected: @@ -103,6 +106,8 @@ public: } void appendColumnName(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } + void updateTreeHashImpl(SipHash & hash_state) const override; String name; @@ -121,6 +126,8 @@ public: } void transform(ASTs & nodes) const override; void appendColumnName(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } + void updateTreeHashImpl(SipHash & hash_state) const override; protected: diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 129d3d60744..3dd301b1b64 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -468,6 +468,27 @@ namespace }; } +bool ASTFunction::hasColumnName() const +{ + if (parameters) + { + for (const auto & child : parameters->children) + { + if (!child->hasColumnName()) + return false; + } + } + + if (arguments) + { + for (const auto & child : arguments->children) + { + if (!child->hasColumnName()) + return false; + } + } + return true; +} void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const { diff --git a/src/Parsers/ASTFunction.h b/src/Parsers/ASTFunction.h index 4a036c5e94a..39867e4fee0 100644 --- a/src/Parsers/ASTFunction.h +++ b/src/Parsers/ASTFunction.h @@ -79,6 +79,8 @@ public: protected: void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; + bool hasColumnName() const override; + private: void finishFormatWithWindow(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; }; diff --git a/src/Parsers/ASTIdentifier.h b/src/Parsers/ASTIdentifier.h index 0e030c797ce..301542ce6e5 100644 --- a/src/Parsers/ASTIdentifier.h +++ b/src/Parsers/ASTIdentifier.h @@ -58,6 +58,7 @@ protected: void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } private: using ASTWithAlias::children; /// ASTIdentifier is child free diff --git a/src/Parsers/ASTLiteral.h b/src/Parsers/ASTLiteral.h index e57bcfcd9d5..84140ade189 100644 --- a/src/Parsers/ASTLiteral.h +++ b/src/Parsers/ASTLiteral.h @@ -47,6 +47,7 @@ protected: void formatImplWithoutAlias(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } private: /// Legacy version of 'appendColumnNameImpl'. It differs only with tuple literals. diff --git a/src/Parsers/ASTQualifiedAsterisk.h b/src/Parsers/ASTQualifiedAsterisk.h index 079b83ae171..0e6c69f2269 100644 --- a/src/Parsers/ASTQualifiedAsterisk.h +++ b/src/Parsers/ASTQualifiedAsterisk.h @@ -31,6 +31,7 @@ public: return clone; } void appendColumnName(WriteBuffer & ostr) const override; + bool hasColumnName() const override { return true; } ASTPtr qualifier; ASTPtr transformers; diff --git a/src/Parsers/ASTWithAlias.h b/src/Parsers/ASTWithAlias.h index ea4419402b0..8b05385cff6 100644 --- a/src/Parsers/ASTWithAlias.h +++ b/src/Parsers/ASTWithAlias.h @@ -21,6 +21,8 @@ public: using IAST::IAST; void appendColumnName(WriteBuffer & ostr) const final; + bool hasColumnName() const override { return true; } + void appendColumnNameWithoutAlias(WriteBuffer & ostr) const final; String getAliasOrColumnName() const override { return alias.empty() ? getColumnName() : alias; } String tryGetAlias() const override { return alias; } diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 5928506aa5b..554484ea737 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -56,6 +56,9 @@ public: throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to get name of not a column: {}", getID()); } + /* This method should be overridden with appendColumnName */ + virtual bool hasColumnName() const { return false; } + /** Get the alias, if any, or the canonical name of the column, if it is not. */ virtual String getAliasOrColumnName() const { return getColumnName(); } diff --git a/tests/queries/0_stateless/02680_mysql_ast_logical_err.reference b/tests/queries/0_stateless/02680_mysql_ast_logical_err.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql b/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql new file mode 100644 index 00000000000..39f743ee332 --- /dev/null +++ b/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql @@ -0,0 +1,2 @@ +SELECT count() FROM mysql(mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', '', SETTINGS connection_pool_size = 1), '127.0.0.1:9004', currentDatabase(), 'foo', '', ''); -- { serverError BAD_ARGUMENTS } + From 4930683aa87467f21a9c25fd067857b818d549fb Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 17 Mar 2023 17:01:40 +0000 Subject: [PATCH 49/69] Revert "Fix logical error in evaluate constant expression" This reverts commit 2958c5f0f130c6308b00af7f3c41c91357dedb72. --- .../evaluateConstantExpression.cpp | 3 --- src/Parsers/ASTAsterisk.h | 1 - src/Parsers/ASTColumnsMatcher.h | 6 ------ src/Parsers/ASTColumnsTransformers.h | 7 ------- src/Parsers/ASTFunction.cpp | 21 ------------------- src/Parsers/ASTFunction.h | 2 -- src/Parsers/ASTIdentifier.h | 1 - src/Parsers/ASTLiteral.h | 1 - src/Parsers/ASTQualifiedAsterisk.h | 1 - src/Parsers/ASTWithAlias.h | 2 -- src/Parsers/IAST.h | 3 --- .../02680_mysql_ast_logical_err.reference | 0 .../02680_mysql_ast_logical_err.sql | 2 -- 13 files changed, 50 deletions(-) delete mode 100644 tests/queries/0_stateless/02680_mysql_ast_logical_err.reference delete mode 100644 tests/queries/0_stateless/02680_mysql_ast_logical_err.sql diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index 3e5684946c2..ebefa0d9ce7 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -44,9 +44,6 @@ static std::pair> getFieldAndDataTypeFro std::pair> evaluateConstantExpression(const ASTPtr & node, const ContextPtr & context) { - if (!node->hasColumnName()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expression '{}' is not a constant expression", node->formatForErrorMessage()); - if (ASTLiteral * literal = node->as()) return getFieldAndDataTypeFromLiteral(literal); diff --git a/src/Parsers/ASTAsterisk.h b/src/Parsers/ASTAsterisk.h index 72fd82fc9a7..840b7996536 100644 --- a/src/Parsers/ASTAsterisk.h +++ b/src/Parsers/ASTAsterisk.h @@ -15,7 +15,6 @@ public: String getID(char) const override { return "Asterisk"; } ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } ASTPtr expression; ASTPtr transformers; diff --git a/src/Parsers/ASTColumnsMatcher.h b/src/Parsers/ASTColumnsMatcher.h index 5e0149756f1..f31a8bd9a22 100644 --- a/src/Parsers/ASTColumnsMatcher.h +++ b/src/Parsers/ASTColumnsMatcher.h @@ -23,8 +23,6 @@ public: ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } - void setPattern(String pattern); const String & getPattern() const; const std::shared_ptr & getMatcher() const; @@ -48,7 +46,6 @@ public: String getID(char) const override { return "ColumnsListMatcher"; } ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } ASTPtr expression; ASTPtr column_list; @@ -65,8 +62,6 @@ public: ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } - const std::shared_ptr & getMatcher() const; void setPattern(String pattern, bool set_matcher = true); void setMatcher(std::shared_ptr matcher); @@ -89,7 +84,6 @@ public: String getID(char) const override { return "QualifiedColumnsListMatcher"; } ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } ASTPtr qualifier; ASTPtr column_list; diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h index ae84f424a8b..e42949ebfd8 100644 --- a/src/Parsers/ASTColumnsTransformers.h +++ b/src/Parsers/ASTColumnsTransformers.h @@ -48,8 +48,6 @@ public: } void transform(ASTs & nodes) const override; void appendColumnName(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } - void updateTreeHashImpl(SipHash & hash_state) const override; // Case 1 APPLY (quantile(0.9)) @@ -82,7 +80,6 @@ public: const std::shared_ptr & getMatcher() const; bool isColumnMatching(const String & column_name) const; void appendColumnName(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } void updateTreeHashImpl(SipHash & hash_state) const override; protected: @@ -106,8 +103,6 @@ public: } void appendColumnName(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } - void updateTreeHashImpl(SipHash & hash_state) const override; String name; @@ -126,8 +121,6 @@ public: } void transform(ASTs & nodes) const override; void appendColumnName(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } - void updateTreeHashImpl(SipHash & hash_state) const override; protected: diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 3dd301b1b64..129d3d60744 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -468,27 +468,6 @@ namespace }; } -bool ASTFunction::hasColumnName() const -{ - if (parameters) - { - for (const auto & child : parameters->children) - { - if (!child->hasColumnName()) - return false; - } - } - - if (arguments) - { - for (const auto & child : arguments->children) - { - if (!child->hasColumnName()) - return false; - } - } - return true; -} void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const { diff --git a/src/Parsers/ASTFunction.h b/src/Parsers/ASTFunction.h index 39867e4fee0..4a036c5e94a 100644 --- a/src/Parsers/ASTFunction.h +++ b/src/Parsers/ASTFunction.h @@ -79,8 +79,6 @@ public: protected: void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; - bool hasColumnName() const override; - private: void finishFormatWithWindow(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; }; diff --git a/src/Parsers/ASTIdentifier.h b/src/Parsers/ASTIdentifier.h index 301542ce6e5..0e030c797ce 100644 --- a/src/Parsers/ASTIdentifier.h +++ b/src/Parsers/ASTIdentifier.h @@ -58,7 +58,6 @@ protected: void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } private: using ASTWithAlias::children; /// ASTIdentifier is child free diff --git a/src/Parsers/ASTLiteral.h b/src/Parsers/ASTLiteral.h index 84140ade189..e57bcfcd9d5 100644 --- a/src/Parsers/ASTLiteral.h +++ b/src/Parsers/ASTLiteral.h @@ -47,7 +47,6 @@ protected: void formatImplWithoutAlias(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } private: /// Legacy version of 'appendColumnNameImpl'. It differs only with tuple literals. diff --git a/src/Parsers/ASTQualifiedAsterisk.h b/src/Parsers/ASTQualifiedAsterisk.h index 0e6c69f2269..079b83ae171 100644 --- a/src/Parsers/ASTQualifiedAsterisk.h +++ b/src/Parsers/ASTQualifiedAsterisk.h @@ -31,7 +31,6 @@ public: return clone; } void appendColumnName(WriteBuffer & ostr) const override; - bool hasColumnName() const override { return true; } ASTPtr qualifier; ASTPtr transformers; diff --git a/src/Parsers/ASTWithAlias.h b/src/Parsers/ASTWithAlias.h index 8b05385cff6..ea4419402b0 100644 --- a/src/Parsers/ASTWithAlias.h +++ b/src/Parsers/ASTWithAlias.h @@ -21,8 +21,6 @@ public: using IAST::IAST; void appendColumnName(WriteBuffer & ostr) const final; - bool hasColumnName() const override { return true; } - void appendColumnNameWithoutAlias(WriteBuffer & ostr) const final; String getAliasOrColumnName() const override { return alias.empty() ? getColumnName() : alias; } String tryGetAlias() const override { return alias; } diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 554484ea737..5928506aa5b 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -56,9 +56,6 @@ public: throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to get name of not a column: {}", getID()); } - /* This method should be overridden with appendColumnName */ - virtual bool hasColumnName() const { return false; } - /** Get the alias, if any, or the canonical name of the column, if it is not. */ virtual String getAliasOrColumnName() const { return getColumnName(); } diff --git a/tests/queries/0_stateless/02680_mysql_ast_logical_err.reference b/tests/queries/0_stateless/02680_mysql_ast_logical_err.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql b/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql deleted file mode 100644 index 39f743ee332..00000000000 --- a/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql +++ /dev/null @@ -1,2 +0,0 @@ -SELECT count() FROM mysql(mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', '', SETTINGS connection_pool_size = 1), '127.0.0.1:9004', currentDatabase(), 'foo', '', ''); -- { serverError BAD_ARGUMENTS } - From dcf8314122ea5804512893693a8f1986e0751217 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 17 Mar 2023 17:04:28 +0000 Subject: [PATCH 50/69] Better fix logical error in evaluate constant expression --- src/Interpreters/evaluateConstantExpression.cpp | 2 +- tests/queries/0_stateless/02680_mysql_ast_logical_err.reference | 0 tests/queries/0_stateless/02680_mysql_ast_logical_err.sql | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02680_mysql_ast_logical_err.reference create mode 100644 tests/queries/0_stateless/02680_mysql_ast_logical_err.sql diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index ebefa0d9ce7..5a333172b14 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -70,7 +70,6 @@ std::pair> evaluateConstantExpression(co if (context->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && context->getSettingsRef().normalize_function_names) FunctionNameNormalizer().visit(ast.get()); - String result_name = ast->getColumnName(); auto syntax_result = TreeRewriter(context).analyze(ast, source_columns); /// AST potentially could be transformed to literal during TreeRewriter analyze. @@ -82,6 +81,7 @@ std::pair> evaluateConstantExpression(co ColumnPtr result_column; DataTypePtr result_type; + String result_name = ast->getColumnName(); for (const auto & action_node : actions->getOutputs()) { if ((action_node->result_name == result_name) && action_node->column) diff --git a/tests/queries/0_stateless/02680_mysql_ast_logical_err.reference b/tests/queries/0_stateless/02680_mysql_ast_logical_err.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql b/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql new file mode 100644 index 00000000000..39f743ee332 --- /dev/null +++ b/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql @@ -0,0 +1,2 @@ +SELECT count() FROM mysql(mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', '', SETTINGS connection_pool_size = 1), '127.0.0.1:9004', currentDatabase(), 'foo', '', ''); -- { serverError BAD_ARGUMENTS } + From 68da4f713e99b18d46ae2193e54aec6cae444938 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 17 Mar 2023 18:28:55 +0000 Subject: [PATCH 51/69] Real better fix logical error in evaluate constant expression --- src/Interpreters/ActionsVisitor.cpp | 12 +++++++++--- .../0_stateless/02680_mysql_ast_logical_err.sql | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 43db25e775f..96765683abc 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -33,6 +33,8 @@ #include +#include + #include #include #include @@ -67,6 +69,7 @@ namespace ErrorCodes extern const int UNKNOWN_IDENTIFIER; extern const int NOT_AN_AGGREGATE; extern const int UNEXPECTED_EXPRESSION; + extern const int UNKNOWN_FUNCTION; extern const int TYPE_MISMATCH; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_ELEMENT_OF_SET; @@ -880,13 +883,16 @@ void ActionsMatcher::visit(const ASTIdentifier & identifier, const ASTPtr &, Dat void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & data) { - auto column_name = ast->getColumnName(); - if (data.hasColumn(column_name)) - return; + if (TableFunctionFactory::instance().isTableFunctionName(node.name)) + throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "Unexpected table function '{}'", node.name); if (node.name == "lambda") throw Exception(ErrorCodes::UNEXPECTED_EXPRESSION, "Unexpected lambda expression"); + auto column_name = ast->getColumnName(); + if (data.hasColumn(column_name)) + return; + /// Function arrayJoin. if (node.name == "arrayJoin") { diff --git a/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql b/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql index 39f743ee332..5b0530e05ae 100644 --- a/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql +++ b/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql @@ -1,2 +1,2 @@ -SELECT count() FROM mysql(mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', '', SETTINGS connection_pool_size = 1), '127.0.0.1:9004', currentDatabase(), 'foo', '', ''); -- { serverError BAD_ARGUMENTS } - +SELECT count() FROM mysql(mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', ''), '127.0.0.1:9004', currentDatabase(), 'foo', '', ''); -- { serverError UNKNOWN_FUNCTION } +-- SELECT count() FROM mysql(mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', '', SETTINGS connection_pool_size = 1), '127.0.0.1:9004', currentDatabase(), 'foo', '', ''); -- { serverError UNKNOWN_FUNCTION } From 8537ee400d36b5b477de9235d8bf9c101962e699 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 20 Mar 2023 11:22:25 +0100 Subject: [PATCH 52/69] Fix kafka test --- src/Storages/Kafka/StorageKafka.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 538b170ca59..3381561eb1b 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -848,8 +848,6 @@ void registerStorageKafka(StorageFactory & factory) } collection_name = assert_cast(args.engine_args[0].get())->name(); } - else if (!has_settings) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Kafka engine must have settings"); if (has_settings) { From a78b817e6fec8ba814e8d2db9d178a2d619811aa Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 20 Mar 2023 09:45:12 +0000 Subject: [PATCH 53/69] Fix logical error in evaluate constant expression --- src/Interpreters/ActionsVisitor.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 96765683abc..766740665a0 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -883,12 +883,18 @@ void ActionsMatcher::visit(const ASTIdentifier & identifier, const ASTPtr &, Dat void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & data) { - if (TableFunctionFactory::instance().isTableFunctionName(node.name)) - throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "Unexpected table function '{}'", node.name); - if (node.name == "lambda") throw Exception(ErrorCodes::UNEXPECTED_EXPRESSION, "Unexpected lambda expression"); + bool is_valid_function = node.name == "arrayJoin" || node.name == "grouping" || node.name == "indexHint" || + AggregateUtils::isAggregateFunction(node) || + node.is_window_function || + FunctionFactory::instance().has(node.name) || + UserDefinedExecutableFunctionFactory::instance().has(node.name, data.getContext()); + + if (!is_valid_function) + throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "Unexpected function '{}'", node.name); + auto column_name = ast->getColumnName(); if (data.hasColumn(column_name)) return; From 9b8d20e4439605522f7a3a41c0efd68775977468 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 20 Mar 2023 12:42:09 +0000 Subject: [PATCH 54/69] Revert better fix logical error in evaluate constant expression --- src/Interpreters/ActionsVisitor.cpp | 18 +++--------------- .../evaluateConstantExpression.cpp | 2 +- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 766740665a0..43db25e775f 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -33,8 +33,6 @@ #include -#include - #include #include #include @@ -69,7 +67,6 @@ namespace ErrorCodes extern const int UNKNOWN_IDENTIFIER; extern const int NOT_AN_AGGREGATE; extern const int UNEXPECTED_EXPRESSION; - extern const int UNKNOWN_FUNCTION; extern const int TYPE_MISMATCH; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_ELEMENT_OF_SET; @@ -883,22 +880,13 @@ void ActionsMatcher::visit(const ASTIdentifier & identifier, const ASTPtr &, Dat void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & data) { - if (node.name == "lambda") - throw Exception(ErrorCodes::UNEXPECTED_EXPRESSION, "Unexpected lambda expression"); - - bool is_valid_function = node.name == "arrayJoin" || node.name == "grouping" || node.name == "indexHint" || - AggregateUtils::isAggregateFunction(node) || - node.is_window_function || - FunctionFactory::instance().has(node.name) || - UserDefinedExecutableFunctionFactory::instance().has(node.name, data.getContext()); - - if (!is_valid_function) - throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "Unexpected function '{}'", node.name); - auto column_name = ast->getColumnName(); if (data.hasColumn(column_name)) return; + if (node.name == "lambda") + throw Exception(ErrorCodes::UNEXPECTED_EXPRESSION, "Unexpected lambda expression"); + /// Function arrayJoin. if (node.name == "arrayJoin") { diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index 5a333172b14..ebefa0d9ce7 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -70,6 +70,7 @@ std::pair> evaluateConstantExpression(co if (context->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && context->getSettingsRef().normalize_function_names) FunctionNameNormalizer().visit(ast.get()); + String result_name = ast->getColumnName(); auto syntax_result = TreeRewriter(context).analyze(ast, source_columns); /// AST potentially could be transformed to literal during TreeRewriter analyze. @@ -81,7 +82,6 @@ std::pair> evaluateConstantExpression(co ColumnPtr result_column; DataTypePtr result_type; - String result_name = ast->getColumnName(); for (const auto & action_node : actions->getOutputs()) { if ((action_node->result_name == result_name) && action_node->column) From 5b036a1a3bc783086265c3070c9b868f53a9fe4c Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 20 Mar 2023 12:55:03 +0000 Subject: [PATCH 55/69] More preparation for libcxx(abi), llvm, clang-tidy 16 (follow-up to #47722) --- base/base/wide_integer_to_string.h | 2 +- src/Dictionaries/RedisDictionarySource.cpp | 10 +++++ src/Dictionaries/RedisDictionarySource.h | 11 +---- src/Interpreters/ActionsVisitor.cpp | 8 ++-- src/Storages/Distributed/DistributedSink.cpp | 4 ++ src/Storages/Distributed/DistributedSink.h | 3 +- src/Storages/HDFS/ReadBufferFromHDFS.cpp | 4 +- src/Storages/MergeTree/MergeTreeSink.cpp | 28 ++++++------- src/Storages/MergeTree/MergeTreeSource.cpp | 44 ++++++++++---------- 9 files changed, 60 insertions(+), 54 deletions(-) diff --git a/base/base/wide_integer_to_string.h b/base/base/wide_integer_to_string.h index 160bf599516..c2cbe8d82e3 100644 --- a/base/base/wide_integer_to_string.h +++ b/base/base/wide_integer_to_string.h @@ -64,6 +64,6 @@ struct fmt::formatter> template auto format(const wide::integer & value, FormatContext & ctx) { - return format_to(ctx.out(), "{}", to_string(value)); + return fmt::format_to(ctx.out(), "{}", to_string(value)); } }; diff --git a/src/Dictionaries/RedisDictionarySource.cpp b/src/Dictionaries/RedisDictionarySource.cpp index bde4d596352..6e4c5d1d5d9 100644 --- a/src/Dictionaries/RedisDictionarySource.cpp +++ b/src/Dictionaries/RedisDictionarySource.cpp @@ -68,6 +68,16 @@ namespace DB factory.registerSource("redis", create_table_source); } + RedisDictionarySource::Connection::Connection(PoolPtr pool_, ClientPtr client_) + : pool(std::move(pool_)), client(std::move(client_)) + { + } + + RedisDictionarySource::Connection::~Connection() + { + pool->returnObject(std::move(client)); + } + static constexpr size_t REDIS_MAX_BLOCK_SIZE = DEFAULT_BLOCK_SIZE; static constexpr size_t REDIS_LOCK_ACQUIRE_TIMEOUT_MS = 5000; diff --git a/src/Dictionaries/RedisDictionarySource.h b/src/Dictionaries/RedisDictionarySource.h index 26f5ab2a613..8fb6f93193b 100644 --- a/src/Dictionaries/RedisDictionarySource.h +++ b/src/Dictionaries/RedisDictionarySource.h @@ -52,15 +52,8 @@ namespace DB struct Connection { - Connection(PoolPtr pool_, ClientPtr client_) - : pool(std::move(pool_)), client(std::move(client_)) - { - } - - ~Connection() - { - pool->returnObject(std::move(client)); - } + Connection(PoolPtr pool_, ClientPtr client_); + ~Connection(); PoolPtr pool; ClientPtr client; diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 43db25e775f..8a5ea1205e7 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -467,10 +467,6 @@ SetPtr makeExplicitSet( return set; } -ScopeStack::Level::~Level() = default; -ScopeStack::Level::Level() = default; -ScopeStack::Level::Level(Level &&) noexcept = default; - class ScopeStack::Index { /// Map column name -> Node. @@ -524,6 +520,10 @@ public: } }; +ScopeStack::Level::~Level() = default; +ScopeStack::Level::Level() = default; +ScopeStack::Level::Level(Level &&) noexcept = default; + ActionsMatcher::Data::Data( ContextPtr context_, SizeLimits set_size_limit_, diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 19eab0b5837..73d3aebe0d0 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -210,6 +210,10 @@ std::string DistributedSink::getCurrentStateDescription() } +DistributedSink::JobReplica::JobReplica(size_t shard_index_, size_t replica_index_, bool is_local_job_, const Block & sample_block) + : shard_index(shard_index_), replica_index(replica_index_), is_local_job(is_local_job_), current_shard_block(sample_block.cloneEmpty()) {} + + void DistributedSink::initWritingJobs(const Block & first_block, size_t start, size_t end) { const Settings & settings = context->getSettingsRef(); diff --git a/src/Storages/Distributed/DistributedSink.h b/src/Storages/Distributed/DistributedSink.h index 325d5859289..1bb4419e1a5 100644 --- a/src/Storages/Distributed/DistributedSink.h +++ b/src/Storages/Distributed/DistributedSink.h @@ -118,8 +118,7 @@ private: struct JobReplica { JobReplica() = default; - JobReplica(size_t shard_index_, size_t replica_index_, bool is_local_job_, const Block & sample_block) - : shard_index(shard_index_), replica_index(replica_index_), is_local_job(is_local_job_), current_shard_block(sample_block.cloneEmpty()) {} + JobReplica(size_t shard_index_, size_t replica_index_, bool is_local_job_, const Block & sample_block); size_t shard_index = 0; size_t replica_index = 0; diff --git a/src/Storages/HDFS/ReadBufferFromHDFS.cpp b/src/Storages/HDFS/ReadBufferFromHDFS.cpp index 3e7c27fe4f2..ee8e0764db0 100644 --- a/src/Storages/HDFS/ReadBufferFromHDFS.cpp +++ b/src/Storages/HDFS/ReadBufferFromHDFS.cpp @@ -29,8 +29,6 @@ namespace ErrorCodes } -ReadBufferFromHDFS::~ReadBufferFromHDFS() = default; - struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory { String hdfs_uri; @@ -166,6 +164,8 @@ ReadBufferFromHDFS::ReadBufferFromHDFS( { } +ReadBufferFromHDFS::~ReadBufferFromHDFS() = default; + size_t ReadBufferFromHDFS::getFileSize() { return impl->getFileSize(); diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 1e607767f86..981eb1af280 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -13,6 +13,20 @@ namespace ProfileEvents namespace DB { +struct MergeTreeSink::DelayedChunk +{ + struct Partition + { + MergeTreeDataWriter::TemporaryPart temp_part; + UInt64 elapsed_ns; + String block_dedup_token; + ProfileEvents::Counters part_counters; + }; + + std::vector partitions; +}; + + MergeTreeSink::~MergeTreeSink() = default; MergeTreeSink::MergeTreeSink( @@ -41,20 +55,6 @@ void MergeTreeSink::onFinish() finishDelayedChunk(); } -struct MergeTreeSink::DelayedChunk -{ - struct Partition - { - MergeTreeDataWriter::TemporaryPart temp_part; - UInt64 elapsed_ns; - String block_dedup_token; - ProfileEvents::Counters part_counters; - }; - - std::vector partitions; -}; - - void MergeTreeSink::consume(Chunk chunk) { auto block = getHeader().cloneWithColumns(chunk.detachColumns()); diff --git a/src/Storages/MergeTree/MergeTreeSource.cpp b/src/Storages/MergeTree/MergeTreeSource.cpp index a37d1d3ec2c..328336ff71a 100644 --- a/src/Storages/MergeTree/MergeTreeSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSource.cpp @@ -7,28 +7,6 @@ namespace DB { -MergeTreeSource::MergeTreeSource(MergeTreeSelectAlgorithmPtr algorithm_) - : ISource(algorithm_->getHeader()) - , algorithm(std::move(algorithm_)) -{ -#if defined(OS_LINUX) - if (algorithm->getSettings().use_asynchronous_read_from_pool) - async_reading_state = std::make_unique(); -#endif -} - -MergeTreeSource::~MergeTreeSource() = default; - -std::string MergeTreeSource::getName() const -{ - return algorithm->getName(); -} - -void MergeTreeSource::onCancel() -{ - algorithm->cancel(); -} - #if defined(OS_LINUX) struct MergeTreeSource::AsyncReadingState { @@ -155,6 +133,28 @@ private: }; #endif +MergeTreeSource::MergeTreeSource(MergeTreeSelectAlgorithmPtr algorithm_) + : ISource(algorithm_->getHeader()) + , algorithm(std::move(algorithm_)) +{ +#if defined(OS_LINUX) + if (algorithm->getSettings().use_asynchronous_read_from_pool) + async_reading_state = std::make_unique(); +#endif +} + +MergeTreeSource::~MergeTreeSource() = default; + +std::string MergeTreeSource::getName() const +{ + return algorithm->getName(); +} + +void MergeTreeSource::onCancel() +{ + algorithm->cancel(); +} + ISource::Status MergeTreeSource::prepare() { #if defined(OS_LINUX) From 19fb88c2c1ac2cd67fd377066251ddb5cce7f3f2 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 20 Mar 2023 12:42:30 +0000 Subject: [PATCH 56/69] Simple fix logical error in evaluate constant expression --- src/Interpreters/evaluateConstantExpression.cpp | 2 +- src/Parsers/ASTFunction.cpp | 7 ++++--- .../queries/0_stateless/01715_table_function_view_fix.sql | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index ebefa0d9ce7..5a333172b14 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -70,7 +70,6 @@ std::pair> evaluateConstantExpression(co if (context->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && context->getSettingsRef().normalize_function_names) FunctionNameNormalizer().visit(ast.get()); - String result_name = ast->getColumnName(); auto syntax_result = TreeRewriter(context).analyze(ast, source_columns); /// AST potentially could be transformed to literal during TreeRewriter analyze. @@ -82,6 +81,7 @@ std::pair> evaluateConstantExpression(co ColumnPtr result_column; DataTypePtr result_type; + String result_name = ast->getColumnName(); for (const auto & action_node : actions->getOutputs()) { if ((action_node->result_name == result_name) && action_node->column) diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 129d3d60744..fa4874d89b6 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -28,8 +28,8 @@ namespace DB namespace ErrorCodes { - extern const int UNEXPECTED_EXPRESSION; extern const int UNEXPECTED_AST_STRUCTURE; + extern const int UNKNOWN_FUNCTION; } @@ -471,8 +471,9 @@ namespace void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const { - if (name == "view") - throw Exception(ErrorCodes::UNEXPECTED_EXPRESSION, "Table function view cannot be used as an expression"); + /// These functions contain some unexpected ASTs in arguments (e.g. SETTINGS or even a SELECT query) + if (name == "view" || name == "viewIfPermitted" || name == "mysql" || name == "postgresql" || name == "mongodb" || name == "s3") + throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "Table function '{}' cannot be used as an expression", name); /// If function can be converted to literal it will be parsed as literal after formatting. /// In distributed query it may lead to mismathed column names. diff --git a/tests/queries/0_stateless/01715_table_function_view_fix.sql b/tests/queries/0_stateless/01715_table_function_view_fix.sql index b96609391b5..5c24131b438 100644 --- a/tests/queries/0_stateless/01715_table_function_view_fix.sql +++ b/tests/queries/0_stateless/01715_table_function_view_fix.sql @@ -1,3 +1,3 @@ SELECT view(SELECT 1); -- { clientError 62 } -SELECT sumIf(dummy, dummy) FROM remote('127.0.0.{1,2}', numbers(2, 100), view(SELECT CAST(NULL, 'Nullable(UInt8)') AS dummy FROM system.one)); -- { serverError 183 } +SELECT sumIf(dummy, dummy) FROM remote('127.0.0.{1,2}', numbers(2, 100), view(SELECT CAST(NULL, 'Nullable(UInt8)') AS dummy FROM system.one)); -- { serverError UNKNOWN_FUNCTION } From bb7a8ed862d6fd9c43f841ae9f6ee339177cba77 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 17 Mar 2023 15:09:38 +0000 Subject: [PATCH 57/69] Apply log_queries_cut_to_length in MergeTreeWhereOptimizer --- src/Common/SensitiveDataMasker.cpp | 7 ++++++- src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp | 3 ++- src/Storages/MergeTree/MergeTreeWhereOptimizer.h | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Common/SensitiveDataMasker.cpp b/src/Common/SensitiveDataMasker.cpp index 5336b48524f..a5de2baca02 100644 --- a/src/Common/SensitiveDataMasker.cpp +++ b/src/Common/SensitiveDataMasker.cpp @@ -202,8 +202,13 @@ std::string wipeSensitiveDataAndCutToLength(const std::string & str, size_t max_ if (auto * masker = SensitiveDataMasker::getInstance()) masker->wipeSensitiveData(res); - if (max_length && (res.length() > max_length)) + size_t length = res.length(); + if (max_length && (length > max_length)) + { + size_t truncated_len = length - max_length; res.resize(max_length); + res += "... (truncated " + std::to_string(truncated_len) + " chars)"; + } return res; } diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index fdddc29048b..c7afe9319f9 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -44,6 +44,7 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( , log{log_} , column_sizes{std::move(column_sizes_)} , move_all_conditions_to_prewhere(context->getSettingsRef().move_all_conditions_to_prewhere) + , log_queries_cut_to_length(context->getSettingsRef().log_queries_cut_to_length) { for (const auto & name : queried_columns) { @@ -310,7 +311,7 @@ void MergeTreeWhereOptimizer::optimize(ASTSelectQuery & select) const select.setExpression(ASTSelectQuery::Expression::WHERE, reconstruct(where_conditions)); select.setExpression(ASTSelectQuery::Expression::PREWHERE, reconstruct(prewhere_conditions)); - LOG_DEBUG(log, "MergeTreeWhereOptimizer: condition \"{}\" moved to PREWHERE", select.prewhere()); + LOG_DEBUG(log, "MergeTreeWhereOptimizer: condition \"{}\" moved to PREWHERE", select.prewhere()->formatForLogging(log_queries_cut_to_length)); } diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h index 8953923542e..ef87018f93e 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h @@ -115,6 +115,7 @@ private: UInt64 total_size_of_queried_columns = 0; NameSet array_joined_names; const bool move_all_conditions_to_prewhere = false; + UInt64 log_queries_cut_to_length = 0; }; From 34b247900ff93e24d979fdfa006ce59011edb06e Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 20 Mar 2023 16:53:54 +0100 Subject: [PATCH 58/69] Better test --- .../test_named_collections/test.py | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index af5aab38264..5f5657fad54 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -199,23 +199,40 @@ def test_granular_access_show_query(cluster): assert 0 == int( node.query("SELECT count() FROM system.named_collections", user="koko") ) + assert "GRANT SELECT ON *.* TO koko" == node.query("SHOW GRANTS FOR koko;").strip() node.query("GRANT show named collections ON * TO koko") + assert ( + "GRANT SELECT ON *.* TO koko\nGRANT SHOW NAMED COLLECTIONS ON * TO koko" + == node.query("SHOW GRANTS FOR koko;").strip() + ) assert ( "collection1\ncollection2" == node.query("select name from system.named_collections", user="koko").strip() ) node.restart_clickhouse() + assert ( + "GRANT SELECT ON *.* TO koko\nGRANT SHOW NAMED COLLECTIONS ON * TO koko" + == node.query("SHOW GRANTS FOR koko;").strip() + ) assert ( "collection1\ncollection2" == node.query("select name from system.named_collections", user="koko").strip() ) node.query("REVOKE show named collections ON collection1 FROM koko;") + assert ( + "GRANT SELECT ON *.* TO koko\nGRANT SHOW NAMED COLLECTIONS ON * TO koko\nREVOKE SHOW NAMED COLLECTIONS ON collection1 FROM koko" + == node.query("SHOW GRANTS FOR koko;").strip() + ) assert ( "collection2" == node.query("select name from system.named_collections", user="koko").strip() ) node.restart_clickhouse() + assert ( + "GRANT SELECT ON *.* TO koko\nGRANT SHOW NAMED COLLECTIONS ON * TO koko\nREVOKE SHOW NAMED COLLECTIONS ON collection1 FROM koko" + == node.query("SHOW GRANTS FOR koko;").strip() + ) assert ( "collection2" == node.query("select name from system.named_collections", user="koko").strip() @@ -224,17 +241,26 @@ def test_granular_access_show_query(cluster): assert ( "" == node.query("select * from system.named_collections", user="koko").strip() ) + assert ( + "GRANT SELECT ON *.* TO koko\nGRANT SHOW NAMED COLLECTIONS ON * TO koko\nREVOKE SHOW NAMED COLLECTIONS ON collection1 FROM koko\nREVOKE SHOW NAMED COLLECTIONS ON collection2 FROM koko" + == node.query("SHOW GRANTS FOR koko;").strip() + ) # check: # GRANT show named collections ON collection # REVOKE show named collections ON * node.query("GRANT show named collections ON collection2 TO koko") + assert ( + "GRANT SELECT ON *.* TO koko\nGRANT SHOW NAMED COLLECTIONS ON * TO koko\nREVOKE SHOW NAMED COLLECTIONS ON collection1 FROM koko" + == node.query("SHOW GRANTS FOR koko;").strip() + ) assert ( "collection2" == node.query("select name from system.named_collections", user="koko").strip() ) node.query("REVOKE show named collections ON * FROM koko;") + assert "GRANT SELECT ON *.* TO koko" == node.query("SHOW GRANTS FOR koko;").strip() assert ( "" == node.query("select * from system.named_collections", user="koko").strip() ) @@ -250,7 +276,7 @@ def test_show_grants(cluster): node.query("GRANT select ON name1.* TO koko") assert ( "GRANT SELECT ON name1.* TO koko\nGRANT CREATE NAMED COLLECTION ON name1 TO koko" - in node.query("SHOW GRANTS FOR koko;").strip() + == node.query("SHOW GRANTS FOR koko;").strip() ) node.query("DROP USER IF EXISTS koko") @@ -259,7 +285,7 @@ def test_show_grants(cluster): node.query("GRANT select ON name1 TO koko") assert ( "GRANT SELECT ON default.name1 TO koko\nGRANT CREATE NAMED COLLECTION ON name1 TO koko" - in node.query("SHOW GRANTS FOR koko;").strip() + == node.query("SHOW GRANTS FOR koko;").strip() ) node.query("DROP USER IF EXISTS koko") @@ -268,7 +294,7 @@ def test_show_grants(cluster): node.query("GRANT CREATE NAMED COLLECTION ON name1 TO koko") assert ( "GRANT SELECT ON default.name1 TO koko\nGRANT CREATE NAMED COLLECTION ON name1 TO koko" - in node.query("SHOW GRANTS FOR koko;").strip() + == node.query("SHOW GRANTS FOR koko;").strip() ) node.query("DROP USER IF EXISTS koko") @@ -277,7 +303,7 @@ def test_show_grants(cluster): node.query("GRANT CREATE NAMED COLLECTION ON * TO koko") assert ( "GRANT SELECT ON *.* TO koko\nGRANT CREATE NAMED COLLECTION ON * TO koko" - in node.query("SHOW GRANTS FOR koko;").strip() + == node.query("SHOW GRANTS FOR koko;").strip() ) node.query("DROP USER IF EXISTS koko") @@ -286,7 +312,7 @@ def test_show_grants(cluster): node.query("GRANT select ON *.* TO koko") assert ( "GRANT SELECT ON *.* TO koko\nGRANT CREATE NAMED COLLECTION ON * TO koko" - in node.query("SHOW GRANTS FOR koko;").strip() + == node.query("SHOW GRANTS FOR koko;").strip() ) node.query("DROP USER IF EXISTS koko") @@ -295,7 +321,7 @@ def test_show_grants(cluster): node.query("GRANT select ON * TO koko") assert ( "GRANT CREATE NAMED COLLECTION ON * TO koko\nGRANT SELECT ON default.* TO koko" - in node.query("SHOW GRANTS FOR koko;").strip() + == node.query("SHOW GRANTS FOR koko;").strip() ) node.query("DROP USER IF EXISTS koko") @@ -304,7 +330,7 @@ def test_show_grants(cluster): node.query("GRANT CREATE NAMED COLLECTION ON * TO koko") assert ( "GRANT CREATE NAMED COLLECTION ON * TO koko\nGRANT SELECT ON default.* TO koko" - in node.query("SHOW GRANTS FOR koko;").strip() + == node.query("SHOW GRANTS FOR koko;").strip() ) From 8c48714236394b1069e4ae743b2d79148733ad77 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 20 Mar 2023 15:54:19 +0000 Subject: [PATCH 59/69] wipeSensitiveDataAndCutToLength take into accound suffix length --- src/Common/SensitiveDataMasker.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Common/SensitiveDataMasker.cpp b/src/Common/SensitiveDataMasker.cpp index a5de2baca02..34db78d00fb 100644 --- a/src/Common/SensitiveDataMasker.cpp +++ b/src/Common/SensitiveDataMasker.cpp @@ -205,9 +205,12 @@ std::string wipeSensitiveDataAndCutToLength(const std::string & str, size_t max_ size_t length = res.length(); if (max_length && (length > max_length)) { - size_t truncated_len = length - max_length; + constexpr size_t max_extra_msg_len = sizeof("... (truncated 18446744073709551615 characters)"); + if (max_length < max_extra_msg_len) + return "(removed " + std::to_string(length) + " characters)"; + max_length -= max_extra_msg_len; res.resize(max_length); - res += "... (truncated " + std::to_string(truncated_len) + " chars)"; + res.append("... (truncated " + std::to_string(length - max_length) + " characters)"); } return res; From b75c92e6b3bff275ee02d60729f03102fc63e737 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 20 Mar 2023 18:38:53 +0100 Subject: [PATCH 60/69] May be fix window view test --- .../0_stateless/01072_window_view_multiple_columns_groupby.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh index 05cb973cdb5..1b66beffe7e 100755 --- a/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh +++ b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh @@ -17,7 +17,7 @@ DROP TABLE IF EXISTS wv; CREATE TABLE dst(time DateTime, colA String, colB String) Engine=MergeTree ORDER BY tuple(); CREATE TABLE mt(colA String, colB String) ENGINE=MergeTree ORDER BY tuple(); -CREATE WINDOW VIEW wv TO dst AS SELECT tumbleStart(w_id) AS time, colA, colB FROM mt GROUP BY tumble(now(), INTERVAL '10' SECOND, 'US/Samoa') AS w_id, colA, colB; +CREATE WINDOW VIEW wv TO dst AS SELECT tumbleStart(w_id) AS time, colA, colB FROM mt GROUP BY tumble(now('US/Samoa'), INTERVAL '10' SECOND, 'US/Samoa') AS w_id, colA, colB; INSERT INTO mt VALUES ('test1', 'test2'); EOF From dabc022289bde9986bacffa4dec8c25fe298e024 Mon Sep 17 00:00:00 2001 From: fyu Date: Tue, 21 Mar 2023 03:27:18 +0800 Subject: [PATCH 61/69] Fixed error print message while Decimal parameters is incorrect --- src/DataTypes/DataTypesDecimal.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DataTypes/DataTypesDecimal.cpp b/src/DataTypes/DataTypesDecimal.cpp index 1ae0d8a6db1..2838f5868d4 100644 --- a/src/DataTypes/DataTypesDecimal.cpp +++ b/src/DataTypes/DataTypesDecimal.cpp @@ -89,12 +89,12 @@ static DataTypePtr createExact(const ASTPtr & arguments) { if (!arguments || arguments->children.size() != 1) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Decimal data type family must have exactly two arguments: precision and scale"); - + "Decimal32 | Decimal64 | Decimal128 | Decimal256 data type family must have exactly one arguments: scale"); const auto * scale_arg = arguments->children[0]->as(); if (!scale_arg || !(scale_arg->value.getType() == Field::Types::Int64 || scale_arg->value.getType() == Field::Types::UInt64)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Decimal data type family must have a two numbers as its arguments"); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Decimal32 | Decimal64 | Decimal128 | Decimal256 data type family must have a one number as its argument"); UInt64 precision = DecimalUtils::max_precision; UInt64 scale = scale_arg->value.get(); From 15d02f77cfcbfe5af0350e4c5b26daa2a68fb8bd Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Mar 2023 20:56:11 +0100 Subject: [PATCH 62/69] Decrease scale_down ratio for faster deflation --- tests/ci/autoscale_runners_lambda/app.py | 4 ++-- tests/ci/autoscale_runners_lambda_test.py | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/ci/autoscale_runners_lambda/app.py b/tests/ci/autoscale_runners_lambda/app.py index 7e3af3f6779..1fcdbc40155 100644 --- a/tests/ci/autoscale_runners_lambda/app.py +++ b/tests/ci/autoscale_runners_lambda/app.py @@ -59,11 +59,11 @@ def get_scales(runner_type: str) -> Tuple[int, int]: "returns the multipliers for scaling down and up ASG by types" # Scaling down is quicker on the lack of running jobs than scaling up on # queue - scale_down = 3 + scale_down = 2 scale_up = 5 if runner_type == "style-checker": # the style checkers have so many noise, so it scales up too quickly - scale_down = 2 + scale_down = 1 scale_up = 10 return scale_down, scale_up diff --git a/tests/ci/autoscale_runners_lambda_test.py b/tests/ci/autoscale_runners_lambda_test.py index 7efa0004745..8e3828f51c0 100644 --- a/tests/ci/autoscale_runners_lambda_test.py +++ b/tests/ci/autoscale_runners_lambda_test.py @@ -70,6 +70,9 @@ class TestSetCapacity(unittest.TestCase): TestCase("w/reserve", 1, 13, 20, [Queue("queued", 17, "w/reserve")], -1), # Increase capacity TestCase("increase", 1, 13, 20, [Queue("queued", 23, "increase")], 15), + TestCase( + "style-checker", 1, 13, 20, [Queue("queued", 33, "style-checker")], 15 + ), TestCase("increase", 1, 13, 20, [Queue("queued", 18, "increase")], 14), TestCase("increase", 1, 13, 20, [Queue("queued", 183, "increase")], 20), TestCase( @@ -85,10 +88,20 @@ class TestSetCapacity(unittest.TestCase): ), TestCase("lower-min", 10, 5, 20, [Queue("queued", 5, "lower-min")], 10), # Decrease capacity - TestCase("w/reserve", 1, 13, 20, [Queue("queued", 5, "w/reserve")], 11), + TestCase("w/reserve", 1, 13, 20, [Queue("queued", 5, "w/reserve")], 9), + TestCase( + "style-checker", 1, 13, 20, [Queue("queued", 5, "style-checker")], 5 + ), TestCase("w/reserve", 1, 23, 20, [Queue("queued", 17, "w/reserve")], 20), - TestCase("decrease", 1, 13, 20, [Queue("in_progress", 3, "decrease")], 10), - TestCase("decrease", 1, 13, 20, [Queue("in_progress", 5, "decrease")], 11), + TestCase("decrease", 1, 13, 20, [Queue("in_progress", 3, "decrease")], 8), + TestCase( + "style-checker", + 1, + 13, + 20, + [Queue("in_progress", 5, "style-checker")], + 5, + ), ) for t in test_cases: self.client.data_helper(t.name, t.min_size, t.desired_capacity, t.max_size) From 7531840fd735465affecb57ed8d6883fb7c1d6fd Mon Sep 17 00:00:00 2001 From: flynn Date: Tue, 21 Mar 2023 02:41:51 +0000 Subject: [PATCH 63/69] Support fuzz Explain query --- src/Client/QueryFuzzer.cpp | 105 ++++++++++++++++++++++++++++++++-- src/Client/QueryFuzzer.h | 4 ++ src/Parsers/ASTExplainQuery.h | 2 + 3 files changed, 107 insertions(+), 4 deletions(-) diff --git a/src/Client/QueryFuzzer.cpp b/src/Client/QueryFuzzer.cpp index e150717db95..4383984539e 100644 --- a/src/Client/QueryFuzzer.cpp +++ b/src/Client/QueryFuzzer.cpp @@ -20,13 +20,11 @@ #include -#include -#include -#include #include #include #include #include +#include #include #include #include @@ -34,17 +32,20 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include #include #include #include +#include +#include +#include namespace DB @@ -681,6 +682,98 @@ void QueryFuzzer::fuzzTableName(ASTTableExpression & table) } } +void QueryFuzzer::fuzzExplainQuery(ASTExplainQuery & explain) +{ + /// Fuzz ExplainKind + if (fuzz_rand() % 20 == 0) + { + /// Do not modify ExplainKind + } + else if (fuzz_rand() % 11 == 0) + { + explain.setExplainKind(ASTExplainQuery::ExplainKind::ParsedAST); + } + else if (fuzz_rand() % 11 == 0) + { + explain.setExplainKind(ASTExplainQuery::ExplainKind::AnalyzedSyntax); + } + else if (fuzz_rand() % 11 == 0) + { + explain.setExplainKind(ASTExplainQuery::ExplainKind::QueryTree); + } + else if (fuzz_rand() % 11 == 0) + { + explain.setExplainKind(ASTExplainQuery::ExplainKind::QueryPlan); + } + else if (fuzz_rand() % 11 == 0) + { + explain.setExplainKind(ASTExplainQuery::ExplainKind::QueryPipeline); + } + else if (fuzz_rand() % 11 == 0) + { + explain.setExplainKind(ASTExplainQuery::ExplainKind::QueryEstimates); + } + else if (fuzz_rand() % 11 == 0) + { + explain.setExplainKind(ASTExplainQuery::ExplainKind::TableOverride); + } + else if (fuzz_rand() % 11 == 0) + { + explain.setExplainKind(ASTExplainQuery::ExplainKind::CurrentTransaction); + } + + static const std::unordered_map> settings_by_kind + = {{ASTExplainQuery::ExplainKind::ParsedAST, {"graph", "optimize"}}, + {ASTExplainQuery::ExplainKind::AnalyzedSyntax, {}}, + {ASTExplainQuery::QueryTree, {"run_passes", "dump_passes", "dump_ast", "passes"}}, + {ASTExplainQuery::ExplainKind::QueryPlan, {"header, description", "actions", "indexes", "optimize", "json", "sorting"}}, + {ASTExplainQuery::ExplainKind::QueryPipeline, {"header", "graph=1", "compact"}}, + {ASTExplainQuery::ExplainKind::QueryEstimates, {}}, + {ASTExplainQuery::ExplainKind::TableOverride, {}}, + {ASTExplainQuery::ExplainKind::CurrentTransaction, {}}}; + + const auto & settings = settings_by_kind.at(explain.getKind()); + bool settings_have_fuzzed = false; + for (auto & child : explain.children) + { + if (auto * settings_ast = typeid_cast(child.get())) + { + fuzzExplainSettings(*settings_ast, settings); + settings_have_fuzzed = true; + } + /// Fuzz other child like Explain Query + else + { + fuzz(child); + } + } + + if (!settings_have_fuzzed && !settings.empty()) + { + auto settings_ast = std::make_shared(); + fuzzExplainSettings(*settings_ast, settings); + explain.setSettings(settings_ast); + } +} + +void QueryFuzzer::fuzzExplainSettings(ASTSetQuery & settings, const std::vector & names) +{ + auto & changes = settings.changes; + + if (fuzz_rand() % 50 == 0 && !changes.empty()) + { + changes.erase(changes.begin() + fuzz_rand() % changes.size()); + } + + for (const auto & name : names) + { + if (fuzz_rand() % 5 == 0) + { + changes.emplace_back(name, true); + } + } +} + static ASTPtr tryParseInsertQuery(const String & full_query) { const char * pos = full_query.data(); @@ -991,6 +1084,10 @@ void QueryFuzzer::fuzz(ASTPtr & ast) { fuzzCreateQuery(*create_query); } + else if (auto * explain_query = typeid_cast(ast.get())) + { + fuzzExplainQuery(*explain_query); + } else { fuzz(ast->children); diff --git a/src/Client/QueryFuzzer.h b/src/Client/QueryFuzzer.h index bdfdeb67663..445533258bb 100644 --- a/src/Client/QueryFuzzer.h +++ b/src/Client/QueryFuzzer.h @@ -22,6 +22,8 @@ class ASTCreateQuery; class ASTInsertQuery; class ASTColumnDeclaration; class ASTDropQuery; +class ASTExplainQuery; +class ASTSetQuery; struct ASTTableExpression; struct ASTWindowDefinition; @@ -86,6 +88,8 @@ struct QueryFuzzer void fuzzColumnLikeExpressionList(IAST * ast); void fuzzWindowFrame(ASTWindowDefinition & def); void fuzzCreateQuery(ASTCreateQuery & create); + void fuzzExplainQuery(ASTExplainQuery & explain); + void fuzzExplainSettings(ASTSetQuery & settings, const std::vector & names); void fuzzColumnDeclaration(ASTColumnDeclaration & column); void fuzzTableName(ASTTableExpression & table); void fuzz(ASTs & asts); diff --git a/src/Parsers/ASTExplainQuery.h b/src/Parsers/ASTExplainQuery.h index 3903cf42269..701bde8cebd 100644 --- a/src/Parsers/ASTExplainQuery.h +++ b/src/Parsers/ASTExplainQuery.h @@ -80,6 +80,8 @@ public: return res; } + void setExplainKind(ExplainKind kind_) { kind = kind_; } + void setExplainedQuery(ASTPtr query_) { children.emplace_back(query_); From d3515dd7ae7fe4eec4afd4cae5304872b755afb1 Mon Sep 17 00:00:00 2001 From: Tanya Bragin Date: Mon, 20 Mar 2023 22:05:10 -0700 Subject: [PATCH 64/69] Update query profiler docs to remove self-managed only Query profiler works in ClickHouse Cloud - update docs to reflect it and add an example. cc @alexey-milovidov --- .../sampling-query-profiler.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/optimizing-performance/sampling-query-profiler.md b/docs/en/operations/optimizing-performance/sampling-query-profiler.md index 72eb655101f..ab42eec4190 100644 --- a/docs/en/operations/optimizing-performance/sampling-query-profiler.md +++ b/docs/en/operations/optimizing-performance/sampling-query-profiler.md @@ -7,11 +7,23 @@ import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.m # Sampling Query Profiler - - ClickHouse runs sampling profiler that allows analyzing query execution. Using profiler you can find source code routines that used the most frequently during query execution. You can trace CPU time and wall-clock time spent including idle time. -To use profiler: +Query profiler is automatically enabled in ClickHouse Cloud and you can run a sample query as follows + +``` sql +SELECT + count(), + arrayStringConcat(arrayMap(x -> concat(demangle(addressToSymbol(x)), '\n ', addressToLine(x)), trace), '\n') AS sym +FROM system.trace_log +WHERE (query_id = 'ebca3574-ad0a-400a-9cbc-dca382f5998c') AND (event_date = today()) +GROUP BY trace +ORDER BY count() DESC +LIMIT 10 +SETTINGS allow_introspection_functions = 1 +``` + +In self-managed deployments, to use query profiler: - Setup the [trace_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) section of the server configuration. From 2c9761bca8821e9fbb19f5aaa497b7097244a9e8 Mon Sep 17 00:00:00 2001 From: rfraposa Date: Mon, 20 Mar 2023 23:09:54 -0600 Subject: [PATCH 65/69] Update sparse-primary-indexes.md --- .../sparse-primary-indexes.md | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md b/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md index 2ab9a85b7e8..54827388013 100644 --- a/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md +++ b/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md @@ -149,10 +149,10 @@ SETTINGS index_granularity = 8192, index_granularity_bytes = 0; [//]: # (
)
- + DDL详情 -

+

为了简化本文后面的讨论，并使图和结果可重现，使用DDL语句有如下说明：

    @@ -164,7 +164,7 @@ SETTINGS index_granularity = 8192, index_granularity_bytes = 0;
  • index_granularity: 显式设置为其默认值8192。这意味着对于每一组8192行，主索引将有一个索引条目，例如，如果表包含16384行，那么索引将有两个索引条目。

    • -
  • index_granularity_bytes: 设置为0表示禁止自适应索引粒度。自适应索引粒度意味着ClickHouse自动为一组n行创建一个索引条目 +
  • index_granularity_bytes: 设置为0表示禁止自适应索引粒度。自适应索引粒度意味着ClickHouse自动为一组n行创建一个索引条目
    • 如果n小于8192，但n行的合并行数据大小大于或等于10MB (index_granularity_bytes的默认值)或
    • n达到8192
      • @@ -446,10 +446,10 @@ ClickHouse客户端的输出显示，没有进行全表扫描，只有8.19千行 我们可以在上面的跟踪日志中看到，1083个现有标记中有一个满足查询。
      - + Trace Log详情 -

      +

      Mark 176 was identified (the 'found left boundary mark' is inclusive, the 'found right boundary mark' is exclusive), and therefore all 8192 rows from granule 176 (which starts at row 1.441.792 - we will see that later on in this article) are then streamed into ClickHouse in order to find the actual rows with a UserID column value of 749927693.

      @@ -520,10 +520,10 @@ LIMIT 10; 如上所述，通过对索引的1083个UserID标记进行二分搜索，确定了第176个标记。因此，它对应的颗粒176可能包含UserID列值为749.927.693的行。
      - + 颗粒选择的具体过程 -

      +

      上图显示，标记176是第一个UserID值小于749.927.693的索引条目，并且下一个标记(标记177)的颗粒177的最小UserID值大于该值的索引条目。因此，只有标记176对应的颗粒176可能包含UserID列值为749.927.693的行。

      @@ -671,15 +671,15 @@ Processed 8.81 million rows, 为了说明，我们给出通用的排除搜索算法的工作原理：
      - + 通用排除搜索算法 -

      +

      -下面将演示当通过第一个列之后的任何列选择颗粒时，当前一个键列具有或高或低的基数时，ClickHouse通用排除搜索算法 是如何工作的。 +下面将演示当通过第一个列之后的任何列选择颗粒时，当前一个键列具有或高或低的基数时，ClickHouse通用排除搜索算法 是如何工作的。 作为这两种情况的例子，我们将假设： - 搜索URL值为"W3"的行。 @@ -736,9 +736,9 @@ Processed 8.81 million rows, 在我们的示例数据集中，两个键列(UserID、URL)都具有类似的高基数，并且，如前所述，当URL列的前一个键列具有较高基数时，通用排除搜索算法不是很有效。 :::note 看下跳数索引 -因为UserID和URL具有较高的基数，[根据URL过滤数据](#query-on-url)不是特别有效，对URL列创建[二级跳数索引](./skipping-indexes.md)同样也不会有太多改善。 +因为UserID和URL具有较高的基数，[根据URL过滤数据](#query-on-url)不是特别有效，对URL列创建[二级跳数索引](./skipping-indexes.md)同样也不会有太多改善。 -例如，这两个语句在我们的表的URL列上创建并填充一个minmax跳数索引。 +例如，这两个语句在我们的表的URL列上创建并填充一个minmax跳数索引。 ```sql ALTER TABLE hits_UserID_URL ADD INDEX url_skipping_index URL TYPE minmax GRANULARITY 4; ALTER TABLE hits_UserID_URL MATERIALIZE INDEX url_skipping_index; @@ -907,10 +907,10 @@ ClickHouse只选择了39个索引标记，而不是使用通用排除搜索时 点击下面了解详情：

      - + 对UserID的查询过滤性能较差 -

      +

      ```sql SELECT URL, count(URL) AS Count From 0b9d84788248ec6ffa1a7cbd172045b4cb3a5df4 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 21 Mar 2023 08:41:21 +0200 Subject: [PATCH 66/69] Fix flaky test 02417_opentelemetry_insert_on_distributed_table Signed-off-by: Azat Khuzhin --- .../02417_opentelemetry_insert_on_distributed_table.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh index 81f3e3f4ea6..edc3d06e5bf 100755 --- a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh +++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh @@ -20,6 +20,7 @@ function insert() -H "tracestate: $4" \ "${CLICKHOUSE_URL}" \ --data @- + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH DISTRIBUTED ${CLICKHOUSE_DATABASE}.dist_opentelemetry" } function check_span() From 83a8da9e507a7c078660174d4919c09970f82eae Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 21 Mar 2023 08:50:42 +0100 Subject: [PATCH 67/69] Make new deb signed working on old deb systems --- docs/en/getting-started/install.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index f5ca44415f9..5b1a3f20a52 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -118,7 +118,10 @@ It is recommended to use official pre-compiled `deb` packages for Debian or Ubun #### Setup the Debian repository ``` bash sudo apt-get install -y apt-transport-https ca-certificates dirmngr -sudo gpg --no-default-keyring --keyring /usr/share/keyrings/clickhouse-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 8919F6BD2B48D754 +GNUPGHOME=$(mktemp -d) +sudo GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring --keyring /usr/share/keyrings/clickhouse-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 8919F6BD2B48D754 +sudo rm -r "$GNUPGHOME" +sudo chmod +r /usr/share/keyrings/clickhouse-keyring.gpg echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main" | sudo tee \ /etc/apt/sources.list.d/clickhouse.list From b9dca4e0dfaa08287a031e2ed85933d0c3e8adb3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 21 Mar 2023 11:29:16 +0100 Subject: [PATCH 68/69] Place short return before big block, improve logging --- tests/ci/merge_pr.py | 81 +++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 42 deletions(-) diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py index 3656f12ed4b..2d6d81a152a 100644 --- a/tests/ci/merge_pr.py +++ b/tests/ci/merge_pr.py @@ -100,55 +100,52 @@ class Reviews: if review.state == "APPROVED" } - if approved: + if not approved: logging.info( - "The following users from %s team approved the PR: %s", + "The PR #%s is not approved by any of %s team member", + self.pr.number, TEAM_NAME, - ", ".join(user.login for user in approved.keys()), ) - # The only reliable place to get the 100% accurate last_modified - # info is when the commit was pushed to GitHub. The info is - # available as a header 'last-modified' of /{org}/{repo}/commits/{sha}. - # Unfortunately, it's formatted as 'Wed, 04 Jan 2023 11:05:13 GMT' - - commit = self.pr.head.repo.get_commit(self.pr.head.sha) - if commit.stats.last_modified is None: - logging.warning( - "Unable to get info about the commit %s", self.pr.head.sha - ) - return False - - last_changed = datetime.strptime( - commit.stats.last_modified, "%a, %d %b %Y %H:%M:%S GMT" - ) - - approved_at = max(review.submitted_at for review in approved.values()) - if approved_at == datetime.fromtimestamp(0): - logging.info( - "Unable to get `datetime.fromtimestamp(0)`, " - "here's debug info about reviews: %s", - "\n".join(pformat(review) for review in self.reviews.values()), - ) - else: - logging.info( - "The PR is approved at %s", - approved_at.isoformat(), - ) - - if approved_at < last_changed: - logging.info( - "There are changes after approve at %s", - approved_at.isoformat(), - ) - return False - return True + return False logging.info( - "The PR #%s is not approved by any of %s team member", - self.pr.number, + "The following users from %s team approved the PR: %s", TEAM_NAME, + ", ".join(user.login for user in approved.keys()), ) - return False + + # The only reliable place to get the 100% accurate last_modified + # info is when the commit was pushed to GitHub. The info is + # available as a header 'last-modified' of /{org}/{repo}/commits/{sha}. + # Unfortunately, it's formatted as 'Wed, 04 Jan 2023 11:05:13 GMT' + commit = self.pr.head.repo.get_commit(self.pr.head.sha) + if commit.stats.last_modified is None: + logging.warning("Unable to get info about the commit %s", self.pr.head.sha) + return False + + last_changed = datetime.strptime( + commit.stats.last_modified, "%a, %d %b %Y %H:%M:%S GMT" + ) + logging.info("The PR is changed at %s", last_changed.isoformat()) + + approved_at = max(review.submitted_at for review in approved.values()) + if approved_at == datetime.fromtimestamp(0): + logging.info( + "Unable to get `datetime.fromtimestamp(0)`, " + "here's debug info about reviews: %s", + "\n".join(pformat(review) for review in self.reviews.values()), + ) + else: + logging.info("The PR is approved at %s", approved_at.isoformat()) + + if approved_at < last_changed: + logging.info( + "There are changes done at %s after approval at %s", + last_changed.isoformat(), + approved_at.isoformat(), + ) + return False + return True def get_workflows_for_head(repo: Repository, head_sha: str) -> List[WorkflowRun]: From 4eeab9fd25d7fee38f4c99b5d9d4c8835060b8e8 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 21 Mar 2023 14:24:53 +0100 Subject: [PATCH 69/69] Revert "Add sanity checks for writing number in variable length format" --- src/IO/VarInt.h | 18 +----------------- src/Server/TCPHandler.cpp | 2 +- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/src/IO/VarInt.h b/src/IO/VarInt.h index d026192cb7d..0869051034a 100644 --- a/src/IO/VarInt.h +++ b/src/IO/VarInt.h @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -15,19 +14,7 @@ namespace ErrorCodes } -/** Variable-Length Quantity (VLQ) Base-128 compression - * - * NOTE: Due to historical reasons, only up to 1<<63-1 are supported, which - * cannot be changed without breaking the backward compatibility. - * Also some drivers may support full 1<<64 range (i.e. python - - * clickhouse-driver), while others has the same limitations as ClickHouse - * (i.e. Rust - clickhouse-rs). - * So implementing VLQ for the whole 1<<64 range will require different set of - * helpers. - */ -constexpr size_t VAR_UINT_MAX = (1ULL<<63) - 1; - -/** Write UInt64 in variable length format (base128) */ +/** Write UInt64 in variable length format (base128) NOTE Only up to 2^63 - 1 are supported. */ void writeVarUInt(UInt64 x, std::ostream & ostr); void writeVarUInt(UInt64 x, WriteBuffer & ostr); char * writeVarUInt(UInt64 x, char * ostr); @@ -199,7 +186,6 @@ inline const char * readVarUInt(UInt64 & x, const char * istr, size_t size) inline void writeVarUInt(UInt64 x, WriteBuffer & ostr) { - chassert(x <= VAR_UINT_MAX); for (size_t i = 0; i < 9; ++i) { uint8_t byte = x & 0x7F; @@ -219,7 +205,6 @@ inline void writeVarUInt(UInt64 x, WriteBuffer & ostr) inline void writeVarUInt(UInt64 x, std::ostream & ostr) { - chassert(x <= VAR_UINT_MAX); for (size_t i = 0; i < 9; ++i) { uint8_t byte = x & 0x7F; @@ -237,7 +222,6 @@ inline void writeVarUInt(UInt64 x, std::ostream & ostr) inline char * writeVarUInt(UInt64 x, char * ostr) { - chassert(x <= VAR_UINT_MAX); for (size_t i = 0; i < 9; ++i) { uint8_t byte = x & 0x7F; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 1f25f127b2f..5da2a9dd169 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1869,7 +1869,7 @@ void TCPHandler::sendData(const Block & block) { --unknown_packet_in_send_data; if (unknown_packet_in_send_data == 0) - writeVarUInt(VAR_UINT_MAX, *out); + writeVarUInt(UInt64(-1), *out); } writeVarUInt(Protocol::Server::Data, *out);