From 7357bc7114b310e64e857f6cc483e346c160bcf9 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 1 Dec 2023 04:26:43 +0100 Subject: [PATCH 001/502] Add syntax ALTER USER {ADD|MODIFY|DROP SETTING}, ALTER USER {ADD|DROP PROFILE}, the same for ALTER ROLE and ALTER PROFILE. --- .../en/sql-reference/statements/alter/role.md | 7 +- .../statements/alter/settings-profile.md | 7 +- .../en/sql-reference/statements/alter/user.md | 7 +- .../ru/sql-reference/statements/alter/role.md | 7 +- .../statements/alter/settings-profile.md | 7 +- .../ru/sql-reference/statements/alter/user.md | 7 +- src/Access/RoleCache.cpp | 2 +- src/Access/SettingsConstraints.cpp | 7 + src/Access/SettingsConstraints.h | 3 +- src/Access/SettingsProfileElement.cpp | 237 ++++++++++++- src/Access/SettingsProfileElement.h | 46 ++- src/Access/SettingsProfilesCache.cpp | 4 +- .../Access/InterpreterCreateRoleQuery.cpp | 22 +- .../InterpreterCreateSettingsProfileQuery.cpp | 22 +- .../Access/InterpreterCreateUserQuery.cpp | 22 +- ...InterpreterShowCreateAccessEntityQuery.cpp | 26 +- src/Interpreters/Context.cpp | 22 +- src/Interpreters/Context.h | 8 +- src/Parsers/Access/ASTCreateRoleQuery.cpp | 13 +- src/Parsers/Access/ASTCreateRoleQuery.h | 9 +- .../Access/ASTCreateSettingsProfileQuery.cpp | 13 +- .../Access/ASTCreateSettingsProfileQuery.h | 10 +- src/Parsers/Access/ASTCreateUserQuery.cpp | 13 +- src/Parsers/Access/ASTCreateUserQuery.h | 9 +- .../Access/ASTSettingsProfileElement.cpp | 180 ++++++++++ .../Access/ASTSettingsProfileElement.h | 33 +- src/Parsers/Access/ParserCreateRoleQuery.cpp | 54 ++- src/Parsers/Access/ParserCreateRoleQuery.h | 7 +- .../ParserCreateSettingsProfileQuery.cpp | 55 ++- .../Access/ParserCreateSettingsProfileQuery.h | 7 +- src/Parsers/Access/ParserCreateUserQuery.cpp | 54 ++- src/Parsers/Access/ParserCreateUserQuery.h | 7 +- .../Access/ParserSettingsProfileElement.cpp | 322 ++++++++++++++---- .../Access/ParserSettingsProfileElement.h | 14 + .../01294_create_settings_profile.reference | 2 +- ...02943_alter_user_modify_profiles.reference | 21 ++ .../02943_alter_user_modify_profiles.sh | 76 +++++ ...ser_modify_profiles_and_settings.reference | 39 +++ ...alter_user_modify_profiles_and_settings.sh | 86 +++++ ...02943_alter_user_modify_settings.reference | 42 +++ .../02943_alter_user_modify_settings.sh | 73 ++++ 41 files changed, 1401 insertions(+), 201 deletions(-) create mode 100644 tests/queries/0_stateless/02943_alter_user_modify_profiles.reference create mode 100755 tests/queries/0_stateless/02943_alter_user_modify_profiles.sh create mode 100644 tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference create mode 100755 tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.sh create mode 100644 tests/queries/0_stateless/02943_alter_user_modify_settings.reference create mode 100755 tests/queries/0_stateless/02943_alter_user_modify_settings.sh diff --git a/docs/en/sql-reference/statements/alter/role.md b/docs/en/sql-reference/statements/alter/role.md index c068d6c4fce..34f98da4722 100644 --- a/docs/en/sql-reference/statements/alter/role.md +++ b/docs/en/sql-reference/statements/alter/role.md @@ -13,5 +13,10 @@ Syntax: ``` sql ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] + [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE|CONST|CHANGEABLE_IN_READONLY] [,...] ] + [DROP SETTINGS variable [,...] ] + [ADD PROFILES 'profile_name' [,...] ] + [DROP PROFILES 'profile_name' [,...] ] + [DROP ALL PROFILES] + [DROP ALL SETTINGS] ``` diff --git a/docs/en/sql-reference/statements/alter/settings-profile.md b/docs/en/sql-reference/statements/alter/settings-profile.md index adfe65e3c80..543b8a63064 100644 --- a/docs/en/sql-reference/statements/alter/settings-profile.md +++ b/docs/en/sql-reference/statements/alter/settings-profile.md @@ -13,5 +13,10 @@ Syntax: ``` sql ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] + [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE|CONST|CHANGEABLE_IN_READONLY] [,...] ] + [DROP SETTINGS variable [,...] ] + [ADD PROFILES 'profile_name' [,...] ] + [DROP PROFILES 'profile_name' [,...] ] + [DROP ALL PROFILES] + [DROP ALL SETTINGS] ``` diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md index fd7da05167c..4525c9c9a65 100644 --- a/docs/en/sql-reference/statements/alter/user.md +++ b/docs/en/sql-reference/statements/alter/user.md @@ -17,7 +17,12 @@ ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [VALID UNTIL datetime] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...] + [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE|CONST|CHANGEABLE_IN_READONLY] [,...] ] + [DROP SETTINGS variable [,...] ] + [ADD PROFILES 'profile_name' [,...] ] + [DROP PROFILES 'profile_name' [,...] ] + [DROP ALL PROFILES] + [DROP ALL SETTINGS] ``` To use `ALTER USER` you must have the [ALTER USER](../../../sql-reference/statements/grant.md#grant-access-management) privilege. diff --git a/docs/ru/sql-reference/statements/alter/role.md b/docs/ru/sql-reference/statements/alter/role.md index 0c117d29be8..489ec6e88ac 100644 --- a/docs/ru/sql-reference/statements/alter/role.md +++ b/docs/ru/sql-reference/statements/alter/role.md @@ -13,5 +13,10 @@ sidebar_label: ROLE ``` sql ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] + [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE|CONST|CHANGEABLE_IN_READONLY] [,...] ] + [DROP SETTINGS variable [,...] ] + [ADD PROFILES 'profile_name' [,...] ] + [DROP PROFILES 'profile_name' [,...] ] + [DROP ALL PROFILES] + [DROP ALL SETTINGS] ``` diff --git a/docs/ru/sql-reference/statements/alter/settings-profile.md b/docs/ru/sql-reference/statements/alter/settings-profile.md index 90d7a7f6202..fe6b0ba456f 100644 --- a/docs/ru/sql-reference/statements/alter/settings-profile.md +++ b/docs/ru/sql-reference/statements/alter/settings-profile.md @@ -13,5 +13,10 @@ sidebar_label: SETTINGS PROFILE ``` sql ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] + [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE|CONST|CHANGEABLE_IN_READONLY] [,...] ] + [DROP SETTINGS variable [,...] ] + [ADD PROFILES 'profile_name' [,...] ] + [DROP PROFILES 'profile_name' [,...] ] + [DROP ALL PROFILES] + [DROP ALL SETTINGS] ``` diff --git a/docs/ru/sql-reference/statements/alter/user.md b/docs/ru/sql-reference/statements/alter/user.md index 1d6ad34ebed..300c9a47021 100644 --- a/docs/ru/sql-reference/statements/alter/user.md +++ b/docs/ru/sql-reference/statements/alter/user.md @@ -17,7 +17,12 @@ ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...] + [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE|CONST|CHANGEABLE_IN_READONLY] [,...] ] + [DROP SETTINGS variable [,...] ] + [ADD PROFILES 'profile_name' [,...] ] + [DROP PROFILES 'profile_name' [,...] ] + [DROP ALL PROFILES] + [DROP ALL SETTINGS] ``` Для выполнения `ALTER USER` необходима привилегия [ALTER USER](../grant.md#grant-access-management). diff --git a/src/Access/RoleCache.cpp b/src/Access/RoleCache.cpp index 2d94df2eea5..d2937107afa 100644 --- a/src/Access/RoleCache.cpp +++ b/src/Access/RoleCache.cpp @@ -45,7 +45,7 @@ namespace roles_info.names_of_roles[role_id] = role->getName(); roles_info.access.makeUnion(role->access); - roles_info.settings_from_enabled_roles.merge(role->settings); + roles_info.settings_from_enabled_roles.merge(role->settings, /* normalize= */ false); for (const auto & granted_role : role->granted_roles.getGranted()) collectRoles(roles_info, skip_ids, get_role_function, granted_role, false, false); diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index db805c83e17..dc02941e1bb 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -133,6 +133,13 @@ void SettingsConstraints::merge(const SettingsConstraints & other) } +void SettingsConstraints::check(const Settings & current_settings, const AlterSettingsProfileElements & profile_elements, SettingSource source) const +{ + check(current_settings, profile_elements.add_settings, source); + check(current_settings, profile_elements.modify_settings, source); + /// We don't check `drop_settings` here. +} + void SettingsConstraints::check(const Settings & current_settings, const SettingsProfileElements & profile_elements, SettingSource source) const { for (const auto & element : profile_elements) diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 5bbff86ff61..8e35f0a9a29 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -74,10 +74,11 @@ public: void merge(const SettingsConstraints & other); /// Checks whether `change` violates these constraints and throws an exception if so. - void check(const Settings & current_settings, const SettingsProfileElements & profile_elements, SettingSource source) const; void check(const Settings & current_settings, const SettingChange & change, SettingSource source) const; void check(const Settings & current_settings, const SettingsChanges & changes, SettingSource source) const; void check(const Settings & current_settings, SettingsChanges & changes, SettingSource source) const; + void check(const Settings & current_settings, const SettingsProfileElements & profile_elements, SettingSource source) const; + void check(const Settings & current_settings, const AlterSettingsProfileElements & profile_elements, SettingSource source) const; /// Checks whether `change` violates these constraints and throws an exception if so. (setting short name is expected inside `changes`) void check(const MergeTreeSettings & current_settings, const SettingChange & change) const; diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 9358391cb93..ed873193fe7 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include namespace DB @@ -19,6 +21,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } + SettingsProfileElement::SettingsProfileElement(const ASTSettingsProfileElement & ast) { init(ast, nullptr); @@ -116,16 +119,20 @@ std::shared_ptr SettingsProfileElement::toASTWithName } -SettingsProfileElements::SettingsProfileElements(const ASTSettingsProfileElements & ast) +SettingsProfileElements::SettingsProfileElements(const ASTSettingsProfileElements & ast, bool normalize_) { for (const auto & ast_element : ast.elements) emplace_back(*ast_element); + if (normalize_) + normalize(); } -SettingsProfileElements::SettingsProfileElements(const ASTSettingsProfileElements & ast, const AccessControl & access_control) +SettingsProfileElements::SettingsProfileElements(const ASTSettingsProfileElements & ast, const AccessControl & access_control, bool normalize_) { for (const auto & ast_element : ast.elements) emplace_back(*ast_element, access_control); + if (normalize_) + normalize(); } @@ -133,7 +140,11 @@ std::shared_ptr SettingsProfileElements::toAST() con { auto res = std::make_shared(); for (const auto & element : *this) - res->elements.push_back(element.toAST()); + { + auto element_ast = element.toAST(); + if (!element_ast->empty()) + res->elements.push_back(element_ast); + } return res; } @@ -141,7 +152,11 @@ std::shared_ptr SettingsProfileElements::toASTWithNa { auto res = std::make_shared(); for (const auto & element : *this) - res->elements.push_back(element.toASTWithNames(access_control)); + { + auto element_ast = element.toASTWithNames(access_control); + if (!element_ast->empty()) + res->elements.push_back(element_ast); + } return res; } @@ -176,9 +191,11 @@ void SettingsProfileElements::replaceDependencies(const std::unordered_map SettingsProfileElements::toProfileIDs() const return res; } + +void SettingsProfileElements::normalize() +{ + /// Ensure that each element represents either a setting or a profile. + { + SettingsProfileElements new_elements; + for (auto & element : *this) + { + if (element.parent_profile && !element.setting_name.empty()) + { + SettingsProfileElement new_element; + new_element.parent_profile = element.parent_profile; + element.parent_profile.reset(); + new_elements.push_back(std::move(new_element)); + } + } + insert(end(), new_elements.begin(), new_elements.end()); + } + + /// Partitioning: first profiles, then settings. + /// We use std::stable_partition() here because we want to preserve the relative order of profiles and the relative order of settings. + /// (We need that order to be preserved to remove duplicates correctly - see below.) + auto profiles_begin = begin(); + auto profiles_end = std::stable_partition(begin(), end(), [](const SettingsProfileElement & element) { return static_cast(element.parent_profile); }); + auto settings_begin = profiles_end; + auto settings_end = end(); + + /// Remove duplicates among profiles. + /// We keep the last position of any used profile. + /// It's important to keep exactly the last position (and not just any position) because profiles can override settings from each other. + /// For example, [pr_A, pr_B, pr_A, pr_C] is always the same as [pr_B, pr_A, pr_C], but can be not the same as [pr_A, pr_B, pr_C] + /// if pr_A and pr_B give different values to same settings. + { + boost::container::flat_set profile_ids; + profile_ids.reserve(profiles_end - profiles_begin); + auto it = profiles_end; + while (it != profiles_begin) + { + --it; + auto & element = *it; + if (element.parent_profile && !profile_ids.emplace(*element.parent_profile).second) + element.parent_profile.reset(); + } + } + + /// Remove duplicates among settings. + /// We keep the first position of any used setting, and merge settings with the same name to that first element. + { + boost::container::flat_map setting_name_to_first_encounter; + setting_name_to_first_encounter.reserve(settings_end - settings_begin); + for (auto it = settings_begin; it != settings_end; ++it) + { + auto & element = *it; + auto first = setting_name_to_first_encounter.emplace(element.setting_name, it).first->second; + if (it != first) + { + auto & first_element = *first; + if (element.value) + first_element.value = element.value; + if (element.min_value) + first_element.min_value = element.min_value; + if (element.max_value) + first_element.max_value = element.max_value; + if (element.writability) + first_element.writability = element.writability; + element.setting_name.clear(); + } + } + } + + /// Remove empty elements. + std::erase_if(*this, [](const SettingsProfileElement & element) { return element.empty(); }); +} + + bool SettingsProfileElements::isBackupAllowed() const { for (const auto & setting : *this) @@ -252,4 +344,139 @@ bool SettingsProfileElements::isAllowBackupSetting(const String & setting_name) return Settings::Traits::resolveName(setting_name) == ALLOW_BACKUP_SETTING_NAME; } + +AlterSettingsProfileElements::AlterSettingsProfileElements(const SettingsProfileElements & ast) +{ + drop_all_settings = true; + drop_all_profiles = true; + add_settings = ast; +} + +AlterSettingsProfileElements::AlterSettingsProfileElements(const ASTSettingsProfileElements & ast) + : AlterSettingsProfileElements(SettingsProfileElements{ast}) +{ +} + +AlterSettingsProfileElements::AlterSettingsProfileElements(const ASTSettingsProfileElements & ast, const AccessControl & access_control) + : AlterSettingsProfileElements(SettingsProfileElements{ast, access_control}) +{ +} + +AlterSettingsProfileElements::AlterSettingsProfileElements(const ASTAlterSettingsProfileElements & ast) +{ + drop_all_settings = ast.drop_all_settings; + drop_all_profiles = ast.drop_all_profiles; + + if (ast.add_settings) + add_settings = SettingsProfileElements{*ast.add_settings, /* normalize= */ false}; /// For "ALTER" the normalization is unnecessary. + + if (ast.modify_settings) + modify_settings = SettingsProfileElements{*ast.modify_settings, /* normalize= */ false}; + + if (ast.drop_settings) + drop_settings = SettingsProfileElements{*ast.drop_settings, /* normalize= */ false}; +} + +AlterSettingsProfileElements::AlterSettingsProfileElements(const ASTAlterSettingsProfileElements & ast, const AccessControl & access_control) +{ + drop_all_settings = ast.drop_all_settings; + drop_all_profiles = ast.drop_all_profiles; + + if (ast.add_settings) + add_settings = SettingsProfileElements{*ast.add_settings, access_control, /* normalize= */ false}; /// For "ALTER" the normalization is unnecessary. + + if (ast.modify_settings) + modify_settings = SettingsProfileElements{*ast.modify_settings, access_control, /* normalize= */ false}; + + if (ast.drop_settings) + drop_settings = SettingsProfileElements{*ast.drop_settings, access_control, /* normalize= */ false}; +} + +void SettingsProfileElements::applyChanges(const AlterSettingsProfileElements & changes) +{ + /// Apply "DROP" changes. + if (changes.drop_all_profiles) + { + for (auto & element : *this) + element.parent_profile.reset(); /// We only make this element empty, the element will be removed in normalizeProfileElements(). + } + + if (changes.drop_all_settings) + { + for (auto & element : *this) + element.setting_name.clear(); /// We only make this element empty, the element will be removed in normalizeProfileElements(). + } + + auto apply_drop_setting = [&](const String & setting_name) + { + for (auto & element : *this) + { + if (element.setting_name == setting_name) + element.setting_name.clear(); + } + }; + + auto apply_drop_profile = [&](const UUID & profile_id) + { + for (auto & element : *this) + { + if (element.parent_profile == profile_id) + element.parent_profile.reset(); + } + }; + + for (const auto & drop : changes.drop_settings) + { + if (drop.parent_profile) + apply_drop_profile(*drop.parent_profile); + if (!drop.setting_name.empty()) + apply_drop_setting(drop.setting_name); + } + + auto apply_modify_setting = [&](const SettingsProfileElement & modify) + { + SettingsProfileElement new_element; + new_element.setting_name = modify.setting_name; + new_element.value = modify.value; + new_element.min_value = modify.min_value; + new_element.max_value = modify.max_value; + new_element.writability = modify.writability; + push_back(new_element); /// normalizeProfileElements() will merge this new element with the previous elements. + }; + + /// Apply "ADD" changes. + auto apply_add_setting = [&](const SettingsProfileElement & add) + { + /// "ADD SETTING" must replace the value and the constraints of a setting, so first we need drop the previous elements for that setting. + apply_drop_setting(add.setting_name); + apply_modify_setting(add); + }; + + auto apply_add_profile = [&](const UUID & profile_id) + { + SettingsProfileElement new_element; + new_element.parent_profile = profile_id; + push_back(new_element); /// We don't care about possible duplicates here, normalizeProfileElements() will remove duplicates. + }; + + for (const auto & add : changes.add_settings) + { + if (add.parent_profile) + apply_add_profile(*add.parent_profile); + if (!add.setting_name.empty()) + apply_add_setting(add); + } + + /// Apply "MODIFY" changes. + for (const auto & modify : changes.modify_settings) + { + chassert(!modify.parent_profile); /// There is no such thing as "MODIFY PROFILE". + if (!modify.setting_name.empty()) + apply_modify_setting(modify); + } + + /// Remove empty elements and duplicates, and sort the result. + normalize(); +} + } diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index 7078f565295..742b7b1e40b 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -13,8 +13,10 @@ namespace DB struct Settings; class SettingsChanges; class SettingsConstraints; +struct AlterSettingsProfileElements; class ASTSettingsProfileElement; class ASTSettingsProfileElements; +class ASTAlterSettingsProfileElements; class AccessControl; @@ -44,6 +46,8 @@ struct SettingsProfileElement std::shared_ptr toAST() const; std::shared_ptr toASTWithNames(const AccessControl & access_control) const; + bool empty() const { return !parent_profile && (setting_name.empty() || (!value && !min_value && !max_value && !writability)); } + bool isConstraint() const; private: @@ -57,24 +61,50 @@ public: SettingsProfileElements() = default; /// The constructor from AST requires the AccessControl if `ast.id_mode == false`. - SettingsProfileElements(const ASTSettingsProfileElements & ast); /// NOLINT - SettingsProfileElements(const ASTSettingsProfileElements & ast, const AccessControl & access_control); + SettingsProfileElements(const ASTSettingsProfileElements & ast, bool normalize_ = true); /// NOLINT + SettingsProfileElements(const ASTSettingsProfileElements & ast, const AccessControl & access_control, bool normalize_ = true); + std::shared_ptr toAST() const; std::shared_ptr toASTWithNames(const AccessControl & access_control) const; - std::vector findDependencies() const; - void replaceDependencies(const std::unordered_map & old_to_new_ids); - - void merge(const SettingsProfileElements & other); - Settings toSettings() const; SettingsChanges toSettingsChanges() const; SettingsConstraints toSettingsConstraints(const AccessControl & access_control) const; std::vector toProfileIDs() const; - bool isBackupAllowed() const; + /// Normalizes this list of profile elements: removes duplicates and empty elements, and also sorts the elements + /// in the following order: first profiles, then settings. + /// The function is called automatically after parsing profile elements from an AST and + /// at the end of an "ALTER PROFILE (USER/ROLE) command". + void normalize(); + /// Appends all the elements of another list of profile elements to this list. + void merge(const SettingsProfileElements & other, bool normalize_ = true); + + /// Applies changes from an "ALTER PROFILE (USER/ROLE)" command. Always normalizes the result. + void applyChanges(const AlterSettingsProfileElements & changes); + + std::vector findDependencies() const; + void replaceDependencies(const std::unordered_map & old_to_new_ids); + + bool isBackupAllowed() const; static bool isAllowBackupSetting(const String & setting_name); }; +struct AlterSettingsProfileElements +{ + bool drop_all_settings = false; + bool drop_all_profiles = false; + SettingsProfileElements add_settings; + SettingsProfileElements modify_settings; + SettingsProfileElements drop_settings; + + AlterSettingsProfileElements() = default; + explicit AlterSettingsProfileElements(const SettingsProfileElements & ast); + explicit AlterSettingsProfileElements(const ASTSettingsProfileElements & ast); + explicit AlterSettingsProfileElements(const ASTAlterSettingsProfileElements & ast); + AlterSettingsProfileElements(const ASTSettingsProfileElements & ast, const AccessControl & access_control); + AlterSettingsProfileElements(const ASTAlterSettingsProfileElements & ast, const AccessControl & access_control); +}; + } diff --git a/src/Access/SettingsProfilesCache.cpp b/src/Access/SettingsProfilesCache.cpp index f03e68ba455..591a87e52a4 100644 --- a/src/Access/SettingsProfilesCache.cpp +++ b/src/Access/SettingsProfilesCache.cpp @@ -135,8 +135,8 @@ void SettingsProfilesCache::mergeSettingsAndConstraintsFor(EnabledSettings & ena merged_settings.emplace_back(new_element); } - merged_settings.merge(enabled.params.settings_from_enabled_roles); - merged_settings.merge(enabled.params.settings_from_user); + merged_settings.merge(enabled.params.settings_from_enabled_roles, /* normalize= */ false); + merged_settings.merge(enabled.params.settings_from_user, /* normalize= */ false); auto info = std::make_shared(access_control); diff --git a/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp b/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp index fef1f285c8b..45d7e0d5b23 100644 --- a/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp @@ -22,7 +22,7 @@ namespace Role & role, const ASTCreateRoleQuery & query, const String & override_name, - const std::optional & override_settings) + const std::optional & override_settings) { if (!override_name.empty()) role.setName(override_name); @@ -32,9 +32,11 @@ namespace role.setName(query.names.front()); if (override_settings) - role.settings = *override_settings; + role.settings.applyChanges(*override_settings); + else if (query.alter_settings) + role.settings.applyChanges(AlterSettingsProfileElements{*query.alter_settings}); else if (query.settings) - role.settings = *query.settings; + role.settings.applyChanges(AlterSettingsProfileElements{*query.settings}); } } @@ -50,14 +52,14 @@ BlockIO InterpreterCreateRoleQuery::execute() else getContext()->checkAccess(AccessType::CREATE_ROLE); - std::optional settings_from_query; - if (query.settings) - { - settings_from_query = SettingsProfileElements{*query.settings, access_control}; + std::optional settings_from_query; + if (query.alter_settings) + settings_from_query = AlterSettingsProfileElements{*query.alter_settings, access_control}; + else if (query.settings) + settings_from_query = AlterSettingsProfileElements{SettingsProfileElements(*query.settings, access_control)}; - if (!query.attach) - getContext()->checkSettingsConstraints(*settings_from_query, SettingSource::ROLE); - } + if (settings_from_query && !query.attach) + getContext()->checkSettingsConstraints(*settings_from_query, SettingSource::ROLE); if (!query.cluster.empty()) return executeDDLQueryOnCluster(updated_query_ptr, getContext()); diff --git a/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp b/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp index 3a96c0a96ff..5514d885ddd 100644 --- a/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp @@ -24,7 +24,7 @@ namespace SettingsProfile & profile, const ASTCreateSettingsProfileQuery & query, const String & override_name, - const std::optional & override_settings, + const std::optional & override_settings, const std::optional & override_to_roles) { if (!override_name.empty()) @@ -35,9 +35,11 @@ namespace profile.setName(query.names.front()); if (override_settings) - profile.elements = *override_settings; + profile.elements.applyChanges(*override_settings); + else if (query.alter_settings) + profile.elements.applyChanges(AlterSettingsProfileElements{*query.alter_settings}); else if (query.settings) - profile.elements = *query.settings; + profile.elements.applyChanges(AlterSettingsProfileElements{*query.settings}); if (override_to_roles) profile.to_roles = *override_to_roles; @@ -58,14 +60,14 @@ BlockIO InterpreterCreateSettingsProfileQuery::execute() else getContext()->checkAccess(AccessType::CREATE_SETTINGS_PROFILE); - std::optional settings_from_query; - if (query.settings) - { - settings_from_query = SettingsProfileElements{*query.settings, access_control}; + std::optional settings_from_query; + if (query.alter_settings) + settings_from_query = AlterSettingsProfileElements{*query.alter_settings, access_control}; + else if (query.settings) + settings_from_query = AlterSettingsProfileElements{SettingsProfileElements(*query.settings, access_control)}; - if (!query.attach) - getContext()->checkSettingsConstraints(*settings_from_query, SettingSource::PROFILE); - } + if (settings_from_query && !query.attach) + getContext()->checkSettingsConstraints(*settings_from_query, SettingSource::PROFILE); if (!query.cluster.empty()) { diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 00e21f27d2e..67df60e5f0c 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -35,7 +35,7 @@ namespace const std::optional auth_data, const std::shared_ptr & override_name, const std::optional & override_default_roles, - const std::optional & override_settings, + const std::optional & override_settings, const std::optional & override_grantees, const std::optional & valid_until, bool allow_implicit_no_password, @@ -104,9 +104,11 @@ namespace user.default_database = query.default_database->database_name; if (override_settings) - user.settings = *override_settings; + user.settings.applyChanges(*override_settings); + else if (query.alter_settings) + user.settings.applyChanges(AlterSettingsProfileElements{*query.alter_settings}); else if (query.settings) - user.settings = *query.settings; + user.settings.applyChanges(AlterSettingsProfileElements{*query.settings}); if (override_grantees) user.grantees = *override_grantees; @@ -162,14 +164,14 @@ BlockIO InterpreterCreateUserQuery::execute() } } - std::optional settings_from_query; - if (query.settings) - { - settings_from_query = SettingsProfileElements{*query.settings, access_control}; + std::optional settings_from_query; + if (query.alter_settings) + settings_from_query = AlterSettingsProfileElements{*query.alter_settings, access_control}; + else if (query.settings) + settings_from_query = AlterSettingsProfileElements{*query.settings, access_control}; - if (!query.attach) - getContext()->checkSettingsConstraints(*settings_from_query, SettingSource::USER); - } + if (settings_from_query && !query.attach) + getContext()->checkSettingsConstraints(*settings_from_query, SettingSource::USER); if (!query.cluster.empty()) return executeDDLQueryOnCluster(updated_query_ptr, getContext()); diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index ec2e60b2ef7..14d455b9150 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -75,10 +75,13 @@ namespace if (!user.settings.empty()) { + std::shared_ptr query_settings; if (attach_mode) - query->settings = user.settings.toAST(); + query_settings = user.settings.toAST(); else - query->settings = user.settings.toASTWithNames(*access_control); + query_settings = user.settings.toASTWithNames(*access_control); + if (!query_settings->empty()) + query->settings = query_settings; } if (user.grantees != RolesOrUsersSet::AllTag{}) @@ -109,10 +112,13 @@ namespace if (!role.settings.empty()) { + std::shared_ptr query_settings; if (attach_mode) - query->settings = role.settings.toAST(); + query_settings = role.settings.toAST(); else - query->settings = role.settings.toASTWithNames(*access_control); + query_settings = role.settings.toASTWithNames(*access_control); + if (!query_settings->empty()) + query->settings = query_settings; } return query; @@ -127,12 +133,16 @@ namespace if (!profile.elements.empty()) { + std::shared_ptr query_settings; if (attach_mode) - query->settings = profile.elements.toAST(); + query_settings = profile.elements.toAST(); else - query->settings = profile.elements.toASTWithNames(*access_control); - if (query->settings) - query->settings->setUseInheritKeyword(true); + query_settings = profile.elements.toASTWithNames(*access_control); + if (!query_settings->empty()) + { + query_settings->setUseInheritKeyword(true); + query->settings = query_settings; + } } if (!profile.to_roles.empty()) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 3da0e2508a0..5c404898239 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1997,11 +1997,6 @@ void Context::applySettingsChanges(const SettingsChanges & changes) applySettingsChangesWithLock(changes, lock); } -void Context::checkSettingsConstraintsWithLock(const SettingsProfileElements & profile_elements, SettingSource source) const -{ - getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(settings, profile_elements, source); -} - void Context::checkSettingsConstraintsWithLock(const SettingChange & change, SettingSource source) const { getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(settings, change, source); @@ -2017,6 +2012,11 @@ void Context::checkSettingsConstraintsWithLock(SettingsChanges & changes, Settin getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(settings, changes, source); } +void Context::checkSettingsConstraintsWithLock(const AlterSettingsProfileElements & profile_elements, SettingSource source) const +{ + getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(settings, profile_elements, source); +} + void Context::clampToSettingsConstraintsWithLock(SettingsChanges & changes, SettingSource source) const { getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.clamp(settings, changes, source); @@ -2027,12 +2027,6 @@ void Context::checkMergeTreeSettingsConstraintsWithLock(const MergeTreeSettings getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(merge_tree_settings, changes); } -void Context::checkSettingsConstraints(const SettingsProfileElements & profile_elements, SettingSource source) const -{ - SharedLockGuard lock(mutex); - checkSettingsConstraintsWithLock(profile_elements, source); -} - void Context::checkSettingsConstraints(const SettingChange & change, SettingSource source) const { SharedLockGuard lock(mutex); @@ -2051,6 +2045,12 @@ void Context::checkSettingsConstraints(SettingsChanges & changes, SettingSource checkSettingsConstraintsWithLock(changes, source); } +void Context::checkSettingsConstraints(const AlterSettingsProfileElements & profile_elements, SettingSource source) const +{ + SharedLockGuard lock(mutex); + checkSettingsConstraintsWithLock(profile_elements, source); +} + void Context::clampToSettingsConstraints(SettingsChanges & changes, SettingSource source) const { SharedLockGuard lock(mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 329dc928eb5..682f8515752 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -128,7 +128,7 @@ class AccessControl; class Credentials; class GSSAcceptorContext; struct SettingsConstraintsAndProfileIDs; -class SettingsProfileElements; +struct AlterSettingsProfileElements; class RemoteHostFilter; struct StorageID; class IDisk; @@ -776,10 +776,10 @@ public: void applySettingsChanges(const SettingsChanges & changes); /// Checks the constraints. - void checkSettingsConstraints(const SettingsProfileElements & profile_elements, SettingSource source) const; void checkSettingsConstraints(const SettingChange & change, SettingSource source) const; void checkSettingsConstraints(const SettingsChanges & changes, SettingSource source) const; void checkSettingsConstraints(SettingsChanges & changes, SettingSource source) const; + void checkSettingsConstraints(const AlterSettingsProfileElements & profile_elements, SettingSource source) const; void clampToSettingsConstraints(SettingsChanges & changes, SettingSource source) const; void checkMergeTreeSettingsConstraints(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const; @@ -1268,14 +1268,14 @@ private: void setCurrentDatabaseWithLock(const String & name, const std::lock_guard & lock); - void checkSettingsConstraintsWithLock(const SettingsProfileElements & profile_elements, SettingSource source) const; - void checkSettingsConstraintsWithLock(const SettingChange & change, SettingSource source) const; void checkSettingsConstraintsWithLock(const SettingsChanges & changes, SettingSource source) const; void checkSettingsConstraintsWithLock(SettingsChanges & changes, SettingSource source) const; + void checkSettingsConstraintsWithLock(const AlterSettingsProfileElements & profile_elements, SettingSource source) const; + void clampToSettingsConstraintsWithLock(SettingsChanges & changes, SettingSource source) const; void checkMergeTreeSettingsConstraintsWithLock(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const; diff --git a/src/Parsers/Access/ASTCreateRoleQuery.cpp b/src/Parsers/Access/ASTCreateRoleQuery.cpp index eeeb34c97e4..2da4f2bf683 100644 --- a/src/Parsers/Access/ASTCreateRoleQuery.cpp +++ b/src/Parsers/Access/ASTCreateRoleQuery.cpp @@ -31,6 +31,12 @@ namespace format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " SETTINGS " << (format.hilite ? IAST::hilite_none : ""); settings.format(format); } + + void formatAlterSettings(const ASTAlterSettingsProfileElements & alter_settings, const IAST::FormatSettings & format) + { + format.ostr << " "; + alter_settings.format(format); + } } @@ -47,6 +53,9 @@ ASTPtr ASTCreateRoleQuery::clone() const if (settings) res->settings = std::static_pointer_cast(settings->clone()); + if (alter_settings) + res->alter_settings = std::static_pointer_cast(alter_settings->clone()); + return res; } @@ -82,7 +91,9 @@ void ASTCreateRoleQuery::formatImpl(const FormatSettings & format, FormatState & if (!new_name.empty()) formatRenameTo(new_name, format); - if (settings && (!settings->empty() || alter)) + if (alter_settings) + formatAlterSettings(*alter_settings, format); + else if (settings) formatSettings(*settings, format); } diff --git a/src/Parsers/Access/ASTCreateRoleQuery.h b/src/Parsers/Access/ASTCreateRoleQuery.h index 4e465553164..ab6ecfe4427 100644 --- a/src/Parsers/Access/ASTCreateRoleQuery.h +++ b/src/Parsers/Access/ASTCreateRoleQuery.h @@ -7,6 +7,7 @@ namespace DB { class ASTSettingsProfileElements; +class ASTAlterSettingsProfileElements; /** CREATE ROLE [IF NOT EXISTS | OR REPLACE] name @@ -14,7 +15,12 @@ class ASTSettingsProfileElements; * * ALTER ROLE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] + * [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] [,...] ] + * [DROP SETTINGS variable [,...] ] + * [ADD PROFILES 'profile_name' [,...] ] + * [DROP PROFILES 'profile_name' [,...] ] + * [DROP ALL PROFILES] + * [DROP ALL SETTINGS] */ class ASTCreateRoleQuery : public IAST, public ASTQueryWithOnCluster { @@ -31,6 +37,7 @@ public: String storage_name; std::shared_ptr settings; + std::shared_ptr alter_settings; String getID(char) const override; ASTPtr clone() const override; diff --git a/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp b/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp index 3379486d789..bf563451a14 100644 --- a/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp +++ b/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp @@ -33,6 +33,12 @@ namespace settings.format(format); } + void formatAlterSettings(const ASTAlterSettingsProfileElements & alter_settings, const IAST::FormatSettings & format) + { + format.ostr << " "; + alter_settings.format(format); + } + void formatToRoles(const ASTRolesOrUsersSet & roles, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); @@ -57,6 +63,9 @@ ASTPtr ASTCreateSettingsProfileQuery::clone() const if (settings) res->settings = std::static_pointer_cast(settings->clone()); + if (alter_settings) + res->alter_settings = std::static_pointer_cast(alter_settings->clone()); + return res; } @@ -92,7 +101,9 @@ void ASTCreateSettingsProfileQuery::formatImpl(const FormatSettings & format, Fo if (!new_name.empty()) formatRenameTo(new_name, format); - if (settings && (!settings->empty() || alter)) + if (alter_settings) + formatAlterSettings(*alter_settings, format); + else if (settings) formatSettings(*settings, format); if (to_roles && (!to_roles->empty() || alter)) diff --git a/src/Parsers/Access/ASTCreateSettingsProfileQuery.h b/src/Parsers/Access/ASTCreateSettingsProfileQuery.h index be01aae1e26..c86badb1392 100644 --- a/src/Parsers/Access/ASTCreateSettingsProfileQuery.h +++ b/src/Parsers/Access/ASTCreateSettingsProfileQuery.h @@ -7,6 +7,7 @@ namespace DB { class ASTSettingsProfileElements; +class ASTAlterSettingsProfileElements; class ASTRolesOrUsersSet; @@ -16,7 +17,12 @@ class ASTRolesOrUsersSet; * * ALTER SETTINGS PROFILE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] + * [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] [,...] ] + * [DROP SETTINGS variable [,...] ] + * [ADD PROFILES 'profile_name' [,...] ] + * [DROP PROFILES 'profile_name' [,...] ] + * [DROP ALL PROFILES] + * [DROP ALL SETTINGS] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] */ class ASTCreateSettingsProfileQuery : public IAST, public ASTQueryWithOnCluster @@ -34,6 +40,7 @@ public: String new_name; std::shared_ptr settings; + std::shared_ptr alter_settings; std::shared_ptr to_roles; @@ -44,4 +51,5 @@ public: ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } QueryKind getQueryKind() const override { return QueryKind::Create; } }; + } diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 96306fa0dd2..e2fbdc57aa7 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -133,13 +133,17 @@ namespace default_roles.format(settings); } - void formatSettings(const ASTSettingsProfileElements & settings, const IAST::FormatSettings & format) { format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " SETTINGS " << (format.hilite ? IAST::hilite_none : ""); settings.format(format); } + void formatAlterSettings(const ASTAlterSettingsProfileElements & alter_settings, const IAST::FormatSettings & format) + { + format.ostr << " "; + alter_settings.format(format); + } void formatGrantees(const ASTRolesOrUsersSet & grantees, const IAST::FormatSettings & settings) { @@ -181,6 +185,9 @@ ASTPtr ASTCreateUserQuery::clone() const if (settings) res->settings = std::static_pointer_cast(settings->clone()); + if (alter_settings) + res->alter_settings = std::static_pointer_cast(alter_settings->clone()); + if (auth_data) { res->auth_data = std::static_pointer_cast(auth_data->clone()); @@ -242,7 +249,9 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & if (default_roles) formatDefaultRoles(*default_roles, format); - if (settings && (!settings->empty() || alter)) + if (alter_settings) + formatAlterSettings(*alter_settings, format); + else if (settings) formatSettings(*settings, format); if (grantees) diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index 4e14d86c425..517bfaa7c58 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -13,6 +13,7 @@ class ASTUserNamesWithHost; class ASTRolesOrUsersSet; class ASTDatabaseOrNone; class ASTSettingsProfileElements; +class ASTAlterSettingsProfileElements; class ASTAuthenticationData; @@ -30,7 +31,12 @@ class ASTAuthenticationData; * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [DEFAULT DATABASE database | NONE] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] + * [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] [,...] ] + * [DROP SETTINGS variable [,...] ] + * [ADD PROFILES 'profile_name' [,...] ] + * [DROP PROFILES 'profile_name' [,...] ] + * [DROP ALL PROFILES] + * [DROP ALL SETTINGS] * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] */ class ASTCreateUserQuery : public IAST, public ASTQueryWithOnCluster @@ -55,6 +61,7 @@ public: std::shared_ptr default_roles; std::shared_ptr settings; + std::shared_ptr alter_settings; std::shared_ptr grantees; std::shared_ptr default_database; diff --git a/src/Parsers/Access/ASTSettingsProfileElement.cpp b/src/Parsers/Access/ASTSettingsProfileElement.cpp index 7b29b15cb29..a61b8ac123b 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.cpp +++ b/src/Parsers/Access/ASTSettingsProfileElement.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -21,8 +22,54 @@ namespace settings.ostr << backQuoteIfNeed(str); } } + + void formatSettingsProfileElementsForAlter(std::string_view kind, const ASTSettingsProfileElements & elements, const IAST::FormatSettings & settings) + { + bool need_comma = false; + + size_t num_profiles = elements.getNumberOfProfiles(); + if (num_profiles > 0) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << kind << " " << (num_profiles == 1 ? "PROFILE" : "PROFILES") + << (settings.hilite ? IAST::hilite_none : "") << " "; + + for (const auto & element : elements.elements) + { + if (!element->parent_profile.empty()) + { + if (need_comma) + settings.ostr << ", "; + formatProfileNameOrID(element->parent_profile, /* is_id= */ false, settings); + need_comma = true; + } + } + } + + size_t num_settings = elements.getNumberOfSettings(); + if (num_settings > 0) + { + if (need_comma) + settings.ostr << ", "; + need_comma = false; + + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << kind << " " << (num_settings == 1 ? "SETTING" : "SETTINGS") + << (settings.hilite ? IAST::hilite_none : "") << " "; + + for (const auto & element : elements.elements) + { + if (!element->setting_name.empty()) + { + if (need_comma) + settings.ostr << ", "; + element->format(settings); + need_comma = true; + } + } + } + } } + void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { if (!parent_profile.empty()) @@ -82,6 +129,35 @@ bool ASTSettingsProfileElements::empty() const return true; } +size_t ASTSettingsProfileElements::getNumberOfSettings() const +{ + size_t count = 0; + for (const auto & element : elements) + if (!element->setting_name.empty()) + ++count; + return count; +} + +size_t ASTSettingsProfileElements::getNumberOfProfiles() const +{ + size_t count = 0; + for (const auto & element : elements) + if (!element->parent_profile.empty()) + ++count; + return count; +} + + +ASTPtr ASTSettingsProfileElements::clone() const +{ + auto res = std::make_shared(*this); + + for (auto & element : res->elements) + element = std::static_pointer_cast(element->clone()); + + return res; +} + void ASTSettingsProfileElements::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { @@ -109,4 +185,108 @@ void ASTSettingsProfileElements::setUseInheritKeyword(bool use_inherit_keyword_) element->use_inherit_keyword = use_inherit_keyword_; } + +void ASTSettingsProfileElements::add(ASTSettingsProfileElements && other) +{ + insertAtEnd(elements, std::move(other.elements)); +} + + +String ASTAlterSettingsProfileElements::getID(char) const +{ + return "AlterSettingsProfileElements"; +} + +ASTPtr ASTAlterSettingsProfileElements::clone() const +{ + auto res = std::make_shared(*this); + + if (add_settings) + res->add_settings = std::static_pointer_cast(add_settings->clone()); + + if (modify_settings) + res->modify_settings = std::static_pointer_cast(modify_settings->clone()); + + if (drop_settings) + res->drop_settings = std::static_pointer_cast(drop_settings->clone()); + + return res; +} + +void ASTAlterSettingsProfileElements::formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const +{ + bool need_comma = false; + + if (drop_all_settings) + { + format.ostr << (format.hilite ? IAST::hilite_keyword : "") << "DROP ALL SETTINGS" << (format.hilite ? IAST::hilite_none : ""); + need_comma = true; + } + + if (drop_all_profiles) + { + if (need_comma) + format.ostr << ", "; + format.ostr << (format.hilite ? IAST::hilite_keyword : "") << "DROP ALL PROFILES" << (format.hilite ? IAST::hilite_none : ""); + need_comma = true; + } + + if (drop_settings && !drop_settings->empty()) + { + if (need_comma) + format.ostr << ", "; + formatSettingsProfileElementsForAlter("DROP", *drop_settings, format); + need_comma = true; + } + + if (add_settings && !add_settings->empty()) + { + if (need_comma) + format.ostr << ", "; + formatSettingsProfileElementsForAlter("ADD", *add_settings, format); + need_comma = true; + } + + if (modify_settings && !modify_settings->empty()) + { + if (need_comma) + format.ostr << ", "; + formatSettingsProfileElementsForAlter("MODIFY", *modify_settings, format); + } +} + +void ASTAlterSettingsProfileElements::add(ASTAlterSettingsProfileElements && other) +{ + drop_all_settings |= other.drop_all_settings; + drop_all_profiles |= other.drop_all_profiles; + + if (other.add_settings) + { + if (!add_settings) + add_settings = std::make_shared(); + add_settings->add(std::move(*other.add_settings)); + } + + if (other.add_settings) + { + if (!add_settings) + add_settings = std::make_shared(); + add_settings->add(std::move(*other.add_settings)); + } + + if (other.modify_settings) + { + if (!modify_settings) + modify_settings = std::make_shared(); + modify_settings->add(std::move(*other.modify_settings)); + } + + if (other.drop_settings) + { + if (!drop_settings) + drop_settings = std::make_shared(); + drop_settings->add(std::move(*other.drop_settings)); + } +} + } diff --git a/src/Parsers/Access/ASTSettingsProfileElement.h b/src/Parsers/Access/ASTSettingsProfileElement.h index 13c1926d9b0..32a0d15e173 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.h +++ b/src/Parsers/Access/ASTSettingsProfileElement.h @@ -39,10 +39,41 @@ public: bool empty() const; + size_t getNumberOfSettings() const; + size_t getNumberOfProfiles() const; + String getID(char) const override { return "SettingsProfileElements"; } - ASTPtr clone() const override { return std::make_shared(*this); } + ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; void setUseInheritKeyword(bool use_inherit_keyword_); + + void add(ASTSettingsProfileElements && other); }; + +/* Represents a clause used to alter settings or profiles assigned to a user or a role or another profile. + * [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] [,...] ] + * [DROP SETTINGS variable [,...] ] + * [ADD PROFILES 'profile_name' [,...] ] + * [DROP PROFILES 'profile_name' [,...] ] + * [DROP ALL PROFILES] + * [DROP ALL SETTINGS] +*/ +class ASTAlterSettingsProfileElements : public IAST +{ +public: + std::shared_ptr add_settings; + std::shared_ptr modify_settings; + std::shared_ptr drop_settings; + + bool drop_all_settings = false; + bool drop_all_profiles = false; + + String getID(char) const override; + ASTPtr clone() const override; + void formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const override; + + void add(ASTAlterSettingsProfileElements && other); +}; + } diff --git a/src/Parsers/Access/ParserCreateRoleQuery.cpp b/src/Parsers/Access/ParserCreateRoleQuery.cpp index 99a97f6901d..4f223d2dce6 100644 --- a/src/Parsers/Access/ParserCreateRoleQuery.cpp +++ b/src/Parsers/Access/ParserCreateRoleQuery.cpp @@ -25,20 +25,31 @@ namespace }); } - bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::vector> & settings) + bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & settings) { return IParserBase::wrapParseImpl(pos, [&] { - if (!ParserKeyword{"SETTINGS"}.ignore(pos, expected)) - return false; - - ASTPtr new_settings_ast; + ASTPtr ast; ParserSettingsProfileElements elements_p; elements_p.useIDMode(id_mode); - if (!elements_p.parse(pos, new_settings_ast, expected)) + if (!elements_p.parse(pos, ast, expected)) return false; - settings = std::move(new_settings_ast->as().elements); + settings = typeid_cast>(ast); + return true; + }); + } + + bool parseAlterSettings(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & alter_settings) + { + return IParserBase::wrapParseImpl(pos, [&] + { + ASTPtr ast; + ParserAlterSettingsProfileElements elements_p; + if (!elements_p.parse(pos, ast, expected)) + return false; + + alter_settings = typeid_cast>(ast); return true; }); } @@ -91,6 +102,7 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec String new_name; std::shared_ptr settings; + std::shared_ptr alter_settings; String cluster; String storage_name; @@ -99,14 +111,27 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (alter && new_name.empty() && (names.size() == 1) && parseRenameTo(pos, expected, new_name)) continue; - std::vector> new_settings; - if (parseSettings(pos, expected, attach_mode, new_settings)) + if (alter) { - if (!settings) - settings = std::make_shared(); - - insertAtEnd(settings->elements, std::move(new_settings)); - continue; + std::shared_ptr new_alter_settings; + if (parseAlterSettings(pos, expected, new_alter_settings)) + { + if (!alter_settings) + alter_settings = std::make_shared(); + alter_settings->add(std::move(*new_alter_settings)); + continue; + } + } + else + { + std::shared_ptr new_settings; + if (parseSettings(pos, expected, attach_mode, new_settings)) + { + if (!settings) + settings = std::make_shared(); + settings->add(std::move(*new_settings)); + continue; + } } if (cluster.empty() && parseOnCluster(pos, expected, cluster)) @@ -130,6 +155,7 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->names = std::move(names); query->new_name = std::move(new_name); query->settings = std::move(settings); + query->alter_settings = std::move(alter_settings); query->storage_name = std::move(storage_name); return true; diff --git a/src/Parsers/Access/ParserCreateRoleQuery.h b/src/Parsers/Access/ParserCreateRoleQuery.h index 883ea89854a..3ab18b47299 100644 --- a/src/Parsers/Access/ParserCreateRoleQuery.h +++ b/src/Parsers/Access/ParserCreateRoleQuery.h @@ -11,7 +11,12 @@ namespace DB * * ALTER ROLE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] + * [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] [,...] ] + * [DROP SETTINGS variable [,...] ] + * [ADD PROFILES 'profile_name' [,...] ] + * [DROP PROFILES 'profile_name' [,...] ] + * [DROP ALL PROFILES] + * [DROP ALL SETTINGS] */ class ParserCreateRoleQuery : public IParserBase { diff --git a/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp b/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp index fcb6ebd8806..ed559c200c7 100644 --- a/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp +++ b/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp @@ -27,20 +27,32 @@ namespace }); } - bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::vector> & settings) + bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & settings) { return IParserBase::wrapParseImpl(pos, [&] { - if (!ParserKeyword{"SETTINGS"}.ignore(pos, expected)) - return false; - - ASTPtr new_settings_ast; + ASTPtr ast; ParserSettingsProfileElements elements_p; elements_p.useInheritKeyword(true).useIDMode(id_mode); - if (!elements_p.parse(pos, new_settings_ast, expected)) + if (!elements_p.parse(pos, ast, expected)) return false; - settings = std::move(new_settings_ast->as().elements); + settings = typeid_cast>(ast); + return true; + }); + } + + bool parseAlterSettings(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & alter_settings) + { + return IParserBase::wrapParseImpl(pos, [&] + { + ASTPtr ast; + ParserAlterSettingsProfileElements elements_p; + elements_p.useInheritKeyword(true); + if (!elements_p.parse(pos, ast, expected)) + return false; + + alter_settings = typeid_cast>(ast); return true; }); } @@ -111,6 +123,7 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec String new_name; std::shared_ptr settings; + std::shared_ptr alter_settings; String cluster; String storage_name; @@ -119,14 +132,27 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec if (alter && new_name.empty() && (names.size() == 1) && parseRenameTo(pos, expected, new_name)) continue; - std::vector> new_settings; - if (parseSettings(pos, expected, attach_mode, new_settings)) + if (alter) { - if (!settings) - settings = std::make_shared(); - - insertAtEnd(settings->elements, std::move(new_settings)); - continue; + std::shared_ptr new_alter_settings; + if (parseAlterSettings(pos, expected, new_alter_settings)) + { + if (!alter_settings) + alter_settings = std::make_shared(); + alter_settings->add(std::move(*new_alter_settings)); + continue; + } + } + else + { + std::shared_ptr new_settings; + if (parseSettings(pos, expected, attach_mode, new_settings)) + { + if (!settings) + settings = std::make_shared(); + settings->add(std::move(*new_settings)); + continue; + } } if (cluster.empty() && parseOnCluster(pos, expected, cluster)) @@ -156,6 +182,7 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec query->names = std::move(names); query->new_name = std::move(new_name); query->settings = std::move(settings); + query->alter_settings = std::move(alter_settings); query->to_roles = std::move(to_roles); query->storage_name = std::move(storage_name); diff --git a/src/Parsers/Access/ParserCreateSettingsProfileQuery.h b/src/Parsers/Access/ParserCreateSettingsProfileQuery.h index bee5bdcb2d1..b48d46c1b36 100644 --- a/src/Parsers/Access/ParserCreateSettingsProfileQuery.h +++ b/src/Parsers/Access/ParserCreateSettingsProfileQuery.h @@ -11,7 +11,12 @@ namespace DB * * ALTER SETTINGS PROFILE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] + * [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] [,...] ] + * [DROP SETTINGS variable [,...] ] + * [ADD PROFILES 'profile_name' [,...] ] + * [DROP PROFILES 'profile_name' [,...] ] + * [DROP ALL PROFILES] + * [DROP ALL SETTINGS] */ class ParserCreateSettingsProfileQuery : public IParserBase { diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 6fec40ec84a..6e276102e26 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -342,20 +342,31 @@ namespace } - bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::vector> & settings) + bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & settings) { return IParserBase::wrapParseImpl(pos, [&] { - if (!ParserKeyword{"SETTINGS"}.ignore(pos, expected)) - return false; - - ASTPtr new_settings_ast; + ASTPtr ast; ParserSettingsProfileElements elements_p; elements_p.useIDMode(id_mode); - if (!elements_p.parse(pos, new_settings_ast, expected)) + if (!elements_p.parse(pos, ast, expected)) return false; - settings = std::move(new_settings_ast->as().elements); + settings = typeid_cast>(ast); + return true; + }); + } + + bool parseAlterSettings(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & alter_settings) + { + return IParserBase::wrapParseImpl(pos, [&] + { + ASTPtr ast; + ParserAlterSettingsProfileElements elements_p; + if (!elements_p.parse(pos, ast, expected)) + return false; + + alter_settings = typeid_cast>(ast); return true; }); } @@ -463,6 +474,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec std::shared_ptr auth_data; std::shared_ptr default_roles; std::shared_ptr settings; + std::shared_ptr alter_settings; std::shared_ptr grantees; std::shared_ptr default_database; ASTPtr valid_until; @@ -495,14 +507,27 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec continue; } - std::vector> new_settings; - if (parseSettings(pos, expected, attach_mode, new_settings)) + if (alter) { - if (!settings) - settings = std::make_shared(); - - insertAtEnd(settings->elements, std::move(new_settings)); - continue; + std::shared_ptr new_alter_settings; + if (parseAlterSettings(pos, expected, new_alter_settings)) + { + if (!alter_settings) + alter_settings = std::make_shared(); + alter_settings->add(std::move(*new_alter_settings)); + continue; + } + } + else + { + std::shared_ptr new_settings; + if (parseSettings(pos, expected, attach_mode, new_settings)) + { + if (!settings) + settings = std::make_shared(); + settings->add(std::move(*new_settings)); + continue; + } } if (!default_roles && parseDefaultRoles(pos, expected, attach_mode, default_roles)) @@ -574,6 +599,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->remove_hosts = std::move(remove_hosts); query->default_roles = std::move(default_roles); query->settings = std::move(settings); + query->alter_settings = std::move(alter_settings); query->grantees = std::move(grantees); query->default_database = std::move(default_database); query->valid_until = std::move(valid_until); diff --git a/src/Parsers/Access/ParserCreateUserQuery.h b/src/Parsers/Access/ParserCreateUserQuery.h index 0cc8c9b6649..4dfff8713d7 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.h +++ b/src/Parsers/Access/ParserCreateUserQuery.h @@ -18,7 +18,12 @@ namespace DB * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] + * [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] [,...] ] + * [DROP SETTINGS variable [,...] ] + * [ADD PROFILES 'profile_name' [,...] ] + * [DROP PROFILES 'profile_name' [,...] ] + * [DROP ALL PROFILES] + * [DROP ALL SETTINGS] * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] */ class ParserCreateUserQuery : public IParserBase diff --git a/src/Parsers/Access/ParserSettingsProfileElement.cpp b/src/Parsers/Access/ParserSettingsProfileElement.cpp index 36330b96622..fbd91835298 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.cpp +++ b/src/Parsers/Access/ParserSettingsProfileElement.cpp @@ -7,27 +7,13 @@ #include #include #include +#include namespace DB { namespace { - bool parseProfileKeyword(IParserBase::Pos & pos, Expected & expected, bool use_inherit_keyword) - { - if (ParserKeyword{"PROFILE"}.ignore(pos, expected)) - return true; - - if (use_inherit_keyword && ParserKeyword{"INHERIT"}.ignore(pos, expected)) - { - ParserKeyword{"PROFILE"}.ignore(pos, expected); - return true; - } - - return false; - } - - bool parseProfileNameOrID(IParserBase::Pos & pos, Expected & expected, bool id_mode, String & res) { return IParserBase::wrapParseImpl(pos, [&] @@ -120,6 +106,17 @@ namespace } + bool parseSettingName(IParserBase::Pos & pos, Expected & expected, String & res) + { + ASTPtr name_ast; + if (!ParserCompoundIdentifier{}.parse(pos, name_ast, expected)) + return false; + + res = getIdentifierName(name_ast); + return true; + } + + bool parseSettingNameWithValueOrConstraints( IParserBase::Pos & pos, Expected & expected, @@ -131,11 +128,10 @@ namespace { return IParserBase::wrapParseImpl(pos, [&] { - ASTPtr name_ast; - if (!ParserCompoundIdentifier{}.parse(pos, name_ast, expected)) + String res_setting_name; + if (!parseSettingName(pos, expected, res_setting_name)) return false; - String res_setting_name = getIdentifierName(name_ast); std::optional res_value; std::optional res_min_value; std::optional res_max_value; @@ -170,52 +166,112 @@ namespace } - bool parseSettingsProfileElement(IParserBase::Pos & pos, - Expected & expected, - bool id_mode, - bool use_inherit_keyword, - bool previous_element_was_parent_profile, - std::shared_ptr & result) + bool parseSettingsProfileElements(IParserBase::Pos & pos, + Expected & expected, + bool id_mode, + bool use_inherit_keyword, + std::vector> & res) { - return IParserBase::wrapParseImpl(pos, [&] + std::vector> elements; + bool found_none = false; + + bool expect_profiles = false; + bool expect_settings = false; + + auto parse_element = [&] { - String parent_profile; - String setting_name; - std::optional value; - std::optional min_value; - std::optional max_value; - std::optional writability; + if (ParserKeyword{"SETTINGS"}.ignore(pos, expected) || ParserKeyword{"SETTING"}.ignore(pos, expected)) + { + expect_settings = true; + } - bool ok = parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, writability); + bool expect_settings_next = expect_settings; - if (!ok && (parseProfileKeyword(pos, expected, use_inherit_keyword) || previous_element_was_parent_profile)) - ok = parseProfileNameOrID(pos, expected, id_mode, parent_profile); + if (ParserKeyword{"PROFILES"}.ignore(pos, expected) || ParserKeyword{"PROFILE"}.ignore(pos, expected)) + { + expect_profiles = true; + expect_settings = false; + } + else if (use_inherit_keyword && ParserKeyword{"INHERIT"}.ignore(pos, expected)) + { + if (!ParserKeyword{"PROFILES"}.ignore(pos, expected)) + ParserKeyword{"PROFILE"}.ignore(pos, expected); + expect_profiles = true; + expect_settings = false; + } - if (!ok) + if (!expect_profiles && !expect_settings) return false; - result = std::make_shared(); - result->parent_profile = std::move(parent_profile); - result->setting_name = std::move(setting_name); - result->value = std::move(value); - result->min_value = std::move(min_value); - result->max_value = std::move(max_value); - result->writability = writability; - result->id_mode = id_mode; - result->use_inherit_keyword = use_inherit_keyword; - return true; - }); + if (ParserKeyword{"NONE"}.ignore(pos, expected)) + { + found_none = true; + expect_settings = expect_settings_next; + return true; + } + + if (expect_settings) + { + String setting_name; + std::optional value; + std::optional min_value; + std::optional max_value; + std::optional writability; + if (parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, writability)) + { + auto element = std::make_shared(); + element->setting_name = std::move(setting_name); + element->value = std::move(value); + element->min_value = std::move(min_value); + element->max_value = std::move(max_value); + element->writability = std::move(writability); + elements.push_back(element); + expect_profiles = false; + expect_settings = expect_settings_next; + return true; + } + } + + if (expect_profiles) + { + String profile_name; + if (parseProfileNameOrID(pos, expected, id_mode, profile_name)) + { + auto element = std::make_shared(); + element->parent_profile = std::move(profile_name); + element->id_mode = id_mode; + element->use_inherit_keyword = use_inherit_keyword; + elements.push_back(element); + expect_settings = expect_settings_next; + return true; + } + } + + return false; + }; + + if (!ParserList::parseUtil(pos, expected, parse_element, false)) + return false; + + if (elements.empty() && !found_none) + return false; + + res = std::move(elements); + return true; } } bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - std::shared_ptr res; - if (!parseSettingsProfileElement(pos, expected, id_mode, use_inherit_keyword, false, res)) + std::vector> elements; + if (!parseSettingsProfileElements(pos, expected, id_mode, use_inherit_keyword, elements)) return false; - node = res; + if (elements.size() != 1) + return false; + + node = elements[0]; return true; } @@ -223,28 +279,8 @@ bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected bool ParserSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { std::vector> elements; - - if (ParserKeyword{"NONE"}.ignore(pos, expected)) - { - } - else - { - bool previous_element_was_parent_profile = false; - - auto parse_element = [&] - { - std::shared_ptr element; - if (!parseSettingsProfileElement(pos, expected, id_mode, use_inherit_keyword, previous_element_was_parent_profile, element)) - return false; - - elements.push_back(element); - previous_element_was_parent_profile = !element->parent_profile.empty(); - return true; - }; - - if (!ParserList::parseUtil(pos, expected, parse_element, false)) - return false; - } + if (!parseSettingsProfileElements(pos, expected, id_mode, use_inherit_keyword, elements)) + return false; auto result = std::make_shared(); result->elements = std::move(elements); @@ -252,4 +288,148 @@ bool ParserSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Expected return true; } + +bool ParserAlterSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + std::vector> add_settings; + std::vector> modify_settings; + std::vector> drop_settings; + bool drop_all_settings = false; + bool drop_all_profiles = false; + + std::vector> old_style_settings; + if (parseSettingsProfileElements(pos, expected, /* id_mode= */ false, use_inherit_keyword, old_style_settings)) + { + /// old style: "SETTINGS ..." replaces all the settings ad profiles. + add_settings = std::move(old_style_settings); + drop_all_settings = true; + drop_all_profiles = true; + } + else + { + /// new style: "MODIFY SETTINGS ..., ADD PROFILES ..., DROP PROFILES ..., DROP SETTINGS ..." + std::string_view mode; + std::string_view submode; + + auto parse_element = [&] + { + if (ParserKeyword{"ADD"}.ignore(pos, expected)) + { + mode = "ADD"; + submode = ""; + } + else if (ParserKeyword{"DROP"}.ignore(pos, expected)) + { + mode = "DROP"; + submode = ""; + } + else if (ParserKeyword{"MODIFY"}.ignore(pos, expected)) + { + mode = "MODIFY"; + submode = ""; + } + + if (!mode.empty()) + { + if (ParserKeyword{"ALL PROFILES"}.ignore(pos, expected)) + submode = "ALL PROFILES"; + else if (ParserKeyword{"ALL SETTINGS"}.ignore(pos, expected)) + submode = "ALL SETTINGS"; + else if (ParserKeyword{"PROFILES"}.ignore(pos, expected) || ParserKeyword{"PROFILE"}.ignore(pos, expected)) + submode = "PROFILES"; + else if (ParserKeyword{"SETTINGS"}.ignore(pos, expected) || ParserKeyword{"SETTING"}.ignore(pos, expected)) + submode = "SETTINGS"; + } + + if (submode.empty()) + return false; + + if (submode == "PROFILES") + { + auto element = std::make_shared(); + if (!parseProfileNameOrID(pos, expected, /* id_mode= */ false, element->parent_profile)) + return false; + if (mode == "ADD") + { + add_settings.push_back(element); + return true; + } + if (mode == "DROP") + { + drop_settings.push_back(element); + return true; + } + return false; + } + + if (submode == "SETTINGS") + { + auto element = std::make_shared(); + if (mode == "ADD" || mode == "MODIFY") + { + if (!parseSettingNameWithValueOrConstraints(pos, expected, element->setting_name, element->value, element->min_value, element->max_value, element->writability)) + return false; + if (mode == "ADD") + add_settings.push_back(element); + else + modify_settings.push_back(element); + return true; + } + if (mode == "DROP") + { + ASTPtr name_ast; + if (!ParserCompoundIdentifier{}.parse(pos, name_ast, expected)) + return false; + element->setting_name = getIdentifierName(name_ast); + drop_settings.push_back(element); + return true; + } + return false; + } + + if (mode == "DROP" && submode == "ALL PROFILES") + { + drop_all_profiles = true; + return true; + } + + if (mode == "DROP" && submode == "ALL SETTINGS") + { + drop_all_settings = true; + return true; + } + + return false; + }; + + if (!ParserList::parseUtil(pos, expected, parse_element, false)) + return false; + } + + if (add_settings.empty() && modify_settings.empty() && drop_settings.empty() && !drop_all_settings && !drop_all_profiles) + return false; + + auto result = std::make_shared(); + if (!add_settings.empty()) + { + result->add_settings = std::make_shared(); + result->add_settings->elements = std::move(add_settings); + } + if (!modify_settings.empty()) + { + result->modify_settings = std::make_shared(); + result->modify_settings->elements = std::move(modify_settings); + } + if (!drop_settings.empty()) + { + result->drop_settings = std::make_shared(); + result->drop_settings->elements = std::move(drop_settings); + } + result->drop_all_settings = drop_all_settings; + result->drop_all_profiles = drop_all_profiles; + + node = result; + return true; +} + } diff --git a/src/Parsers/Access/ParserSettingsProfileElement.h b/src/Parsers/Access/ParserSettingsProfileElement.h index 082fc66625f..0761311e833 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.h +++ b/src/Parsers/Access/ParserSettingsProfileElement.h @@ -39,4 +39,18 @@ private: bool use_inherit_keyword = false; }; + +class ParserAlterSettingsProfileElements : public IParserBase +{ +public: + ParserAlterSettingsProfileElements & useInheritKeyword(bool use_inherit_keyword_ = true) { use_inherit_keyword = use_inherit_keyword_; return *this; } + +protected: + const char * getName() const override { return "AlterSettingsProfileElements"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +private: + bool use_inherit_keyword = false; +}; + } diff --git a/tests/queries/0_stateless/01294_create_settings_profile.reference b/tests/queries/0_stateless/01294_create_settings_profile.reference index a10d5758752..ab67ee24a43 100644 --- a/tests/queries/0_stateless/01294_create_settings_profile.reference +++ b/tests/queries/0_stateless/01294_create_settings_profile.reference @@ -49,7 +49,7 @@ CREATE SETTINGS PROFILE s4_01294 TO r1_01294 CREATE SETTINGS PROFILE s1_01294 SETTINGS readonly = 1 CREATE SETTINGS PROFILE s2_01294 SETTINGS readonly CONST CREATE SETTINGS PROFILE s3_01294 SETTINGS INHERIT readonly -CREATE SETTINGS PROFILE s4_01294 SETTINGS INHERIT readonly, INHERIT readonly +CREATE SETTINGS PROFILE s4_01294 SETTINGS INHERIT readonly CREATE SETTINGS PROFILE s5_01294 SETTINGS INHERIT readonly, readonly = 1 CREATE SETTINGS PROFILE s6_01294 SETTINGS INHERIT readonly, readonly CONST -- system.settings_profiles diff --git a/tests/queries/0_stateless/02943_alter_user_modify_profiles.reference b/tests/queries/0_stateless/02943_alter_user_modify_profiles.reference new file mode 100644 index 00000000000..8c435e31354 --- /dev/null +++ b/tests/queries/0_stateless/02943_alter_user_modify_profiles.reference @@ -0,0 +1,21 @@ +CREATE USER test_user +CREATE USER test_user SETTINGS PROFILE profile_a +CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_b +CREATE USER test_user SETTINGS PROFILE profile_b, PROFILE profile_a, PROFILE profile_c +CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_d +CREATE USER test_user SETTINGS PROFILE profile_e +CREATE USER test_user +CREATE ROLE test_role +CREATE ROLE test_role SETTINGS PROFILE profile_a +CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_b +CREATE ROLE test_role SETTINGS PROFILE profile_b, PROFILE profile_a, PROFILE profile_c +CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_d +CREATE ROLE test_role SETTINGS PROFILE profile_e +CREATE ROLE test_role +CREATE SETTINGS PROFILE test_profile +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_b +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_b, INHERIT profile_a, INHERIT profile_c +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_d +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_e +CREATE SETTINGS PROFILE test_profile diff --git a/tests/queries/0_stateless/02943_alter_user_modify_profiles.sh b/tests/queries/0_stateless/02943_alter_user_modify_profiles.sh new file mode 100755 index 00000000000..8a16cacbbf7 --- /dev/null +++ b/tests/queries/0_stateless/02943_alter_user_modify_profiles.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +test_user="test_user_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +test_role="test_role_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +test_profile="test_profile_02932_${CLICKHOUSE_DATABASE}_$RANDOM" + +profile_a="profile_a_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +profile_b="profile_b_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +profile_c="profile_c_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +profile_d="profile_d_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +profile_e="profile_e_02932_${CLICKHOUSE_DATABASE}_$RANDOM" + +$CLICKHOUSE_CLIENT -q "DROP PROFILE IF EXISTS ${profile_a};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE IF EXISTS ${profile_b};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE IF EXISTS ${profile_c};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE IF EXISTS ${profile_d};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE IF EXISTS ${profile_e};" + +$CLICKHOUSE_CLIENT -q "CREATE PROFILE ${profile_a};" +$CLICKHOUSE_CLIENT -q "CREATE PROFILE ${profile_b};" +$CLICKHOUSE_CLIENT -q "CREATE PROFILE ${profile_c};" +$CLICKHOUSE_CLIENT -q "CREATE PROFILE ${profile_d};" +$CLICKHOUSE_CLIENT -q "CREATE PROFILE ${profile_e};" + +function show_create() +{ + local type="$1" + local name="$2" + + $CLICKHOUSE_CLIENT -q "SHOW CREATE $type ${name};" | sed -e "s/${name}/test_${type}/g" -e "s/${profile_a}/profile_a/g" -e "s/${profile_b}/profile_b/g" -e "s/${profile_c}/profile_c/g" -e "s/${profile_d}/profile_d/g" -e "s/${profile_e}/profile_e/g" +} + +function run_test() +{ + local type="$1" + local name="$2" + + $CLICKHOUSE_CLIENT -q "DROP $type IF EXISTS ${name};" + + $CLICKHOUSE_CLIENT -q "CREATE $type ${name};" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD PROFILE ${profile_a};" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD PROFILE ${profile_b};" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD PROFILES ${profile_a}, ${profile_c};" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD PROFILE ${profile_d} DROP PROFILES ${profile_b}, ${profile_c};" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} DROP ALL PROFILES, ADD PROFILE ${profile_e};" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} DROP ALL PROFILES;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "DROP $type ${name};" +} + +run_test user ${test_user} +run_test role ${test_role} +run_test profile ${test_profile} + +$CLICKHOUSE_CLIENT -q "DROP PROFILE ${profile_a};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE ${profile_b};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE ${profile_c};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE ${profile_d};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE ${profile_e};" diff --git a/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference b/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference new file mode 100644 index 00000000000..7ae78818372 --- /dev/null +++ b/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference @@ -0,0 +1,39 @@ +CREATE USER test_user +CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE USER test_user +CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE USER test_user +CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE USER test_user +CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE USER test_user +CREATE USER test_user SETTINGS custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE USER test_user SETTINGS PROFILE profile_a, custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE USER test_user SETTINGS PROFILE profile_b, PROFILE profile_a, custom_x = 321, custom_s = \'str\' CONST +CREATE USER test_user +CREATE ROLE test_role +CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE ROLE test_role +CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE ROLE test_role +CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE ROLE test_role +CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE ROLE test_role +CREATE ROLE test_role SETTINGS custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE ROLE test_role SETTINGS PROFILE profile_a, custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE ROLE test_role SETTINGS PROFILE profile_b, PROFILE profile_a, custom_x = 321, custom_s = \'str\' CONST +CREATE ROLE test_role +CREATE SETTINGS PROFILE test_profile +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE test_profile +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE test_profile +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE test_profile +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE test_profile +CREATE SETTINGS PROFILE test_profile SETTINGS custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_b, INHERIT profile_a, custom_x = 321, custom_s = \'str\' CONST +CREATE SETTINGS PROFILE test_profile diff --git a/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.sh b/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.sh new file mode 100755 index 00000000000..2e75d7025be --- /dev/null +++ b/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +test_user="test_user_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +test_role="test_role_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +test_profile="test_profile_02932_${CLICKHOUSE_DATABASE}_$RANDOM" + +profile_a="profile_a_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +profile_b="profile_b_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +profile_c="profile_c_02932_${CLICKHOUSE_DATABASE}_$RANDOM" + +$CLICKHOUSE_CLIENT -q "DROP PROFILE IF EXISTS ${profile_a};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE IF EXISTS ${profile_b};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE IF EXISTS ${profile_c};" + +$CLICKHOUSE_CLIENT -q "CREATE PROFILE ${profile_a};" +$CLICKHOUSE_CLIENT -q "CREATE PROFILE ${profile_b};" +$CLICKHOUSE_CLIENT -q "CREATE PROFILE ${profile_c};" + +function show_create() +{ + local type="$1" + local name="$2" + + $CLICKHOUSE_CLIENT -q "SHOW CREATE $type ${name};" | sed -e "s/${name}/test_${type}/g" -e "s/${profile_a}/profile_a/g" -e "s/${profile_b}/profile_b/g" -e "s/${profile_c}/profile_c/g" +} + +function run_test() +{ + local type="$1" + local name="$2" + + $CLICKHOUSE_CLIENT -q "DROP $type IF EXISTS ${name};" + + $CLICKHOUSE_CLIENT -q "CREATE $type ${name};" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD SETTINGS custom_x = 123, custom_y=56.5, custom_w='www', ADD PROFILES ${profile_a}, ${profile_b}" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} DROP ALL SETTINGS, DROP ALL PROFILES" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD SETTINGS custom_x = 123, custom_y=56.5, custom_w='www', PROFILES ${profile_a}, ${profile_b}" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} DROP ALL SETTINGS, ALL PROFILES" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD SETTING custom_x = 123, ADD SETTING custom_y=56.5, ADD SETTING custom_w='www', ADD PROFILE ${profile_a}, ADD PROFILE ${profile_b}" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} DROP ALL PROFILES, DROP ALL SETTINGS" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD SETTING custom_x = 123 ADD SETTING custom_y=56.5 ADD SETTING custom_w='www' ADD PROFILE ${profile_a} ADD PROFILE ${profile_b}" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} DROP ALL SETTINGS DROP ALL PROFILES" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD SETTING custom_x = 123, custom_t = 7, custom_s = 'str'" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD PROFILE ${profile_a}" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD PROFILE ${profile_b}, ${profile_a}, DROP SETTING custom_t, MODIFY SETTING custom_s READONLY, custom_x = 321" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} DROP ALL SETTINGS, ALL PROFILES" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "DROP $type ${name};" +} + +run_test user ${test_user} +run_test role ${test_role} +run_test profile ${test_profile} + +$CLICKHOUSE_CLIENT -q "DROP PROFILE ${profile_a};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE ${profile_b};" +$CLICKHOUSE_CLIENT -q "DROP PROFILE ${profile_c};" diff --git a/tests/queries/0_stateless/02943_alter_user_modify_settings.reference b/tests/queries/0_stateless/02943_alter_user_modify_settings.reference new file mode 100644 index 00000000000..8ed61ef77c7 --- /dev/null +++ b/tests/queries/0_stateless/02943_alter_user_modify_settings.reference @@ -0,0 +1,42 @@ +CREATE USER test_user +CREATE USER test_user SETTINGS custom_a = 100 +CREATE USER test_user SETTINGS custom_a = 100, custom_b = 200 +CREATE USER test_user SETTINGS custom_a = 100, custom_b = 300, custom_c = 400 +CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500 +CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 1000 +CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 2000 +CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 500 MAX 2000 +CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 700 MIN 500 MAX 2000 +CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 800 MIN 150 +CREATE USER test_user SETTINGS custom_c = 400, custom_d = 500 +CREATE USER test_user SETTINGS custom_c = 400, custom_d = 500, custom_x = 1, custom_y = 2 +CREATE USER test_user SETTINGS custom_z = 3 +CREATE USER test_user +CREATE ROLE test_role +CREATE ROLE test_role SETTINGS custom_a = 100 +CREATE ROLE test_role SETTINGS custom_a = 100, custom_b = 200 +CREATE ROLE test_role SETTINGS custom_a = 100, custom_b = 300, custom_c = 400 +CREATE ROLE test_role SETTINGS custom_b = 300, custom_c = 400, custom_d = 500 +CREATE ROLE test_role SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 1000 +CREATE ROLE test_role SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 2000 +CREATE ROLE test_role SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 500 MAX 2000 +CREATE ROLE test_role SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 700 MIN 500 MAX 2000 +CREATE ROLE test_role SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 800 MIN 150 +CREATE ROLE test_role SETTINGS custom_c = 400, custom_d = 500 +CREATE ROLE test_role SETTINGS custom_c = 400, custom_d = 500, custom_x = 1, custom_y = 2 +CREATE ROLE test_role SETTINGS custom_z = 3 +CREATE ROLE test_role +CREATE SETTINGS PROFILE test_profile +CREATE SETTINGS PROFILE test_profile SETTINGS custom_a = 100 +CREATE SETTINGS PROFILE test_profile SETTINGS custom_a = 100, custom_b = 200 +CREATE SETTINGS PROFILE test_profile SETTINGS custom_a = 100, custom_b = 300, custom_c = 400 +CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500 +CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 1000 +CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 2000 +CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 500 MAX 2000 +CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 700 MIN 500 MAX 2000 +CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 800 MIN 150 +CREATE SETTINGS PROFILE test_profile SETTINGS custom_c = 400, custom_d = 500 +CREATE SETTINGS PROFILE test_profile SETTINGS custom_c = 400, custom_d = 500, custom_x = 1, custom_y = 2 +CREATE SETTINGS PROFILE test_profile SETTINGS custom_z = 3 +CREATE SETTINGS PROFILE test_profile diff --git a/tests/queries/0_stateless/02943_alter_user_modify_settings.sh b/tests/queries/0_stateless/02943_alter_user_modify_settings.sh new file mode 100755 index 00000000000..d9153e8b4e1 --- /dev/null +++ b/tests/queries/0_stateless/02943_alter_user_modify_settings.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +test_user="test_user_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +test_role="test_role_02932_${CLICKHOUSE_DATABASE}_$RANDOM" +test_profile="test_profile_02932_${CLICKHOUSE_DATABASE}_$RANDOM" + +function show_create() +{ + local type="$1" + local name="$2" + + $CLICKHOUSE_CLIENT -q "SHOW CREATE $type ${name};" | sed "s/${name}/test_${type}/g" +} + +function run_test() +{ + local type="$1" + local name="$2" + + $CLICKHOUSE_CLIENT -q "DROP $type IF EXISTS ${name};" + + $CLICKHOUSE_CLIENT -q "CREATE $type ${name};" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD SETTING custom_a=100;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD SETTING custom_b=200;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD SETTINGS custom_b=300, custom_c=400;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD SETTING custom_d=500 DROP SETTING custom_a;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD SETTING custom_e=600 MIN 0 MAX 1000;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} MODIFY SETTING custom_e MAX 2000;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} MODIFY SETTING custom_e MIN 500;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} MODIFY SETTING custom_e=700;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} ADD SETTING custom_e=800 MIN 150;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} DROP SETTINGS custom_b, custom_e;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} MODIFY SETTINGS custom_x=1, custom_y=2;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} DROP ALL SETTINGS, ADD SETTING custom_z=3;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "ALTER $type ${name} DROP ALL SETTINGS;" + show_create $type ${name} + + $CLICKHOUSE_CLIENT -q "DROP $type ${name};" +} + +run_test user ${test_user} +run_test role ${test_role} +run_test profile ${test_profile} From 54aadd8bd23fbc9738ff254e2b0090853142e407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Mon, 29 Jul 2024 01:15:15 +0300 Subject: [PATCH 002/502] Split removeSharedRecursive operation to smaller transactions --- src/Core/Settings.h | 1 + src/Disks/IDisk.cpp | 1 + src/Disks/IDisk.h | 4 + src/Disks/IDiskTransaction.h | 7 + .../ObjectStorages/DiskObjectStorage.cpp | 12 +- .../DiskObjectStorageTransaction.cpp | 42 ++- .../DiskObjectStorageTransaction.h | 13 + src/IO/S3Defines.h | 1 + .../test_disks_app_func/config.xml | 1 + tests/integration/test_disks_app_func/test.py | 342 +++++++----------- 10 files changed, 207 insertions(+), 217 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 3d181e33001..3d89bf6c78d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -891,6 +891,7 @@ class IColumn; M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \ M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \ M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \ + M(UInt64, object_storage_remove_recursive_batch_size, S3::DEFAULT_REMOVE_SHARED_RECURSIVE_BATCH_SIZE, "Max number of files to collect for removal in one transaction. Used to reduce memory usage.", 0) \ \ \ /* ###################################### */ \ diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index 4bbefad5290..cc04ebd5fd2 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -251,6 +251,7 @@ catch (Exception & e) void IDisk::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr /*context*/, const String & config_prefix, const DisksMap & /*map*/) { copying_thread_pool.setMaxThreads(config.getInt(config_prefix + ".thread_pool_size", 16)); + remove_shared_recursive_batch_size = config.getInt(config_prefix + ".remove_shared_recursive_batch_size", S3::DEFAULT_REMOVE_SHARED_RECURSIVE_BATCH_SIZE); } } diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 412ad27e94f..10e185739ef 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -115,6 +115,7 @@ public: /// Default constructor. IDisk(const String & name_, const Poco::Util::AbstractConfiguration & config, const String & config_prefix) : name(name_) + , remove_shared_recursive_batch_size(config.getUInt64(config_prefix + ".remove_shared_recursive_batch_size", S3::DEFAULT_REMOVE_SHARED_RECURSIVE_BATCH_SIZE)) , copying_thread_pool( CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, @@ -125,6 +126,7 @@ public: explicit IDisk(const String & name_) : name(name_) + , remove_shared_recursive_batch_size(S3::DEFAULT_REMOVE_SHARED_RECURSIVE_BATCH_SIZE) , copying_thread_pool( CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, CurrentMetrics::IDiskCopierThreadsScheduled, 16) { @@ -502,6 +504,8 @@ protected: virtual void checkAccessImpl(const String & path); + UInt64 remove_shared_recursive_batch_size; + private: ThreadPool copying_thread_pool; bool is_custom_disk = false; diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h index fc84281baea..9eef5d2266f 100644 --- a/src/Disks/IDiskTransaction.h +++ b/src/Disks/IDiskTransaction.h @@ -32,6 +32,10 @@ using RemoveBatchRequest = std::vector; struct IDiskTransaction : private boost::noncopyable { public: + IDiskTransaction(): remove_shared_recursive_batch_size(S3::DEFAULT_REMOVE_SHARED_RECURSIVE_BATCH_SIZE) {} + + explicit IDiskTransaction(UInt64 remove_shared_recursive_batch_size_): remove_shared_recursive_batch_size(remove_shared_recursive_batch_size_) {} + /// Tries to commit all accumulated operations simultaneously. /// If something fails rollback and throw exception. virtual void commit() = 0; @@ -131,6 +135,9 @@ public: /// Truncate file to the target size. virtual void truncateFile(const std::string & src_path, size_t target_size) = 0; + +protected: + UInt64 remove_shared_recursive_batch_size; }; using DiskTransactionPtr = std::shared_ptr; diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 4de6d78e952..e04318b1a5a 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -41,7 +41,8 @@ DiskTransactionPtr DiskObjectStorage::createObjectStorageTransaction() return std::make_shared( *object_storage, *metadata_storage, - send_metadata ? metadata_helper.get() : nullptr); + send_metadata ? metadata_helper.get() : nullptr, + remove_shared_recursive_batch_size); } DiskTransactionPtr DiskObjectStorage::createObjectStorageTransactionToAnotherDisk(DiskObjectStorage& to_disk) @@ -373,9 +374,12 @@ void DiskObjectStorage::removeSharedFileIfExists(const String & path, bool delet void DiskObjectStorage::removeSharedRecursive( const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) { - auto transaction = createObjectStorageTransaction(); - transaction->removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only); - transaction->commit(); + while (exists(path)) + { + auto transaction = createObjectStorageTransaction(); + transaction->removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only); + transaction->commit(); + } } bool DiskObjectStorage::tryReserve(UInt64 bytes) diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index e7c85bea1c6..0db2c4911df 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -37,6 +37,17 @@ DiskObjectStorageTransaction::DiskObjectStorageTransaction( , metadata_helper(metadata_helper_) {} +DiskObjectStorageTransaction::DiskObjectStorageTransaction( + IObjectStorage & object_storage_, + IMetadataStorage & metadata_storage_, + DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, + UInt64 remove_shared_recursive_batch_size_) + : IDiskTransaction(remove_shared_recursive_batch_size_) + , object_storage(object_storage_) + , metadata_storage(metadata_storage_) + , metadata_transaction(metadata_storage.createTransaction()) + , metadata_helper(metadata_helper_) +{} DiskObjectStorageTransaction::DiskObjectStorageTransaction( IObjectStorage & object_storage_, @@ -49,6 +60,19 @@ DiskObjectStorageTransaction::DiskObjectStorageTransaction( , metadata_helper(metadata_helper_) {} +DiskObjectStorageTransaction::DiskObjectStorageTransaction( + IObjectStorage & object_storage_, + IMetadataStorage & metadata_storage_, + DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, + MetadataTransactionPtr metadata_transaction_, + UInt64 remove_shared_recursive_batch_size_) + : IDiskTransaction(remove_shared_recursive_batch_size_) + , object_storage(object_storage_) + , metadata_storage(metadata_storage_) + , metadata_transaction(metadata_transaction_) + , metadata_helper(metadata_helper_) +{} + MultipleDisksObjectStorageTransaction::MultipleDisksObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, @@ -292,6 +316,7 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp const bool keep_all_batch_data; /// paths inside the 'this->path' const NameSet file_names_remove_metadata_only; + const UInt64 batch_size; /// map from local_path to its remote objects with hardlinks counter /// local_path is the path inside 'this->path' @@ -302,11 +327,13 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp IMetadataStorage & metadata_storage_, const std::string & path_, bool keep_all_batch_data_, - const NameSet & file_names_remove_metadata_only_) + const NameSet & file_names_remove_metadata_only_, + UInt64 batch_size_) : IDiskObjectStorageOperation(object_storage_, metadata_storage_) , path(path_) , keep_all_batch_data(keep_all_batch_data_) , file_names_remove_metadata_only(file_names_remove_metadata_only_) + , batch_size(batch_size_) {} std::string getInfoForLog() const override @@ -317,6 +344,8 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp void removeMetadataRecursive(MetadataTransactionPtr tx, const std::string & path_to_remove) { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. + if (objects_to_remove_by_path.size() == batch_size) + return; if (metadata_storage.isFile(path_to_remove)) { @@ -358,9 +387,14 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp else { for (auto it = metadata_storage.iterateDirectory(path_to_remove); it->isValid(); it->next()) + { + if (objects_to_remove_by_path.size() == batch_size) + return; removeMetadataRecursive(tx, it->path()); - - tx->removeDirectory(path_to_remove); + } + /// Do not delete in case directory contains >= batch_size files + if (objects_to_remove_by_path.size() < batch_size) + tx->removeDirectory(path_to_remove); } } @@ -683,7 +717,7 @@ void DiskObjectStorageTransaction::removeSharedRecursive( const std::string & path, bool keep_all_shared_data, const NameSet & file_names_remove_metadata_only) { auto operation = std::make_unique( - object_storage, metadata_storage, path, keep_all_shared_data, file_names_remove_metadata_only); + object_storage, metadata_storage, path, keep_all_shared_data, file_names_remove_metadata_only, remove_shared_recursive_batch_size); operations_to_execute.emplace_back(std::move(operation)); } diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h index 23f66990d54..8a782cbd2e9 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h @@ -69,12 +69,25 @@ protected: DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, MetadataTransactionPtr metadata_transaction_); + DiskObjectStorageTransaction( + IObjectStorage & object_storage_, + IMetadataStorage & metadata_storage_, + DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, + MetadataTransactionPtr metadata_transaction_, + UInt64 remove_shared_recursive_batch_size); + public: DiskObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_); + DiskObjectStorageTransaction( + IObjectStorage & object_storage_, + IMetadataStorage & metadata_storage_, + DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, + UInt64 remove_shared_recursive_batch_size); + void commit() override; void undo() override; diff --git a/src/IO/S3Defines.h b/src/IO/S3Defines.h index 332ebcfea92..2db93369a95 100644 --- a/src/IO/S3Defines.h +++ b/src/IO/S3Defines.h @@ -34,6 +34,7 @@ inline static constexpr uint64_t DEFAULT_MAX_SINGLE_READ_TRIES = 4; inline static constexpr uint64_t DEFAULT_MAX_UNEXPECTED_WRITE_ERROR_RETRIES = 4; inline static constexpr uint64_t DEFAULT_MAX_REDIRECTS = 10; inline static constexpr uint64_t DEFAULT_RETRY_ATTEMPTS = 100; +inline static constexpr uint64_t DEFAULT_REMOVE_SHARED_RECURSIVE_BATCH_SIZE = 1000; inline static constexpr bool DEFAULT_ALLOW_NATIVE_COPY = true; inline static constexpr bool DEFAULT_CHECK_OBJECTS_AFTER_UPLOAD = false; diff --git a/tests/integration/test_disks_app_func/config.xml b/tests/integration/test_disks_app_func/config.xml index c7cbe51f208..fb9fb34a4d1 100644 --- a/tests/integration/test_disks_app_func/config.xml +++ b/tests/integration/test_disks_app_func/config.xml @@ -15,6 +15,7 @@ http://minio1:9001/root/data/ minio minio123 + 3 diff --git a/tests/integration/test_disks_app_func/test.py b/tests/integration/test_disks_app_func/test.py index 56ea5c8846a..48c2fc6fef2 100644 --- a/tests/integration/test_disks_app_func/test.py +++ b/tests/integration/test_disks_app_func/test.py @@ -21,6 +21,69 @@ def started_cluster(): cluster.shutdown() +def write(source, disk, path): + source.exec_in_container( + [ + "bash", + "-c", + "echo 'tester' |" + + " ".join( + [ + "/usr/bin/clickhouse", + "disks", + "--save-logs", + "--disk", + f"{disk}", + "--query", + f"'write {path}'", + ] + ), + ] + ) + + +def mkdir(source, disk, path): + source.exec_in_container( + [ + "/usr/bin/clickhouse", + "disks", + "--save-logs", + "--disk", + f"{disk}", + "--query", + f"mkdir {path}", + ] + ) + + +def ls(source, disk, path): + return source.exec_in_container( + [ + "/usr/bin/clickhouse", + "disks", + "--save-logs", + "--disk", + f"{disk}", + "--query", + f"list {path}", + ] + ) + + +def remove(source, disk, path): + return source.exec_in_container( + [ + "/usr/bin/clickhouse", + "disks", + "--save-logs", + "--disk", + f"{disk}", + "--query", + f"remove {path}", + ] + ) + + def init_data(source): source.query("DROP TABLE IF EXISTS test_table") @@ -45,6 +108,30 @@ def init_data_s3(source): source.query("INSERT INTO test_table_s3(*) VALUES ('test1', 2)") +def init_data_s3_rm_rec(source): + # a / a / a, b, c + # / b / a, b, c, d, e + # / c / + # / d / a + mkdir(source, "test3", "a") + mkdir(source, "test3", "a/a") + mkdir(source, "test3", "a/b") + mkdir(source, "test3", "a/c") + mkdir(source, "test3", "a/d") + + write(source, "test3", "a/a/a") + write(source, "test3", "a/a/b") + write(source, "test3", "a/a/c") + + write(source, "test3", "a/b/a") + write(source, "test3", "a/b/b") + write(source, "test3", "a/b/c") + write(source, "test3", "a/b/d") + write(source, "test3", "a/b/e") + + write(source, "test3", "d/a") + + def test_disks_app_func_ld(started_cluster): source = cluster.instances["disks_app_test"] @@ -68,33 +155,13 @@ def test_disks_app_func_ls(started_cluster): init_data(source) - out = source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test1", - "--query", - "list .", - ] - ) + out = ls(source, "test1", ".") files = out.split("\n") assert files[0] == "store" - out = source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test1", - "--query", - "list . --recursive", - ] - ) + out = ls(source, "test1", ". --recursive") assert ".:\nstore\n" in out assert "\n./store:\n" in out @@ -105,24 +172,7 @@ def test_disks_app_func_cp(started_cluster): init_data(source) - source.exec_in_container( - [ - "bash", - "-c", - "echo 'tester' |" - + " ".join( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test1", - "--query", - "'write path1'", - ] - ), - ] - ) + write(source, "test1", "path1") source.exec_in_container( [ @@ -133,71 +183,20 @@ def test_disks_app_func_cp(started_cluster): ] ) - out = source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test2", - "--query", - "list .", - ] - ) + out = ls(source, "test2", ".") assert "path1" in out - source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test2", - "--query", - "remove path1", - ] - ) - - source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test1", - "--query", - "remove path1", - ] - ) + remove(source, "test2", "path1") + remove(source, "test1", "path1") # alesapin: Why we need list one more time? # kssenii: it is an assertion that the file is indeed deleted - out = source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test2", - "--query", - "list .", - ] - ) + out = ls(source, "test2", ".") assert "path1" not in out - out = source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test1", - "--query", - "list .", - ] - ) + out = ls(source, "test1", ".") assert "path1" not in out @@ -230,82 +229,54 @@ def test_disks_app_func_rm(started_cluster): init_data(source) - source.exec_in_container( - [ - "bash", - "-c", - "echo 'tester' |" - + " ".join( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test2", - "--query", - "'write path3'", - ] - ), - ] - ) + write(source, "test2", "path3") - out = source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test2", - "--query", - "list .", - ] - ) + out = ls(source, "test2", ".") assert "path3" in out - source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test2", - "--query", - "remove path3", - ] - ) + remove(source, "test2", "path3") - out = source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test2", - "--query", - "list .", - ] - ) + out = ls(source, "test2", ".") assert "path3" not in out +def test_disks_app_func_rm_shared_recursive(started_cluster): + source = cluster.instances["disks_app_test"] + + init_data_s3_rm_rec(source) + out = ls(source, "test3", ". --recursive") + assert ( + out + == ".:\na\n\n./a:\na\nb\nc\nd\n\n./a/a:\na\nb\nc\n\n./a/b:\na\nb\nc\nd\ne\n\n./a/c:\n\n./a/d:\n\n" + ) + + remove(source, "test3", "a/a --recursive") + out = ls(source, "test3", ". --recursive") + assert ( + out == ".:\na\n\n./a:\nb\nc\nd\n\n./a/b:\na\nb\nc\nd\ne\n\n./a/c:\n\n./a/d:\n\n" + ) + + remove(source, "test3", "a/b --recursive") + out = ls(source, "test3", ". --recursive") + assert out == ".:\na\n\n./a:\nc\nd\n\n./a/c:\n\n./a/d:\n\n" + + remove(source, "test3", "a/c --recursive") + out = ls(source, "test3", ". --recursive") + assert out == ".:\na\n\n./a:\nd\n\n./a/d:\n\n" + + remove(source, "test3", "a --recursive") + out = ls(source, "test3", ". --recursive") + assert out == ".:\n\n" + + def test_disks_app_func_mv(started_cluster): source = cluster.instances["disks_app_test"] init_data(source) - out = source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test1", - "--query", - "list .", - ] - ) + out = ls(source, "test1", ".") files = out.split("\n") assert "old_store" not in files @@ -322,17 +293,7 @@ def test_disks_app_func_mv(started_cluster): ] ) - out = source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test1", - "--query", - "list .", - ] - ) + out = ls(source, "test1", ".") files = out.split("\n") assert "old_store" in files @@ -342,24 +303,7 @@ def test_disks_app_func_mv(started_cluster): def test_disks_app_func_read_write(started_cluster): source = cluster.instances["disks_app_test"] - source.exec_in_container( - [ - "bash", - "-c", - "echo 'tester' |" - + " ".join( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test1", - "--query", - "'write 5.txt'", - ] - ), - ] - ) + write(source, "test1", "5.txt") out = source.exec_in_container( [ @@ -382,33 +326,13 @@ def test_remote_disk_list(started_cluster): source = cluster.instances["disks_app_test"] init_data_s3(source) - out = source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test3", - "--query", - "list .", - ] - ) + out = ls(source, "test3", ".") files = out.split("\n") assert files[0] == "store" - out = source.exec_in_container( - [ - "/usr/bin/clickhouse", - "disks", - "--save-logs", - "--disk", - "test3", - "--query", - "list . --recursive", - ] - ) + out = ls(source, "test3", ". --recursive") assert ".:\nstore\n" in out assert "\n./store:\n" in out From 3c1a09fea33ed97e249742a9d977b45857d046da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Mon, 29 Jul 2024 01:28:32 +0300 Subject: [PATCH 003/502] Update SettingsChangesHistory --- src/Core/SettingsChangesHistory.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index de4725dc350..9a488b3c680 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -80,7 +80,8 @@ static std::initializer_list Date: Mon, 29 Jul 2024 01:43:57 +0300 Subject: [PATCH 004/502] Replace map for vectors --- .../DiskObjectStorageTransaction.cpp | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 0db2c4911df..e75bf14a9a3 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -318,9 +318,10 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp const NameSet file_names_remove_metadata_only; const UInt64 batch_size; - /// map from local_path to its remote objects with hardlinks counter + /// local_path and related remote objects with hardlinks counter /// local_path is the path inside 'this->path' - std::unordered_map objects_to_remove_by_path; + std::vector local_paths_to_remove; + std::vector objects_to_remove; RemoveRecursiveObjectStorageOperation( IObjectStorage & object_storage_, @@ -344,7 +345,7 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp void removeMetadataRecursive(MetadataTransactionPtr tx, const std::string & path_to_remove) { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. - if (objects_to_remove_by_path.size() == batch_size) + if (local_paths_to_remove.size() == batch_size) return; if (metadata_storage.isFile(path_to_remove)) @@ -359,8 +360,8 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp if (unlink_outcome && !file_names_remove_metadata_only.contains(rel_path)) { - objects_to_remove_by_path[std::move(rel_path)] - = ObjectsToRemove{std::move(objects_paths), std::move(unlink_outcome)}; + local_paths_to_remove.push_back(std::move(rel_path)); + objects_to_remove.push_back({std::move(objects_paths), std::move(unlink_outcome)}); } } catch (const Exception & e) @@ -388,12 +389,12 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp { for (auto it = metadata_storage.iterateDirectory(path_to_remove); it->isValid(); it->next()) { - if (objects_to_remove_by_path.size() == batch_size) + if (local_paths_to_remove.size() == batch_size) return; removeMetadataRecursive(tx, it->path()); } /// Do not delete in case directory contains >= batch_size files - if (objects_to_remove_by_path.size() < batch_size) + if (local_paths_to_remove.size() < batch_size) tx->removeDirectory(path_to_remove); } } @@ -414,15 +415,17 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp if (!keep_all_batch_data) { std::vector total_removed_paths; - total_removed_paths.reserve(objects_to_remove_by_path.size()); + total_removed_paths.reserve(local_paths_to_remove.size()); StoredObjects remove_from_remote; - for (auto && [local_path, objects_to_remove] : objects_to_remove_by_path) + for (size_t i = 0; i < local_paths_to_remove.size(); ++i) { + std::string local_path = local_paths_to_remove[i]; + ObjectsToRemove objects_to_remove_for_path = objects_to_remove[i]; chassert(!file_names_remove_metadata_only.contains(local_path)); - if (objects_to_remove.unlink_outcome->num_hardlinks == 0) + if (objects_to_remove_for_path.unlink_outcome->num_hardlinks == 0) { - std::move(objects_to_remove.objects.begin(), objects_to_remove.objects.end(), std::back_inserter(remove_from_remote)); + std::move(objects_to_remove_for_path.objects.begin(), objects_to_remove_for_path.objects.end(), std::back_inserter(remove_from_remote)); total_removed_paths.push_back(local_path); } } From 1acabed13e3dc1e0c396a67599c52499d14e00af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Mon, 29 Jul 2024 12:30:44 +0300 Subject: [PATCH 005/502] Fix include --- src/Disks/IDisk.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 10e185739ef..cb55c7287d9 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include From 3405745f22ef7dc9a32ba732e50be18c602eb5d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Mon, 29 Jul 2024 14:05:18 +0300 Subject: [PATCH 006/502] Update SettingsChangesHistory 2 --- src/Core/SettingsChangesHistory.cpp | 5 ++--- src/Disks/IDisk.cpp | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 9a488b3c680..eb2aabb1fc3 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -57,6 +57,7 @@ String ClickHouseVersion::toString() const /// Note: please check if the key already exists to prevent duplicate entries. static std::initializer_list> settings_changes_history_initializer = { + {"24.8", {{"object_storage_remove_recursive_batch_size", UINT64_MAX, 1000, "Added new setting to limit number of files stored in memory while removing from object storage."}}}, {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."}, {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"}, {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"}, @@ -80,9 +81,7 @@ static std::initializer_list Date: Mon, 29 Jul 2024 16:36:52 +0300 Subject: [PATCH 007/502] Revert "Replace map for vectors" This reverts commit b1e7d85df3037ff3e6316134b61356a438c9c4b9. --- .../DiskObjectStorageTransaction.cpp | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index e75bf14a9a3..0db2c4911df 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -318,10 +318,9 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp const NameSet file_names_remove_metadata_only; const UInt64 batch_size; - /// local_path and related remote objects with hardlinks counter + /// map from local_path to its remote objects with hardlinks counter /// local_path is the path inside 'this->path' - std::vector local_paths_to_remove; - std::vector objects_to_remove; + std::unordered_map objects_to_remove_by_path; RemoveRecursiveObjectStorageOperation( IObjectStorage & object_storage_, @@ -345,7 +344,7 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp void removeMetadataRecursive(MetadataTransactionPtr tx, const std::string & path_to_remove) { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. - if (local_paths_to_remove.size() == batch_size) + if (objects_to_remove_by_path.size() == batch_size) return; if (metadata_storage.isFile(path_to_remove)) @@ -360,8 +359,8 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp if (unlink_outcome && !file_names_remove_metadata_only.contains(rel_path)) { - local_paths_to_remove.push_back(std::move(rel_path)); - objects_to_remove.push_back({std::move(objects_paths), std::move(unlink_outcome)}); + objects_to_remove_by_path[std::move(rel_path)] + = ObjectsToRemove{std::move(objects_paths), std::move(unlink_outcome)}; } } catch (const Exception & e) @@ -389,12 +388,12 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp { for (auto it = metadata_storage.iterateDirectory(path_to_remove); it->isValid(); it->next()) { - if (local_paths_to_remove.size() == batch_size) + if (objects_to_remove_by_path.size() == batch_size) return; removeMetadataRecursive(tx, it->path()); } /// Do not delete in case directory contains >= batch_size files - if (local_paths_to_remove.size() < batch_size) + if (objects_to_remove_by_path.size() < batch_size) tx->removeDirectory(path_to_remove); } } @@ -415,17 +414,15 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp if (!keep_all_batch_data) { std::vector total_removed_paths; - total_removed_paths.reserve(local_paths_to_remove.size()); + total_removed_paths.reserve(objects_to_remove_by_path.size()); StoredObjects remove_from_remote; - for (size_t i = 0; i < local_paths_to_remove.size(); ++i) + for (auto && [local_path, objects_to_remove] : objects_to_remove_by_path) { - std::string local_path = local_paths_to_remove[i]; - ObjectsToRemove objects_to_remove_for_path = objects_to_remove[i]; chassert(!file_names_remove_metadata_only.contains(local_path)); - if (objects_to_remove_for_path.unlink_outcome->num_hardlinks == 0) + if (objects_to_remove.unlink_outcome->num_hardlinks == 0) { - std::move(objects_to_remove_for_path.objects.begin(), objects_to_remove_for_path.objects.end(), std::back_inserter(remove_from_remote)); + std::move(objects_to_remove.objects.begin(), objects_to_remove.objects.end(), std::back_inserter(remove_from_remote)); total_removed_paths.push_back(local_path); } } From 66489ef6d616cc92b8570a50bf46f9e7bef13ea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Tue, 30 Jul 2024 16:09:50 +0300 Subject: [PATCH 008/502] Rename and move setting to ObjectStorage classes --- src/Core/Defines.h | 2 + src/Core/Settings.h | 2 +- src/Core/SettingsChangesHistory.cpp | 6 +- src/Disks/IDisk.cpp | 1 - src/Disks/IDisk.h | 5 -- src/Disks/IDiskTransaction.h | 7 --- .../ObjectStorages/DiskObjectStorage.cpp | 10 +++- src/Disks/ObjectStorages/DiskObjectStorage.h | 2 + .../DiskObjectStorageTransaction.cpp | 59 +++++++------------ .../DiskObjectStorageTransaction.h | 18 ++---- src/IO/S3Defines.h | 1 - .../test_disks_app_func/config.xml | 2 +- 12 files changed, 45 insertions(+), 70 deletions(-) diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 6df335a9c8f..bc76ca9bf2f 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -121,4 +121,6 @@ static constexpr auto QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS = 1000000000; static constexpr auto QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS = 0; #endif +static constexpr auto DEFAULT_REMOVE_SHARED_RECURSIVE_FILE_LIMIT = 1000uz; + } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 3d89bf6c78d..71984e81807 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -891,7 +891,7 @@ class IColumn; M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \ M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \ M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \ - M(UInt64, object_storage_remove_recursive_batch_size, S3::DEFAULT_REMOVE_SHARED_RECURSIVE_BATCH_SIZE, "Max number of files to collect for removal in one transaction. Used to reduce memory usage.", 0) \ + M(UInt64, object_storage_remove_recursive_file_limit, DEFAULT_REMOVE_SHARED_RECURSIVE_FILE_LIMIT, "Max number of files to store in memory during remove. Zero value means unlimited. Used to reduce memory usage.", 0) \ \ \ /* ###################################### */ \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index eb2aabb1fc3..936880c2e00 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -57,7 +57,8 @@ String ClickHouseVersion::toString() const /// Note: please check if the key already exists to prevent duplicate entries. static std::initializer_list> settings_changes_history_initializer = { - {"24.8", {{"object_storage_remove_recursive_batch_size", UINT64_MAX, 1000, "Added new setting to limit number of files stored in memory while removing from object storage."}}}, + {"24.8", {{"object_storage_remove_recursive_file_limit", 0, 1000, "Added new setting to limit number of files stored in memory while removing from object storage. Zero value means unlimited."} + }}, {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."}, {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"}, {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"}, @@ -81,7 +82,8 @@ static std::initializer_list #include #include -#include #include #include #include @@ -116,7 +115,6 @@ public: /// Default constructor. IDisk(const String & name_, const Poco::Util::AbstractConfiguration & config, const String & config_prefix) : name(name_) - , remove_shared_recursive_batch_size(config.getUInt64(config_prefix + ".remove_shared_recursive_batch_size", S3::DEFAULT_REMOVE_SHARED_RECURSIVE_BATCH_SIZE)) , copying_thread_pool( CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, @@ -127,7 +125,6 @@ public: explicit IDisk(const String & name_) : name(name_) - , remove_shared_recursive_batch_size(S3::DEFAULT_REMOVE_SHARED_RECURSIVE_BATCH_SIZE) , copying_thread_pool( CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, CurrentMetrics::IDiskCopierThreadsScheduled, 16) { @@ -505,8 +502,6 @@ protected: virtual void checkAccessImpl(const String & path); - UInt64 remove_shared_recursive_batch_size; - private: ThreadPool copying_thread_pool; bool is_custom_disk = false; diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h index 9eef5d2266f..fc84281baea 100644 --- a/src/Disks/IDiskTransaction.h +++ b/src/Disks/IDiskTransaction.h @@ -32,10 +32,6 @@ using RemoveBatchRequest = std::vector; struct IDiskTransaction : private boost::noncopyable { public: - IDiskTransaction(): remove_shared_recursive_batch_size(S3::DEFAULT_REMOVE_SHARED_RECURSIVE_BATCH_SIZE) {} - - explicit IDiskTransaction(UInt64 remove_shared_recursive_batch_size_): remove_shared_recursive_batch_size(remove_shared_recursive_batch_size_) {} - /// Tries to commit all accumulated operations simultaneously. /// If something fails rollback and throw exception. virtual void commit() = 0; @@ -135,9 +131,6 @@ public: /// Truncate file to the target size. virtual void truncateFile(const std::string & src_path, size_t target_size) = 0; - -protected: - UInt64 remove_shared_recursive_batch_size; }; using DiskTransactionPtr = std::shared_ptr; diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index e04318b1a5a..af2f9bf4258 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -42,7 +42,7 @@ DiskTransactionPtr DiskObjectStorage::createObjectStorageTransaction() *object_storage, *metadata_storage, send_metadata ? metadata_helper.get() : nullptr, - remove_shared_recursive_batch_size); + remove_shared_recursive_file_limit); } DiskTransactionPtr DiskObjectStorage::createObjectStorageTransactionToAnotherDisk(DiskObjectStorage& to_disk) @@ -52,7 +52,8 @@ DiskTransactionPtr DiskObjectStorage::createObjectStorageTransactionToAnotherDis *metadata_storage, *to_disk.getObjectStorage(), *to_disk.getMetadataStorage(), - send_metadata ? metadata_helper.get() : nullptr); + send_metadata ? metadata_helper.get() : nullptr, + remove_shared_recursive_file_limit); } @@ -72,6 +73,7 @@ DiskObjectStorage::DiskObjectStorage( , read_resource_name(config.getString(config_prefix + ".read_resource", "")) , write_resource_name(config.getString(config_prefix + ".write_resource", "")) , metadata_helper(std::make_unique(this, ReadSettings{}, WriteSettings{})) + , remove_shared_recursive_file_limit(config.getUInt64(config_prefix + ".remove_shared_recursive_file_limit", DEFAULT_REMOVE_SHARED_RECURSIVE_FILE_LIMIT)) { data_source_description = DataSourceDescription{ .type = DataSourceType::ObjectStorage, @@ -374,6 +376,8 @@ void DiskObjectStorage::removeSharedFileIfExists(const String & path, bool delet void DiskObjectStorage::removeSharedRecursive( const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) { + /// At most remove_shared_recursive_file_limit files are removed in one transaction + /// Retry until all of them are removed while (exists(path)) { auto transaction = createObjectStorageTransaction(); @@ -558,6 +562,8 @@ void DiskObjectStorage::applyNewSettings( write_resource_name = new_write_resource_name; } + remove_shared_recursive_file_limit = config.getUInt64(config_prefix + ".remove_shared_recursive_file_limit", DEFAULT_REMOVE_SHARED_RECURSIVE_FILE_LIMIT); + IDisk::applyNewSettings(config, context_, config_prefix, disk_map); } diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index 5c45a258806..2759eba2964 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -246,6 +246,8 @@ private: String write_resource_name; std::unique_ptr metadata_helper; + + UInt64 remove_shared_recursive_file_limit; }; using DiskObjectStoragePtr = std::shared_ptr; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 0db2c4911df..4bf67805219 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -30,34 +30,13 @@ namespace ErrorCodes DiskObjectStorageTransaction::DiskObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, - DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_) + DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, + UInt64 remove_shared_recursive_file_limit_) : object_storage(object_storage_) , metadata_storage(metadata_storage_) , metadata_transaction(metadata_storage.createTransaction()) , metadata_helper(metadata_helper_) -{} - -DiskObjectStorageTransaction::DiskObjectStorageTransaction( - IObjectStorage & object_storage_, - IMetadataStorage & metadata_storage_, - DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, - UInt64 remove_shared_recursive_batch_size_) - : IDiskTransaction(remove_shared_recursive_batch_size_) - , object_storage(object_storage_) - , metadata_storage(metadata_storage_) - , metadata_transaction(metadata_storage.createTransaction()) - , metadata_helper(metadata_helper_) -{} - -DiskObjectStorageTransaction::DiskObjectStorageTransaction( - IObjectStorage & object_storage_, - IMetadataStorage & metadata_storage_, - DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, - MetadataTransactionPtr metadata_transaction_) - : object_storage(object_storage_) - , metadata_storage(metadata_storage_) - , metadata_transaction(metadata_transaction_) - , metadata_helper(metadata_helper_) + , remove_shared_recursive_file_limit(remove_shared_recursive_file_limit_) {} DiskObjectStorageTransaction::DiskObjectStorageTransaction( @@ -65,12 +44,12 @@ DiskObjectStorageTransaction::DiskObjectStorageTransaction( IMetadataStorage & metadata_storage_, DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, MetadataTransactionPtr metadata_transaction_, - UInt64 remove_shared_recursive_batch_size_) - : IDiskTransaction(remove_shared_recursive_batch_size_) - , object_storage(object_storage_) + UInt64 remove_shared_recursive_file_limit_) + : object_storage(object_storage_) , metadata_storage(metadata_storage_) , metadata_transaction(metadata_transaction_) , metadata_helper(metadata_helper_) + , remove_shared_recursive_file_limit(remove_shared_recursive_file_limit_) {} MultipleDisksObjectStorageTransaction::MultipleDisksObjectStorageTransaction( @@ -78,8 +57,9 @@ MultipleDisksObjectStorageTransaction::MultipleDisksObjectStorageTransaction( IMetadataStorage & metadata_storage_, IObjectStorage& destination_object_storage_, IMetadataStorage& destination_metadata_storage_, - DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_) - : DiskObjectStorageTransaction(object_storage_, metadata_storage_, metadata_helper_, destination_metadata_storage_.createTransaction()) + DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, + UInt64 remove_shared_recursive_file_limit_) + : DiskObjectStorageTransaction(object_storage_, metadata_storage_, metadata_helper_, destination_metadata_storage_.createTransaction(), remove_shared_recursive_file_limit_) , destination_object_storage(destination_object_storage_) , destination_metadata_storage(destination_metadata_storage_) {} @@ -316,7 +296,7 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp const bool keep_all_batch_data; /// paths inside the 'this->path' const NameSet file_names_remove_metadata_only; - const UInt64 batch_size; + const UInt64 limit; /// map from local_path to its remote objects with hardlinks counter /// local_path is the path inside 'this->path' @@ -328,12 +308,12 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp const std::string & path_, bool keep_all_batch_data_, const NameSet & file_names_remove_metadata_only_, - UInt64 batch_size_) + UInt64 limit_) : IDiskObjectStorageOperation(object_storage_, metadata_storage_) , path(path_) , keep_all_batch_data(keep_all_batch_data_) , file_names_remove_metadata_only(file_names_remove_metadata_only_) - , batch_size(batch_size_) + , limit(limit_) {} std::string getInfoForLog() const override @@ -341,10 +321,15 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp return fmt::format("RemoveRecursiveObjectStorageOperation (path: {})", path); } + bool checkLimitReached() const + { + return limit > 0 && objects_to_remove_by_path.size() == limit; + } + void removeMetadataRecursive(MetadataTransactionPtr tx, const std::string & path_to_remove) { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. - if (objects_to_remove_by_path.size() == batch_size) + if (checkLimitReached()) return; if (metadata_storage.isFile(path_to_remove)) @@ -388,12 +373,12 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp { for (auto it = metadata_storage.iterateDirectory(path_to_remove); it->isValid(); it->next()) { - if (objects_to_remove_by_path.size() == batch_size) + if (checkLimitReached()) return; removeMetadataRecursive(tx, it->path()); } - /// Do not delete in case directory contains >= batch_size files - if (objects_to_remove_by_path.size() < batch_size) + /// Do not delete in case directory contains >= limit files + if (objects_to_remove_by_path.size() < limit) tx->removeDirectory(path_to_remove); } } @@ -717,7 +702,7 @@ void DiskObjectStorageTransaction::removeSharedRecursive( const std::string & path, bool keep_all_shared_data, const NameSet & file_names_remove_metadata_only) { auto operation = std::make_unique( - object_storage, metadata_storage, path, keep_all_shared_data, file_names_remove_metadata_only, remove_shared_recursive_batch_size); + object_storage, metadata_storage, path, keep_all_shared_data, file_names_remove_metadata_only, remove_shared_recursive_file_limit); operations_to_execute.emplace_back(std::move(operation)); } diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h index 8a782cbd2e9..06be42f5881 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h @@ -63,30 +63,21 @@ protected: DiskObjectStorageOperations operations_to_execute; - DiskObjectStorageTransaction( - IObjectStorage & object_storage_, - IMetadataStorage & metadata_storage_, - DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, - MetadataTransactionPtr metadata_transaction_); + UInt64 remove_shared_recursive_file_limit; DiskObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, MetadataTransactionPtr metadata_transaction_, - UInt64 remove_shared_recursive_batch_size); + UInt64 remove_shared_recursive_file_limit); public: - DiskObjectStorageTransaction( - IObjectStorage & object_storage_, - IMetadataStorage & metadata_storage_, - DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_); - DiskObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, - UInt64 remove_shared_recursive_batch_size); + UInt64 remove_shared_recursive_file_limit); void commit() override; void undo() override; @@ -149,7 +140,8 @@ struct MultipleDisksObjectStorageTransaction final : public DiskObjectStorageTra IMetadataStorage & metadata_storage_, IObjectStorage& destination_object_storage, IMetadataStorage& destination_metadata_storage, - DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_); + DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, + UInt64 remove_shared_recursive_file_limit_); void copyFile(const std::string & from_file_path, const std::string & to_file_path, const ReadSettings & read_settings, const WriteSettings &) override; }; diff --git a/src/IO/S3Defines.h b/src/IO/S3Defines.h index 2db93369a95..332ebcfea92 100644 --- a/src/IO/S3Defines.h +++ b/src/IO/S3Defines.h @@ -34,7 +34,6 @@ inline static constexpr uint64_t DEFAULT_MAX_SINGLE_READ_TRIES = 4; inline static constexpr uint64_t DEFAULT_MAX_UNEXPECTED_WRITE_ERROR_RETRIES = 4; inline static constexpr uint64_t DEFAULT_MAX_REDIRECTS = 10; inline static constexpr uint64_t DEFAULT_RETRY_ATTEMPTS = 100; -inline static constexpr uint64_t DEFAULT_REMOVE_SHARED_RECURSIVE_BATCH_SIZE = 1000; inline static constexpr bool DEFAULT_ALLOW_NATIVE_COPY = true; inline static constexpr bool DEFAULT_CHECK_OBJECTS_AFTER_UPLOAD = false; diff --git a/tests/integration/test_disks_app_func/config.xml b/tests/integration/test_disks_app_func/config.xml index fb9fb34a4d1..99dc3e17089 100644 --- a/tests/integration/test_disks_app_func/config.xml +++ b/tests/integration/test_disks_app_func/config.xml @@ -15,7 +15,7 @@ http://minio1:9001/root/data/ minio minio123 - 3 + 3 From b601d6b64d9b77f92a75c95ae994e7052c8bc1bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Thu, 1 Aug 2024 23:57:55 +0300 Subject: [PATCH 009/502] Move all logic to operation --- .../ObjectStorages/DiskObjectStorage.cpp | 11 +-- .../DiskObjectStorageTransaction.cpp | 75 ++++++++++--------- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index af2f9bf4258..a48bbbaf69c 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -376,14 +376,9 @@ void DiskObjectStorage::removeSharedFileIfExists(const String & path, bool delet void DiskObjectStorage::removeSharedRecursive( const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) { - /// At most remove_shared_recursive_file_limit files are removed in one transaction - /// Retry until all of them are removed - while (exists(path)) - { - auto transaction = createObjectStorageTransaction(); - transaction->removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only); - transaction->commit(); - } + auto transaction = createObjectStorageTransaction(); + transaction->removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only); + transaction->commit(); } bool DiskObjectStorage::tryReserve(UInt64 bytes) diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 17a2a4ebdd1..875fb380b93 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -326,11 +326,46 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp return limit > 0 && objects_to_remove_by_path.size() == limit; } + void removeObjects() + { + if (!keep_all_batch_data) + { + std::vector total_removed_paths; + total_removed_paths.reserve(objects_to_remove_by_path.size()); + + StoredObjects remove_from_remote; + for (auto && [local_path, objects_to_remove] : objects_to_remove_by_path) + { + chassert(!file_names_remove_metadata_only.contains(local_path)); + if (objects_to_remove.unlink_outcome->num_hardlinks == 0) + { + std::move(objects_to_remove.objects.begin(), objects_to_remove.objects.end(), std::back_inserter(remove_from_remote)); + total_removed_paths.push_back(local_path); + } + } + + /// Read comment inside RemoveObjectStorageOperation class + /// TL;DR Don't pay any attention to 404 status code + object_storage.removeObjectsIfExist(remove_from_remote); + + objects_to_remove_by_path.clear(); + + LOG_DEBUG( + getLogger("RemoveRecursiveObjectStorageOperation"), + "Recursively remove path {}: " + "metadata and objects were removed for [{}], " + "only metadata were removed for [{}].", + path, + boost::algorithm::join(total_removed_paths, ", "), + boost::algorithm::join(file_names_remove_metadata_only, ", ")); + } + } + void removeMetadataRecursive(MetadataTransactionPtr tx, const std::string & path_to_remove) { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. if (checkLimitReached()) - return; + removeObjects(); if (metadata_storage.isFile(path_to_remove)) { @@ -373,13 +408,11 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp { for (auto it = metadata_storage.iterateDirectory(path_to_remove); it->isValid(); it->next()) { - if (checkLimitReached()) - return; removeMetadataRecursive(tx, it->path()); + if (checkLimitReached()) + removeObjects(); } - /// Do not delete in case directory contains >= limit files - if (objects_to_remove_by_path.size() < limit) - tx->removeDirectory(path_to_remove); + tx->removeDirectory(path_to_remove); } } @@ -396,35 +429,7 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp void finalize() override { - if (!keep_all_batch_data) - { - std::vector total_removed_paths; - total_removed_paths.reserve(objects_to_remove_by_path.size()); - - StoredObjects remove_from_remote; - for (auto && [local_path, objects_to_remove] : objects_to_remove_by_path) - { - chassert(!file_names_remove_metadata_only.contains(local_path)); - if (objects_to_remove.unlink_outcome->num_hardlinks == 0) - { - std::move(objects_to_remove.objects.begin(), objects_to_remove.objects.end(), std::back_inserter(remove_from_remote)); - total_removed_paths.push_back(local_path); - } - } - - /// Read comment inside RemoveObjectStorageOperation class - /// TL;DR Don't pay any attention to 404 status code - object_storage.removeObjectsIfExist(remove_from_remote); - - LOG_DEBUG( - getLogger("RemoveRecursiveObjectStorageOperation"), - "Recursively remove path {}: " - "metadata and objects were removed for [{}], " - "only metadata were removed for [{}].", - path, - boost::algorithm::join(total_removed_paths, ", "), - boost::algorithm::join(file_names_remove_metadata_only, ", ")); - } + removeObjects(); } }; From 24e021a1301d0e6519633b7d12c9c204f978b16f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Mon, 5 Aug 2024 19:19:50 +0300 Subject: [PATCH 010/502] Move to new method, call it only from disk client --- programs/disks/CommandRemove.cpp | 2 +- src/Disks/IDisk.h | 5 + .../ObjectStorages/DiskObjectStorage.cpp | 62 +++++++++++- src/Disks/ObjectStorages/DiskObjectStorage.h | 4 + .../DiskObjectStorageTransaction.cpp | 97 +++++++------------ .../DiskObjectStorageTransaction.h | 11 +-- .../test_disks_app_func/config.xml | 2 +- 7 files changed, 108 insertions(+), 75 deletions(-) diff --git a/programs/disks/CommandRemove.cpp b/programs/disks/CommandRemove.cpp index 4388f6c0b14..8e86691497a 100644 --- a/programs/disks/CommandRemove.cpp +++ b/programs/disks/CommandRemove.cpp @@ -40,7 +40,7 @@ public: } else { - disk.getDisk()->removeRecursive(path); + disk.getDisk()->removeRecursiveWithLimit(path); } } else diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 412ad27e94f..d2bd8c4532d 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -243,6 +243,11 @@ public: /// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists. virtual void removeRecursive(const String & path) = 0; + /// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists. + /// Differs from removeRecursive for S3/HDFS disks + /// Removes files every time limit is reached and reduces memory consumption. + virtual void removeRecursiveWithLimit(const String & path) { removeRecursive(path); } + /// Remove file. Throws exception if file doesn't exists or if directory is not empty. /// Differs from removeFile for S3/HDFS disks /// Second bool param is a flag to remove (true) or keep (false) shared data on S3 diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index a48bbbaf69c..d34b94744a2 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -41,8 +42,7 @@ DiskTransactionPtr DiskObjectStorage::createObjectStorageTransaction() return std::make_shared( *object_storage, *metadata_storage, - send_metadata ? metadata_helper.get() : nullptr, - remove_shared_recursive_file_limit); + send_metadata ? metadata_helper.get() : nullptr); } DiskTransactionPtr DiskObjectStorage::createObjectStorageTransactionToAnotherDisk(DiskObjectStorage& to_disk) @@ -52,8 +52,7 @@ DiskTransactionPtr DiskObjectStorage::createObjectStorageTransactionToAnotherDis *metadata_storage, *to_disk.getObjectStorage(), *to_disk.getMetadataStorage(), - send_metadata ? metadata_helper.get() : nullptr, - remove_shared_recursive_file_limit); + send_metadata ? metadata_helper.get() : nullptr); } @@ -381,6 +380,61 @@ void DiskObjectStorage::removeSharedRecursive( transaction->commit(); } +void DiskObjectStorage::removeSharedRecursiveWithLimit( + const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) +{ + if (remove_shared_recursive_file_limit == 0) + return removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only); + + RemoveBatchRequest local_paths; + std::vector directories; + + auto check_limit_reached = [&]() + { + const auto path_count = local_paths.size() + directories.size(); + chassert(path_count <= this->remove_shared_recursive_file_limit); + return local_paths.size() + directories.size() == this->remove_shared_recursive_file_limit; + }; + + auto remove = [&]() + { + auto transaction = createObjectStorageTransaction(); + if (!local_paths.empty()) + transaction->removeSharedFiles(local_paths, keep_all_batch_data, file_names_remove_metadata_only); + for (auto & directory : directories) + transaction->removeDirectory(directory); + transaction->commit(); + local_paths.clear(); + directories.clear(); + }; + + std::function traverse_metadata_recursive = [&](const std::string & path_to_remove) + { + checkStackSize(); + if (check_limit_reached()) + remove(); + + if (metadata_storage->isFile(path_to_remove)) + { + chassert(path_to_remove.starts_with(path)); + local_paths.emplace_back(path_to_remove); + } + else + { + for (auto it = metadata_storage->iterateDirectory(path_to_remove); it->isValid(); it->next()) + { + traverse_metadata_recursive(it->path()); + if (check_limit_reached()) + remove(); + } + directories.push_back(path_to_remove); + } + }; + + traverse_metadata_recursive(path); + remove(); +} + bool DiskObjectStorage::tryReserve(UInt64 bytes) { std::lock_guard lock(reservation_mutex); diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index 2759eba2964..ba5cffbc5d2 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -78,12 +78,16 @@ public: void removeRecursive(const String & path) override { removeSharedRecursive(path, false, {}); } + void removeRecursiveWithLimit(const String & path) override { removeSharedRecursiveWithLimit(path, false, {}); } + void removeSharedFile(const String & path, bool delete_metadata_only) override; void removeSharedFileIfExists(const String & path, bool delete_metadata_only) override; void removeSharedRecursive(const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) override; + void removeSharedRecursiveWithLimit(const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only); + void removeSharedFiles(const RemoveBatchRequest & files, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) override; void truncateFile(const String & path, size_t size) override; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 875fb380b93..10b7393a9f2 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -30,26 +30,22 @@ namespace ErrorCodes DiskObjectStorageTransaction::DiskObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, - DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, - UInt64 remove_shared_recursive_file_limit_) + DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_) : object_storage(object_storage_) , metadata_storage(metadata_storage_) , metadata_transaction(metadata_storage.createTransaction()) , metadata_helper(metadata_helper_) - , remove_shared_recursive_file_limit(remove_shared_recursive_file_limit_) {} DiskObjectStorageTransaction::DiskObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, - MetadataTransactionPtr metadata_transaction_, - UInt64 remove_shared_recursive_file_limit_) + MetadataTransactionPtr metadata_transaction_) : object_storage(object_storage_) , metadata_storage(metadata_storage_) , metadata_transaction(metadata_transaction_) , metadata_helper(metadata_helper_) - , remove_shared_recursive_file_limit(remove_shared_recursive_file_limit_) {} MultipleDisksObjectStorageTransaction::MultipleDisksObjectStorageTransaction( @@ -57,9 +53,8 @@ MultipleDisksObjectStorageTransaction::MultipleDisksObjectStorageTransaction( IMetadataStorage & metadata_storage_, IObjectStorage& destination_object_storage_, IMetadataStorage& destination_metadata_storage_, - DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, - UInt64 remove_shared_recursive_file_limit_) - : DiskObjectStorageTransaction(object_storage_, metadata_storage_, metadata_helper_, destination_metadata_storage_.createTransaction(), remove_shared_recursive_file_limit_) + DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_) + : DiskObjectStorageTransaction(object_storage_, metadata_storage_, metadata_helper_, destination_metadata_storage_.createTransaction()) , destination_object_storage(destination_object_storage_) , destination_metadata_storage(destination_metadata_storage_) {} @@ -296,7 +291,6 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp const bool keep_all_batch_data; /// paths inside the 'this->path' const NameSet file_names_remove_metadata_only; - const UInt64 limit; /// map from local_path to its remote objects with hardlinks counter /// local_path is the path inside 'this->path' @@ -307,13 +301,11 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp IMetadataStorage & metadata_storage_, const std::string & path_, bool keep_all_batch_data_, - const NameSet & file_names_remove_metadata_only_, - UInt64 limit_) + const NameSet & file_names_remove_metadata_only_) : IDiskObjectStorageOperation(object_storage_, metadata_storage_) , path(path_) , keep_all_batch_data(keep_all_batch_data_) , file_names_remove_metadata_only(file_names_remove_metadata_only_) - , limit(limit_) {} std::string getInfoForLog() const override @@ -321,51 +313,9 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp return fmt::format("RemoveRecursiveObjectStorageOperation (path: {})", path); } - bool checkLimitReached() const - { - return limit > 0 && objects_to_remove_by_path.size() == limit; - } - - void removeObjects() - { - if (!keep_all_batch_data) - { - std::vector total_removed_paths; - total_removed_paths.reserve(objects_to_remove_by_path.size()); - - StoredObjects remove_from_remote; - for (auto && [local_path, objects_to_remove] : objects_to_remove_by_path) - { - chassert(!file_names_remove_metadata_only.contains(local_path)); - if (objects_to_remove.unlink_outcome->num_hardlinks == 0) - { - std::move(objects_to_remove.objects.begin(), objects_to_remove.objects.end(), std::back_inserter(remove_from_remote)); - total_removed_paths.push_back(local_path); - } - } - - /// Read comment inside RemoveObjectStorageOperation class - /// TL;DR Don't pay any attention to 404 status code - object_storage.removeObjectsIfExist(remove_from_remote); - - objects_to_remove_by_path.clear(); - - LOG_DEBUG( - getLogger("RemoveRecursiveObjectStorageOperation"), - "Recursively remove path {}: " - "metadata and objects were removed for [{}], " - "only metadata were removed for [{}].", - path, - boost::algorithm::join(total_removed_paths, ", "), - boost::algorithm::join(file_names_remove_metadata_only, ", ")); - } - } - void removeMetadataRecursive(MetadataTransactionPtr tx, const std::string & path_to_remove) { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. - if (checkLimitReached()) - removeObjects(); if (metadata_storage.isFile(path_to_remove)) { @@ -407,11 +357,8 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp else { for (auto it = metadata_storage.iterateDirectory(path_to_remove); it->isValid(); it->next()) - { removeMetadataRecursive(tx, it->path()); - if (checkLimitReached()) - removeObjects(); - } + tx->removeDirectory(path_to_remove); } } @@ -429,7 +376,35 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp void finalize() override { - removeObjects(); + if (!keep_all_batch_data) + { + std::vector total_removed_paths; + total_removed_paths.reserve(objects_to_remove_by_path.size()); + + StoredObjects remove_from_remote; + for (auto && [local_path, objects_to_remove] : objects_to_remove_by_path) + { + chassert(!file_names_remove_metadata_only.contains(local_path)); + if (objects_to_remove.unlink_outcome->num_hardlinks == 0) + { + std::move(objects_to_remove.objects.begin(), objects_to_remove.objects.end(), std::back_inserter(remove_from_remote)); + total_removed_paths.push_back(local_path); + } + } + + /// Read comment inside RemoveObjectStorageOperation class + /// TL;DR Don't pay any attention to 404 status code + object_storage.removeObjectsIfExist(remove_from_remote); + + LOG_DEBUG( + getLogger("RemoveRecursiveObjectStorageOperation"), + "Recursively remove path {}: " + "metadata and objects were removed for [{}], " + "only metadata were removed for [{}].", + path, + boost::algorithm::join(total_removed_paths, ", "), + boost::algorithm::join(file_names_remove_metadata_only, ", ")); + } } }; @@ -707,7 +682,7 @@ void DiskObjectStorageTransaction::removeSharedRecursive( const std::string & path, bool keep_all_shared_data, const NameSet & file_names_remove_metadata_only) { auto operation = std::make_unique( - object_storage, metadata_storage, path, keep_all_shared_data, file_names_remove_metadata_only, remove_shared_recursive_file_limit); + object_storage, metadata_storage, path, keep_all_shared_data, file_names_remove_metadata_only); operations_to_execute.emplace_back(std::move(operation)); } diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h index 06be42f5881..23f66990d54 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h @@ -63,21 +63,17 @@ protected: DiskObjectStorageOperations operations_to_execute; - UInt64 remove_shared_recursive_file_limit; - DiskObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, - MetadataTransactionPtr metadata_transaction_, - UInt64 remove_shared_recursive_file_limit); + MetadataTransactionPtr metadata_transaction_); public: DiskObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, - DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, - UInt64 remove_shared_recursive_file_limit); + DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_); void commit() override; void undo() override; @@ -140,8 +136,7 @@ struct MultipleDisksObjectStorageTransaction final : public DiskObjectStorageTra IMetadataStorage & metadata_storage_, IObjectStorage& destination_object_storage, IMetadataStorage& destination_metadata_storage, - DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_, - UInt64 remove_shared_recursive_file_limit_); + DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_); void copyFile(const std::string & from_file_path, const std::string & to_file_path, const ReadSettings & read_settings, const WriteSettings &) override; }; diff --git a/tests/integration/test_disks_app_func/config.xml b/tests/integration/test_disks_app_func/config.xml index 99dc3e17089..1054f3866fa 100644 --- a/tests/integration/test_disks_app_func/config.xml +++ b/tests/integration/test_disks_app_func/config.xml @@ -15,7 +15,7 @@ http://minio1:9001/root/data/ minio minio123 - 3 + 4 From 030f58f79e2574be7ccd2b22d09ff88c58c82657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Tue, 6 Aug 2024 15:09:39 +0300 Subject: [PATCH 011/502] Add cleanup after test to fix flaky tests --- tests/integration/test_disks_app_func/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_disks_app_func/test.py b/tests/integration/test_disks_app_func/test.py index 48c2fc6fef2..b461a15d942 100644 --- a/tests/integration/test_disks_app_func/test.py +++ b/tests/integration/test_disks_app_func/test.py @@ -170,8 +170,6 @@ def test_disks_app_func_ls(started_cluster): def test_disks_app_func_cp(started_cluster): source = cluster.instances["disks_app_test"] - init_data(source) - write(source, "test1", "path1") source.exec_in_container( @@ -227,8 +225,6 @@ def test_disks_app_func_ln(started_cluster): def test_disks_app_func_rm(started_cluster): source = cluster.instances["disks_app_test"] - init_data(source) - write(source, "test2", "path3") out = ls(source, "test2", ".") @@ -270,6 +266,8 @@ def test_disks_app_func_rm_shared_recursive(started_cluster): out = ls(source, "test3", ". --recursive") assert out == ".:\n\n" + remove(source, "test3", ". --recursive") + def test_disks_app_func_mv(started_cluster): source = cluster.instances["disks_app_test"] @@ -299,6 +297,8 @@ def test_disks_app_func_mv(started_cluster): assert "old_store" in files assert "store" not in files + remove(source, "test1", "./old_store --recursive") + def test_disks_app_func_read_write(started_cluster): source = cluster.instances["disks_app_test"] From b76606016cebf7c905539f94fde93c1a5d14b3df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Tue, 6 Aug 2024 15:12:50 +0300 Subject: [PATCH 012/502] Comment fix --- src/Disks/IDisk.h | 2 +- src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index d2bd8c4532d..18d9456f9db 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -245,7 +245,7 @@ public: /// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists. /// Differs from removeRecursive for S3/HDFS disks - /// Removes files every time limit is reached and reduces memory consumption. + /// Limits the number of removing files in batches to prevent high memory consumption. virtual void removeRecursiveWithLimit(const String & path) { removeRecursive(path); } /// Remove file. Throws exception if file doesn't exists or if directory is not empty. diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 99159953ec5..880911b9958 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -37,6 +37,7 @@ DiskObjectStorageTransaction::DiskObjectStorageTransaction( , metadata_helper(metadata_helper_) {} + DiskObjectStorageTransaction::DiskObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, From c2bed0d8993927390488be0770571387e09fd362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Thu, 8 Aug 2024 01:40:21 +0300 Subject: [PATCH 013/502] More test case isolation --- tests/integration/test_disks_app_func/config.xml | 13 +++++++++++++ tests/integration/test_disks_app_func/test.py | 10 +++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_disks_app_func/config.xml b/tests/integration/test_disks_app_func/config.xml index 1054f3866fa..b3b1a157290 100644 --- a/tests/integration/test_disks_app_func/config.xml +++ b/tests/integration/test_disks_app_func/config.xml @@ -17,6 +17,12 @@ minio123 4 + + s3 + http://minio1:9001/root/data/ + minio + minio123 + @@ -40,6 +46,13 @@ + + +
+ test4 +
+ + diff --git a/tests/integration/test_disks_app_func/test.py b/tests/integration/test_disks_app_func/test.py index b461a15d942..f6269af8ea1 100644 --- a/tests/integration/test_disks_app_func/test.py +++ b/tests/integration/test_disks_app_func/test.py @@ -96,13 +96,13 @@ def init_data(source): source.query("INSERT INTO test_table(*) VALUES ('test1', 2)") -def init_data_s3(source): +def init_data_s3(source, disk="test3"): source.query("DROP TABLE IF EXISTS test_table_s3") source.query( "CREATE TABLE test_table_s3(word String, value UInt64) " "ENGINE=MergeTree() " - "ORDER BY word SETTINGS storage_policy = 'test3'" + f"ORDER BY word SETTINGS storage_policy = '{disk}'" ) source.query("INSERT INTO test_table_s3(*) VALUES ('test1', 2)") @@ -324,15 +324,15 @@ def test_disks_app_func_read_write(started_cluster): def test_remote_disk_list(started_cluster): source = cluster.instances["disks_app_test"] - init_data_s3(source) + init_data_s3(source, "test4") - out = ls(source, "test3", ".") + out = ls(source, "test4", ".") files = out.split("\n") assert files[0] == "store" - out = ls(source, "test3", ". --recursive") + out = ls(source, "test4", ". --recursive") assert ".:\nstore\n" in out assert "\n./store:\n" in out From df615cc85bf382f4410fc94198d116d8eb5dd48f Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 26 Aug 2024 16:07:26 +0000 Subject: [PATCH 014/502] Some intermediate draft done --- src/Core/SettingsChangesHistory.cpp | 2 +- .../ObjectStorage/DataLakes/HudiMetadata.h | 2 +- .../DataLakes/IDataLakeMetadata.h | 14 +- .../DataLakes/IcebergMetadata.cpp | 150 ++++++++++-------- .../ObjectStorage/DataLakes/IcebergMetadata.h | 10 +- 5 files changed, 108 insertions(+), 70 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 4ab3b329c84..930eaf5661c 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -91,7 +91,7 @@ static std::initializer_list -#include #include +#include +#include +#include "Interpreters/ActionsDAG.h" #include "PartitionColumns.h" namespace DB { +struct DataFileInfo +{ + String data_path; + std::optional initial_schema; + std::optional schema_transform; +}; + class IDataLakeMetadata : boost::noncopyable { public: virtual ~IDataLakeMetadata() = default; - virtual Strings getDataFiles() const = 0; + virtual std::vector getDataFilesWithSchemaTransform() const = 0; virtual NamesAndTypesList getTableSchema() const = 0; virtual bool operator==(const IDataLakeMetadata & other) const = 0; virtual const DataLakePartitionColumns & getPartitionColumns() const = 0; diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 9b9d92e282c..e22b2333a24 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -52,7 +52,8 @@ IcebergMetadata::IcebergMetadata( Int32 format_version_, String manifest_list_file_, Int32 current_schema_id_, - DB::NamesAndTypesList schema_) + DB::NamesAndTypesList schema_, + std::map relevant_schemas_by_ids_) : WithContext(context_) , object_storage(object_storage_) , configuration(configuration_) @@ -61,6 +62,7 @@ IcebergMetadata::IcebergMetadata( , manifest_list_file(std::move(manifest_list_file_)) , current_schema_id(current_schema_id_) , schema(std::move(schema_)) + , relevant_schemas_by_ids(std::move(relevant_schemas_by_ids_)) , log(getLogger("IcebergMetadata")) { } @@ -243,7 +245,32 @@ DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & t } -std::pair parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, int format_version, bool ignore_schema_evolution) +std::map getRelevantSchemasByIds(const Poco::JSON::Object::Ptr & metadata_object, int format_version) +{ + std::map relevant_schemas_by_ids schemas_by_id; + if (format_version == 2) + { + auto schemas = metadata_object->get("schemas").extract(); + if (schemas->size() == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: schemas field is empty"); + + for (uint32_t i = 0; i != schemas->size(); ++i) + { + auto current_schema = schemas->getObject(i); + schemas_by_id[current_schema->getValue("schema-id")] = current_schema; + } + } + else + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported format"); + } + return schemas_by_id; +} + +std::pair parseTableSchema( + const Poco::JSON::Object::Ptr & metadata_object, + Int32 format_version, + const std::map & relevant_schema_by_id) { Poco::JSON::Object::Ptr schema; Int32 current_schema_id; @@ -255,54 +282,26 @@ std::pair parseTableSchema(const Poco::JSON::Object::P if (format_version == 2) { current_schema_id = metadata_object->getValue("current-schema-id"); - auto schemas = metadata_object->get("schemas").extract(); - if (schemas->size() == 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: schemas field is empty"); - - if (ignore_schema_evolution) - { - /// If we ignore schema evolution, we will just use latest schema for all data files. - /// Find schema with 'schema-id' equal to 'current_schema_id'. - for (uint32_t i = 0; i != schemas->size(); ++i) - { - auto current_schema = schemas->getObject(i); - if (current_schema->getValue("schema-id") == current_schema_id) - { - schema = current_schema; - break; - } - } - - if (!schema) - throw Exception(ErrorCodes::BAD_ARGUMENTS, R"(There is no schema with "schema-id" that matches "current-schema-id" in metadata)"); - } - else - { - if (schemas->size() != 1) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Cannot read Iceberg table: the table schema has been changed at least 1 time, reading tables with evolved schema is " - "supported. If you want to ignore schema evolution and read all files using latest schema saved on table creation, enable setting " - "iceberg_engine_ignore_schema_evolution (Note: enabling this setting can lead to incorrect result)"); - - /// Now we sure that there is only one schema. - schema = schemas->getObject(0); - if (schema->getValue("schema-id") != current_schema_id) - throw Exception(ErrorCodes::BAD_ARGUMENTS, R"(Field "schema-id" of the schema doesn't match "current-schema-id" in metadata)"); - } } else { - schema = metadata_object->getObject("schema"); current_schema_id = schema->getValue("schema-id"); - /// Field "schemas" is optional for version 1, but after version 2 was introduced, - /// in most cases this field is added for new tables in version 1 as well. - if (!ignore_schema_evolution && metadata_object->has("schemas") && metadata_object->get("schemas").extract()->size() > 1) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Cannot read Iceberg table: the table schema has been changed at least 1 time, reading tables with evolved schema is not " - "supported. If you want to ignore schema evolution and read all files using latest schema saved on table creation, enable setting " - "iceberg_engine_ignore_schema_evolution (Note: enabling this setting can lead to incorrect result)"); } + try + { + schema = relevant_schema_by_id.at(current_schema_id); + } + catch (const std::exception &) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, R"(There is no schema with "schema-id" that matches "current-schema-id" in metadata)"); + } + + return {getClickhouseSchemaFromIcebergSchema(schema), current_schema_id}; +} + +NamesAndTypesList getClickhouseSchemaFromIcebergSchema(const Poco::JSON::Object::Ptr & schema) +{ NamesAndTypesList names_and_types; auto fields = schema->get("fields").extract(); for (size_t i = 0; i != fields->size(); ++i) @@ -313,13 +312,15 @@ std::pair parseTableSchema(const Poco::JSON::Object::P names_and_types.push_back({name, getFieldType(field, "type", required)}); } - return {std::move(names_and_types), current_schema_id}; + return std::move(names_and_types); } -MutableColumns parseAvro( - avro::DataFileReaderBase & file_reader, - const Block & header, - const FormatSettings & settings) +ActionsDag getSchemaTransformDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) +{ + return ActionsDAG{}; +} + +MutableColumns parseAvro(avro::DataFileReaderBase & file_reader, const Block & header, const FormatSettings & settings) { auto deserializer = std::make_unique(header, file_reader.dataSchema(), true, true, settings); MutableColumns columns = header.cloneEmptyColumns(); @@ -374,7 +375,6 @@ std::pair getMetadataFileAndVersion( /// Get the latest version of metadata file: v.metadata.json return *std::max_element(metadata_files_with_versions.begin(), metadata_files_with_versions.end()); } - } DataLakeMetadataPtr IcebergMetadata::create( @@ -394,7 +394,10 @@ DataLakeMetadataPtr IcebergMetadata::create( Poco::JSON::Object::Ptr object = json.extract(); auto format_version = object->getValue("format-version"); - auto [schema, schema_id] = parseTableSchema(object, format_version, local_context->getSettingsRef().iceberg_engine_ignore_schema_evolution); + + auto relevant_schemas_by_ids = getRelevantSchemasByIds(object, format_version); + + auto [schema, schema_id] = parseTableSchema(object, format_version); auto current_snapshot_id = object->getValue("current-snapshot-id"); auto snapshots = object->get("snapshots").extract(); @@ -411,7 +414,9 @@ DataLakeMetadataPtr IcebergMetadata::create( } } - return std::make_unique(object_storage, configuration, local_context, metadata_version, format_version, manifest_list_file, schema_id, schema); + return std::make_unique( + object_storage, configuration, local_context, metadata_version, format_version, manifest_list_file, schema_id, schema), + ; } /** @@ -439,7 +444,7 @@ DataLakeMetadataPtr IcebergMetadata::create( * │ 1 │ 2252246380142525104 │ ('/iceberg_data/db/table_name/data/a=2/00000-1-c9535a00-2f4f-405c-bcfa-6d4f9f477235-00003.parquet','PARQUET',(2),1,631,67108864,[(1,46),(2,48)],[(1,1),(2,1)],[(1,0),(2,0)],[],[(1,'\0\0\0\0\0\0\0'),(2,'3')],[(1,'\0\0\0\0\0\0\0'),(2,'3')],NULL,[4],0) │ * └────────┴─────────────────────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ */ -Strings IcebergMetadata::getDataFiles() const +Strings IcebergMetadata::getDataFilesInfo() const { if (!data_files.empty()) return data_files; @@ -476,7 +481,7 @@ Strings IcebergMetadata::getDataFiles() const manifest_files.emplace_back(std::filesystem::path(configuration->getPath()) / "metadata" / filename); } - NameSet files; + std::set> files; LOG_TEST(log, "Collect data files"); for (const auto & manifest_file : manifest_files) { @@ -493,13 +498,7 @@ Strings IcebergMetadata::getDataFiles() const Poco::JSON::Parser parser; Poco::Dynamic::Var json = parser.parse(schema_json_string); Poco::JSON::Object::Ptr schema_object = json.extract(); - if (!context->getSettingsRef().iceberg_engine_ignore_schema_evolution && schema_object->getValue("schema-id") != current_schema_id) - throw Exception( - ErrorCodes::UNSUPPORTED_METHOD, - "Cannot read Iceberg table: the table schema has been changed at least 1 time, reading tables with evolved schema is not " - "supported. If you want to ignore schema evolution and read all files using latest schema saved on table creation, enable setting " - "iceberg_engine_ignore_schema_evolution (Note: enabling this setting can lead to incorrect result)"); - + Int32 schema_object_id = schema_object->getValue("schema-id"); avro::NodePtr root_node = manifest_file_reader->dataSchema().root(); size_t leaves_num = root_node->leaves(); size_t expected_min_num = format_version == 1 ? 3 : 2; @@ -620,11 +619,38 @@ Strings IcebergMetadata::getDataFiles() const else { LOG_TEST(log, "Processing data file for path: {}", file_path); - files.insert(file_path); + files.insert({file_path, schema_object_id}); } } } + std::vector data_file_infos; + + + for (const auto & [file_path, schema_object_id] : files) + { + if (current_schema_id == schema_object_id) + { + data_file_infos.emplace_back(file_path, std::nullopt, std::nullopt); + } + else + { + if (!relevant_schemas_by_ids.count(schema_object_id)) + { + relevant_schemas_by_ids[schema_object_id] = schema_object; + } + if (!transform_dags_by_ids.count(schema_object_id)) + { + transform_dags_by_ids[schema_object_id] + = getSchemaTransformDag(relevant_schemas_by_ids[schema_object_id], relevant_schemas_by_ids[current_schema_id]); + } + data_file_infos.emplace_back( + file_path, + getClickhouseSchemaFromIcebergSchema(relevant_schemas_by_ids[schema_object_id]), + transform_dags_by_ids[schema_object_id]) + } + } + data_files = std::vector(files.begin(), files.end()); return data_files; } diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 7b0deab91c3..ac9cc208926 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -8,6 +8,7 @@ #include #include + namespace DB { @@ -73,11 +74,12 @@ public: Int32 format_version_, String manifest_list_file_, Int32 current_schema_id_, - NamesAndTypesList schema_); + NamesAndTypesList schema_, + std::map relevant_schemas_by_ids_); /// Get data files. On first request it reads manifest_list file and iterates through manifest files to find all data files. /// All subsequent calls will return saved list of files (because it cannot be changed without changing metadata file) - Strings getDataFiles() const override; + std::vector getDataFilesInfo() const override; /// Get table schema parsed from metadata. NamesAndTypesList getTableSchema() const override { return schema; } @@ -105,11 +107,13 @@ private: Int32 metadata_version; Int32 format_version; String manifest_list_file; - Int32 current_schema_id; + const Int32 current_schema_id; NamesAndTypesList schema; mutable Strings data_files; std::unordered_map column_name_to_physical_name; DataLakePartitionColumns partition_columns; + std::map relevant_schemas_by_ids; + std::map transform_dags_by_ids; LoggerPtr log; }; From 55c460bf50e6595f29a5338d848db1254e27703a Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 27 Aug 2024 09:32:09 +0000 Subject: [PATCH 015/502] Adding schema transform --- .../ObjectStorage/Azure/Configuration.cpp | 4 +-- .../ObjectStorage/Azure/Configuration.h | 6 ++-- src/Storages/ObjectStorage/DataFileInfo.h | 20 +++++++++++ .../DataLakes/DeltaLakeMetadata.cpp | 5 ++- .../DataLakes/DeltaLakeMetadata.h | 6 ++-- .../ObjectStorage/DataLakes/HudiMetadata.cpp | 9 +++-- .../ObjectStorage/DataLakes/HudiMetadata.h | 15 ++++---- .../DataLakes/IDataLakeMetadata.h | 13 ++----- .../DataLakes/IStorageDataLake.h | 2 +- .../DataLakes/IcebergMetadata.cpp | 34 +++++++++---------- .../ObjectStorage/DataLakes/IcebergMetadata.h | 5 ++- .../ObjectStorage/HDFS/Configuration.cpp | 2 +- .../ObjectStorage/HDFS/Configuration.h | 6 ++-- .../ObjectStorage/Local/Configuration.cpp | 4 +-- .../ObjectStorage/Local/Configuration.h | 6 ++-- .../ObjectStorage/S3/Configuration.cpp | 4 +-- src/Storages/ObjectStorage/S3/Configuration.h | 8 +++-- .../ObjectStorage/StorageObjectStorage.cpp | 2 +- .../ObjectStorage/StorageObjectStorage.h | 8 +++-- .../StorageObjectStorageSource.cpp | 2 +- 20 files changed, 97 insertions(+), 64 deletions(-) create mode 100644 src/Storages/ObjectStorage/DataFileInfo.h diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index 9730391d429..8bee4fa8f34 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -140,7 +140,7 @@ void StorageAzureConfiguration::fromNamedCollection(const NamedCollection & coll format = collection.getOrDefault("format", format); compression_method = collection.getOrDefault("compression_method", collection.getOrDefault("compression", "auto")); - blobs_paths = {blob_path}; + blobs_paths = {blob_path, std::nullopt, std::nullopt}; connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context); } @@ -268,7 +268,7 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, structure = checkAndGetLiteralArgument(engine_args[7], "structure"); } - blobs_paths = {blob_path}; + blobs_paths = {blob_path, std::nullopt, std::nullopt}; connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context); } diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index 4e6bfbc0745..09df4722760 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -32,8 +32,8 @@ public: Path getPath() const override { return blob_path; } void setPath(const Path & path) override { blob_path = path; } - const Paths & getPaths() const override { return blobs_paths; } - void setPaths(const Paths & paths) override { blobs_paths = paths; } + const DataFileInfos & getPaths() const override { return blobs_paths; } + void setPaths(const DataFileInfos & paths) override { blobs_paths = paths; } String getNamespace() const override { return connection_params.getContainer(); } String getDataSourceDescription() const override { return std::filesystem::path(connection_params.getConnectionURL()) / connection_params.getContainer(); } @@ -55,7 +55,7 @@ protected: void fromAST(ASTs & args, ContextPtr context, bool with_structure) override; std::string blob_path; - std::vector blobs_paths; + DataFileInfos blobs_paths; AzureBlobStorage::ConnectionParams connection_params; }; diff --git a/src/Storages/ObjectStorage/DataFileInfo.h b/src/Storages/ObjectStorage/DataFileInfo.h new file mode 100644 index 00000000000..b1f01c8abd1 --- /dev/null +++ b/src/Storages/ObjectStorage/DataFileInfo.h @@ -0,0 +1,20 @@ +#include +#include + +#include "Interpreters/ActionsDAG.h" + +namespace DB +{ + +struct DataFileInfo +{ + String data_path; + std::optional initial_schema; + std::optional schema_transform; + + bool operator==(DataFileInfo & other_info) { return data_path == other_info.data_path; } +}; + +using DataFileInfos = std::vector; + +} \ No newline at end of file diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp index d357beeba47..34540b38709 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp @@ -664,7 +664,10 @@ DeltaLakeMetadata::DeltaLakeMetadata( { auto impl = DeltaLakeMetadataImpl(object_storage_, configuration_, context_); auto result = impl.processMetadataFiles(); - data_files = result.data_files; + for (const auto & data_file_name : result.data_files) + { + data_files.emplace_back(data_file_name, std::nullopt, std::nullopt); + } schema = result.schema; partition_columns = result.partition_columns; diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h index a479a3dd293..75ac4f566e1 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h @@ -6,6 +6,8 @@ #include #include +#include + namespace DB { @@ -20,7 +22,7 @@ public: ConfigurationPtr configuration_, ContextPtr context_); - Strings getDataFiles() const override { return data_files; } + std::vector getDataFilesInfo() const override { return data_files; } NamesAndTypesList getTableSchema() const override { return schema; } @@ -45,7 +47,7 @@ public: } private: - mutable Strings data_files; + mutable std::vector data_files; NamesAndTypesList schema; std::unordered_map column_name_to_physical_name; DataLakePartitionColumns partition_columns; diff --git a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp index 91a586ccbf9..128d87a5e43 100644 --- a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp @@ -96,10 +96,15 @@ HudiMetadata::HudiMetadata( { } -Strings HudiMetadata::getDataFiles() const +Strings HudiMetadata::getDataFilesInfo() const { if (data_files.empty()) - data_files = getDataFilesImpl(); + { + for (const auto & data_file_name : getDataFilesImpl) + { + data_files.push_back({data_file_name, std::nullopt, std::nullopt}); + } + } return data_files; } diff --git a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h index df2714c6690..1ac632b15c6 100644 --- a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h @@ -1,11 +1,12 @@ #pragma once -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include "IDataLakeMetadata.h" namespace DB { @@ -22,7 +23,7 @@ public: ConfigurationPtr configuration_, ContextPtr context_); - Strings getDataFilesWithSchemaTransform() const override; + std::vector getDataFilesInfo() const override; NamesAndTypesList getTableSchema() const override { return {}; } @@ -49,7 +50,7 @@ public: private: const ObjectStoragePtr object_storage; const ConfigurationPtr configuration; - mutable Strings data_files; + mutable std::vector data_files; std::unordered_map column_name_to_physical_name; DataLakePartitionColumns partition_columns; diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h index 778a149ae92..3ff19607486 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h @@ -2,24 +2,17 @@ #include #include #include -#include "Interpreters/ActionsDAG.h" #include "PartitionColumns.h" +#include "Storages/ObjectStorage/DataFileInfo.h" + namespace DB { - -struct DataFileInfo -{ - String data_path; - std::optional initial_schema; - std::optional schema_transform; -}; - class IDataLakeMetadata : boost::noncopyable { public: virtual ~IDataLakeMetadata() = default; - virtual std::vector getDataFilesWithSchemaTransform() const = 0; + virtual std::vector getDataFilesInfo() const = 0; virtual NamesAndTypesList getTableSchema() const = 0; virtual bool operator==(const IDataLakeMetadata & other) const = 0; virtual const DataLakePartitionColumns & getPartitionColumns() const = 0; diff --git a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h index 4b21bdb0494..b100016b37d 100644 --- a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h +++ b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h @@ -107,7 +107,7 @@ public: current_metadata = std::move(new_metadata); auto updated_configuration = base_configuration->clone(); - updated_configuration->setPaths(current_metadata->getDataFiles()); + updated_configuration->setPaths(current_metadata->getDataFileInfos()); updated_configuration->setPartitionColumns(current_metadata->getPartitionColumns()); Storage::configuration = updated_configuration; diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index e22b2333a24..7315dede03e 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -247,7 +247,7 @@ DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & t std::map getRelevantSchemasByIds(const Poco::JSON::Object::Ptr & metadata_object, int format_version) { - std::map relevant_schemas_by_ids schemas_by_id; + std::map schemas_by_id; if (format_version == 2) { auto schemas = metadata_object->get("schemas").extract(); @@ -267,6 +267,21 @@ std::map getRelevantSchemasByIds(const Poco::JSO return schemas_by_id; } +NamesAndTypesList getClickhouseSchemaFromIcebergSchema(const Poco::JSON::Object::Ptr & schema) +{ + NamesAndTypesList names_and_types; + auto fields = schema->get("fields").extract(); + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + auto name = field->getValue("name"); + bool required = field->getValue("required"); + names_and_types.push_back({name, getFieldType(field, "type", required)}); + } + + return std::move(names_and_types); +} + std::pair parseTableSchema( const Poco::JSON::Object::Ptr & metadata_object, Int32 format_version, @@ -300,22 +315,7 @@ std::pair parseTableSchema( return {getClickhouseSchemaFromIcebergSchema(schema), current_schema_id}; } -NamesAndTypesList getClickhouseSchemaFromIcebergSchema(const Poco::JSON::Object::Ptr & schema) -{ - NamesAndTypesList names_and_types; - auto fields = schema->get("fields").extract(); - for (size_t i = 0; i != fields->size(); ++i) - { - auto field = fields->getObject(static_cast(i)); - auto name = field->getValue("name"); - bool required = field->getValue("required"); - names_and_types.push_back({name, getFieldType(field, "type", required)}); - } - - return std::move(names_and_types); -} - -ActionsDag getSchemaTransformDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) +ActionsDAG getSchemaTransformDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) { return ActionsDAG{}; } diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index ac9cc208926..29673875a12 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -8,6 +8,9 @@ #include #include +# include +# include +# include namespace DB { @@ -113,7 +116,7 @@ private: std::unordered_map column_name_to_physical_name; DataLakePartitionColumns partition_columns; std::map relevant_schemas_by_ids; - std::map transform_dags_by_ids; + std::map transform_dags_by_ids; LoggerPtr log; }; diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.cpp b/src/Storages/ObjectStorage/HDFS/Configuration.cpp index 85eb29a3868..74f5d885922 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.cpp +++ b/src/Storages/ObjectStorage/HDFS/Configuration.cpp @@ -153,7 +153,7 @@ void StorageHDFSConfiguration::setURL(const std::string & url_) path = '/' + path; url = url_.substr(0, pos); - paths = {path}; + paths = {path, std::nullopt, std::nullopt}; LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path); } diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.h b/src/Storages/ObjectStorage/HDFS/Configuration.h index 04884542908..bd13c4dcf09 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.h +++ b/src/Storages/ObjectStorage/HDFS/Configuration.h @@ -26,8 +26,8 @@ public: Path getPath() const override { return path; } void setPath(const Path & path_) override { path = path_; } - const Paths & getPaths() const override { return paths; } - void setPaths(const Paths & paths_) override { paths = paths_; } + const DataFileInfos & getPaths() const override { return paths; } + void setPaths(const DataFileInfos & paths_) override { paths = paths_; } std::string getPathWithoutGlobs() const override; String getNamespace() const override { return ""; } @@ -52,7 +52,7 @@ private: String url; String path; - std::vector paths; + DataFileInfos paths; }; } diff --git a/src/Storages/ObjectStorage/Local/Configuration.cpp b/src/Storages/ObjectStorage/Local/Configuration.cpp index d64b22769ad..0ce93136b4a 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.cpp +++ b/src/Storages/ObjectStorage/Local/Configuration.cpp @@ -20,7 +20,7 @@ void StorageLocalConfiguration::fromNamedCollection(const NamedCollection & coll format = collection.getOrDefault("format", "auto"); compression_method = collection.getOrDefault("compression_method", collection.getOrDefault("compression", "auto")); structure = collection.getOrDefault("structure", "auto"); - paths = {path}; + paths = {path, std::nullopt, std::nullopt}; } @@ -57,7 +57,7 @@ void StorageLocalConfiguration::fromAST(ASTs & args, ContextPtr context, bool wi { compression_method = checkAndGetLiteralArgument(args[2], "compression_method"); } - paths = {path}; + paths = {path, std::nullopt, std::nullopt}; } StorageObjectStorage::QuerySettings StorageLocalConfiguration::getQuerySettings(const ContextPtr & context) const diff --git a/src/Storages/ObjectStorage/Local/Configuration.h b/src/Storages/ObjectStorage/Local/Configuration.h index ba4de63ac47..8936b147a6a 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.h +++ b/src/Storages/ObjectStorage/Local/Configuration.h @@ -29,8 +29,8 @@ public: Path getPath() const override { return path; } void setPath(const Path & path_) override { path = path_; } - const Paths & getPaths() const override { return paths; } - void setPaths(const Paths & paths_) override { paths = paths_; } + const DataFileInfos & getPaths() const override { return paths; } + void setPaths(const DataFileInfos & paths_) override { paths = paths_; } String getNamespace() const override { return ""; } String getDataSourceDescription() const override { return ""; } @@ -46,7 +46,7 @@ private: void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override; void fromAST(ASTs & args, ContextPtr context, bool with_structure) override; Path path; - Paths paths; + DataFileInfos paths; }; } diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 7542f59dcc4..f6abfc69138 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -165,7 +165,7 @@ void StorageS3Configuration::fromNamedCollection(const NamedCollection & collect static_configuration = !auth_settings.access_key_id.value.empty() || auth_settings.no_sign_request.changed; - keys = {url.key}; + keys = {url.key, std::nullopt, std::nullopt}; } void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_structure) @@ -362,7 +362,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ static_configuration = !auth_settings.access_key_id.value.empty() || auth_settings.no_sign_request.changed; auth_settings.no_sign_request = no_sign_request; - keys = {url.key}; + keys = {url.key, std::nullopt, std::nullopt}; } void StorageS3Configuration::addStructureAndFormatToArgs( diff --git a/src/Storages/ObjectStorage/S3/Configuration.h b/src/Storages/ObjectStorage/S3/Configuration.h index 39a646c7df2..e853755799f 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.h +++ b/src/Storages/ObjectStorage/S3/Configuration.h @@ -6,6 +6,8 @@ #include #include +# include "Storages/ObjectStorage/DataFileInfo.h" + namespace DB { @@ -27,8 +29,8 @@ public: Path getPath() const override { return url.key; } void setPath(const Path & path) override { url.key = path; } - const Paths & getPaths() const override { return keys; } - void setPaths(const Paths & paths) override { keys = paths; } + const DataFileInfos & getPaths() const override { return keys; } + void setPaths(const DataFileInfos & paths) override { keys = paths; } String getNamespace() const override { return url.bucket; } String getDataSourceDescription() const override; @@ -55,7 +57,7 @@ private: void fromAST(ASTs & args, ContextPtr context, bool with_structure) override; S3::URI url; - std::vector keys; + DataFileInfos keys; S3::AuthSettings auth_settings; S3::RequestSettings request_settings; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index f0686c144ba..0a85ca1af03 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -334,7 +334,7 @@ SinkToStoragePtr StorageObjectStorage::write( if (auto new_key = checkAndGetNewFileOnInsertIfNeeded( *object_storage, *configuration, settings, paths.front(), paths.size())) { - paths.push_back(*new_key); + paths.emplace_back(*new_key, std::nullopt, std::nullopt); } configuration->setPaths(paths); diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 562ca259089..7d7d90af178 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -8,6 +8,9 @@ #include #include +#include "Storages/ObjectStorage/DataFileInfo.h" + + namespace DB { @@ -15,6 +18,7 @@ class ReadBufferIterator; class SchemaCache; class NamedCollection; + /** * A general class containing implementation for external table engines * such as StorageS3, StorageAzure, StorageHDFS. @@ -173,8 +177,8 @@ public: virtual Path getPath() const = 0; virtual void setPath(const Path & path) = 0; - virtual const Paths & getPaths() const = 0; - virtual void setPaths(const Paths & paths) = 0; + virtual const DataFileInfos & getPaths() const = 0; + virtual void setPaths(const DataFileInfos & paths) = 0; virtual String getDataSourceDescription() const = 0; virtual String getNamespace() const = 0; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 3ae884bf98c..ad99879d07d 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -138,7 +138,7 @@ std::shared_ptr StorageObjectStorageSourc std::vector paths; paths.reserve(keys.size()); for (const auto & key : keys) - paths.push_back(fs::path(configuration->getNamespace()) / key); + paths.push_back(fs::path(configuration->getNamespace()) / key.data_path); VirtualColumnUtils::buildSetsForDAG(*filter_dag, local_context); auto actions = std::make_shared(std::move(*filter_dag)); From 3f3a52b490f9aecb3f09f52cbda0100d0ef67db9 Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 28 Aug 2024 10:10:54 +0000 Subject: [PATCH 016/502] Intermediate state --- .../ObjectStorage/Azure/Configuration.cpp | 4 +- src/Storages/ObjectStorage/DataFileInfo.h | 18 ++++-- .../DataLakes/DeltaLakeMetadata.cpp | 2 +- .../DataLakes/DeltaLakeMetadata.h | 2 +- .../ObjectStorage/DataLakes/HudiMetadata.cpp | 6 +- .../ObjectStorage/DataLakes/HudiMetadata.h | 4 +- .../DataLakes/IDataLakeMetadata.h | 2 +- .../DataLakes/IStorageDataLake.h | 4 +- .../DataLakes/IcebergMetadata.cpp | 64 ++++++++++--------- .../ObjectStorage/DataLakes/IcebergMetadata.h | 8 +-- .../ObjectStorage/HDFS/Configuration.cpp | 2 +- .../ObjectStorage/Local/Configuration.cpp | 4 +- .../ObjectStorage/S3/Configuration.cpp | 4 +- .../ObjectStorage/StorageObjectStorage.cpp | 7 +- .../ObjectStorage/StorageObjectStorage.h | 32 ++++++++-- .../StorageObjectStorageSink.cpp | 2 +- .../StorageObjectStorageSource.cpp | 34 +++++----- .../StorageObjectStorageSource.h | 2 +- src/Storages/VirtualColumnUtils.h | 2 +- 19 files changed, 122 insertions(+), 81 deletions(-) diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index 8bee4fa8f34..7c7e37b7255 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -140,7 +140,7 @@ void StorageAzureConfiguration::fromNamedCollection(const NamedCollection & coll format = collection.getOrDefault("format", format); compression_method = collection.getOrDefault("compression_method", collection.getOrDefault("compression", "auto")); - blobs_paths = {blob_path, std::nullopt, std::nullopt}; + blobs_paths = {DataFileInfo(blob_path)}; connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context); } @@ -268,7 +268,7 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, structure = checkAndGetLiteralArgument(engine_args[7], "structure"); } - blobs_paths = {blob_path, std::nullopt, std::nullopt}; + blobs_paths = {DataFileInfo(blob_path)}; connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context); } diff --git a/src/Storages/ObjectStorage/DataFileInfo.h b/src/Storages/ObjectStorage/DataFileInfo.h index b1f01c8abd1..794815f2020 100644 --- a/src/Storages/ObjectStorage/DataFileInfo.h +++ b/src/Storages/ObjectStorage/DataFileInfo.h @@ -1,3 +1,5 @@ +#pragma once + #include #include @@ -9,12 +11,20 @@ namespace DB struct DataFileInfo { String data_path; - std::optional initial_schema; - std::optional schema_transform; + std::shared_ptr initial_schema; + std::shared_ptr schema_transform; - bool operator==(DataFileInfo & other_info) { return data_path == other_info.data_path; } + DataFileInfo( + String data_path_, + std::shared_ptr initial_schema_ = nullptr, + std::shared_ptr schema_transform_ = nullptr) + : data_path(data_path_), initial_schema(initial_schema_), schema_transform(schema_transform_) + { + } + + bool operator==(const DataFileInfo & other_info) const { return data_path == other_info.data_path; } }; using DataFileInfos = std::vector; -} \ No newline at end of file +} diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp index 34540b38709..5631f494d6a 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp @@ -666,7 +666,7 @@ DeltaLakeMetadata::DeltaLakeMetadata( auto result = impl.processMetadataFiles(); for (const auto & data_file_name : result.data_files) { - data_files.emplace_back(data_file_name, std::nullopt, std::nullopt); + data_files.emplace_back(DataFileInfo{data_file_name}); } schema = result.schema; partition_columns = result.partition_columns; diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h index 75ac4f566e1..dbcf3a4ec0b 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h @@ -22,7 +22,7 @@ public: ConfigurationPtr configuration_, ContextPtr context_); - std::vector getDataFilesInfo() const override { return data_files; } + std::vector getDataFileInfos() const override { return data_files; } NamesAndTypesList getTableSchema() const override { return schema; } diff --git a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp index 128d87a5e43..016d6255b8d 100644 --- a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp @@ -96,13 +96,13 @@ HudiMetadata::HudiMetadata( { } -Strings HudiMetadata::getDataFilesInfo() const +DataFileInfos HudiMetadata::getDataFileInfos() const { if (data_files.empty()) { - for (const auto & data_file_name : getDataFilesImpl) + for (const auto & data_file_name : getDataFilesImpl()) { - data_files.push_back({data_file_name, std::nullopt, std::nullopt}); + data_files.push_back({DataFileInfo(data_file_name)}); } } return data_files; diff --git a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h index 1ac632b15c6..f2308085dca 100644 --- a/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/HudiMetadata.h @@ -23,7 +23,7 @@ public: ConfigurationPtr configuration_, ContextPtr context_); - std::vector getDataFilesInfo() const override; + DataFileInfos getDataFileInfos() const override; NamesAndTypesList getTableSchema() const override { return {}; } @@ -50,7 +50,7 @@ public: private: const ObjectStoragePtr object_storage; const ConfigurationPtr configuration; - mutable std::vector data_files; + mutable DataFileInfos data_files; std::unordered_map column_name_to_physical_name; DataLakePartitionColumns partition_columns; diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h index 3ff19607486..d2b26cca8c5 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h @@ -12,7 +12,7 @@ class IDataLakeMetadata : boost::noncopyable { public: virtual ~IDataLakeMetadata() = default; - virtual std::vector getDataFilesInfo() const = 0; + virtual std::vector getDataFileInfos() const = 0; virtual NamesAndTypesList getTableSchema() const = 0; virtual bool operator==(const IDataLakeMetadata & other) const = 0; virtual const DataLakePartitionColumns & getPartitionColumns() const = 0; diff --git a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h index b100016b37d..d39a662d265 100644 --- a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h +++ b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h @@ -51,7 +51,7 @@ public: try { metadata = DataLakeMetadata::create(object_storage, base_configuration, context); - configuration->setPaths(metadata->getDataFiles()); + configuration->setPaths(metadata->getDataFileInfos()); if (use_schema_from_metadata) schema_from_metadata = metadata->getTableSchema(); } @@ -90,7 +90,7 @@ public: else { ConfigurationPtr configuration = base_configuration->clone(); - configuration->setPaths(metadata->getDataFiles()); + configuration->setPaths(metadata->getDataFileInfos()); std::string sample_path; return Storage::resolveSchemaFromData( object_storage_, configuration, format_settings_, sample_path, local_context); diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 7315dede03e..9c78af86030 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -279,7 +279,7 @@ NamesAndTypesList getClickhouseSchemaFromIcebergSchema(const Poco::JSON::Object: names_and_types.push_back({name, getFieldType(field, "type", required)}); } - return std::move(names_and_types); + return names_and_types; } std::pair parseTableSchema( @@ -315,9 +315,10 @@ std::pair parseTableSchema( return {getClickhouseSchemaFromIcebergSchema(schema), current_schema_id}; } -ActionsDAG getSchemaTransformDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) +std::shared_ptr getSchemaTransformDag( + [[maybe_unused]] const Poco::JSON::Object::Ptr & old_schema, [[maybe_unused]] const Poco::JSON::Object::Ptr & new_schema) { - return ActionsDAG{}; + return std::make_shared(); } MutableColumns parseAvro(avro::DataFileReaderBase & file_reader, const Block & header, const FormatSettings & settings) @@ -397,7 +398,7 @@ DataLakeMetadataPtr IcebergMetadata::create( auto relevant_schemas_by_ids = getRelevantSchemasByIds(object, format_version); - auto [schema, schema_id] = parseTableSchema(object, format_version); + auto [schema, schema_id] = parseTableSchema(object, format_version, relevant_schemas_by_ids); auto current_snapshot_id = object->getValue("current-snapshot-id"); auto snapshots = object->get("snapshots").extract(); @@ -415,8 +416,15 @@ DataLakeMetadataPtr IcebergMetadata::create( } return std::make_unique( - object_storage, configuration, local_context, metadata_version, format_version, manifest_list_file, schema_id, schema), - ; + object_storage, + configuration, + local_context, + metadata_version, + format_version, + manifest_list_file, + schema_id, + schema, + relevant_schemas_by_ids); } /** @@ -444,14 +452,14 @@ DataLakeMetadataPtr IcebergMetadata::create( * │ 1 │ 2252246380142525104 │ ('/iceberg_data/db/table_name/data/a=2/00000-1-c9535a00-2f4f-405c-bcfa-6d4f9f477235-00003.parquet','PARQUET',(2),1,631,67108864,[(1,46),(2,48)],[(1,1),(2,1)],[(1,0),(2,0)],[],[(1,'\0\0\0\0\0\0\0'),(2,'3')],[(1,'\0\0\0\0\0\0\0'),(2,'3')],NULL,[4],0) │ * └────────┴─────────────────────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ */ -Strings IcebergMetadata::getDataFilesInfo() const +DataFileInfos IcebergMetadata::getDataFileInfos() const { - if (!data_files.empty()) - return data_files; + if (!data_file_infos.empty()) + return data_file_infos; Strings manifest_files; if (manifest_list_file.empty()) - return data_files; + return data_file_infos; LOG_TEST(log, "Collect manifest files from manifest list {}", manifest_list_file); @@ -481,7 +489,7 @@ Strings IcebergMetadata::getDataFilesInfo() const manifest_files.emplace_back(std::filesystem::path(configuration->getPath()) / "metadata" / filename); } - std::set> files; + std::map files; LOG_TEST(log, "Collect data files"); for (const auto & manifest_file : manifest_files) { @@ -614,45 +622,43 @@ Strings IcebergMetadata::getDataFilesInfo() const if (ManifestEntryStatus(status) == ManifestEntryStatus::DELETED) { LOG_TEST(log, "Processing delete file for path: {}", file_path); - chassert(!files.contains(file_path)); + chassert(files.count(file_path) == 0); } else { LOG_TEST(log, "Processing data file for path: {}", file_path); - files.insert({file_path, schema_object_id}); + files[file_path] = schema_object_id; } } - } - std::vector data_file_infos; + if (!relevant_schemas_by_ids.count(schema_object_id)) + { + relevant_schemas_by_ids[schema_object_id] = schema_object; + } + if (!transform_dags_by_ids.count(schema_object_id)) + { + transform_dags_by_ids[schema_object_id] + = getSchemaTransformDag(relevant_schemas_by_ids[schema_object_id], relevant_schemas_by_ids[current_schema_id]); + } + } for (const auto & [file_path, schema_object_id] : files) { if (current_schema_id == schema_object_id) { - data_file_infos.emplace_back(file_path, std::nullopt, std::nullopt); + data_file_infos.emplace_back(DataFileInfo{file_path}); } else { - if (!relevant_schemas_by_ids.count(schema_object_id)) - { - relevant_schemas_by_ids[schema_object_id] = schema_object; - } - if (!transform_dags_by_ids.count(schema_object_id)) - { - transform_dags_by_ids[schema_object_id] - = getSchemaTransformDag(relevant_schemas_by_ids[schema_object_id], relevant_schemas_by_ids[current_schema_id]); - } data_file_infos.emplace_back( file_path, - getClickhouseSchemaFromIcebergSchema(relevant_schemas_by_ids[schema_object_id]), - transform_dags_by_ids[schema_object_id]) + std::make_shared(getClickhouseSchemaFromIcebergSchema(relevant_schemas_by_ids[schema_object_id])), + transform_dags_by_ids[schema_object_id]); } } - data_files = std::vector(files.begin(), files.end()); - return data_files; + return data_file_infos; } } diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 29673875a12..b10a06d0639 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -82,7 +82,7 @@ public: /// Get data files. On first request it reads manifest_list file and iterates through manifest files to find all data files. /// All subsequent calls will return saved list of files (because it cannot be changed without changing metadata file) - std::vector getDataFilesInfo() const override; + DataFileInfos getDataFileInfos() const override; /// Get table schema parsed from metadata. NamesAndTypesList getTableSchema() const override { return schema; } @@ -112,11 +112,11 @@ private: String manifest_list_file; const Int32 current_schema_id; NamesAndTypesList schema; - mutable Strings data_files; + mutable DataFileInfos data_file_infos; std::unordered_map column_name_to_physical_name; DataLakePartitionColumns partition_columns; - std::map relevant_schemas_by_ids; - std::map transform_dags_by_ids; + mutable std::map relevant_schemas_by_ids; + mutable std::map> transform_dags_by_ids; LoggerPtr log; }; diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.cpp b/src/Storages/ObjectStorage/HDFS/Configuration.cpp index 74f5d885922..038c2813978 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.cpp +++ b/src/Storages/ObjectStorage/HDFS/Configuration.cpp @@ -153,7 +153,7 @@ void StorageHDFSConfiguration::setURL(const std::string & url_) path = '/' + path; url = url_.substr(0, pos); - paths = {path, std::nullopt, std::nullopt}; + paths = {DataFileInfo(path)}; LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path); } diff --git a/src/Storages/ObjectStorage/Local/Configuration.cpp b/src/Storages/ObjectStorage/Local/Configuration.cpp index 0ce93136b4a..0d49be2abe4 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.cpp +++ b/src/Storages/ObjectStorage/Local/Configuration.cpp @@ -20,7 +20,7 @@ void StorageLocalConfiguration::fromNamedCollection(const NamedCollection & coll format = collection.getOrDefault("format", "auto"); compression_method = collection.getOrDefault("compression_method", collection.getOrDefault("compression", "auto")); structure = collection.getOrDefault("structure", "auto"); - paths = {path, std::nullopt, std::nullopt}; + paths = {DataFileInfo(path)}; } @@ -57,7 +57,7 @@ void StorageLocalConfiguration::fromAST(ASTs & args, ContextPtr context, bool wi { compression_method = checkAndGetLiteralArgument(args[2], "compression_method"); } - paths = {path, std::nullopt, std::nullopt}; + paths = {DataFileInfo(path)}; } StorageObjectStorage::QuerySettings StorageLocalConfiguration::getQuerySettings(const ContextPtr & context) const diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index f6abfc69138..1a1a35a497e 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -165,7 +165,7 @@ void StorageS3Configuration::fromNamedCollection(const NamedCollection & collect static_configuration = !auth_settings.access_key_id.value.empty() || auth_settings.no_sign_request.changed; - keys = {url.key, std::nullopt, std::nullopt}; + keys = {DataFileInfo(url.key)}; } void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_structure) @@ -362,7 +362,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ static_configuration = !auth_settings.access_key_id.value.empty() || auth_settings.no_sign_request.changed; auth_settings.no_sign_request = no_sign_request; - keys = {url.key, std::nullopt, std::nullopt}; + keys = {DataFileInfo(url.key)}; } void StorageS3Configuration::addStructureAndFormatToArgs( diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 0a85ca1af03..88d71a295c5 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -331,10 +331,9 @@ SinkToStoragePtr StorageObjectStorage::write( } auto paths = configuration->getPaths(); - if (auto new_key = checkAndGetNewFileOnInsertIfNeeded( - *object_storage, *configuration, settings, paths.front(), paths.size())) + if (auto new_key = checkAndGetNewFileOnInsertIfNeeded(*object_storage, *configuration, settings, paths.front().data_path, paths.size())) { - paths.emplace_back(*new_key, std::nullopt, std::nullopt); + paths.emplace_back(*new_key); } configuration->setPaths(paths); @@ -369,7 +368,7 @@ void StorageObjectStorage::truncate( StoredObjects objects; for (const auto & key : configuration->getPaths()) - objects.emplace_back(key); + objects.emplace_back(key.data_path); object_storage->removeObjectsIfExist(objects); } diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 7d7d90af178..b69347991b9 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -1,15 +1,18 @@ #pragma once -#include -#include +#include #include -#include +#include #include -#include #include +#include #include +#include +#include #include "Storages/ObjectStorage/DataFileInfo.h" +#include + namespace DB { @@ -29,7 +32,26 @@ class StorageObjectStorage : public IStorage public: class Configuration; using ConfigurationPtr = std::shared_ptr; - using ObjectInfo = RelativePathWithMetadata; + + struct ObjectInfo : public RelativePathWithMetadata + { + ObjectInfo( + String relative_path_, + std::optional metadata_ = std::nullopt, + std::shared_ptr initial_schema_ = nullptr, + std::shared_ptr schema_transformer_ = nullptr) + : RelativePathWithMetadata(std::move(relative_path_), std::move(metadata_)) + , initial_schema(std::move(initial_schema_)) + , schema_transformer(std::move(schema_transformer_)) + { + } + + ObjectInfo() = default; + + public: + std::shared_ptr initial_schema; + std::shared_ptr schema_transformer; + }; using ObjectInfoPtr = std::shared_ptr; using ObjectInfos = std::vector; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp index 33c96fdfae9..8d88719028d 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp @@ -25,7 +25,7 @@ StorageObjectStorageSink::StorageObjectStorageSink( , sample_block(sample_block_) { const auto & settings = context->getSettingsRef(); - const auto path = blob_path.empty() ? configuration->getPaths().back() : blob_path; + const auto path = blob_path.empty() ? configuration->getPaths().back().data_path : blob_path; const auto chosen_compression_method = chooseCompressionMethod(path, configuration->compression_method); auto buffer = object_storage->writeObject( diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index ad99879d07d..67e10b944d2 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1,19 +1,20 @@ #include "StorageObjectStorageSource.h" -#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include #include #include -#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include #include -#include namespace fs = std::filesystem; @@ -569,7 +570,10 @@ StorageObjectStorage::ObjectInfoPtr StorageObjectStorageSource::GlobIterator::ne return {}; } - new_batch = std::move(result.value()); + for (const auto & relative_metadata : *result) + { + new_batch.emplace_back(std::make_shared(relative_metadata->relative_path, relative_metadata->metadata)); + } for (auto it = new_batch.begin(); it != new_batch.end();) { if (!recursive && !re2::RE2::FullMatch((*it)->getPath(), *matcher)) @@ -639,7 +643,7 @@ StorageObjectStorageSource::KeysIterator::KeysIterator( /// TODO: should we add metadata if we anyway fetch it if file_progress_callback is passed? for (auto && key : keys) { - auto object_info = std::make_shared(key); + auto object_info = std::make_shared(key.data_path, std::nullopt, key.initial_schema, key.schema_transform); read_keys_->emplace_back(object_info); } } @@ -658,17 +662,17 @@ StorageObjectStorage::ObjectInfoPtr StorageObjectStorageSource::KeysIterator::ne ObjectMetadata object_metadata{}; if (ignore_non_existent_files) { - auto metadata = object_storage->tryGetObjectMetadata(key); + auto metadata = object_storage->tryGetObjectMetadata(key.data_path); if (!metadata) continue; } else - object_metadata = object_storage->getObjectMetadata(key); + object_metadata = object_storage->getObjectMetadata(key.data_path); if (file_progress_callback) file_progress_callback(FileProgress(0, object_metadata.size_bytes)); - return std::make_shared(key, object_metadata); + return std::make_shared(key.data_path, object_metadata); } } diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.h b/src/Storages/ObjectStorage/StorageObjectStorageSource.h index 7ae7a2358e9..02e3dd06b38 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.h @@ -246,7 +246,7 @@ private: const ConfigurationPtr configuration; const NamesAndTypesList virtual_columns; const std::function file_progress_callback; - const std::vector keys; + const DataFileInfos keys; std::atomic index = 0; bool ignore_non_existent_files; }; diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h index 6aa08b2aef2..d7bad8de39f 100644 --- a/src/Storages/VirtualColumnUtils.h +++ b/src/Storages/VirtualColumnUtils.h @@ -28,7 +28,7 @@ namespace VirtualColumnUtils /// Otherwise calling filter*() outside applyFilters() will throw "Not-ready Set is passed" /// if there are subqueries. /// -/// Similar to filterBlockWithExpression(buildFilterExpression(splitFilterDagForAllowedInputs(...)))./// Similar to filterBlockWithQuery, but uses ActionsDAG as a predicate. +/// Similar to yPalockWithExpression(buildFilterExpression(splitFilterDagForAllowedInputs(...)))./// Similar to filterBlockWithQuery, but uses ActionsDAG as a predicate. /// Basically it is filterBlockWithDAG(splitFilterDagForAllowedInputs). /// If allow_filtering_with_partial_predicate is true, then the filtering will be done even if some part of the predicate /// cannot be evaluated using the columns from the block. From 789f8697d1dc888d506c5e5bb92b6f2547404432 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 29 Aug 2024 12:06:03 +0000 Subject: [PATCH 017/502] Add DAG creation --- .../DataLakes/IcebergMetadata.cpp | 51 ++++++++++++++++++- .../StorageObjectStorageSource.cpp | 12 +++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 9c78af86030..2d05d139892 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -318,7 +318,56 @@ std::pair parseTableSchema( std::shared_ptr getSchemaTransformDag( [[maybe_unused]] const Poco::JSON::Object::Ptr & old_schema, [[maybe_unused]] const Poco::JSON::Object::Ptr & new_schema) { - return std::make_shared(); + std::map old_schema_entries; + std::map new_schema_entries; + auto fields = old_schema->get("fields").extract(); + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + auto name = field->getValue("name"); + bool required = field->getValue("required"); + size_t id = field->getValue("id"); + old_schema_entries[id] = {name, getFieldType(field, "type", required)}; + } + fields = new_schema->get("fields").extract(); + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + auto name = field->getValue("name"); + bool required = field->getValue("required"); + size_t id = field->getValue("id"); + new_schema_entries[id] = {name, getFieldType(field, "type", required)}; + } + std::shared_ptr dag; + auto& outputs = dag->getOutputs(); + for (const auto& [id, name_and_type] : old_schema_entries) { + const ActionsDAG::Node & input_node = dag->addInput(name_and_type.name, name_and_type.type); + if (new_schema_entries.count(id)) { + if (new_schema_entries[id].type != name_and_type.type) { + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, + "Type promotion for schema evolution is not implemeted yet"); + } + if (name_and_type.name != new_schema_entries[id].name) { + const ActionsDAG::Node & alias_node = dag->addAlias(input_node, new_schema_entries[id].name); + outputs.push_back(&alias_node); + } else { + outputs.push_back(&input_node); + } + new_schema_entries.erase(id); + } + } + for (const auto& [_id, name_and_type] : new_schema_entries) { + if (!name_and_type.type->isNullable()) { + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, + "Can't add a column with required values to the table"); + } + ColumnPtr default_type_column = name_and_type.type->createColumnConstWithDefaultValue(0); + const auto & constant = dag->addColumn({default_type_column, name_and_type.type, name_and_type.name}); + outputs.push_back(&constant); + } + return dag; } MutableColumns parseAvro(avro::DataFileReaderBase & file_reader, const Block & header, const FormatSettings & settings) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 67e10b944d2..77d8e09c79b 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -15,6 +15,9 @@ #include #include #include +#include +#include + namespace fs = std::filesystem; @@ -384,6 +387,15 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade builder.init(Pipe(input_format)); + if (object_info->schema_transformer) { + auto schema_modifying_actions = std::make_shared(object_info->schema_transformer->clone()); + builder.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, schema_modifying_actions); + }); + } + + if (read_from_format_info.columns_description.hasDefaults()) { builder.addSimpleTransform( From c71303686d451f856451ea19c05f7bce92ccf783 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 2 Sep 2024 13:39:42 +0000 Subject: [PATCH 018/502] To checkout --- src/Interpreters/ActionsDAG.h | 2 +- .../DataLakes/IcebergMetadata.cpp | 227 ++++++++++++++---- .../ObjectStorage/DataLakes/IcebergMetadata.h | 53 ++++ .../StorageObjectStorageSource.cpp | 39 ++- 4 files changed, 276 insertions(+), 45 deletions(-) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index ee2b3fbf4f2..1ded275fc98 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -63,7 +63,7 @@ public: struct Node; using NodeRawPtrs = std::vector; - using NodeRawConstPtrs = std::vector; + using NodeRawConstPtrs = std::vector<`>; struct Node { diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 2d05d139892..30447e23dcf 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -136,7 +136,20 @@ enum class DataFileContent : uint8_t * } */ -DataTypePtr getSimpleTypeByName(const String & type_name) +std::pair parse_decimal(const String & type_name) +{ + ReadBufferFromString buf(std::string_view(type_name.begin() + 8, type_name.end() - 1)); + size_t precision; + size_t scale; + readIntText(precision, buf); + skipWhitespaceIfAny(buf); + assertChar(',', buf); + skipWhitespaceIfAny(buf); + tryReadIntText(scale, buf); + return {precision, scale}; +} + +DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name) { if (type_name == "boolean") return DataTypeFactory::instance().get("Bool"); @@ -174,22 +187,19 @@ DataTypePtr getSimpleTypeByName(const String & type_name) if (type_name.starts_with("decimal(") && type_name.ends_with(')')) { ReadBufferFromString buf(std::string_view(type_name.begin() + 8, type_name.end() - 1)); - size_t precision; - size_t scale; - readIntText(precision, buf); - skipWhitespaceIfAny(buf); - assertChar(',', buf); - skipWhitespaceIfAny(buf); - tryReadIntText(scale, buf); + auto [precision, scale] = parse_decimal(type_name); return createDecimal(precision, scale); } throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Iceberg type: {}", type_name); } -DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & type_key, bool required); +bool allowedDecimalConversion(const string & old_type, const string & new_type) +{ + if (!(old_type.starts_with("decimal(") && old_type.ends_with(')') && new_type.starts_with("decimal(")) +} -DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type) +DataTypePtr IcebergSchemaProcessor::getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type) { String type_name = type->getValue("type"); if (type_name == "list") @@ -228,7 +238,7 @@ DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Iceberg type: {}", type_name); } -DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & type_key, bool required) +DataTypePtr IcebergSchemaProcessor::getFieldType(const Poco::JSON::Object::Ptr & field, const String & type_key, bool required) { if (field->isObject(type_key)) return getComplexTypeFromObject(field->getObject(type_key)); @@ -245,43 +255,92 @@ DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & t } -std::map getRelevantSchemasByIds(const Poco::JSON::Object::Ptr & metadata_object, int format_version) +bool IcebergSchemaProcessor::allowPrimitiveTypeConversion(const String & old_type, const String & new_type) { - std::map schemas_by_id; - if (format_version == 2) + bool allowed_type_conversion = (old_type == new_type); + allowed |= (old_type == "int") && (new_type == "long"); + allowed |= (old_type == "float") && (new_type == "double"); + if (old_type.starts_with("decimal(") && old_type.ends_with(')') && new_type.starts_with("decimal(") && new_type.ends_with(")")) { - auto schemas = metadata_object->get("schemas").extract(); - if (schemas->size() == 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: schemas field is empty"); + auto [old_precision, old_scale] = parse_decimal(old_type); + auto [new_precision, new_scale] = parse_decimal(new_type); + allowed |= (old_precision <= new_precision) && (old_scale == new_scale); + } + return allowed; +} - for (uint32_t i = 0; i != schemas->size(); ++i) +const Node * output getRemappingForStructField( + const Poco::JSON::Array::Ptr & old_node, + const Poco::JSON::Array::Ptr & new_node, + const Node * input_node, + const String & type_name, + const String & required_name, + const std::optional & name_name) +{ + bool old_required = old_node->getValue(required_name); + bool new_required = new_node->getValue(required_name); + if (old_node->isObject("type")) + { + auto remapped = getRemappingForComplexType(old_node, new_node); + if (remapped == true) { - auto current_schema = schemas->getObject(i); - schemas_by_id[current_schema->getValue("schema-id")] = current_schema; } } - else + chassert(!new_node->isObject("type")); + assert(new_node->get("type").isString()); + String old_type = old_node->getValue("type"); + String new_type = new_node->getValue("type"); + if (!allowPrimitiveTypeConversion(old_type, new_type)) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported format"); - } - return schemas_by_id; -} - -NamesAndTypesList getClickhouseSchemaFromIcebergSchema(const Poco::JSON::Object::Ptr & schema) -{ - NamesAndTypesList names_and_types; - auto fields = schema->get("fields").extract(); - for (size_t i = 0; i != fields->size(); ++i) - { - auto field = fields->getObject(static_cast(i)); - auto name = field->getValue("name"); - bool required = field->getValue("required"); - names_and_types.push_back({name, getFieldType(field, "type", required)}); + throw undefined; } - return names_and_types; + auto old_clickhouse_type = old_required ? getSimpleType(old_type) : makeNullable(getSimpleType(old_type)); + auto new_clickhouse_type = old_required ? getSimpleType(old_type) : makeNullable(getSimpleType(old_type)); + + auto new_name = name_name.has_value() ? new_node->getValue(name_name.value()) : input_node.name; + return current_actions_dag->addCast(input_node, new_clickhouse_type, new_name); + return getPrimitiveTypeConversed(input_node, old_type, new_type, old_required, new_required); } + +// std::map getRelevantSchemasByIds(const Poco::JSON::Object::Ptr & metadata_object, int format_version) +// { +// std::map schemas_by_id; +// if (format_version == 2) +// { +// auto schemas = metadata_object->get("schemas").extract(); +// if (schemas->size() == 0) +// throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: schemas field is empty"); + +// for (uint32_t i = 0; i != schemas->size(); ++i) +// { +// auto current_schema = schemas->getObject(i); +// schemas_by_id[current_schema->getValue("schema-id")] = current_schema; +// } +// } +// else +// { +// throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported format"); +// } +// return schemas_by_id; +// } + +// NamesAndTypesList getClickhouseSchemaFromIcebergSchema(const Poco::JSON::Object::Ptr & schema) +// { +// NamesAndTypesList names_and_types; +// auto fields = schema->get("fields").extract(); +// for (size_t i = 0; i != fields->size(); ++i) +// { +// auto field = fields->getObject(static_cast(i)); +// auto name = field->getValue("name"); +// bool required = field->getValue("required"); +// names_and_types.push_back({name, getFieldType(field, "type", required)}); +// } + +// return names_and_types; +// } + std::pair parseTableSchema( const Poco::JSON::Object::Ptr & metadata_object, Int32 format_version, @@ -314,6 +373,61 @@ std::pair parseTableSchema( return {getClickhouseSchemaFromIcebergSchema(schema), current_schema_id}; } +/* +bool isPrimitiveType(const Poco::JSON::Object::Ptr & type_key) { + return !field->isObject(type_key); +} + +bool isStructType(constPoco::JSON::Object::Ptr & type_key) { + return (!isPrimitiveType(type_key)) && type->getValue("type") == "struct"; +} + +*/ + +pair, vector<>> getStructRemapping(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) +{ + Int32 old_id = old_schema->getValue("id"); + Int32 new_id = new_schema->getValue("id"); + chassert(old_id == new_id); + auto name = old_schema->getValue("name"); + bool required = old_schema->getValue("required"); + auto old_type = {name, getFieldType(old_schema, "type", required)}; + + auto old_fields = old_schema->get("fields").extract(); + auto new_fiels = new_schema->get("fields").extract(); + std::vector old_schema_elements; + std::vector const ActionsDAG::Node & input_node = dag->addInput(name_and_type.name, name_and_type.type); + + + ColumnPtr default_type_column = name_and_type.type->createColumnConstWithDefaultValue(0); + const auto & constant = dag->addColumn({default_type_column, name_and_type.type, name_and_type.name}); +} + +const Node * IcebergSchemaProcessor::getDefaultNodeForField(const Poco::JSON::Object::Ptr & field) +{ +} + + +NodeRawConstPtrs reconstructStruct( + ActionDAG & action_dag, + const Poco::JSON::Object::Ptr & old_schema, + const Poco::JSON::Object::Ptr & new_schema, + const NodeRawConstPtrs & input_struct_nodes) +{ + auto name = old_schema->getValue("name"); + bool required = old_schema->getValue("required"); + auto old_type = {name, getFieldType(old_schema, "type", required)}; + + auto old_fields = old_schema->get("fields").extract(); + auto new_fiels = new_schema->get("fields").extract(); + std::vector old_schema_elements; + std::vector const ActionsDAG::Node & input_node = dag->addInput(name_and_type.name, name_and_type.type); + + + ColumnPtr default_type_column = name_and_type.type->createColumnConstWithDefaultValue(0); + const auto & constant = dag->addColumn({default_type_column, name_and_type.type, name_and_type.name}); +} + std::shared_ptr getSchemaTransformDag( [[maybe_unused]] const Poco::JSON::Object::Ptr & old_schema, [[maybe_unused]] const Poco::JSON::Object::Ptr & new_schema) @@ -328,6 +442,8 @@ std::shared_ptr getSchemaTransformDag( bool required = field->getValue("required"); size_t id = field->getValue("id"); old_schema_entries[id] = {name, getFieldType(field, "type", required)}; + LOG_DEBUG( + &Poco::Logger::get("Old schema"), "Id: {}, Name: {}, type: {}", id, name, getFieldType(field, "type", required)->getName()); } fields = new_schema->get("fields").extract(); for (size_t i = 0; i != fields->size(); ++i) @@ -337,27 +453,45 @@ std::shared_ptr getSchemaTransformDag( bool required = field->getValue("required"); size_t id = field->getValue("id"); new_schema_entries[id] = {name, getFieldType(field, "type", required)}; + LOG_DEBUG( + &Poco::Logger::get("New schema"), "Id: {}, Name: {}, type: {}", id, name, getFieldType(field, "type", required)->getName()); } - std::shared_ptr dag; + std::shared_ptr dag = std::make_shared(); auto& outputs = dag->getOutputs(); for (const auto& [id, name_and_type] : old_schema_entries) { + LOG_DEBUG(&Poco::Logger::get("Adding Input"), "Name: {}, type: {}", name_and_type.name, name_and_type.type->getName()); const ActionsDAG::Node & input_node = dag->addInput(name_and_type.name, name_and_type.type); if (new_schema_entries.count(id)) { - if (new_schema_entries[id].type != name_and_type.type) { + if (new_schema_entries[id].type->getTypeId() + != name_and_type.type->getTypeId()) // If this exists it should be written in another manner + { throw Exception( - ErrorCodes::UNSUPPORTED_METHOD, - "Type promotion for schema evolution is not implemeted yet"); + ErrorCodes::UNSUPPORTED_METHOD, + "Type promotion for schema evolution is not implemeted yet: old_type: {}, new_type: {}", + name_and_type.type->getName(), + new_schema_entries[id].type->getName()); } if (name_and_type.name != new_schema_entries[id].name) { const ActionsDAG::Node & alias_node = dag->addAlias(input_node, new_schema_entries[id].name); + LOG_DEBUG( + &Poco::Logger::get("Adding Alias as Output"), + "Name: {}, type: {}", + alias_node.result_name, + alias_node.result_type->getName()); outputs.push_back(&alias_node); } else { + LOG_DEBUG( + &Poco::Logger::get("Adding Input as Ouput"), + "Name: {}, type: {}", + input_node.result_name, + input_node.result_type->getName()); outputs.push_back(&input_node); } new_schema_entries.erase(id); } } for (const auto& [_id, name_and_type] : new_schema_entries) { + LOG_DEBUG(&Poco::Logger::get("Adding default"), "Name: {}, type: {}", name_and_type.name, name_and_type.type->getName()); if (!name_and_type.type->isNullable()) { throw Exception( ErrorCodes::UNSUPPORTED_METHOD, @@ -700,6 +834,17 @@ DataFileInfos IcebergMetadata::getDataFileInfos() const } else { + LOG_DEBUG( + &Poco::Logger::get("Adding not empty stuff"), + "File path: {}, Old schema: {}, New schema: {}", + file_path, + schema_object_id, + current_schema_id); + if (!transform_dags_by_ids.count(schema_object_id)) + { + transform_dags_by_ids[schema_object_id] + = getSchemaTransformDag(relevant_schemas_by_ids[schema_object_id], relevant_schemas_by_ids[current_schema_id]); + } data_file_infos.emplace_back( file_path, std::make_shared(getClickhouseSchemaFromIcebergSchema(relevant_schemas_by_ids[schema_object_id])), diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index b10a06d0639..7593944124c 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -62,6 +62,59 @@ namespace DB * "metadata-log" : [ ] * } */ + +class IcebergSchemaProcessor +{ +public: + void addIcebergTableSchema(const Poco::JSON::Object::Ptr & ptr); + NamesAndTypesList getClickhouseTableSchemaById(Int32 id); + std::shared_ptr getTransformationDagByIds(Int32 old_id, Int32 new_id); + +private: + std::map iceberg_table_schemas_by_ids; + std::map clickhouse_table_schemas_by_ids; + std::map, std::shared_ptr> transform_dags_by_ids; + ActionsDag * current_actions_dag; + + + NamesAndTypeList getSchemaType(const Poco::JSON::Object::Ptr & schema); + DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type); + DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & type_key, bool required); + DataTypePtr getSimpleType(const String & type_name); + + // DataTypePtr getStructType(const Poco::JSON::Object::Ptr & node); + // DataTypePtr getListType(const Poco::JSON::Object::Ptr & node); + // DataTypePtr getMapType(const Poco::JSON::Object::Ptr & node); + // DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & node); + // DataTypePtr getSimpleType(const String & type); + // DataTypePtr getElementType(const Poco::JSON::Object::Ptr & node); + // DataTypePtr getKeyType(const) + + const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); + const Node * getDefaultNodeForList(const Poco::JSON::Object::Ptr & field); + const Node * getDefaultNodeForMap(const Poco::JSON::Object::Ptr & field); + const Node * getDefaultNodeForStruct(const Poco::JSON::Object::Ptr & field); + + std::pair + getRemappingForList(const Poco::JSON::Object::Ptr & old_node, const Poco::JSON::Object::Ptr & new_node); + + std::pair + getRemappingForMap(const Poco::JSON::Object::Ptr & old_node, const Poco::JSON::Object::Ptr & new_node); + + std::pair + getRemappingForStruct(const Poco::JSON::Object::Ptr & old_node, const Poco::JSON::Object::Ptr & new_node); + + std::pair + getRemappingForStructField(const Poco::JSON::Array::Ptr & old_node, const Poco::JSON::Array::Ptr & new_node, const Node * input_node); + + NodeRawConstPtrs getRemappingForFields( + const Poco::JSON::Array::Ptr & old_fields, + const Poco::JSON::Array::Ptr & new_fields, + const NodeRawConstPtrs & input_action_dag_nodes); + + DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type_field); +}; + class IcebergMetadata : public IDataLakeMetadata, private WithContext { public: diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 77d8e09c79b..d52e5318ba3 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -366,16 +366,30 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade read_buf = createReadBuffer(*object_info, object_storage, context_, log); } + /*Temporary desicion. Need better anstraction here.*/ + Block initial_header = read_from_format_info.format_header; + + if (object_info->schema_transformer) + { + Block sample_header; + for (const auto & [name, type] : *object_info->initial_schema) + { + sample_header.insert({type->createColumn(), type, name}); + } + initial_header = sample_header; + } + + auto input_format = FormatFactory::instance().getInput( configuration->format, *read_buf, - read_from_format_info.format_header, + initial_header, context_, max_block_size, format_settings, need_only_count ? 1 : max_parsing_threads, std::nullopt, - true/* is_remote_fs */, + true /* is_remote_fs */, compression_method, need_only_count); @@ -387,6 +401,11 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade builder.init(Pipe(input_format)); + LOG_DEBUG( + &Poco::Logger::get("Transformer exists"), + "File name: {}, Exists: {}", + object_info->relative_path, + object_info->schema_transformer != nullptr); if (object_info->schema_transformer) { auto schema_modifying_actions = std::make_shared(object_info->schema_transformer->clone()); builder.addSimpleTransform([&](const Block & header) @@ -650,12 +669,26 @@ StorageObjectStorageSource::KeysIterator::KeysIterator( , keys(configuration->getPaths()) , ignore_non_existent_files(ignore_non_existent_files_) { + for (const auto & key : keys) + { + LOG_DEBUG( + &Poco::Logger::get("Copying data_info to object_info"), + "Key data path: {}, Key schema_transform exists: {}", + key.data_path, + key.schema_transform != nullptr); + } + if (read_keys_) { /// TODO: should we add metadata if we anyway fetch it if file_progress_callback is passed? for (auto && key : keys) { auto object_info = std::make_shared(key.data_path, std::nullopt, key.initial_schema, key.schema_transform); + LOG_DEBUG( + &Poco::Logger::get("Copying data_info to object_info"), + "Key data path: {}, Key schema_transform exists: {}", + key.data_path, + key.schema_transform != nullptr); read_keys_->emplace_back(object_info); } } @@ -684,7 +717,7 @@ StorageObjectStorage::ObjectInfoPtr StorageObjectStorageSource::KeysIterator::ne if (file_progress_callback) file_progress_callback(FileProgress(0, object_metadata.size_bytes)); - return std::make_shared(key.data_path, object_metadata); + return std::make_shared(key.data_path, object_metadata, key.initial_schema, key.schema_transform); } } From 644be9fb713a1e4ff84c9983be2d9775b2c59995 Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 4 Sep 2024 10:12:21 +0000 Subject: [PATCH 019/502] To checkout --- src/Interpreters/ActionsDAG.h | 2 +- .../DataLakes/IcebergMetadata.cpp | 121 +++++++++++++++++- .../ObjectStorage/DataLakes/IcebergMetadata.h | 26 ++-- 3 files changed, 125 insertions(+), 24 deletions(-) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 1ded275fc98..ee2b3fbf4f2 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -63,7 +63,7 @@ public: struct Node; using NodeRawPtrs = std::vector; - using NodeRawConstPtrs = std::vector<`>; + using NodeRawConstPtrs = std::vector; struct Node { diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 30447e23dcf..6d208814cda 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -32,6 +32,8 @@ #include #include +# include + namespace DB { @@ -149,6 +151,15 @@ std::pair parse_decimal(const String & type_name) return {precision, scale}; } +bool operator==(const Poco::JSON::Object::Ptr & first, const Poco::JSON::Object::Ptr & second) +{ + std::stringstream ss_first; + std::stringstream ss_second; + first->stringify(ss_first); + second->stringify(ss_second); + return first.str() == second.str(); +} + DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name) { if (type_name == "boolean") @@ -428,12 +439,109 @@ NodeRawConstPtrs reconstructStruct( const auto & constant = dag->addColumn({default_type_column, name_and_type.type, name_and_type.name}); } - -std::shared_ptr getSchemaTransformDag( - [[maybe_unused]] const Poco::JSON::Object::Ptr & old_schema, [[maybe_unused]] const Poco::JSON::Object::Ptr & new_schema) +std::shared_ptr +IcebergSchemaProcessor::getSchemaTransformDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) { - std::map old_schema_entries; - std::map new_schema_entries; + std::map> old_schema_entries; + auto fields = old_schema->get("fields").extract(); + auto & outputs = dag->getOutputs(); + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + size_t id = field->getValue("id"); + auto name = field->getValue("name"); + bool required = field->getValue("required"); + old_schema_entries[id] = {field, dag->addInput(name, getFieldType(field, "type", required))}; + } + fields = new_schema->get("fields").extract(); + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + size_t id = field->getValue("id"); + auto name = field->getValue("name"); + bool required = field->getValue("required"); + auto type = getFieldType(field, "type", required); + if (old_schema_entries.count(id)) + { + } + else + { + if (field->isObject(type_key)) + { + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, "Adding a default column with id {} and complex type is not supported yet", id); + } + if (!type->isNullable()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Can't add a column with id {} with required values to the table during schema evolution", + id); + } + ColumnPtr default_type_column = name_and_type.type->createColumnConstWithDefaultValue(0); + const auto & constant = dag->addColumn({default_type_column, name_and_type.type, name_and_type.name}); + outputs.push_back(&constant); + } + LOG_DEBUG( + &Poco::Logger::get("New schema"), "Id: {}, Name: {}, type: {}", id, name, getFieldType(field, "type", required)->getName()); + } + std::shared_ptr dag = std::make_shared(); + for (const auto & [id, name_and_type] : old_schema_entries) + { + LOG_DEBUG(&Poco::Logger::get("Adding Input"), "Name: {}, type: {}", name_and_type.name, name_and_type.type->getName()); + if (new_schema_entries.count(id)) + { + if (new_schema_entries[id].type->getTypeId() + != name_and_type.type->getTypeId()) // If this exists it should be written in another manner + { + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, + "Type promotion for schema evolution is not implemeted yet: old_type: {}, new_type: {}", + name_and_type.type->getName(), + new_schema_entries[id].type->getName()); + } + if (name_and_type.name != new_schema_entries[id].name) + { + const ActionsDAG::Node & alias_node = dag->addAlias(input_node, new_schema_entries[id].name); + LOG_DEBUG( + &Poco::Logger::get("Adding Alias as Output"), + "Name: {}, type: {}", + alias_node.result_name, + alias_node.result_type->getName()); + outputs.push_back(&alias_node); + } + else + { + LOG_DEBUG( + &Poco::Logger::get("Adding Input as Ouput"), + "Name: {}, type: {}", + input_node.result_name, + input_node.result_type->getName()); + outputs.push_back(&input_node); + } + new_schema_entries.erase(id); + } + } + for (const auto & [_id, name_and_type] : new_schema_entries) + { + LOG_DEBUG(&Poco::Logger::get("Adding default"), "Name: {}, type: {}", name_and_type.name, name_and_type.type->getName()); + if (!name_and_type.type->isNullable()) + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Can't add a column with required values to the table"); + } + ColumnPtr default_type_column = name_and_type.type->createColumnConstWithDefaultValue(0); + const auto & constant = dag->addColumn({default_type_column, name_and_type.type, name_and_type.name}); + outputs.push_back(&constant); + } + return dag; +} + + +std::shared_ptr +IcebergSchemaProcessor::getSchemaTransformDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) +{ + std::map old_schema_entries; + std::map new_schema_entries; auto fields = old_schema->get("fields").extract(); for (size_t i = 0; i != fields->size(); ++i) { @@ -442,6 +550,8 @@ std::shared_ptr getSchemaTransformDag( bool required = field->getValue("required"); size_t id = field->getValue("id"); old_schema_entries[id] = {name, getFieldType(field, "type", required)}; + const ActionsDAG::Node & input_node = dag->addInput(name_and_type.name, name_and_type.type); + LOG_DEBUG( &Poco::Logger::get("Old schema"), "Id: {}, Name: {}, type: {}", id, name, getFieldType(field, "type", required)->getName()); } @@ -460,7 +570,6 @@ std::shared_ptr getSchemaTransformDag( auto& outputs = dag->getOutputs(); for (const auto& [id, name_and_type] : old_schema_entries) { LOG_DEBUG(&Poco::Logger::get("Adding Input"), "Name: {}, type: {}", name_and_type.name, name_and_type.type->getName()); - const ActionsDAG::Node & input_node = dag->addInput(name_and_type.name, name_and_type.type); if (new_schema_entries.count(id)) { if (new_schema_entries[id].type->getTypeId() != name_and_type.type->getTypeId()) // If this exists it should be written in another manner diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 7593944124c..93ffaead373 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -63,6 +63,9 @@ namespace DB * } */ +bool operator==(const Poco::JSON::Object::Ptr & first, const Poco::JSON::Object::Ptr & second); + + class IcebergSchemaProcessor { public: @@ -76,11 +79,12 @@ private: std::map, std::shared_ptr> transform_dags_by_ids; ActionsDag * current_actions_dag; - NamesAndTypeList getSchemaType(const Poco::JSON::Object::Ptr & schema); DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type); DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & type_key, bool required); DataTypePtr getSimpleType(const String & type_name); + std::shared_ptr getSchemaTransformationDag( + [[maybe_unused]] const Poco::JSON::Object::Ptr & old_schema, [[maybe_unused]] const Poco::JSON::Object::Ptr & new_schema); // DataTypePtr getStructType(const Poco::JSON::Object::Ptr & node); // DataTypePtr getListType(const Poco::JSON::Object::Ptr & node); @@ -91,26 +95,14 @@ private: // DataTypePtr getKeyType(const) const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); - const Node * getDefaultNodeForList(const Poco::JSON::Object::Ptr & field); - const Node * getDefaultNodeForMap(const Poco::JSON::Object::Ptr & field); - const Node * getDefaultNodeForStruct(const Poco::JSON::Object::Ptr & field); - - std::pair - getRemappingForList(const Poco::JSON::Object::Ptr & old_node, const Poco::JSON::Object::Ptr & new_node); - - std::pair - getRemappingForMap(const Poco::JSON::Object::Ptr & old_node, const Poco::JSON::Object::Ptr & new_node); - - std::pair - getRemappingForStruct(const Poco::JSON::Object::Ptr & old_node, const Poco::JSON::Object::Ptr & new_node); std::pair getRemappingForStructField(const Poco::JSON::Array::Ptr & old_node, const Poco::JSON::Array::Ptr & new_node, const Node * input_node); - NodeRawConstPtrs getRemappingForFields( - const Poco::JSON::Array::Ptr & old_fields, - const Poco::JSON::Array::Ptr & new_fields, - const NodeRawConstPtrs & input_action_dag_nodes); + // NodeRawConstPtrs getRemappingForFields( + // const Poco::JSON::Array::Ptr & old_fields, + // const Poco::JSON::Array::Ptr & new_fields, + // const NodeRawConstPtrs & input_action_dag_nodes); DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type_field); }; From 5ea0d16eb1c60bb6d15a0d427d3bdd1fdf9bdb26 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 5 Sep 2024 13:15:46 +0200 Subject: [PATCH 020/502] fix clang-tidy --- src/Disks/ObjectStorages/DiskObjectStorage.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index d34b94744a2..b707759a1c2 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -384,7 +384,11 @@ void DiskObjectStorage::removeSharedRecursiveWithLimit( const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) { if (remove_shared_recursive_file_limit == 0) - return removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only); + { + removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only); + return; + } + RemoveBatchRequest local_paths; std::vector directories; From e6424e432aded3008e471d4cf31f4044f8201d59 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 9 Sep 2024 15:53:26 +0000 Subject: [PATCH 021/502] Refactor code --- contrib/annoy | 1 + contrib/robin-map | 1 + .../DataLakes/IcebergMetadata.cpp | 341 +++++------------- .../ObjectStorage/DataLakes/IcebergMetadata.h | 17 +- 4 files changed, 85 insertions(+), 275 deletions(-) create mode 160000 contrib/annoy create mode 160000 contrib/robin-map diff --git a/contrib/annoy b/contrib/annoy new file mode 160000 index 00000000000..f2ac8e7b48f --- /dev/null +++ b/contrib/annoy @@ -0,0 +1 @@ +Subproject commit f2ac8e7b48f9a9cf676d3b58286e5455aba8e956 diff --git a/contrib/robin-map b/contrib/robin-map new file mode 160000 index 00000000000..851a59e0e30 --- /dev/null +++ b/contrib/robin-map @@ -0,0 +1 @@ +Subproject commit 851a59e0e3063ee0e23089062090a73fd3de482d diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 6d208814cda..4e439df6831 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -151,7 +151,7 @@ std::pair parse_decimal(const String & type_name) return {precision, scale}; } -bool operator==(const Poco::JSON::Object::Ptr & first, const Poco::JSON::Object::Ptr & second) +bool operator==(const Poco::JSON::Object::Ptr first, const Poco::JSON::Object::Ptr second) { std::stringstream ss_first; std::stringstream ss_second; @@ -205,11 +205,6 @@ DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown Iceberg type: {}", type_name); } -bool allowedDecimalConversion(const string & old_type, const string & new_type) -{ - if (!(old_type.starts_with("decimal(") && old_type.ends_with(')') && new_type.starts_with("decimal(")) -} - DataTypePtr IcebergSchemaProcessor::getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type) { String type_name = type->getValue("type"); @@ -280,86 +275,13 @@ bool IcebergSchemaProcessor::allowPrimitiveTypeConversion(const String & old_typ return allowed; } -const Node * output getRemappingForStructField( - const Poco::JSON::Array::Ptr & old_node, - const Poco::JSON::Array::Ptr & new_node, - const Node * input_node, - const String & type_name, - const String & required_name, - const std::optional & name_name) -{ - bool old_required = old_node->getValue(required_name); - bool new_required = new_node->getValue(required_name); - if (old_node->isObject("type")) - { - auto remapped = getRemappingForComplexType(old_node, new_node); - if (remapped == true) - { - } - } - chassert(!new_node->isObject("type")); - assert(new_node->get("type").isString()); - String old_type = old_node->getValue("type"); - String new_type = new_node->getValue("type"); - if (!allowPrimitiveTypeConversion(old_type, new_type)) - { - throw undefined; - } - - auto old_clickhouse_type = old_required ? getSimpleType(old_type) : makeNullable(getSimpleType(old_type)); - auto new_clickhouse_type = old_required ? getSimpleType(old_type) : makeNullable(getSimpleType(old_type)); - - auto new_name = name_name.has_value() ? new_node->getValue(name_name.value()) : input_node.name; - return current_actions_dag->addCast(input_node, new_clickhouse_type, new_name); - return getPrimitiveTypeConversed(input_node, old_type, new_type, old_required, new_required); -} - - -// std::map getRelevantSchemasByIds(const Poco::JSON::Object::Ptr & metadata_object, int format_version) -// { -// std::map schemas_by_id; -// if (format_version == 2) -// { -// auto schemas = metadata_object->get("schemas").extract(); -// if (schemas->size() == 0) -// throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: schemas field is empty"); - -// for (uint32_t i = 0; i != schemas->size(); ++i) -// { -// auto current_schema = schemas->getObject(i); -// schemas_by_id[current_schema->getValue("schema-id")] = current_schema; -// } -// } -// else -// { -// throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported format"); -// } -// return schemas_by_id; -// } - -// NamesAndTypesList getClickhouseSchemaFromIcebergSchema(const Poco::JSON::Object::Ptr & schema) -// { -// NamesAndTypesList names_and_types; -// auto fields = schema->get("fields").extract(); -// for (size_t i = 0; i != fields->size(); ++i) -// { -// auto field = fields->getObject(static_cast(i)); -// auto name = field->getValue("name"); -// bool required = field->getValue("required"); -// names_and_types.push_back({name, getFieldType(field, "type", required)}); -// } - -// return names_and_types; -// } - -std::pair parseTableSchema( - const Poco::JSON::Object::Ptr & metadata_object, - Int32 format_version, - const std::map & relevant_schema_by_id) +std::pair parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, Int32 format_version) { Poco::JSON::Object::Ptr schema; Int32 current_schema_id; + + /// First check if schema was evolved, because we don't support it yet. /// For version 2 we can check it by using field schemas, but in version 1 /// this field is optional and we will check it later during parsing manifest files @@ -384,67 +306,14 @@ std::pair parseTableSchema( return {getClickhouseSchemaFromIcebergSchema(schema), current_schema_id}; } -/* -bool isPrimitiveType(const Poco::JSON::Object::Ptr & type_key) { - return !field->isObject(type_key); -} - -bool isStructType(constPoco::JSON::Object::Ptr & type_key) { - return (!isPrimitiveType(type_key)) && type->getValue("type") == "struct"; -} - -*/ - -pair, vector<>> getStructRemapping(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) -{ - Int32 old_id = old_schema->getValue("id"); - Int32 new_id = new_schema->getValue("id"); - chassert(old_id == new_id); - auto name = old_schema->getValue("name"); - bool required = old_schema->getValue("required"); - auto old_type = {name, getFieldType(old_schema, "type", required)}; - - auto old_fields = old_schema->get("fields").extract(); - auto new_fiels = new_schema->get("fields").extract(); - std::vector old_schema_elements; - std::vector const ActionsDAG::Node & input_node = dag->addInput(name_and_type.name, name_and_type.type); - - - ColumnPtr default_type_column = name_and_type.type->createColumnConstWithDefaultValue(0); - const auto & constant = dag->addColumn({default_type_column, name_and_type.type, name_and_type.name}); -} - -const Node * IcebergSchemaProcessor::getDefaultNodeForField(const Poco::JSON::Object::Ptr & field) -{ -} - - -NodeRawConstPtrs reconstructStruct( - ActionDAG & action_dag, - const Poco::JSON::Object::Ptr & old_schema, - const Poco::JSON::Object::Ptr & new_schema, - const NodeRawConstPtrs & input_struct_nodes) -{ - auto name = old_schema->getValue("name"); - bool required = old_schema->getValue("required"); - auto old_type = {name, getFieldType(old_schema, "type", required)}; - - auto old_fields = old_schema->get("fields").extract(); - auto new_fiels = new_schema->get("fields").extract(); - std::vector old_schema_elements; - std::vector const ActionsDAG::Node & input_node = dag->addInput(name_and_type.name, name_and_type.type); - - - ColumnPtr default_type_column = name_and_type.type->createColumnConstWithDefaultValue(0); - const auto & constant = dag->addColumn({default_type_column, name_and_type.type, name_and_type.name}); -} std::shared_ptr -IcebergSchemaProcessor::getSchemaTransformDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) +IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) { std::map> old_schema_entries; auto fields = old_schema->get("fields").extract(); auto & outputs = dag->getOutputs(); + std::shared_ptr dag = std::make_shared(); for (size_t i = 0; i != fields->size(); ++i) { auto field = fields->getObject(static_cast(i)); @@ -463,6 +332,37 @@ IcebergSchemaProcessor::getSchemaTransformDag(const Poco::JSON::Object::Ptr & ol auto type = getFieldType(field, "type", required); if (old_schema_entries.count(id)) { + auto [old_json, old_node] = *old_schema_entries.find(id); + if (field->isObject("type")) + { + if (!(old_json == field)) + { + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, "Schema evolution is not supported for complex types yet, field id is {}", id); + } + } + else + { + if (old_json->isObject("type")) + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Can't cast primitive type to the complex type, field id is {}", id); + } + String old_type = old_json->getValue("type"); + String new_type = field->getValue("type"); + const ActionsDAG::Node & node = old_node; + if (old_type == new_type) + { + if (old_name != new_name) + { + node = dag->addAlias(old_node, name); + } + } + else if (allowPrimitiveTypeConversion(old_type, new_type)) + { + node = dag->addCast(old_node, new_type, name); + } + outputs.push_back(&node); + } } else { @@ -478,139 +378,65 @@ IcebergSchemaProcessor::getSchemaTransformDag(const Poco::JSON::Object::Ptr & ol "Can't add a column with id {} with required values to the table during schema evolution", id); } - ColumnPtr default_type_column = name_and_type.type->createColumnConstWithDefaultValue(0); - const auto & constant = dag->addColumn({default_type_column, name_and_type.type, name_and_type.name}); + ColumnPtr default_type_column = type->createColumnConstWithDefaultValue(0); + const auto & constant = dag->addColumn({default_type_column, type, name}); outputs.push_back(&constant); } LOG_DEBUG( &Poco::Logger::get("New schema"), "Id: {}, Name: {}, type: {}", id, name, getFieldType(field, "type", required)->getName()); } - std::shared_ptr dag = std::make_shared(); - for (const auto & [id, name_and_type] : old_schema_entries) - { - LOG_DEBUG(&Poco::Logger::get("Adding Input"), "Name: {}, type: {}", name_and_type.name, name_and_type.type->getName()); - if (new_schema_entries.count(id)) - { - if (new_schema_entries[id].type->getTypeId() - != name_and_type.type->getTypeId()) // If this exists it should be written in another manner - { - throw Exception( - ErrorCodes::UNSUPPORTED_METHOD, - "Type promotion for schema evolution is not implemeted yet: old_type: {}, new_type: {}", - name_and_type.type->getName(), - new_schema_entries[id].type->getName()); - } - if (name_and_type.name != new_schema_entries[id].name) - { - const ActionsDAG::Node & alias_node = dag->addAlias(input_node, new_schema_entries[id].name); - LOG_DEBUG( - &Poco::Logger::get("Adding Alias as Output"), - "Name: {}, type: {}", - alias_node.result_name, - alias_node.result_type->getName()); - outputs.push_back(&alias_node); - } - else - { - LOG_DEBUG( - &Poco::Logger::get("Adding Input as Ouput"), - "Name: {}, type: {}", - input_node.result_name, - input_node.result_type->getName()); - outputs.push_back(&input_node); - } - new_schema_entries.erase(id); - } - } - for (const auto & [_id, name_and_type] : new_schema_entries) - { - LOG_DEBUG(&Poco::Logger::get("Adding default"), "Name: {}, type: {}", name_and_type.name, name_and_type.type->getName()); - if (!name_and_type.type->isNullable()) - { - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Can't add a column with required values to the table"); - } - ColumnPtr default_type_column = name_and_type.type->createColumnConstWithDefaultValue(0); - const auto & constant = dag->addColumn({default_type_column, name_and_type.type, name_and_type.name}); - outputs.push_back(&constant); - } return dag; } - -std::shared_ptr -IcebergSchemaProcessor::getSchemaTransformDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) +std::shared_ptr getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id) { - std::map old_schema_entries; - std::map new_schema_entries; - auto fields = old_schema->get("fields").extract(); - for (size_t i = 0; i != fields->size(); ++i) + Poco::JSON::Object::Ptr old_schema, new_schema; + try { - auto field = fields->getObject(static_cast(i)); - auto name = field->getValue("name"); - bool required = field->getValue("required"); - size_t id = field->getValue("id"); - old_schema_entries[id] = {name, getFieldType(field, "type", required)}; - const ActionsDAG::Node & input_node = dag->addInput(name_and_type.name, name_and_type.type); + old_schema = iceberg_table_schemas_by_ids.at(old_id); + } + catch (std::exception &) + { + throw Exception(ErrorCodes::BAD_ARGUMENT, "Schema with schema-id {} is unknown", old_id); + } + try + { + new_schema = iceberg_table_schemas_by_ids.at(new_id); + } + catch (std::exception &) + { + throw Exception(ErrorCodes::BAD_ARGUMENT, "Schema with schema-id {} is unknown", new_id); + } + if (old_schema == new_schema) + { + return nullptr; + } + return getSchemaTransformationDagByIds(old_schema, new_schema); +} - LOG_DEBUG( - &Poco::Logger::get("Old schema"), "Id: {}, Name: {}, type: {}", id, name, getFieldType(field, "type", required)->getName()); - } - fields = new_schema->get("fields").extract(); - for (size_t i = 0; i != fields->size(); ++i) +void addIcebergTableSchema(Poco::JSON::Object::Ptr schema) +{ + Int32 schema_id = schema->get("schema-id").extract(); + if (fields.count(schema_id)) { - auto field = fields->getObject(static_cast(i)); - auto name = field->getValue("name"); - bool required = field->getValue("required"); - size_t id = field->getValue("id"); - new_schema_entries[id] = {name, getFieldType(field, "type", required)}; - LOG_DEBUG( - &Poco::Logger::get("New schema"), "Id: {}, Name: {}, type: {}", id, name, getFieldType(field, "type", required)->getName()); + chassert(fields.at(schema_id) == schema); } - std::shared_ptr dag = std::make_shared(); - auto& outputs = dag->getOutputs(); - for (const auto& [id, name_and_type] : old_schema_entries) { - LOG_DEBUG(&Poco::Logger::get("Adding Input"), "Name: {}, type: {}", name_and_type.name, name_and_type.type->getName()); - if (new_schema_entries.count(id)) { - if (new_schema_entries[id].type->getTypeId() - != name_and_type.type->getTypeId()) // If this exists it should be written in another manner - { - throw Exception( - ErrorCodes::UNSUPPORTED_METHOD, - "Type promotion for schema evolution is not implemeted yet: old_type: {}, new_type: {}", - name_and_type.type->getName(), - new_schema_entries[id].type->getName()); - } - if (name_and_type.name != new_schema_entries[id].name) { - const ActionsDAG::Node & alias_node = dag->addAlias(input_node, new_schema_entries[id].name); - LOG_DEBUG( - &Poco::Logger::get("Adding Alias as Output"), - "Name: {}, type: {}", - alias_node.result_name, - alias_node.result_type->getName()); - outputs.push_back(&alias_node); - } else { - LOG_DEBUG( - &Poco::Logger::get("Adding Input as Ouput"), - "Name: {}, type: {}", - input_node.result_name, - input_node.result_type->getName()); - outputs.push_back(&input_node); - } - new_schema_entries.erase(id); - } + else + { + fields[schema_id] = schema; } - for (const auto& [_id, name_and_type] : new_schema_entries) { - LOG_DEBUG(&Poco::Logger::get("Adding default"), "Name: {}, type: {}", name_and_type.name, name_and_type.type->getName()); - if (!name_and_type.type->isNullable()) { - throw Exception( - ErrorCodes::UNSUPPORTED_METHOD, - "Can't add a column with required values to the table"); - } - ColumnPtr default_type_column = name_and_type.type->createColumnConstWithDefaultValue(0); - const auto & constant = dag->addColumn({default_type_column, name_and_type.type, name_and_type.name}); - outputs.push_back(&constant); +} + +NamesAndTypesList getClickhouseTableSchemaById(Int32 id) +{ + try + { + return iceberg_table_schemas_by_ids.at(id); + } + catch (std::exception &) + { + throw Exception(ErrorCodes::BAD_ARGUMENT, "Schema with id {} is unknown", id); } - return dag; } MutableColumns parseAvro(avro::DataFileReaderBase & file_reader, const Block & header, const FormatSettings & settings) @@ -676,7 +502,6 @@ DataLakeMetadataPtr IcebergMetadata::create( ContextPtr local_context) { const auto [metadata_version, metadata_file_path] = getMetadataFileAndVersion(object_storage, *configuration); - LOG_DEBUG(getLogger("IcebergMetadata"), "Parse metadata {}", metadata_file_path); auto read_settings = local_context->getReadSettings(); auto buf = object_storage->readObject(StoredObject(metadata_file_path), read_settings); String json_str; @@ -686,11 +511,9 @@ DataLakeMetadataPtr IcebergMetadata::create( Poco::Dynamic::Var json = parser.parse(json_str); Poco::JSON::Object::Ptr object = json.extract(); - auto format_version = object->getValue("format-version"); - auto relevant_schemas_by_ids = getRelevantSchemasByIds(object, format_version); - auto [schema, schema_id] = parseTableSchema(object, format_version, relevant_schemas_by_ids); + auto schema = parseTableSchema(object); auto current_snapshot_id = object->getValue("current-snapshot-id"); auto snapshots = object->get("snapshots").extract(); diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 93ffaead373..777ded03e1b 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -77,7 +77,6 @@ private: std::map iceberg_table_schemas_by_ids; std::map clickhouse_table_schemas_by_ids; std::map, std::shared_ptr> transform_dags_by_ids; - ActionsDag * current_actions_dag; NamesAndTypeList getSchemaType(const Poco::JSON::Object::Ptr & schema); DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type); @@ -86,24 +85,11 @@ private: std::shared_ptr getSchemaTransformationDag( [[maybe_unused]] const Poco::JSON::Object::Ptr & old_schema, [[maybe_unused]] const Poco::JSON::Object::Ptr & new_schema); - // DataTypePtr getStructType(const Poco::JSON::Object::Ptr & node); - // DataTypePtr getListType(const Poco::JSON::Object::Ptr & node); - // DataTypePtr getMapType(const Poco::JSON::Object::Ptr & node); - // DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & node); - // DataTypePtr getSimpleType(const String & type); - // DataTypePtr getElementType(const Poco::JSON::Object::Ptr & node); - // DataTypePtr getKeyType(const) - const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); std::pair getRemappingForStructField(const Poco::JSON::Array::Ptr & old_node, const Poco::JSON::Array::Ptr & new_node, const Node * input_node); - // NodeRawConstPtrs getRemappingForFields( - // const Poco::JSON::Array::Ptr & old_fields, - // const Poco::JSON::Array::Ptr & new_fields, - // const NodeRawConstPtrs & input_action_dag_nodes); - DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type_field); }; @@ -160,8 +146,7 @@ private: mutable DataFileInfos data_file_infos; std::unordered_map column_name_to_physical_name; DataLakePartitionColumns partition_columns; - mutable std::map relevant_schemas_by_ids; - mutable std::map> transform_dags_by_ids; + mutable IcebergSchemaProcessor schema_processor{}; LoggerPtr log; }; From ad2f5b55650a116eace5f2be1e58493f109e744f Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 10 Sep 2024 14:02:56 +0000 Subject: [PATCH 022/502] Deliver feature --- src/Storages/ObjectStorage/DataFileInfo.h | 4 +- .../DataLakes/IcebergMetadata.cpp | 227 +++++++++--------- .../ObjectStorage/DataLakes/IcebergMetadata.h | 30 ++- .../ObjectStorage/StorageObjectStorage.h | 4 +- .../StorageObjectStorageSource.cpp | 2 +- 5 files changed, 136 insertions(+), 131 deletions(-) diff --git a/src/Storages/ObjectStorage/DataFileInfo.h b/src/Storages/ObjectStorage/DataFileInfo.h index 794815f2020..c9d854d90f2 100644 --- a/src/Storages/ObjectStorage/DataFileInfo.h +++ b/src/Storages/ObjectStorage/DataFileInfo.h @@ -12,12 +12,12 @@ struct DataFileInfo { String data_path; std::shared_ptr initial_schema; - std::shared_ptr schema_transform; + std::shared_ptr schema_transform; DataFileInfo( String data_path_, std::shared_ptr initial_schema_ = nullptr, - std::shared_ptr schema_transform_ = nullptr) + std::shared_ptr schema_transform_ = nullptr) : data_path(data_path_), initial_schema(initial_schema_), schema_transform(schema_transform_) { } diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 4e439df6831..f7225c18b28 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -54,8 +54,7 @@ IcebergMetadata::IcebergMetadata( Int32 format_version_, String manifest_list_file_, Int32 current_schema_id_, - DB::NamesAndTypesList schema_, - std::map relevant_schemas_by_ids_) + IcebergSchemaProcessor schema_processor_) : WithContext(context_) , object_storage(object_storage_) , configuration(configuration_) @@ -63,8 +62,8 @@ IcebergMetadata::IcebergMetadata( , format_version(format_version_) , manifest_list_file(std::move(manifest_list_file_)) , current_schema_id(current_schema_id_) - , schema(std::move(schema_)) - , relevant_schemas_by_ids(std::move(relevant_schemas_by_ids_)) + , schema_processor(schema_processor_) + , schema(*schema_processor.getClickhouseTableSchemaById(current_schema_id)) , log(getLogger("IcebergMetadata")) { } @@ -151,13 +150,14 @@ std::pair parse_decimal(const String & type_name) return {precision, scale}; } -bool operator==(const Poco::JSON::Object::Ptr first, const Poco::JSON::Object::Ptr second) +bool operator==(const Poco::JSON::Object & first, const Poco::JSON::Object & second) { std::stringstream ss_first; std::stringstream ss_second; - first->stringify(ss_first); - second->stringify(ss_second); - return first.str() == second.str(); + first.stringify(ss_first); + second.stringify(ss_second); + return ss_first.str() == ss_second.str(); +} } DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name) @@ -174,8 +174,6 @@ DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name) return std::make_shared(); if (type_name == "date") return std::make_shared(); - /// Time type represents time of the day in microseconds since midnight. - /// We don't have similar type for it, let's use just Int64. if (type_name == "time") return std::make_shared(); if (type_name == "timestamp") @@ -253,7 +251,7 @@ DataTypePtr IcebergSchemaProcessor::getFieldType(const Poco::JSON::Object::Ptr & if (type.isString()) { const String & type_name = type.extract(); - auto data_type = getSimpleTypeByName(type_name); + auto data_type = getSimpleType(type_name); return required ? data_type : makeNullable(data_type); } @@ -264,63 +262,52 @@ DataTypePtr IcebergSchemaProcessor::getFieldType(const Poco::JSON::Object::Ptr & bool IcebergSchemaProcessor::allowPrimitiveTypeConversion(const String & old_type, const String & new_type) { bool allowed_type_conversion = (old_type == new_type); - allowed |= (old_type == "int") && (new_type == "long"); - allowed |= (old_type == "float") && (new_type == "double"); + allowed_type_conversion |= (old_type == "int") && (new_type == "long"); + allowed_type_conversion |= (old_type == "float") && (new_type == "double"); if (old_type.starts_with("decimal(") && old_type.ends_with(')') && new_type.starts_with("decimal(") && new_type.ends_with(")")) { auto [old_precision, old_scale] = parse_decimal(old_type); auto [new_precision, new_scale] = parse_decimal(new_type); - allowed |= (old_precision <= new_precision) && (old_scale == new_scale); + allowed_type_conversion |= (old_precision <= new_precision) && (old_scale == new_scale); } - return allowed; + return allowed_type_conversion; } -std::pair parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, Int32 format_version) +Int32 parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor) { - Poco::JSON::Object::Ptr schema; - Int32 current_schema_id; - - - - /// First check if schema was evolved, because we don't support it yet. - /// For version 2 we can check it by using field schemas, but in version 1 - /// this field is optional and we will check it later during parsing manifest files - /// (we will compare schema id from manifest file and currently used schema). + Int32 format_version = metadata_object->getValue("format-version"); if (format_version == 2) { - current_schema_id = metadata_object->getValue("current-schema-id"); + auto fields = metadata_object->get("schemas").extract(); + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + schema_processor.addIcebergTableSchema(field); + } + return metadata_object->getValue("current-schema-id"); } else { - current_schema_id = schema->getValue("schema-id"); + auto schema = metadata_object->getObject("schema"); + schema_processor.addIcebergTableSchema(schema); + return schema->getValue("schema-id"); } - - try - { - schema = relevant_schema_by_id.at(current_schema_id); - } - catch (const std::exception &) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, R"(There is no schema with "schema-id" that matches "current-schema-id" in metadata)"); - } - - return {getClickhouseSchemaFromIcebergSchema(schema), current_schema_id}; } std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) { - std::map> old_schema_entries; + std::map> old_schema_entries; auto fields = old_schema->get("fields").extract(); - auto & outputs = dag->getOutputs(); std::shared_ptr dag = std::make_shared(); + auto & outputs = dag->getOutputs(); for (size_t i = 0; i != fields->size(); ++i) { auto field = fields->getObject(static_cast(i)); size_t id = field->getValue("id"); auto name = field->getValue("name"); bool required = field->getValue("required"); - old_schema_entries[id] = {field, dag->addInput(name, getFieldType(field, "type", required))}; + old_schema_entries[id] = {field, &dag->addInput(name, getFieldType(field, "type", required))}; } fields = new_schema->get("fields").extract(); for (size_t i = 0; i != fields->size(); ++i) @@ -332,51 +319,72 @@ IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr auto type = getFieldType(field, "type", required); if (old_schema_entries.count(id)) { - auto [old_json, old_node] = *old_schema_entries.find(id); + auto [old_json, old_node] = old_schema_entries.find(id)->second; if (field->isObject("type")) { - if (!(old_json == field)) + if (!(*old_json == *field)) { throw Exception( - ErrorCodes::UNSUPPORTED_METHOD, "Schema evolution is not supported for complex types yet, field id is {}", id); + ErrorCodes::UNSUPPORTED_METHOD, + "Schema evolution is not supported for complex types yet, field id is {}, old schema id is {}, new schema id is {}", + id, + current_old_id, + current_new_id); + } + else + { + outputs.push_back(old_node); } } else { if (old_json->isObject("type")) { - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Can't cast primitive type to the complex type, field id is {}", id); + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, + "Can't cast primitive type to the complex type, field id is {}, old schema id is {}, new schema id is {}", + id, + current_old_id, + current_new_id); } String old_type = old_json->getValue("type"); String new_type = field->getValue("type"); - const ActionsDAG::Node & node = old_node; + + const ActionsDAG::Node * node = old_node; if (old_type == new_type) { - if (old_name != new_name) + if (old_json->getValue("name") != name) { - node = dag->addAlias(old_node, name); + node = &dag->addAlias(*old_node, name); } } else if (allowPrimitiveTypeConversion(old_type, new_type)) { - node = dag->addCast(old_node, new_type, name); + node = &dag->addCast(*old_node, getSimpleType(new_type), name); } - outputs.push_back(&node); + outputs.push_back(node); } } else { - if (field->isObject(type_key)) + if (field->isObject("type")) { throw Exception( - ErrorCodes::UNSUPPORTED_METHOD, "Adding a default column with id {} and complex type is not supported yet", id); + ErrorCodes::UNSUPPORTED_METHOD, + "Adding a default column with id {} and complex type is not supported yet, old schema id is {}, new schema id is {}", + id, + current_old_id, + current_new_id); } if (!type->isNullable()) { throw Exception( ErrorCodes::BAD_ARGUMENTS, - "Can't add a column with id {} with required values to the table during schema evolution", - id); + "Can't add a column with id {} with required values to the table during schema evolution, old schema id is {}, new " + "schema id is {}", + id, + current_old_id, + current_new_id); } ColumnPtr default_type_column = type->createColumnConstWithDefaultValue(0); const auto & constant = dag->addColumn({default_type_column, type, name}); @@ -388,16 +396,26 @@ IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr return dag; } -std::shared_ptr getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id) +std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id) { + current_old_id = old_id; + current_new_id = new_id; Poco::JSON::Object::Ptr old_schema, new_schema; + if (transform_dags_by_ids.count({old_id, new_id})) + { + return transform_dags_by_ids.at({old_id, new_id}); + } try { old_schema = iceberg_table_schemas_by_ids.at(old_id); } catch (std::exception &) { - throw Exception(ErrorCodes::BAD_ARGUMENT, "Schema with schema-id {} is unknown", old_id); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", old_id); + } + if (old_id == new_id) + { + return nullptr; } try { @@ -405,37 +423,46 @@ std::shared_ptr getSchemaTransformationDagByIds(Int32 old_id, } catch (std::exception &) { - throw Exception(ErrorCodes::BAD_ARGUMENT, "Schema with schema-id {} is unknown", new_id); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", new_id); } - if (old_schema == new_schema) - { - return nullptr; - } - return getSchemaTransformationDagByIds(old_schema, new_schema); + current_old_id = -1; + current_new_id = -1; + return transform_dags_by_ids[{old_id, new_id}] = getSchemaTransformationDag(old_schema, new_schema); } -void addIcebergTableSchema(Poco::JSON::Object::Ptr schema) +void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schema) { - Int32 schema_id = schema->get("schema-id").extract(); - if (fields.count(schema_id)) + Int32 schema_id = schema->getValue("schema-id"); + if (iceberg_table_schemas_by_ids.count(schema_id)) { - chassert(fields.at(schema_id) == schema); + chassert(clickhouse_table_schemas_by_ids.count(schema_id) > 0); + chassert(*iceberg_table_schemas_by_ids.at(schema_id) == *schema); } else { - fields[schema_id] = schema; + iceberg_table_schemas_by_ids[schema_id] = schema; + auto fields = schema->get("fields").extract(); + auto clickhouse_schema = std::make_shared(); + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + auto name = field->getValue("name"); + bool required = field->getValue("required"); + clickhouse_schema->push_back(NameAndTypePair{name, getFieldType(field, "type", required)}); + } + clickhouse_table_schemas_by_ids[schema_id] = clickhouse_schema; } } -NamesAndTypesList getClickhouseTableSchemaById(Int32 id) +std::shared_ptr IcebergSchemaProcessor::getClickhouseTableSchemaById(Int32 id) { try { - return iceberg_table_schemas_by_ids.at(id); + return clickhouse_table_schemas_by_ids.at(id); } catch (std::exception &) { - throw Exception(ErrorCodes::BAD_ARGUMENT, "Schema with id {} is unknown", id); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with id {} is unknown", id); } } @@ -494,7 +521,7 @@ std::pair getMetadataFileAndVersion( /// Get the latest version of metadata file: v.metadata.json return *std::max_element(metadata_files_with_versions.begin(), metadata_files_with_versions.end()); } -} + DataLakeMetadataPtr IcebergMetadata::create( ObjectStoragePtr object_storage, @@ -511,9 +538,9 @@ DataLakeMetadataPtr IcebergMetadata::create( Poco::Dynamic::Var json = parser.parse(json_str); Poco::JSON::Object::Ptr object = json.extract(); - auto relevant_schemas_by_ids = getRelevantSchemasByIds(object, format_version); + IcebergSchemaProcessor schema_processor; - auto schema = parseTableSchema(object); + auto schema_id = parseTableSchema(object, schema_processor); auto current_snapshot_id = object->getValue("current-snapshot-id"); auto snapshots = object->get("snapshots").extract(); @@ -530,16 +557,10 @@ DataLakeMetadataPtr IcebergMetadata::create( } } + Int32 format_version = object->getValue("format-version"); + return std::make_unique( - object_storage, - configuration, - local_context, - metadata_version, - format_version, - manifest_list_file, - schema_id, - schema, - relevant_schemas_by_ids); + object_storage, configuration, local_context, metadata_version, format_version, manifest_list_file, schema_id, schema_processor); } /** @@ -746,42 +767,22 @@ DataFileInfos IcebergMetadata::getDataFileInfos() const } } - if (!relevant_schemas_by_ids.count(schema_object_id)) - { - relevant_schemas_by_ids[schema_object_id] = schema_object; - } - if (!transform_dags_by_ids.count(schema_object_id)) - { - transform_dags_by_ids[schema_object_id] - = getSchemaTransformDag(relevant_schemas_by_ids[schema_object_id], relevant_schemas_by_ids[current_schema_id]); - } + schema_processor.addIcebergTableSchema(schema_object); } for (const auto & [file_path, schema_object_id] : files) { - if (current_schema_id == schema_object_id) - { - data_file_infos.emplace_back(DataFileInfo{file_path}); - } - else - { - LOG_DEBUG( - &Poco::Logger::get("Adding not empty stuff"), - "File path: {}, Old schema: {}, New schema: {}", - file_path, - schema_object_id, - current_schema_id); - if (!transform_dags_by_ids.count(schema_object_id)) - { - transform_dags_by_ids[schema_object_id] - = getSchemaTransformDag(relevant_schemas_by_ids[schema_object_id], relevant_schemas_by_ids[current_schema_id]); - } - data_file_infos.emplace_back( - file_path, - std::make_shared(getClickhouseSchemaFromIcebergSchema(relevant_schemas_by_ids[schema_object_id])), - transform_dags_by_ids[schema_object_id]); - } + LOG_DEBUG( + &Poco::Logger::get("Adding not empty stuff"), + "File path: {}, Old schema: {}, New schema: {}", + file_path, + schema_object_id, + current_schema_id); + data_file_infos.emplace_back( + file_path, + schema_processor.getClickhouseTableSchemaById(schema_object_id), + schema_processor.getSchemaTransformationDagByIds(schema_object_id, current_schema_id)); } return data_file_infos; diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 777ded03e1b..bc6adfd9ac4 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -63,34 +63,39 @@ namespace DB * } */ -bool operator==(const Poco::JSON::Object::Ptr & first, const Poco::JSON::Object::Ptr & second); +// bool operator==(const Poco::JSON::Object::Ptr & first, const Poco::JSON::Object::Ptr & second); class IcebergSchemaProcessor { + using Node = ActionsDAG::Node; + public: - void addIcebergTableSchema(const Poco::JSON::Object::Ptr & ptr); - NamesAndTypesList getClickhouseTableSchemaById(Int32 id); - std::shared_ptr getTransformationDagByIds(Int32 old_id, Int32 new_id); + void addIcebergTableSchema(Poco::JSON::Object::Ptr ptr); + std::shared_ptr getClickhouseTableSchemaById(Int32 id); + std::shared_ptr getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id); private: std::map iceberg_table_schemas_by_ids; - std::map clickhouse_table_schemas_by_ids; + std::map> clickhouse_table_schemas_by_ids; std::map, std::shared_ptr> transform_dags_by_ids; - NamesAndTypeList getSchemaType(const Poco::JSON::Object::Ptr & schema); + NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr & schema); DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type); DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & type_key, bool required); DataTypePtr getSimpleType(const String & type_name); std::shared_ptr getSchemaTransformationDag( [[maybe_unused]] const Poco::JSON::Object::Ptr & old_schema, [[maybe_unused]] const Poco::JSON::Object::Ptr & new_schema); + bool allowPrimitiveTypeConversion(const String & old_type, const String & new_type); + const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); - std::pair - getRemappingForStructField(const Poco::JSON::Array::Ptr & old_node, const Poco::JSON::Array::Ptr & new_node, const Node * input_node); + Int32 current_old_id = -1; + Int32 current_new_id = -1; - DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type_field); + // std::pair + // getRemappingForStructField(const Poco::JSON::Array::Ptr & old_node, const Poco::JSON::Array::Ptr & new_node, const Node * input_node); }; class IcebergMetadata : public IDataLakeMetadata, private WithContext @@ -108,8 +113,7 @@ public: Int32 format_version_, String manifest_list_file_, Int32 current_schema_id_, - NamesAndTypesList schema_, - std::map relevant_schemas_by_ids_); + IcebergSchemaProcessor schema_processor); /// Get data files. On first request it reads manifest_list file and iterates through manifest files to find all data files. /// All subsequent calls will return saved list of files (because it cannot be changed without changing metadata file) @@ -142,11 +146,11 @@ private: Int32 format_version; String manifest_list_file; const Int32 current_schema_id; - NamesAndTypesList schema; mutable DataFileInfos data_file_infos; std::unordered_map column_name_to_physical_name; DataLakePartitionColumns partition_columns; - mutable IcebergSchemaProcessor schema_processor{}; + mutable IcebergSchemaProcessor schema_processor; + NamesAndTypesList schema; LoggerPtr log; }; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index b69347991b9..0171a0ef27a 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -39,7 +39,7 @@ public: String relative_path_, std::optional metadata_ = std::nullopt, std::shared_ptr initial_schema_ = nullptr, - std::shared_ptr schema_transformer_ = nullptr) + std::shared_ptr schema_transformer_ = nullptr) : RelativePathWithMetadata(std::move(relative_path_), std::move(metadata_)) , initial_schema(std::move(initial_schema_)) , schema_transformer(std::move(schema_transformer_)) @@ -50,7 +50,7 @@ public: public: std::shared_ptr initial_schema; - std::shared_ptr schema_transformer; + std::shared_ptr schema_transformer; }; using ObjectInfoPtr = std::shared_ptr; using ObjectInfos = std::vector; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index d52e5318ba3..56bb77dc224 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -369,7 +369,7 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade /*Temporary desicion. Need better anstraction here.*/ Block initial_header = read_from_format_info.format_header; - if (object_info->schema_transformer) + if (object_info->initial_schema) { Block sample_header; for (const auto & [name, type] : *object_info->initial_schema) From 7b4721be34b9fb356cde10f0a2d18f90c5f723f3 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 10 Sep 2024 14:15:09 +0000 Subject: [PATCH 023/502] Remove unnecessary changes --- src/Core/SettingsChangesHistory.cpp | 8 ++++---- src/Storages/VirtualColumnUtils.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 9666ea0850d..19f2d5ccdf0 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -76,12 +76,14 @@ static std::initializer_list Date: Tue, 10 Sep 2024 14:17:34 +0000 Subject: [PATCH 024/502] Remove submodules --- contrib/annoy | 1 - contrib/robin-map | 1 - 2 files changed, 2 deletions(-) delete mode 160000 contrib/annoy delete mode 160000 contrib/robin-map diff --git a/contrib/annoy b/contrib/annoy deleted file mode 160000 index f2ac8e7b48f..00000000000 --- a/contrib/annoy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f2ac8e7b48f9a9cf676d3b58286e5455aba8e956 diff --git a/contrib/robin-map b/contrib/robin-map deleted file mode 160000 index 851a59e0e30..00000000000 --- a/contrib/robin-map +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 851a59e0e3063ee0e23089062090a73fd3de482d From d06532734a7fd4ac42bf7386055112a612b8992a Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 11 Sep 2024 15:02:25 +0000 Subject: [PATCH 025/502] Finish the feature --- .../table-engines/integrations/iceberg.md | 9 + .../sql-reference/table-functions/iceberg.md | 8 + .../DataLakes/IStorageDataLake.h | 1 + .../DataLakes/IcebergMetadata.cpp | 99 ++-- .../ObjectStorage/DataLakes/IcebergMetadata.h | 121 +++-- .../StorageObjectStorageSource.cpp | 25 +- .../integration/test_storage_iceberg/test.py | 489 ++++++++++++++++-- 7 files changed, 581 insertions(+), 171 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md index 94468066372..e4a5de327bc 100644 --- a/docs/en/engines/table-engines/integrations/iceberg.md +++ b/docs/en/engines/table-engines/integrations/iceberg.md @@ -60,6 +60,15 @@ CREATE TABLE iceberg_table ENGINE=IcebergS3(iceberg_conf, filename = 'test_table Table engine `Iceberg` is an alias to `IcebergS3` now. +**Schema Evolution** +At the moment, with the help of CH, you can read iceberg tables, the schema of which has changed over time. We currently support reading tables where columns have been added and removed, and their order has changed. You can also change a column where a value is required to one where NULL is allowed. Additionally, we support permitted type casting for simple types, namely:   +* int -> long +* float -> double +* decimal(P, S) -> decimal(P', S) where P' > P. + +Currently, it is not possible to change nested structures or the types of elements within arrays and maps. + + ## See also - [iceberg table function](/docs/en/sql-reference/table-functions/iceberg.md) diff --git a/docs/en/sql-reference/table-functions/iceberg.md b/docs/en/sql-reference/table-functions/iceberg.md index 784fd646860..83e083f96cd 100644 --- a/docs/en/sql-reference/table-functions/iceberg.md +++ b/docs/en/sql-reference/table-functions/iceberg.md @@ -62,6 +62,14 @@ SELECT * FROM icebergS3(iceberg_conf, filename = 'test_table') DESCRIBE icebergS3(iceberg_conf, filename = 'test_table') ``` +**Schema Evolution** +At the moment, with the help of CH, you can read iceberg tables, the schema of which has changed over time. We currently support reading tables where columns have been added and removed, and their order has changed. You can also change a column where a value is required to one where NULL is allowed. Additionally, we support permitted type casting for simple types, namely:   +* int -> long +* float -> double +* decimal(P, S) -> decimal(P', S) where P' > P. + +Currently, it is not possible to change nested structures or the types of elements within arrays and maps. + **Aliases** Table function `iceberg` is an alias to `icebergS3` now. diff --git a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h index d39a662d265..ccb273fd25e 100644 --- a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h +++ b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h @@ -110,6 +110,7 @@ public: updated_configuration->setPaths(current_metadata->getDataFileInfos()); updated_configuration->setPartitionColumns(current_metadata->getPartitionColumns()); + Storage::configuration = updated_configuration; } diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index f7225c18b28..ac9dd019ca5 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -44,6 +44,7 @@ extern const int FILE_DOESNT_EXIST; extern const int ILLEGAL_COLUMN; extern const int BAD_ARGUMENTS; extern const int UNSUPPORTED_METHOD; +extern const int LOGICAL_ERROR: } IcebergMetadata::IcebergMetadata( @@ -85,58 +86,6 @@ enum class DataFileContent : uint8_t EQUALITY_DELETES = 2, }; -/** - * Iceberg supports the next data types (see https://iceberg.apache.org/spec/#schemas-and-data-types): - * - Primitive types: - * - boolean - * - int - * - long - * - float - * - double - * - decimal(P, S) - * - date - * - time (time of day in microseconds since midnight) - * - timestamp (in microseconds since 1970-01-01) - * - timestamptz (timestamp with timezone, stores values in UTC timezone) - * - string - * - uuid - * - fixed(L) (fixed-length byte array of length L) - * - binary - * - Complex types: - * - struct(field1: Type1, field2: Type2, ...) (tuple of typed values) - * - list(nested_type) - * - map(Key, Value) - * - * Example of table schema in metadata: - * { - * "type" : "struct", - * "schema-id" : 0, - * "fields" : [ - * { - * "id" : 1, - * "name" : "id", - * "required" : false, - * "type" : "long" - * }, - * { - * "id" : 2, - * "name" : "array", - * "required" : false, - * "type" : { - * "type" : "list", - * "element-id" : 5, - * "element" : "int", - * "element-required" : false - * }, - * { - * "id" : 3, - * "name" : "data", - * "required" : false, - * "type" : "binary" - * } - * } - */ - std::pair parse_decimal(const String & type_name) { ReadBufferFromString buf(std::string_view(type_name.begin() + 8, type_name.end() - 1)); @@ -152,11 +101,19 @@ std::pair parse_decimal(const String & type_name) bool operator==(const Poco::JSON::Object & first, const Poco::JSON::Object & second) { - std::stringstream ss_first; - std::stringstream ss_second; - first.stringify(ss_first); - second.stringify(ss_second); - return ss_first.str() == ss_second.str(); + std::stringstream first_string_stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + std::stringstream second_string_stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + first.stringify(first_string_stream); + if (!first_string_stream) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "JSON Parsing failed"); + } + second.stringify(second_string_stream); + if (!second_string_stream) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "JSON Parsing failed"); + } + return first_string_stream.str() == second_string_stream.str(); } } @@ -259,6 +216,14 @@ DataTypePtr IcebergSchemaProcessor::getFieldType(const Poco::JSON::Object::Ptr & } + +/** +* Iceberg allows only three types of primitive type conversion: +* int -> long +* float -> double +* decimal(P, S) -> decimal(P', S) where P' > P +* This function checks if `old_type` and `new_type` satisfy to one of these conditions. +**/ bool IcebergSchemaProcessor::allowPrimitiveTypeConversion(const String & old_type, const String & new_type) { bool allowed_type_conversion = (old_type == new_type); @@ -298,21 +263,21 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) { std::map> old_schema_entries; - auto fields = old_schema->get("fields").extract(); + auto old_schema_fields = old_schema->get("fields").extract(); std::shared_ptr dag = std::make_shared(); auto & outputs = dag->getOutputs(); - for (size_t i = 0; i != fields->size(); ++i) + for (size_t i = 0; i != old_schema_fields->size(); ++i) { - auto field = fields->getObject(static_cast(i)); + auto field = old_schema_fields->getObject(static_cast(i)); size_t id = field->getValue("id"); auto name = field->getValue("name"); bool required = field->getValue("required"); old_schema_entries[id] = {field, &dag->addInput(name, getFieldType(field, "type", required))}; } - fields = new_schema->get("fields").extract(); - for (size_t i = 0; i != fields->size(); ++i) + auto new_schema_fields = new_schema->get("fields").extract(); + for (size_t i = 0; i != new_schema_fields->size(); ++i) { - auto field = fields->getObject(static_cast(i)); + auto field = new_schema_fields->getObject(static_cast(i)); size_t id = field->getValue("id"); auto name = field->getValue("name"); bool required = field->getValue("required"); @@ -380,7 +345,7 @@ IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr { throw Exception( ErrorCodes::BAD_ARGUMENTS, - "Can't add a column with id {} with required values to the table during schema evolution, old schema id is {}, new " + "Canßnot add a column with id {} with required values to the table during schema evolution, old schema id is {}, new " "schema id is {}", id, current_old_id, @@ -425,8 +390,10 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformatio { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", new_id); } - current_old_id = -1; - current_new_id = -1; + SCOPE_EXIT({ + current_old_id = -1; + current_new_id = -1; + }); return transform_dags_by_ids[{old_id, new_id}] = getSchemaTransformationDag(old_schema, new_schema); } diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index bc6adfd9ac4..2d93dea9fc9 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -15,6 +15,91 @@ namespace DB { + +/** + * Iceberg supports the next data types (see https://iceberg.apache.org/spec/#schemas-and-data-types): + * - Primitive types: + * - boolean + * - int + * - long + * - float + * - double + * - decimal(P, S) + * - date + * - time (time of day in microseconds since midnight) + * - timestamp (in microseconds since 1970-01-01) + * - timestamptz (timestamp with timezone, stores values in UTC timezone) + * - string + * - uuid + * - fixed(L) (fixed-length byte array of length L) + * - binary + * - Complex types: + * - struct(field1: Type1, field2: Type2, ...) (tuple of typed values) + * - list(nested_type) + * - map(Key, Value) + * + * Example of table schema in metadata: + * { + * "type" : "struct", + * "schema-id" : 0, + * "fields" : [ + * { + * "id" : 1, + * "name" : "id", + * "required" : false, + * "type" : "long" + * }, + * { + * "id" : 2, + * "name" : "array", + * "required" : false, + * "type" : { + * "type" : "list", + * "element-id" : 5, + * "element" : "int", + * "element-required" : false + * }, + * { + * "id" : 3, + * "name" : "data", + * "required" : false, + * "type" : "binary" + * } + * } + */ +class IcebergSchemaProcessor +{ + using Node = ActionsDAG::Node; + +public: + void addIcebergTableSchema(Poco::JSON::Object::Ptr ptr); + std::shared_ptr getClickhouseTableSchemaById(Int32 id); + std::shared_ptr getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id); + +private: + std::map iceberg_table_schemas_by_ids; + std::map> clickhouse_table_schemas_by_ids; + std::map, std::shared_ptr> transform_dags_by_ids; + + NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr & schema); + DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type); + DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & type_key, bool required); + DataTypePtr getSimpleType(const String & type_name); + std::shared_ptr getSchemaTransformationDag( + [[maybe_unused]] const Poco::JSON::Object::Ptr & old_schema, [[maybe_unused]] const Poco::JSON::Object::Ptr & new_schema); + + bool allowPrimitiveTypeConversion(const String & old_type, const String & new_type); + + const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); + + Int32 current_old_id = -1; + Int32 current_new_id = -1; + + // std::pair + // getRemappingForStructField(const Poco::JSON::Array::Ptr & old_node, const Poco::JSON::Array::Ptr & new_node, const Node * input_node); +}; + + /** * Useful links: * - https://iceberg.apache.org/spec/ @@ -62,42 +147,6 @@ namespace DB * "metadata-log" : [ ] * } */ - -// bool operator==(const Poco::JSON::Object::Ptr & first, const Poco::JSON::Object::Ptr & second); - - -class IcebergSchemaProcessor -{ - using Node = ActionsDAG::Node; - -public: - void addIcebergTableSchema(Poco::JSON::Object::Ptr ptr); - std::shared_ptr getClickhouseTableSchemaById(Int32 id); - std::shared_ptr getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id); - -private: - std::map iceberg_table_schemas_by_ids; - std::map> clickhouse_table_schemas_by_ids; - std::map, std::shared_ptr> transform_dags_by_ids; - - NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr & schema); - DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type); - DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & type_key, bool required); - DataTypePtr getSimpleType(const String & type_name); - std::shared_ptr getSchemaTransformationDag( - [[maybe_unused]] const Poco::JSON::Object::Ptr & old_schema, [[maybe_unused]] const Poco::JSON::Object::Ptr & new_schema); - - bool allowPrimitiveTypeConversion(const String & old_type, const String & new_type); - - const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); - - Int32 current_old_id = -1; - Int32 current_new_id = -1; - - // std::pair - // getRemappingForStructField(const Poco::JSON::Array::Ptr & old_node, const Poco::JSON::Array::Ptr & new_node, const Node * input_node); -}; - class IcebergMetadata : public IDataLakeMetadata, private WithContext { public: diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 56bb77dc224..e58f357cf47 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -366,7 +366,7 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade read_buf = createReadBuffer(*object_info, object_storage, context_, log); } - /*Temporary desicion. Need better anstraction here.*/ + /*Temporary decision. Need better anstraction here.*/ Block initial_header = read_from_format_info.format_header; if (object_info->initial_schema) @@ -401,12 +401,8 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade builder.init(Pipe(input_format)); - LOG_DEBUG( - &Poco::Logger::get("Transformer exists"), - "File name: {}, Exists: {}", - object_info->relative_path, - object_info->schema_transformer != nullptr); - if (object_info->schema_transformer) { + if (object_info->schema_transformer) + { auto schema_modifying_actions = std::make_shared(object_info->schema_transformer->clone()); builder.addSimpleTransform([&](const Block & header) { @@ -669,26 +665,11 @@ StorageObjectStorageSource::KeysIterator::KeysIterator( , keys(configuration->getPaths()) , ignore_non_existent_files(ignore_non_existent_files_) { - for (const auto & key : keys) - { - LOG_DEBUG( - &Poco::Logger::get("Copying data_info to object_info"), - "Key data path: {}, Key schema_transform exists: {}", - key.data_path, - key.schema_transform != nullptr); - } - if (read_keys_) { /// TODO: should we add metadata if we anyway fetch it if file_progress_callback is passed? for (auto && key : keys) { - auto object_info = std::make_shared(key.data_path, std::nullopt, key.initial_schema, key.schema_transform); - LOG_DEBUG( - &Poco::Logger::get("Copying data_info to object_info"), - "Key data path: {}, Key schema_transform exists: {}", - key.data_path, - key.schema_transform != nullptr); read_keys_->emplace_back(object_info); } } diff --git a/tests/integration/test_storage_iceberg/test.py b/tests/integration/test_storage_iceberg/test.py index 176c7e209bd..ac6324727ec 100644 --- a/tests/integration/test_storage_iceberg/test.py +++ b/tests/integration/test_storage_iceberg/test.py @@ -210,6 +210,27 @@ def get_creation_expression( raise Exception(f"Unknown iceberg storage type: {storage_type}") +def check_schema_and_data( + instance, table_function_expression, expected_schema, expected_data +): + schema = instance.query(f"DESC {table_function_expression}") + data = instance.query(f"SELECT * FROM {table_function_expression} ORDER BY ALL") + schema = list( + map( + lambda x: x.split("\t")[:2], + filter(lambda x: len(x) > 0, schema.strip().split("\n")), + ) + ) + data = list( + map( + lambda x: x.split("\t"), + filter(lambda x: len(x) > 0, data.strip().split("\n")), + ) + ) + assert expected_schema == schema + assert expected_data == data + + def get_uuid_str(): return str(uuid.uuid4()).replace("-", "_") @@ -282,7 +303,7 @@ def test_single_iceberg_file(started_cluster, format_version, storage_type): write_iceberg_from_df(spark, generate_data(spark, 0, 100), TABLE_NAME) - files = default_upload_directory( + default_upload_directory( started_cluster, storage_type, f"/iceberg_data/default/{TABLE_NAME}/", @@ -336,8 +357,6 @@ def test_partition_by(started_cluster, format_version, storage_type): def test_multiple_iceberg_files(started_cluster, format_version, storage_type): instance = started_cluster.instances["node1"] spark = started_cluster.spark_session - minio_client = started_cluster.minio_client - bucket = started_cluster.minio_bucket TABLE_NAME = ( "test_multiple_iceberg_files_" + format_version @@ -464,8 +483,6 @@ def test_types(started_cluster, format_version, storage_type): def test_delete_files(started_cluster, format_version, storage_type): instance = started_cluster.instances["node1"] spark = started_cluster.spark_session - minio_client = started_cluster.minio_client - bucket = started_cluster.minio_bucket TABLE_NAME = ( "test_delete_files_" + format_version @@ -483,7 +500,7 @@ def test_delete_files(started_cluster, format_version, storage_type): format_version=format_version, ) - files = default_upload_directory( + default_upload_directory( started_cluster, storage_type, f"/iceberg_data/default/{TABLE_NAME}/", @@ -494,7 +511,7 @@ def test_delete_files(started_cluster, format_version, storage_type): assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100 spark.sql(f"DELETE FROM {TABLE_NAME} WHERE a >= 0") - files = default_upload_directory( + default_upload_directory( started_cluster, storage_type, f"/iceberg_data/default/{TABLE_NAME}/", @@ -512,7 +529,7 @@ def test_delete_files(started_cluster, format_version, storage_type): format_version=format_version, ) - files = default_upload_directory( + default_upload_directory( started_cluster, storage_type, f"/iceberg_data/default/{TABLE_NAME}/", @@ -522,7 +539,7 @@ def test_delete_files(started_cluster, format_version, storage_type): assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100 spark.sql(f"DELETE FROM {TABLE_NAME} WHERE a >= 150") - files = default_upload_directory( + default_upload_directory( started_cluster, storage_type, f"/iceberg_data/default/{TABLE_NAME}/", @@ -534,13 +551,11 @@ def test_delete_files(started_cluster, format_version, storage_type): @pytest.mark.parametrize("format_version", ["1", "2"]) @pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) -def test_evolved_schema(started_cluster, format_version, storage_type): +def test_evolved_schema_simple(started_cluster, format_version, storage_type): instance = started_cluster.instances["node1"] spark = started_cluster.spark_session - minio_client = started_cluster.minio_client - bucket = started_cluster.minio_bucket TABLE_NAME = ( - "test_evolved_schema_" + "test_evolved_schema_simple_" + format_version + "_" + storage_type @@ -548,50 +563,430 @@ def test_evolved_schema(started_cluster, format_version, storage_type): + get_uuid_str() ) - write_iceberg_from_df( - spark, - generate_data(spark, 0, 100), - TABLE_NAME, - mode="overwrite", - format_version=format_version, + def execute_spark_query(query: str): + spark.sql(query) + default_upload_directory( + started_cluster, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + return + + execute_spark_query( + f""" + DROP TABLE IF EXISTS {TABLE_NAME}; + """ ) - files = default_upload_directory( - started_cluster, - storage_type, - f"/iceberg_data/default/{TABLE_NAME}/", - f"/iceberg_data/default/{TABLE_NAME}/", + execute_spark_query( + f""" + CREATE TABLE IF NOT EXISTS {TABLE_NAME} ( + a int NOT NULL, + b float, + c decimal(9,2) NOT NULL, + d array + ) + USING iceberg + OPTIONS ('format-version'='{format_version}') + """ ) - create_iceberg_table(storage_type, instance, TABLE_NAME, started_cluster) - - assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100 - - expected_data = instance.query(f"SELECT * FROM {TABLE_NAME} order by a, b") - - spark.sql(f"ALTER TABLE {TABLE_NAME} ADD COLUMNS (x bigint)") - files = default_upload_directory( - started_cluster, - storage_type, - f"/iceberg_data/default/{TABLE_NAME}/", - "", + table_function = get_creation_expression( + storage_type, TABLE_NAME, started_cluster, table_function=True ) - error = instance.query_and_get_error(f"SELECT * FROM {TABLE_NAME}") + check_schema_and_data( + instance, + table_function, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [], + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES (4, 3.0, 7.12, ARRAY(5, 6, 7)); + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [["4", "3", "7.12", "[5,6,7]"]], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN b TYPE double; + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["a", "Int32"], + ["b", "Nullable(Float64)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [["4", "3", "7.12", "[5,6,7]"]], + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES (7, 5.0, 18.1, ARRAY(6, 7, 9)); + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["a", "Int32"], + ["b", "Nullable(Float64)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [["4", "3", "7.12", "[5,6,7]"], ["7", "5", "18.1", "[6,7,9]"]], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN d FIRST; + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["d", "Array(Nullable(Int32))"], + ["a", "Int32"], + ["b", "Nullable(Float64)"], + ["c", "Decimal(9, 2)"], + ], + [["[5,6,7]", "4", "3", "7.12"], ["[6,7,9]", "7", "5", "18.1"]], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN b AFTER d; + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["d", "Array(Nullable(Int32))"], + ["b", "Nullable(Float64)"], + ["a", "Int32"], + ["c", "Decimal(9, 2)"], + ], + [["[5,6,7]", "3", "4", "7.12"], ["[6,7,9]", "5", "7", "18.1"]], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} + ADD COLUMNS ( + e string + ); + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["d", "Array(Nullable(Int32))"], + ["b", "Nullable(Float64)"], + ["a", "Int32"], + ["c", "Decimal(9, 2)"], + ["e", "Nullable(String)"], + ], + [ + ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[6,7,9]", "5", "7", "18.1", "\\N"], + ], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN c TYPE decimal(12, 2); + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["d", "Array(Nullable(Int32))"], + ["b", "Nullable(Float64)"], + ["a", "Int32"], + ["c", "Decimal(12, 2)"], + ["e", "Nullable(String)"], + ], + [ + ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[6,7,9]", "5", "7", "18.1", "\\N"], + ], + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES (ARRAY(5, 6, 7), 3, -30, 7.12, 'AAA'); + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["d", "Array(Nullable(Int32))"], + ["b", "Nullable(Float64)"], + ["a", "Int32"], + ["c", "Decimal(12, 2)"], + ["e", "Nullable(String)"], + ], + [ + ["[5,6,7]", "3", "-30", "7.12", "AAA"], + ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[6,7,9]", "5", "7", "18.1", "\\N"], + ], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN a TYPE BIGINT; + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["d", "Array(Nullable(Int32))"], + ["b", "Nullable(Float64)"], + ["a", "Int64"], + ["c", "Decimal(12, 2)"], + ["e", "Nullable(String)"], + ], + [ + ["[5,6,7]", "3", "-30", "7.12", "AAA"], + ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[6,7,9]", "5", "7", "18.1", "\\N"], + ], + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES (ARRAY(), 3.0, 12, -9.13, 'BBB'); + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["d", "Array(Nullable(Int32))"], + ["b", "Nullable(Float64)"], + ["a", "Int64"], + ["c", "Decimal(12, 2)"], + ["e", "Nullable(String)"], + ], + [ + ["[]", "3", "12", "-9.13", "BBB"], + ["[5,6,7]", "3", "-30", "7.12", "AAA"], + ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[6,7,9]", "5", "7", "18.1", "\\N"], + ], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN a DROP NOT NULL;; + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["d", "Array(Nullable(Int32))"], + ["b", "Nullable(Float64)"], + ["a", "Nullable(Int64)"], + ["c", "Decimal(12, 2)"], + ["e", "Nullable(String)"], + ], + [ + ["[]", "3", "12", "-9.13", "BBB"], + ["[5,6,7]", "3", "-30", "7.12", "AAA"], + ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[6,7,9]", "5", "7", "18.1", "\\N"], + ], + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES (NULL, 3.4, NULL, -9.13, NULL); + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["d", "Array(Nullable(Int32))"], + ["b", "Nullable(Float64)"], + ["a", "Nullable(Int64)"], + ["c", "Decimal(12, 2)"], + ["e", "Nullable(String)"], + ], + [ + ["[]", "3", "12", "-9.13", "BBB"], + ["[]", "3.4", "\\N", "-9.13", "\\N"], + ["[5,6,7]", "3", "-30", "7.12", "AAA"], + ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[6,7,9]", "5", "7", "18.1", "\\N"], + ], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} DROP COLUMN d; + """ + ) + + check_schema_and_data( + instance, + table_function, + [ + ["b", "Nullable(Float64)"], + ["a", "Nullable(Int64)"], + ["c", "Decimal(12, 2)"], + ["e", "Nullable(String)"], + ], + [ + ["3", "-30", "7.12", "AAA"], + ["3", "4", "7.12", "\\N"], + ["3", "12", "-9.13", "BBB"], + ["3.4", "\\N", "-9.13", "\\N"], + ["5", "7", "18.1", "\\N"], + ], + ) + + +@pytest.mark.parametrize("format_version", ["1", "2"]) +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_evolved_schema_complex(started_cluster, format_version, storage_type): + instance = started_cluster.instances["node1"] + spark = started_cluster.spark_session + TABLE_NAME = ( + "test_evolved_schema_complex_" + + format_version + + "_" + + storage_type + + "_" + + get_uuid_str() + ) + + def execute_spark_query(query: str): + spark.sql(query) + default_upload_directory( + started_cluster, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + return + + execute_spark_query( + f""" + DROP TABLE IF EXISTS {TABLE_NAME}; + """ + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + address STRUCT< + house_number : DOUBLE, + city: STRING, + zip: INT + >, + animals ARRAY + ) + USING iceberg + OPTIONS ('format-version'='{format_version}') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES (named_struct('house_number', 3, 'city', 'Singapore', 'zip', 12345), ARRAY(4, 7)); + """ + ) + + table_function = get_creation_expression( + storage_type, TABLE_NAME, started_cluster, table_function=True + ) + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ADD COLUMNS ( address.appartment INT ); + """ + ) + + error = instance.query_and_get_error(f"SELECT * FROM {table_function} ORDER BY ALL") + assert "UNSUPPORTED_METHOD" in error - data = instance.query( - f"SELECT * FROM {TABLE_NAME} SETTINGS iceberg_engine_ignore_schema_evolution=1" + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} DROP COLUMN address.appartment; + """ ) - assert data == expected_data + + check_schema_and_data( + instance, + table_function, + [ + [ + "address", + "Tuple(\\n house_number Nullable(Float64),\\n city Nullable(String),\\n zip Nullable(Int32))", + ], + ["animals", "Array(Nullable(Int32))"], + ], + [["(3,'Singapore',12345)", "[4,7]"]], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN animals.element TYPE BIGINT + """ + ) + + error = instance.query_and_get_error(f"SELECT * FROM {table_function} ORDER BY ALL") + + assert "UNSUPPORTED_METHOD" in error @pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) def test_row_based_deletes(started_cluster, storage_type): instance = started_cluster.instances["node1"] spark = started_cluster.spark_session - minio_client = started_cluster.minio_client - bucket = started_cluster.minio_bucket TABLE_NAME = "test_row_based_deletes_" + storage_type + "_" + get_uuid_str() spark.sql( @@ -601,7 +996,7 @@ def test_row_based_deletes(started_cluster, storage_type): f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range(100)" ) - files = default_upload_directory( + default_upload_directory( started_cluster, storage_type, f"/iceberg_data/default/{TABLE_NAME}/", @@ -613,7 +1008,7 @@ def test_row_based_deletes(started_cluster, storage_type): assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100 spark.sql(f"DELETE FROM {TABLE_NAME} WHERE id < 10") - files = default_upload_directory( + default_upload_directory( started_cluster, storage_type, f"/iceberg_data/default/{TABLE_NAME}/", @@ -649,7 +1044,7 @@ def test_schema_inference(started_cluster, format_version, storage_type): spark.sql( f"insert into {TABLE_NAME} select 42, 4242, 42.42, 4242.4242, decimal(42.42), decimal(42.42), decimal(42.42), date('2020-01-01'), timestamp('2020-01-01 20:00:00'), 'hello', binary('hello'), array(1,2,3), map('key', 'value'), struct(42, 'hello'), array(struct(map('key', array(map('key', 42))), struct(42, 'hello')))" ) - files = default_upload_directory( + default_upload_directory( started_cluster, storage_type, f"/iceberg_data/default/{TABLE_NAME}/", @@ -715,7 +1110,7 @@ def test_metadata_file_selection(started_cluster, format_version, storage_type): f"INSERT INTO {TABLE_NAME} select id, char(id + ascii('a')) from range(10)" ) - files = default_upload_directory( + default_upload_directory( started_cluster, storage_type, f"/iceberg_data/default/{TABLE_NAME}/", @@ -756,7 +1151,7 @@ def test_metadata_file_format_with_uuid(started_cluster, format_version, storage f"/iceberg_data/default/{TABLE_NAME}/metadata/{str(i).zfill(5)}-{get_uuid_str()}.metadata.json", ) - files = default_upload_directory( + default_upload_directory( started_cluster, storage_type, f"/iceberg_data/default/{TABLE_NAME}/", From 7b6cf8bc8e8f3ef741b7d42465e01366b17ca27e Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 12 Sep 2024 15:35:25 +0000 Subject: [PATCH 026/502] Fix some problems with changing schema --- src/Analyzer/TableNode.h | 2 + src/Interpreters/InterpreterSelectQuery.cpp | 6 ++ src/Planner/PlannerJoinTree.cpp | 6 ++ src/Storages/IStorage.h | 9 +++ .../DataLakes/IStorageDataLake.h | 80 ++++++++++++++++--- .../DataLakes/IcebergMetadata.cpp | 10 +-- .../ObjectStorage/DataLakes/IcebergMetadata.h | 2 +- .../ObjectStorage/StorageObjectStorage.cpp | 22 ++++- .../ObjectStorage/StorageObjectStorage.h | 2 + .../StorageObjectStorageSource.cpp | 5 +- 10 files changed, 119 insertions(+), 25 deletions(-) diff --git a/src/Analyzer/TableNode.h b/src/Analyzer/TableNode.h index 30ee19b84a3..9e0aeb0e02b 100644 --- a/src/Analyzer/TableNode.h +++ b/src/Analyzer/TableNode.h @@ -37,6 +37,8 @@ public: */ void updateStorage(StoragePtr storage_value, const ContextPtr & context); + void updateStorage(const ContextPtr & context); + /// Get storage const StoragePtr & getStorage() const { diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ca0e84a5267..f65d17aeab0 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -465,6 +465,12 @@ InterpreterSelectQuery::InterpreterSelectQuery( { table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); table_id = storage->getStorageID(); + if (storage->hasExternalDynamicMetadata()) + { + storage->updateExternalDynamicMetadata(context); + metadata_snapshot = storage->getInMemoryMetadataPtr(); + } + if (!metadata_snapshot) metadata_snapshot = storage->getInMemoryMetadataPtr(); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index bc31af32a20..8a6ad1543a8 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -643,6 +643,12 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres if (table_node || table_function_node) { const auto & storage = table_node ? table_node->getStorage() : table_function_node->getStorage(); + if (table_node && storage->hasExternalDynamicMetadata()) + { + storage->updateExternalDynamicMetadata(query_context); + table_node->updateStorage(storage, query_context); + } + const auto & storage_snapshot = table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot(); auto table_expression_query_info = select_query_info; diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 07058dfb5df..8c2d78a5d1d 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -120,6 +120,9 @@ public: /// Returns true if the storage is dictionary virtual bool isDictionary() const { return false; } + /// Returns true if the metadata of a table can be changed normally by other processes + virtual bool hasExternalDynamicMetadata() const { return false; } + /// Returns true if the storage supports queries with the SAMPLE section. virtual bool supportsSampling() const { return getInMemoryMetadataPtr()->hasSamplingKey(); } @@ -496,6 +499,12 @@ public: */ virtual void alter(const AlterCommands & params, ContextPtr context, AlterLockHolder & alter_lock_holder); + /// Updates metadata that can be changed by other processes + virtual void updateExternalDynamicMetadata(ContextPtr) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method updateExternalDynamicMetadata is not supported by storage {}", getName()); + } + /** Checks that alter commands can be applied to storage. For example, columns can be modified, * or primary key can be changes, etc. */ diff --git a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h index ccb273fd25e..5d1c64a176a 100644 --- a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h +++ b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h @@ -18,6 +18,11 @@ namespace DB { +namespace ErrorCodes +{ +extern const int FORMAT_VERSION_TOO_OLD; +} + /// Storage for read-only integration with Apache Iceberg tables in Amazon S3 (see https://iceberg.apache.org/) /// Right now it's implemented on top of StorageS3 and right now it doesn't support /// many Iceberg features like schema evolution, partitioning, positional and equality deletes. @@ -38,8 +43,7 @@ public: std::optional format_settings_, LoadingStrictnessLevel mode) { - auto object_storage = base_configuration->createObjectStorage(context, /* is_readonly */true); - DataLakeMetadataPtr metadata; + auto object_storage = base_configuration->createObjectStorage(context, /* is_readonly */ true); NamesAndTypesList schema_from_metadata; const bool use_schema_from_metadata = columns_.empty(); @@ -48,28 +52,36 @@ public: ConfigurationPtr configuration = base_configuration->clone(); + DataLakeMetadataPtr datalake_metadata; + try { - metadata = DataLakeMetadata::create(object_storage, base_configuration, context); - configuration->setPaths(metadata->getDataFileInfos()); + datalake_metadata = DataLakeMetadata::create(object_storage, base_configuration, context); + configuration->setPaths(datalake_metadata->getDataFileInfos()); if (use_schema_from_metadata) - schema_from_metadata = metadata->getTableSchema(); + schema_from_metadata = datalake_metadata->getTableSchema(); } catch (...) { if (mode <= LoadingStrictnessLevel::CREATE) throw; - metadata.reset(); + datalake_metadata.reset(); configuration->setPaths({}); tryLogCurrentException(__PRETTY_FUNCTION__); } return std::make_shared>( - base_configuration, std::move(metadata), configuration, object_storage, - context, table_id_, + base_configuration, + std::move(datalake_metadata), + configuration, + object_storage, + context, + table_id_, use_schema_from_metadata ? ColumnsDescription(schema_from_metadata) : columns_, - constraints_, comment_, format_settings_); + constraints_, + comment_, + format_settings_); } String getName() const override { return DataLakeMetadata::name; } @@ -102,15 +114,30 @@ public: Storage::updateConfiguration(local_context); auto new_metadata = DataLakeMetadata::create(Storage::object_storage, base_configuration, local_context); - if (current_metadata && *current_metadata == *new_metadata) - return; + if (!current_metadata || (*current_metadata != *new_metadata)) + { + if constexpr (std::is_same_v) + { + throw Exception( + ErrorCodes::FORMAT_VERSION_TOO_OLD, + "Storage thinks that actual metadata version is {}, but actual metadata version is {} current", + (dynamic_cast(current_metadata.get()) != nullptr) + ? dynamic_cast(current_metadata.get())->getVersion() + : -1, + (dynamic_cast(new_metadata.get()) != nullptr) + ? dynamic_cast(new_metadata.get())->getVersion() + : -1); + } + else + { + current_metadata = std::move(new_metadata); + } + } - current_metadata = std::move(new_metadata); auto updated_configuration = base_configuration->clone(); updated_configuration->setPaths(current_metadata->getDataFileInfos()); updated_configuration->setPartitionColumns(current_metadata->getPartitionColumns()); - Storage::configuration = updated_configuration; } @@ -136,9 +163,36 @@ public: } } + void updateExternalDynamicMetadata(ContextPtr context) override + { + if constexpr (std::is_same_v) + { + current_metadata = DataLakeMetadata::create(Storage::object_storage, base_configuration, context); + ColumnsDescription column_description{current_metadata->getTableSchema()}; + StorageInMemoryMetadata metadata; + metadata.setColumns(std::move(column_description)); + setInMemoryMetadata(metadata); + return; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method updateExternalDynamicMetadata is not supported by storage {}", getName()); + } + + bool hasExternalDynamicMetadata() const override { return std::is_same_v; } + + private: ConfigurationPtr base_configuration; DataLakeMetadataPtr current_metadata; + std::optional current_version; + + void modifyColumnNames(Names & column_names) const override + { + if constexpr (std::is_same_v) + { + column_names = current_metadata->getTableSchema().getNames(); + } + } + ReadFromFormatInfo prepareReadingFromFormat( const Strings & requested_columns, diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index ac9dd019ca5..670680d5ea3 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -44,7 +44,7 @@ extern const int FILE_DOESNT_EXIST; extern const int ILLEGAL_COLUMN; extern const int BAD_ARGUMENTS; extern const int UNSUPPORTED_METHOD; -extern const int LOGICAL_ERROR: +extern const int LOGICAL_ERROR; } IcebergMetadata::IcebergMetadata( @@ -365,6 +365,10 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformatio { current_old_id = old_id; current_new_id = new_id; + SCOPE_EXIT({ + current_old_id = -1; + current_new_id = -1; + }); Poco::JSON::Object::Ptr old_schema, new_schema; if (transform_dags_by_ids.count({old_id, new_id})) { @@ -390,10 +394,6 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformatio { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", new_id); } - SCOPE_EXIT({ - current_old_id = -1; - current_new_id = -1; - }); return transform_dags_by_ids[{old_id, new_id}] = getSchemaTransformationDag(old_schema, new_schema); } diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 2d93dea9fc9..d5091f8695c 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -186,9 +186,9 @@ public: ConfigurationPtr configuration, ContextPtr local_context); -private: size_t getVersion() const { return metadata_version; } +private: const ObjectStoragePtr object_storage; const ConfigurationPtr configuration; Int32 metadata_version; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 88d71a295c5..6b6c9b37c05 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -263,9 +263,25 @@ void StorageObjectStorage::read( "Reading from a partitioned {} storage is not implemented yet", getName()); } + for (const auto & column_name : column_names) + { + LOG_DEBUG(&Poco::Logger::get("Column Names"), "Column Name: {}", column_name); + } - const auto read_from_format_info = prepareReadingFromFormat( - column_names, storage_snapshot, supportsSubsetOfColumns(local_context), local_context); + + LOG_DEBUG(&Poco::Logger::get("Stack trace of read"), "{}", StackTrace().toString()); + + auto names = column_names; + modifyColumnNames(names); + + for (const auto & column_name : names) + { + LOG_DEBUG(&Poco::Logger::get("New Column Names"), "Column Name: {}", column_name); + } + + + const auto read_from_format_info + = prepareReadingFromFormat(names, storage_snapshot, supportsSubsetOfColumns(local_context), local_context); const bool need_only_count = (query_info.optimize_trivial_count || read_from_format_info.requested_columns.empty()) && local_context->getSettingsRef().optimize_count_from_files; @@ -273,7 +289,7 @@ void StorageObjectStorage::read( object_storage, configuration, getName(), - column_names, + names, getVirtualsList(), query_info, storage_snapshot, diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 0171a0ef27a..43d6acd2312 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -124,6 +124,8 @@ public: static SchemaCache & getSchemaCache(const ContextPtr & context, const std::string & storage_type_name); + virtual void modifyColumnNames(Names &) const { } + static ColumnsDescription resolveSchemaFromData( const ObjectStoragePtr & object_storage, const ConfigurationPtr & configuration, diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index e58f357cf47..21890a2e311 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -20,7 +20,6 @@ namespace fs = std::filesystem; - namespace ProfileEvents { extern const Event EngineFileLikeReadFiles; @@ -31,8 +30,7 @@ namespace CurrentMetrics extern const Metric StorageObjectStorageThreads; extern const Metric StorageObjectStorageThreadsActive; extern const Metric StorageObjectStorageThreadsScheduled; -} - + } namespace DB { @@ -670,6 +668,7 @@ StorageObjectStorageSource::KeysIterator::KeysIterator( /// TODO: should we add metadata if we anyway fetch it if file_progress_callback is passed? for (auto && key : keys) { + auto object_info = std::make_shared(key.data_path, std::nullopt, key.initial_schema, key.schema_transform); read_keys_->emplace_back(object_info); } } From d5173d770e75d98aa880e5be25f04553b0d7b665 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 12 Sep 2024 16:23:32 +0000 Subject: [PATCH 027/502] Change doc --- docs/en/engines/table-engines/integrations/iceberg.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md index e4a5de327bc..939cacfe5f0 100644 --- a/docs/en/engines/table-engines/integrations/iceberg.md +++ b/docs/en/engines/table-engines/integrations/iceberg.md @@ -6,6 +6,10 @@ sidebar_label: Iceberg # Iceberg Table Engine +:::warning +We strongly advise against using the Iceberg Table Engine right now because ClickHouse wasn’t originally designed to support tables that have externally changing schemas. As a result, some features that work with regular tables may be unavailable or may not function correctly. We recommend using the [Iceberg Table Function]((/docs/en/sql-reference/table-functions/iceberg.md)) instead." +::: + This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure and locally stored tables. ## Create Table From ae5a70e16cf9d217ee1c35f55f4ce874fb162ef7 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 12 Sep 2024 17:09:54 +0000 Subject: [PATCH 028/502] Remove debug logs --- src/Analyzer/TableNode.h | 2 -- .../DataLakes/DeltaLakeMetadata.h | 4 ++-- .../DataLakes/IDataLakeMetadata.h | 2 +- .../DataLakes/IStorageDataLake.h | 11 +--------- .../DataLakes/IcebergMetadata.cpp | 8 -------- src/Storages/ObjectStorage/S3/Configuration.h | 2 +- .../ObjectStorage/StorageObjectStorage.cpp | 20 ++----------------- .../ObjectStorage/StorageObjectStorage.h | 2 -- .../StorageObjectStorageSource.cpp | 1 - 9 files changed, 7 insertions(+), 45 deletions(-) diff --git a/src/Analyzer/TableNode.h b/src/Analyzer/TableNode.h index 9e0aeb0e02b..30ee19b84a3 100644 --- a/src/Analyzer/TableNode.h +++ b/src/Analyzer/TableNode.h @@ -37,8 +37,6 @@ public: */ void updateStorage(StoragePtr storage_value, const ContextPtr & context); - void updateStorage(const ContextPtr & context); - /// Get storage const StoragePtr & getStorage() const { diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h index dbcf3a4ec0b..c0e206a0d4a 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.h @@ -22,7 +22,7 @@ public: ConfigurationPtr configuration_, ContextPtr context_); - std::vector getDataFileInfos() const override { return data_files; } + DataFileInfos getDataFileInfos() const override { return data_files; } NamesAndTypesList getTableSchema() const override { return schema; } @@ -47,7 +47,7 @@ public: } private: - mutable std::vector data_files; + mutable DataFileInfos data_files; NamesAndTypesList schema; std::unordered_map column_name_to_physical_name; DataLakePartitionColumns partition_columns; diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h index d2b26cca8c5..055c8f32906 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h @@ -12,7 +12,7 @@ class IDataLakeMetadata : boost::noncopyable { public: virtual ~IDataLakeMetadata() = default; - virtual std::vector getDataFileInfos() const = 0; + virtual DataFileInfos getDataFileInfos() const = 0; virtual NamesAndTypesList getTableSchema() const = 0; virtual bool operator==(const IDataLakeMetadata & other) const = 0; virtual const DataLakePartitionColumns & getPartitionColumns() const = 0; diff --git a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h index 5d1c64a176a..84adb893b34 100644 --- a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h +++ b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h @@ -21,6 +21,7 @@ namespace DB namespace ErrorCodes { extern const int FORMAT_VERSION_TOO_OLD; +extern const int NOT_IMPLEMENTED; } /// Storage for read-only integration with Apache Iceberg tables in Amazon S3 (see https://iceberg.apache.org/) @@ -183,16 +184,6 @@ public: private: ConfigurationPtr base_configuration; DataLakeMetadataPtr current_metadata; - std::optional current_version; - - void modifyColumnNames(Names & column_names) const override - { - if constexpr (std::is_same_v) - { - column_names = current_metadata->getTableSchema().getNames(); - } - } - ReadFromFormatInfo prepareReadingFromFormat( const Strings & requested_columns, diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 670680d5ea3..1ce7522a80c 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -355,8 +355,6 @@ IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr const auto & constant = dag->addColumn({default_type_column, type, name}); outputs.push_back(&constant); } - LOG_DEBUG( - &Poco::Logger::get("New schema"), "Id: {}, Name: {}, type: {}", id, name, getFieldType(field, "type", required)->getName()); } return dag; } @@ -740,12 +738,6 @@ DataFileInfos IcebergMetadata::getDataFileInfos() const for (const auto & [file_path, schema_object_id] : files) { - LOG_DEBUG( - &Poco::Logger::get("Adding not empty stuff"), - "File path: {}, Old schema: {}, New schema: {}", - file_path, - schema_object_id, - current_schema_id); data_file_infos.emplace_back( file_path, schema_processor.getClickhouseTableSchemaById(schema_object_id), diff --git a/src/Storages/ObjectStorage/S3/Configuration.h b/src/Storages/ObjectStorage/S3/Configuration.h index e853755799f..82e93288660 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.h +++ b/src/Storages/ObjectStorage/S3/Configuration.h @@ -6,7 +6,7 @@ #include #include -# include "Storages/ObjectStorage/DataFileInfo.h" +#include "Storages/ObjectStorage/DataFileInfo.h" namespace DB { diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 6b6c9b37c05..a06e9bf2703 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -263,25 +263,9 @@ void StorageObjectStorage::read( "Reading from a partitioned {} storage is not implemented yet", getName()); } - for (const auto & column_name : column_names) - { - LOG_DEBUG(&Poco::Logger::get("Column Names"), "Column Name: {}", column_name); - } - - - LOG_DEBUG(&Poco::Logger::get("Stack trace of read"), "{}", StackTrace().toString()); - - auto names = column_names; - modifyColumnNames(names); - - for (const auto & column_name : names) - { - LOG_DEBUG(&Poco::Logger::get("New Column Names"), "Column Name: {}", column_name); - } - const auto read_from_format_info - = prepareReadingFromFormat(names, storage_snapshot, supportsSubsetOfColumns(local_context), local_context); + = prepareReadingFromFormat(column_names, storage_snapshot, supportsSubsetOfColumns(local_context), local_context); const bool need_only_count = (query_info.optimize_trivial_count || read_from_format_info.requested_columns.empty()) && local_context->getSettingsRef().optimize_count_from_files; @@ -289,7 +273,7 @@ void StorageObjectStorage::read( object_storage, configuration, getName(), - names, + column_names, getVirtualsList(), query_info, storage_snapshot, diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 43d6acd2312..0171a0ef27a 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -124,8 +124,6 @@ public: static SchemaCache & getSchemaCache(const ContextPtr & context, const std::string & storage_type_name); - virtual void modifyColumnNames(Names &) const { } - static ColumnsDescription resolveSchemaFromData( const ObjectStoragePtr & object_storage, const ConfigurationPtr & configuration, diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 21890a2e311..99dd104a39a 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -364,7 +364,6 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade read_buf = createReadBuffer(*object_info, object_storage, context_, log); } - /*Temporary decision. Need better anstraction here.*/ Block initial_header = read_from_format_info.format_header; if (object_info->initial_schema) From 6bf5c092bb3e94f26d8c47886582cec7d7fceef6 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 13 Sep 2024 10:18:42 +0000 Subject: [PATCH 029/502] Fix some small issues --- docs/en/engines/table-engines/integrations/iceberg.md | 2 +- src/Analyzer/Resolve/IdentifierResolver.cpp | 5 +++++ src/Interpreters/InterpreterDescribeQuery.cpp | 5 +++++ src/Interpreters/InterpreterSelectQuery.cpp | 11 ++++++----- src/Planner/PlannerJoinTree.cpp | 6 +----- src/Storages/ObjectStorage/DataFileInfo.h | 2 +- src/Storages/ObjectStorage/StorageObjectStorage.h | 2 +- 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md index 939cacfe5f0..774dd5fcf0a 100644 --- a/docs/en/engines/table-engines/integrations/iceberg.md +++ b/docs/en/engines/table-engines/integrations/iceberg.md @@ -7,7 +7,7 @@ sidebar_label: Iceberg # Iceberg Table Engine :::warning -We strongly advise against using the Iceberg Table Engine right now because ClickHouse wasn’t originally designed to support tables that have externally changing schemas. As a result, some features that work with regular tables may be unavailable or may not function correctly. We recommend using the [Iceberg Table Function]((/docs/en/sql-reference/table-functions/iceberg.md)) instead." +We strongly advise against using the Iceberg Table Engine right now because ClickHouse wasn’t originally designed to support tables that have externally changing schemas. As a result, some features that work with regular tables may be unavailable or may not function correctly (especially for the old analyzer). We recommend using the [Iceberg Table Function](/docs/en/sql-reference/table-functions/iceberg.md) instead (right now it is sufficient, because CH supports only partial read-only interface for Iceberg tables)." ::: This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure and locally stored tables. diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index 80e7d1e4445..737346b4a2c 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -420,6 +420,11 @@ QueryTreeNodePtr IdentifierResolver::tryResolveTableIdentifierFromDatabaseCatalo if (!storage) return {}; + if (storage->hasExternalDynamicMetadata()) + { + storage->updateExternalDynamicMetadata(context); + } + auto storage_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); auto storage_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr(), context); auto result = std::make_shared(std::move(storage), std::move(storage_lock), std::move(storage_snapshot)); diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp index 39fc85a5e23..582efb58dd5 100644 --- a/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/src/Interpreters/InterpreterDescribeQuery.cpp @@ -172,6 +172,11 @@ void InterpreterDescribeQuery::fillColumnsFromTable(const ASTTableExpression & t auto table_id = getContext()->resolveStorageID(table_expression.database_and_table_name); getContext()->checkAccess(AccessType::SHOW_COLUMNS, table_id); auto table = DatabaseCatalog::instance().getTable(table_id, getContext()); + if (table->hasExternalDynamicMetadata()) + { + table->updateExternalDynamicMetadata(getContext()); + } + auto table_lock = table->lockForShare(getContext()->getInitialQueryId(), settings.lock_acquire_timeout); auto metadata_snapshot = table->getInMemoryMetadataPtr(); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index f65d17aeab0..6810c752fba 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -409,6 +409,12 @@ InterpreterSelectQuery::InterpreterSelectQuery( { checkStackSize(); + if (storage->hasExternalDynamicMetadata()) + { + storage->updateExternalDynamicMetadata(context); + metadata_snapshot = storage->getInMemoryMetadataPtr(); + } + if (!prepared_sets) prepared_sets = std::make_shared(); @@ -465,11 +471,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( { table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); table_id = storage->getStorageID(); - if (storage->hasExternalDynamicMetadata()) - { - storage->updateExternalDynamicMetadata(context); - metadata_snapshot = storage->getInMemoryMetadataPtr(); - } if (!metadata_snapshot) metadata_snapshot = storage->getInMemoryMetadataPtr(); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 8a6ad1543a8..bf5875336bc 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -643,11 +643,6 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres if (table_node || table_function_node) { const auto & storage = table_node ? table_node->getStorage() : table_function_node->getStorage(); - if (table_node && storage->hasExternalDynamicMetadata()) - { - storage->updateExternalDynamicMetadata(query_context); - table_node->updateStorage(storage, query_context); - } const auto & storage_snapshot = table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot(); @@ -1768,6 +1763,7 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, * then left most table expression is responsible for providing valid JOIN TREE part of final query plan. * * Examples: Distributed, LiveView, Merge storages. + * Examples: Distributed, LiveView, Merge storages. */ auto left_table_expression = table_expressions_stack.front(); auto left_table_expression_query_plan = buildQueryPlanForTableExpression(left_table_expression, diff --git a/src/Storages/ObjectStorage/DataFileInfo.h b/src/Storages/ObjectStorage/DataFileInfo.h index c9d854d90f2..fa477fe0424 100644 --- a/src/Storages/ObjectStorage/DataFileInfo.h +++ b/src/Storages/ObjectStorage/DataFileInfo.h @@ -14,7 +14,7 @@ struct DataFileInfo std::shared_ptr initial_schema; std::shared_ptr schema_transform; - DataFileInfo( + explicit DataFileInfo( String data_path_, std::shared_ptr initial_schema_ = nullptr, std::shared_ptr schema_transform_ = nullptr) diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 0171a0ef27a..7ac4bb2769f 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -35,7 +35,7 @@ public: struct ObjectInfo : public RelativePathWithMetadata { - ObjectInfo( + explicit ObjectInfo( String relative_path_, std::optional metadata_ = std::nullopt, std::shared_ptr initial_schema_ = nullptr, From 0c3679c76ef34abebeb8992735c8617e6709ba85 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 13 Sep 2024 12:23:19 +0000 Subject: [PATCH 030/502] Fix potential seg fault with nullptr storage --- src/Interpreters/InterpreterSelectQuery.cpp | 11 +++++------ src/Planner/PlannerJoinTree.cpp | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6810c752fba..d3ebf2b5532 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -409,12 +409,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( { checkStackSize(); - if (storage->hasExternalDynamicMetadata()) - { - storage->updateExternalDynamicMetadata(context); - metadata_snapshot = storage->getInMemoryMetadataPtr(); - } - if (!prepared_sets) prepared_sets = std::make_shared(); @@ -469,6 +463,11 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage) { + if (storage->hasExternalDynamicMetadata()) + { + storage->updateExternalDynamicMetadata(context); + metadata_snapshot = storage->getInMemoryMetadataPtr(); + } table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); table_id = storage->getStorageID(); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index ab4ff4ec47e..bdd15dfc908 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1783,7 +1783,6 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, * then left most table expression is responsible for providing valid JOIN TREE part of final query plan. * * Examples: Distributed, LiveView, Merge storages. - * Examples: Distributed, LiveView, Merge storages. */ auto left_table_expression = table_expressions_stack.front(); auto left_table_expression_query_plan = buildQueryPlanForTableExpression(left_table_expression, From a90852742127e1aa1eb307a11544d9431604dd5a Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 16 Sep 2024 11:59:43 +0000 Subject: [PATCH 031/502] Fix some binary_tidy mistakes --- contrib/libarchive | 2 +- .../DataLakes/IcebergMetadata.cpp | 20 +++++++++---------- .../ObjectStorage/DataLakes/IcebergMetadata.h | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/contrib/libarchive b/contrib/libarchive index 313aa1fa10b..0c21691b177 160000 --- a/contrib/libarchive +++ b/contrib/libarchive @@ -1 +1 @@ -Subproject commit 313aa1fa10b657de791e3202c168a6c833bc3543 +Subproject commit 0c21691b177fac5f4cceca2a1ff2ddfa5d60f51c diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 1ce7522a80c..e99f5576cdb 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -282,7 +282,7 @@ IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr auto name = field->getValue("name"); bool required = field->getValue("required"); auto type = getFieldType(field, "type", required); - if (old_schema_entries.count(id)) + if (old_schema_entries.contains(id)) { auto [old_json, old_node] = old_schema_entries.find(id)->second; if (field->isObject("type")) @@ -368,7 +368,7 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformatio current_new_id = -1; }); Poco::JSON::Object::Ptr old_schema, new_schema; - if (transform_dags_by_ids.count({old_id, new_id})) + if (transform_dags_by_ids.contains({old_id, new_id})) { return transform_dags_by_ids.at({old_id, new_id}); } @@ -395,18 +395,18 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformatio return transform_dags_by_ids[{old_id, new_id}] = getSchemaTransformationDag(old_schema, new_schema); } -void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schema) +void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schema_ptr) { - Int32 schema_id = schema->getValue("schema-id"); - if (iceberg_table_schemas_by_ids.count(schema_id)) + Int32 schema_id = schema_ptr->getValue("schema-id"); + if (iceberg_table_schemas_by_ids.contains(schema_id)) { - chassert(clickhouse_table_schemas_by_ids.count(schema_id) > 0); - chassert(*iceberg_table_schemas_by_ids.at(schema_id) == *schema); + chassert(clickhouse_table_schemas_by_ids.contains(schema_id) > 0); + chassert(*iceberg_table_schemas_by_ids.at(schema_id) == *schema_ptr); } else { - iceberg_table_schemas_by_ids[schema_id] = schema; - auto fields = schema->get("fields").extract(); + iceberg_table_schemas_by_ids[schema_id] = schema_ptr; + auto fields = schema_ptr->get("fields").extract(); auto clickhouse_schema = std::make_shared(); for (size_t i = 0; i != fields->size(); ++i) { @@ -723,7 +723,7 @@ DataFileInfos IcebergMetadata::getDataFileInfos() const if (ManifestEntryStatus(status) == ManifestEntryStatus::DELETED) { LOG_TEST(log, "Processing delete file for path: {}", file_path); - chassert(files.count(file_path) == 0); + chassert(files.contains(file_path) == 0); } else { diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index d5091f8695c..a9537fcdc2c 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -72,7 +72,7 @@ class IcebergSchemaProcessor using Node = ActionsDAG::Node; public: - void addIcebergTableSchema(Poco::JSON::Object::Ptr ptr); + void addIcebergTableSchema(Poco::JSON::Object::Ptr schema_ptr); std::shared_ptr getClickhouseTableSchemaById(Int32 id); std::shared_ptr getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id); From 51225bf6f69afee6be63c7452dfebeac6b22bb53 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 17 Sep 2024 09:57:56 +0000 Subject: [PATCH 032/502] Revert submodules --- contrib/libarchive | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libarchive b/contrib/libarchive index 0c21691b177..313aa1fa10b 160000 --- a/contrib/libarchive +++ b/contrib/libarchive @@ -1 +1 @@ -Subproject commit 0c21691b177fac5f4cceca2a1ff2ddfa5d60f51c +Subproject commit 313aa1fa10b657de791e3202c168a6c833bc3543 From fb4f8f2d68eecee0d34562c02c4fde48c9eaf852 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 17 Sep 2024 10:03:23 +0000 Subject: [PATCH 033/502] Rewrite docs --- docs/en/engines/table-engines/integrations/iceberg.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md index 774dd5fcf0a..015aad4e83e 100644 --- a/docs/en/engines/table-engines/integrations/iceberg.md +++ b/docs/en/engines/table-engines/integrations/iceberg.md @@ -7,7 +7,11 @@ sidebar_label: Iceberg # Iceberg Table Engine :::warning -We strongly advise against using the Iceberg Table Engine right now because ClickHouse wasn’t originally designed to support tables that have externally changing schemas. As a result, some features that work with regular tables may be unavailable or may not function correctly (especially for the old analyzer). We recommend using the [Iceberg Table Function](/docs/en/sql-reference/table-functions/iceberg.md) instead (right now it is sufficient, because CH supports only partial read-only interface for Iceberg tables)." +We recommend using the Iceberg Table Function for working with Iceberg data in ClickHouse. The Iceberg Table Function currently provides sufficient functionality, offering a partial read-only interface for Iceberg tables. + +The Iceberg Table Engine is available but may have limitations. ClickHouse wasn't originally designed to support tables with externally changing schemas, which can affect the functionality of the Iceberg Table Engine. As a result, some features that work with regular tables may be unavailable or may not function correctly, especially when using the old analyzer. + +For optimal performance and compatibility, we suggest using the Iceberg Table Function while we continue to improve support for the Iceberg Table Engine. ::: This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure and locally stored tables. From 2dd9d31d8fd4996b91cda9163a8a99aba00eec08 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 17 Sep 2024 10:05:55 +0000 Subject: [PATCH 034/502] Return the link --- docs/en/engines/table-engines/integrations/iceberg.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md index 015aad4e83e..74835067f13 100644 --- a/docs/en/engines/table-engines/integrations/iceberg.md +++ b/docs/en/engines/table-engines/integrations/iceberg.md @@ -7,11 +7,11 @@ sidebar_label: Iceberg # Iceberg Table Engine :::warning -We recommend using the Iceberg Table Function for working with Iceberg data in ClickHouse. The Iceberg Table Function currently provides sufficient functionality, offering a partial read-only interface for Iceberg tables. +We recommend using the [Iceberg Table Function](/docs/en/sql-reference/table-functions/iceberg.md) for working with Iceberg data in ClickHouse. The Iceberg Table Function currently provides sufficient functionality, offering a partial read-only interface for Iceberg tables. The Iceberg Table Engine is available but may have limitations. ClickHouse wasn't originally designed to support tables with externally changing schemas, which can affect the functionality of the Iceberg Table Engine. As a result, some features that work with regular tables may be unavailable or may not function correctly, especially when using the old analyzer. -For optimal performance and compatibility, we suggest using the Iceberg Table Function while we continue to improve support for the Iceberg Table Engine. +For optimal compatibility, we suggest using the Iceberg Table Function while we continue to improve support for the Iceberg Table Engine. ::: This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure and locally stored tables. From 2b15ddcbe29fbb92054aa9e7458abf48b1c78f5e Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 20 Sep 2024 16:51:09 +0200 Subject: [PATCH 035/502] setting --- src/Core/Settings.cpp | 1 + src/Core/SettingsChangesHistory.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index f6617fd3981..23f688a2c29 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -918,6 +918,7 @@ namespace ErrorCodes M(Bool, restore_replace_external_engines_to_null, false, "Replace all the external table engines to Null on restore. Useful for testing purposes", 0) \ M(Bool, restore_replace_external_table_functions_to_null, false, "Replace all table functions to Null on restore. Useful for testing purposes", 0) \ M(Bool, create_if_not_exists, false, "Enable IF NOT EXISTS for CREATE statements by default", 0) \ + M(UInt64, object_storage_remove_recursive_file_limit, DEFAULT_REMOVE_SHARED_RECURSIVE_FILE_LIMIT, "Max number of files to store in memory during remove. Zero value means unlimited. Used to reduce memory usage.", 0) \ \ \ /* ###################################### */ \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index dd3c14e39e0..0950cad2bdc 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -67,6 +67,7 @@ static std::initializer_list Date: Mon, 23 Sep 2024 20:31:33 +0200 Subject: [PATCH 036/502] init --- programs/server/Server.cpp | 5 + src/Interpreters/CancellationChecker.h | 147 +++++++++++++++++++++++++ src/Interpreters/ProcessList.cpp | 25 +++-- 3 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 src/Interpreters/CancellationChecker.h diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index f0c9719051f..6ec357a9927 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -59,6 +59,7 @@ #include #include #include +#include #include #include #include @@ -1100,6 +1101,10 @@ try PlacementInfo::PlacementInfo::instance().initialize(config()); + auto cancellation_task_holder = global_context->getSchedulePool().createTask("CancellationChecker", []{ CancellationChecker::getInstance().workerFunction(); }); + auto cancellation_task = std::make_unique(std::move(cancellation_task_holder)); + (*cancellation_task)->activateAndSchedule(); + zkutil::validateZooKeeperConfig(config()); bool has_zookeeper = zkutil::hasZooKeeperConfig(config()); diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h new file mode 100644 index 00000000000..434ae15dea8 --- /dev/null +++ b/src/Interpreters/CancellationChecker.h @@ -0,0 +1,147 @@ +#include +#include +#include +#include +#include +#include "base/types.h" + +namespace DB +{ + +struct CompareEndTime { + bool operator()(const std::pair, UInt64>& a, const std::pair, UInt64>& b) const { + return a.second > b.second; + } +}; + +/* +A Singleton class that check if tasks are cancelled or timed out. +Has a priority queue ordered by end time. Checker waits until the +first task in the list is done, then check if this task needs to be cancelled. +If yes, sets a cancellation flag on this task, otherwise removes the task from the queue. +*/ +class CancellationChecker +{ +private: + // Private constructor for Singleton pattern + CancellationChecker() : stop_thread(false) + { + } + + ~CancellationChecker() + { + stop_thread = true; + } + + // Priority queue to manage tasks based on endTime + std::priority_queue< + std::pair, UInt64>, + std::vector, UInt64>>, + CompareEndTime + > minPriorityQueue; + + std::vector> done_tasks; + + std::atomic stop_thread; + std::mutex m; + std::condition_variable cond_var; + + // Function to execute when a task's endTime is reached + void cancelTask(std::shared_ptr query) + { + query->cancelQuery(/*kill=*/false); + } + +public: + // Singleton instance retrieval + static CancellationChecker& getInstance() + { + static CancellationChecker instance; + return instance; + } + + // Deleted copy constructor and assignment operator + CancellationChecker(const CancellationChecker&) = delete; + CancellationChecker& operator=(const CancellationChecker&) = delete; + + // Method to add a new task to the priority queue + void appendTask(std::shared_ptr query, UInt64 end_time) + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added. query: {}, timeout: {} milliseconds", query->getInfo().query, end_time); + std::unique_lock lock(m); + minPriorityQueue.emplace(query, end_time); + cond_var.notify_all(); + } + + void appendDoneTasks(std::shared_ptr query) + { + std::unique_lock lock(m); + done_tasks.push_back(query); + cond_var.notify_all(); + } + + // Worker thread function + void workerFunction() + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "workerFunction()"); + std::unique_lock lock(m); + + while (!stop_thread) + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Iteration"); + if (minPriorityQueue.empty()) + { + // Wait until a new task is added or the thread is stopped + cond_var.wait(lock, [this]() { return stop_thread || !minPriorityQueue.empty(); }); + } + else if (!done_tasks.empty()) + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Something is done"); + for (size_t i = 0; i < done_tasks.size(); i++) + { + if (done_tasks[i] == minPriorityQueue.top().first) + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "removing {}", done_tasks[i]->getInfo().query); + minPriorityQueue.pop(); + done_tasks.erase(done_tasks.begin() + i); + } + } + } + else + { + auto next_task = minPriorityQueue.top(); + UInt64 end_time_ms = next_task.second; // UInt64 milliseconds + auto now = std::chrono::steady_clock::now(); + UInt64 now_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + + // Convert UInt64 end_time_ms to std::chrono::steady_clock::time_point + const std::chrono::steady_clock::duration duration_milliseconds = std::chrono::milliseconds(end_time_ms); + const std::chrono::time_point end_time_point(duration_milliseconds); + + if (end_time_ms <= now_ms) + { + if (end_time_ms != 0) + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancel task, end_time_ms: {}, now_ms: {}", end_time_ms, now_ms); + // Time to execute func1() + cancelTask(next_task.first); + // Remove task from queue + minPriorityQueue.pop(); + } + } + else + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Set on cond var"); + // Wait until the nearest endTime or until a new task is added that might have an earlier endTime + cond_var.wait_for(lock, duration_milliseconds, [this, end_time_ms]() + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cond var condition is done"); + return stop_thread || (!minPriorityQueue.empty() && minPriorityQueue.top().second < end_time_ms); + }); + } + } + } + } +}; + +} diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 402cbee3c7c..4469d4fedbd 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -104,6 +105,7 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q throw Exception(ErrorCodes::LOGICAL_ERROR, "Query id cannot be empty"); bool is_unlimited_query = isUnlimitedQuery(ast); + std::shared_ptr query; { auto [lock, overcommit_blocker] = safeLock(); // To avoid deadlock in case of OOM @@ -285,17 +287,21 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q /// since allocation and deallocation could happen in different threads } + query = std::make_shared( + query_context, + query_, + client_info, + priorities.insert(static_cast(settings.priority)), + std::move(thread_group), + query_kind, + settings, + watch_start_nanoseconds); + auto process_it = processes.emplace( processes.end(), - std::make_shared( - query_context, - query_, - client_info, - priorities.insert(settings[Setting::priority]), - std::move(thread_group), - query_kind, - settings, - watch_start_nanoseconds)); + query); + + CancellationChecker::getInstance().appendTask(query, query_context->getSettingsRef().max_execution_time.totalMilliseconds()); increaseQueryKindAmount(query_kind); @@ -327,6 +333,7 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q } } + CancellationChecker::getInstance().appendDoneTasks(query); return res; } From ac00df6b58a2eb52233529e9cd292652340a107a Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 25 Sep 2024 20:55:49 +0200 Subject: [PATCH 037/502] add more code --- src/Interpreters/CancellationChecker.h | 151 +++++++++++++++++-------- src/Interpreters/ProcessList.cpp | 2 + src/Server/TCPHandler.cpp | 4 + 3 files changed, 111 insertions(+), 46 deletions(-) diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index 434ae15dea8..e79a2bde5ea 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -8,9 +9,22 @@ namespace DB { -struct CompareEndTime { - bool operator()(const std::pair, UInt64>& a, const std::pair, UInt64>& b) const { - return a.second > b.second; +struct QueryToTrack +{ + QueryToTrack (std::shared_ptr query_, UInt64 timeout_, UInt64 endtime_): + query(query_), + timeout(timeout_), + endtime(endtime_) {} + std::shared_ptr query; + UInt64 timeout; + UInt64 endtime; +}; + +struct CompareEndTime +{ + bool operator()(const QueryToTrack& a, const QueryToTrack& b) const + { + return a.timeout > b.timeout; } }; @@ -35,12 +49,13 @@ private: // Priority queue to manage tasks based on endTime std::priority_queue< - std::pair, UInt64>, - std::vector, UInt64>>, + QueryToTrack, + std::vector, CompareEndTime > minPriorityQueue; std::vector> done_tasks; + std::vector> cancelled_tasks; std::atomic stop_thread; std::mutex m; @@ -65,18 +80,28 @@ public: CancellationChecker& operator=(const CancellationChecker&) = delete; // Method to add a new task to the priority queue - void appendTask(std::shared_ptr query, UInt64 end_time) + void appendTask(std::shared_ptr query, UInt64 timeout) { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added. query: {}, timeout: {} milliseconds", query->getInfo().query, end_time); - std::unique_lock lock(m); - minPriorityQueue.emplace(query, end_time); + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added. query: {}, timeout: {} milliseconds", query->getInfo().query, timeout); + auto now = std::chrono::steady_clock::now(); + UInt64 end_time = std::chrono::duration_cast(now.time_since_epoch()).count() + timeout; + minPriorityQueue.emplace(query, timeout, end_time); cond_var.notify_all(); } void appendDoneTasks(std::shared_ptr query) { - std::unique_lock lock(m); + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added to done tasks, query: {}", query->getInfo().query); done_tasks.push_back(query); + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "done tasks size: {}", done_tasks.size()); + cond_var.notify_all(); + } + + void addToCancelledTasks(std::shared_ptr query) + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added to cancelled tasks, query: {}", query->getInfo().query); + cancelled_tasks.push_back(query); + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancelled tasks size: {}", cancelled_tasks.size()); cond_var.notify_all(); } @@ -91,53 +116,87 @@ public: LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Iteration"); if (minPriorityQueue.empty()) { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "minPriorityQueue.empty()"); // Wait until a new task is added or the thread is stopped cond_var.wait(lock, [this]() { return stop_thread || !minPriorityQueue.empty(); }); } - else if (!done_tasks.empty()) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Something is done"); - for (size_t i = 0; i < done_tasks.size(); i++) - { - if (done_tasks[i] == minPriorityQueue.top().first) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "removing {}", done_tasks[i]->getInfo().query); - minPriorityQueue.pop(); - done_tasks.erase(done_tasks.begin() + i); - } - } - } else { - auto next_task = minPriorityQueue.top(); - UInt64 end_time_ms = next_task.second; // UInt64 milliseconds - auto now = std::chrono::steady_clock::now(); - UInt64 now_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); - - // Convert UInt64 end_time_ms to std::chrono::steady_clock::time_point - const std::chrono::steady_clock::duration duration_milliseconds = std::chrono::milliseconds(end_time_ms); - const std::chrono::time_point end_time_point(duration_milliseconds); - - if (end_time_ms <= now_ms) + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "else"); + if (!done_tasks.empty()) { - if (end_time_ms != 0) + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Something is done"); + for (size_t i = 0; i < done_tasks.size(); i++) + { + if (done_tasks[i] == minPriorityQueue.top().query) + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "removing {}", done_tasks[i]->getInfo().query); + minPriorityQueue.pop(); + done_tasks.erase(done_tasks.begin() + i); + } + } + if (minPriorityQueue.empty()) + continue; + } + + if (!cancelled_tasks.empty()) + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Something needs to be cancelled"); + for (size_t i = 0; i < cancelled_tasks.size(); i++) + { + if (cancelled_tasks[i] == minPriorityQueue.top().query) + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancelling task {}", cancelled_tasks[i]->getInfo().query); + cancelTask(minPriorityQueue.top().query); + minPriorityQueue.pop(); + cancelled_tasks.erase(cancelled_tasks.begin() + i); + } + } + if (minPriorityQueue.empty()) + continue; + } + + const auto next_task = minPriorityQueue.top(); + + { + const UInt64 end_time_ms = next_task.endtime; // UInt64 milliseconds + const auto now = std::chrono::steady_clock::now(); + const UInt64 now_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + + // Convert UInt64 timeout to std::chrono::steady_clock::time_point + const std::chrono::steady_clock::duration duration_milliseconds = std::chrono::milliseconds(next_task.timeout); + const std::chrono::time_point end_time_point(duration_milliseconds); + + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "end_time_ms: {}, now_ms: {}, duration: {}, isKilled: {}", end_time_ms, now_ms, duration_milliseconds.count(), next_task.query->isKilled()); + if ((end_time_ms <= now_ms && duration_milliseconds.count() != 0)) { LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancel task, end_time_ms: {}, now_ms: {}", end_time_ms, now_ms); - // Time to execute func1() - cancelTask(next_task.first); - // Remove task from queue + cancelTask(next_task.query); minPriorityQueue.pop(); } - } - else - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Set on cond var"); - // Wait until the nearest endTime or until a new task is added that might have an earlier endTime - cond_var.wait_for(lock, duration_milliseconds, [this, end_time_ms]() + else { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cond var condition is done"); - return stop_thread || (!minPriorityQueue.empty() && minPriorityQueue.top().second < end_time_ms); - }); + if (duration_milliseconds.count()) + { + const size_t cancelled_size = cancelled_tasks.size(); + // Wait until the nearest endTime or until a new task is added that might have an earlier endTime or maybe some task cancelled + cond_var.wait_for(lock, duration_milliseconds, [this, end_time_ms, cancelled_size]() + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cond var condition is: {}, size of queue: {}", stop_thread || (!minPriorityQueue.empty() && minPriorityQueue.top().endtime < end_time_ms), minPriorityQueue.size()); + return stop_thread || (!minPriorityQueue.empty() && minPriorityQueue.top().endtime < end_time_ms) || cancelled_tasks.size() != cancelled_size; + }); + } + else + { + const size_t cancelled_size = cancelled_tasks.size(); + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "doesn't have duration, done_size: {}", cancelled_size); + cond_var.wait(lock, [this, cancelled_size]() + { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cond var condition is: {}, done_size: {}", stop_thread || cancelled_tasks.size() != cancelled_size, cancelled_size); + return stop_thread || cancelled_tasks.size() != cancelled_size; + }); + } + } } } } diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 4469d4fedbd..dd4e2d3c194 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -374,6 +374,8 @@ ProcessListEntry::~ProcessListEntry() if (auto query_user = parent.queries_to_user.find(query_id); query_user != parent.queries_to_user.end()) parent.queries_to_user.erase(query_user); + CancellationChecker::getInstance().appendDoneTasks(*it); + /// This removes the memory_tracker of one request. parent.processes.erase(it); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index dadaf614f92..124a52ac59b 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -2293,6 +2294,9 @@ void TCPHandler::decreaseCancellationStatus(const std::string & log_message) state.cancellation_status = CancellationStatus::FULLY_CANCELLED; } + if (state.cancellation_status == CancellationStatus::FULLY_CANCELLED) + CancellationChecker::getInstance().addToCancelledTasks(state.io.process_list_entry->getQueryStatus()); + auto current_status = magic_enum::enum_name(state.cancellation_status); LOG_INFO(log, "Change cancellation status from {} to {}. Log message: {}", prev_status, current_status, log_message); } From f64c56bebc1f3fd10a8e53ba048b9728d1a781b9 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:47:27 +0200 Subject: [PATCH 038/502] refactor the checker --- src/Interpreters/CancellationChecker.h | 156 +++++++++++-------------- 1 file changed, 69 insertions(+), 87 deletions(-) diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index e79a2bde5ea..6c5e987a804 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -1,10 +1,8 @@ #include #include #include -#include #include -#include -#include "base/types.h" +#include namespace DB { @@ -24,7 +22,7 @@ struct CompareEndTime { bool operator()(const QueryToTrack& a, const QueryToTrack& b) const { - return a.timeout > b.timeout; + return a.endtime < b.endtime; } }; @@ -48,11 +46,7 @@ private: } // Priority queue to manage tasks based on endTime - std::priority_queue< - QueryToTrack, - std::vector, - CompareEndTime - > minPriorityQueue; + std::multiset querySet; std::vector> done_tasks; std::vector> cancelled_tasks; @@ -67,6 +61,21 @@ private: query->cancelQuery(/*kill=*/false); } + bool removeQueryFromSet(std::shared_ptr query) + { + for (auto it = querySet.begin(); it != querySet.end();) + { + if (it->query == query) + { + it = querySet.erase(it); + return true; + } + else + ++it; + } + return false; + } + public: // Singleton instance retrieval static CancellationChecker& getInstance() @@ -79,123 +88,96 @@ public: CancellationChecker(const CancellationChecker&) = delete; CancellationChecker& operator=(const CancellationChecker&) = delete; - // Method to add a new task to the priority queue - void appendTask(std::shared_ptr query, UInt64 timeout) + // Method to add a new task to the multiset + void appendTask(const std::shared_ptr & query, const UInt64 & timeout) { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added. query: {}, timeout: {} milliseconds", query->getInfo().query, timeout); - auto now = std::chrono::steady_clock::now(); - UInt64 end_time = std::chrono::duration_cast(now.time_since_epoch()).count() + timeout; - minPriorityQueue.emplace(query, timeout, end_time); + const auto & now = std::chrono::steady_clock::now(); + const UInt64 & end_time = std::chrono::duration_cast(now.time_since_epoch()).count() + timeout; + querySet.emplace(query, timeout, end_time); cond_var.notify_all(); } - void appendDoneTasks(std::shared_ptr query) + // Used when some task is done + void appendDoneTasks(const std::shared_ptr & query) { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added to done tasks, query: {}", query->getInfo().query); done_tasks.push_back(query); - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "done tasks size: {}", done_tasks.size()); cond_var.notify_all(); } - void addToCancelledTasks(std::shared_ptr query) + // Used when some task is cancelled + void addToCancelledTasks(const std::shared_ptr & query) { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added to cancelled tasks, query: {}", query->getInfo().query); cancelled_tasks.push_back(query); - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancelled tasks size: {}", cancelled_tasks.size()); cond_var.notify_all(); } // Worker thread function void workerFunction() { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "workerFunction()"); std::unique_lock lock(m); while (!stop_thread) { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Iteration"); - if (minPriorityQueue.empty()) + if (querySet.empty()) { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "minPriorityQueue.empty()"); // Wait until a new task is added or the thread is stopped - cond_var.wait(lock, [this]() { return stop_thread || !minPriorityQueue.empty(); }); + cond_var.wait(lock, [this]() { return stop_thread || !querySet.empty(); }); } else { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "else"); - if (!done_tasks.empty()) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Something is done"); - for (size_t i = 0; i < done_tasks.size(); i++) - { - if (done_tasks[i] == minPriorityQueue.top().query) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "removing {}", done_tasks[i]->getInfo().query); - minPriorityQueue.pop(); - done_tasks.erase(done_tasks.begin() + i); - } - } - if (minPriorityQueue.empty()) - continue; - } - if (!cancelled_tasks.empty()) { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Something needs to be cancelled"); - for (size_t i = 0; i < cancelled_tasks.size(); i++) + for (auto it = cancelled_tasks.begin(); it != cancelled_tasks.end();) { - if (cancelled_tasks[i] == minPriorityQueue.top().query) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancelling task {}", cancelled_tasks[i]->getInfo().query); - cancelTask(minPriorityQueue.top().query); - minPriorityQueue.pop(); - cancelled_tasks.erase(cancelled_tasks.begin() + i); - } + cancelTask(*it); + removeQueryFromSet(*it); + cancelled_tasks.erase(it); } - if (minPriorityQueue.empty()) + if (querySet.empty()) continue; } - const auto next_task = minPriorityQueue.top(); - + if (!done_tasks.empty()) { - const UInt64 end_time_ms = next_task.endtime; // UInt64 milliseconds - const auto now = std::chrono::steady_clock::now(); - const UInt64 now_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); - - // Convert UInt64 timeout to std::chrono::steady_clock::time_point - const std::chrono::steady_clock::duration duration_milliseconds = std::chrono::milliseconds(next_task.timeout); - const std::chrono::time_point end_time_point(duration_milliseconds); - - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "end_time_ms: {}, now_ms: {}, duration: {}, isKilled: {}", end_time_ms, now_ms, duration_milliseconds.count(), next_task.query->isKilled()); - if ((end_time_ms <= now_ms && duration_milliseconds.count() != 0)) + for (auto it = done_tasks.begin(); it != done_tasks.end();) { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancel task, end_time_ms: {}, now_ms: {}", end_time_ms, now_ms); - cancelTask(next_task.query); - minPriorityQueue.pop(); + removeQueryFromSet(*it); + done_tasks.erase(it); + } + if (querySet.empty()) + continue; + } + + const auto next_task = (*querySet.begin()); + + const UInt64 end_time_ms = next_task.endtime; + const auto now = std::chrono::steady_clock::now(); + const UInt64 now_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + + // Convert UInt64 timeout to std::chrono::steady_clock::time_point + const std::chrono::steady_clock::duration duration_milliseconds = std::chrono::milliseconds(next_task.timeout); + + if ((end_time_ms <= now_ms && duration_milliseconds.count() != 0)) + { + cancelTask(next_task.query); + querySet.erase(next_task); + } + else + { + // Wait until the nearest endTime or until a new task is added that might have an earlier endTime or maybe some task cancelled + if (duration_milliseconds.count()) + { + cond_var.wait_for(lock, duration_milliseconds, [this, end_time_ms]() + { + return stop_thread || (!querySet.empty() && (*querySet.begin()).endtime < end_time_ms) || !done_tasks.empty() || !cancelled_tasks.empty(); + }); } else { - if (duration_milliseconds.count()) + cond_var.wait(lock, [this, end_time_ms]() { - const size_t cancelled_size = cancelled_tasks.size(); - // Wait until the nearest endTime or until a new task is added that might have an earlier endTime or maybe some task cancelled - cond_var.wait_for(lock, duration_milliseconds, [this, end_time_ms, cancelled_size]() - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cond var condition is: {}, size of queue: {}", stop_thread || (!minPriorityQueue.empty() && minPriorityQueue.top().endtime < end_time_ms), minPriorityQueue.size()); - return stop_thread || (!minPriorityQueue.empty() && minPriorityQueue.top().endtime < end_time_ms) || cancelled_tasks.size() != cancelled_size; - }); - } - else - { - const size_t cancelled_size = cancelled_tasks.size(); - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "doesn't have duration, done_size: {}", cancelled_size); - cond_var.wait(lock, [this, cancelled_size]() - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cond var condition is: {}, done_size: {}", stop_thread || cancelled_tasks.size() != cancelled_size, cancelled_size); - return stop_thread || cancelled_tasks.size() != cancelled_size; - }); - } + return stop_thread || (!querySet.empty() && (*querySet.begin()).endtime < end_time_ms) || !done_tasks.empty() || !cancelled_tasks.empty(); + }); } } } From 4a77e1d94180d123595758a09d9f48c4452ca4e0 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:54:46 +0200 Subject: [PATCH 039/502] fix style check --- src/Interpreters/CancellationChecker.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index 6c5e987a804..0589923254a 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include From f3150e8e57c3c5be398715c9f18d416b767514b1 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:20:08 +0200 Subject: [PATCH 040/502] fix settings reaching --- src/Interpreters/ProcessList.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index dd4e2d3c194..326ccbe5ae9 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -291,7 +291,7 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q query_context, query_, client_info, - priorities.insert(static_cast(settings.priority)), + priorities.insert(settings[Setting::priority]), std::move(thread_group), query_kind, settings, @@ -301,7 +301,7 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q processes.end(), query); - CancellationChecker::getInstance().appendTask(query, query_context->getSettingsRef().max_execution_time.totalMilliseconds()); + CancellationChecker::getInstance().appendTask(query, query_context->getSettingsRef()[Setting::max_execution_time].totalMilliseconds()); increaseQueryKindAmount(query_kind); From 49df0f95fe13ec29dc00a37c83f460caef9d96df Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:40:03 +0200 Subject: [PATCH 041/502] add logs --- src/Interpreters/CancellationChecker.h | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index 0589923254a..4a870d9122e 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -1,9 +1,8 @@ -#pragma once - #include #include #include #include +#include #include namespace DB @@ -93,6 +92,7 @@ public: // Method to add a new task to the multiset void appendTask(const std::shared_ptr & query, const UInt64 & timeout) { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added. query: {}, timeout: {} milliseconds", query->getInfo().query, timeout); const auto & now = std::chrono::steady_clock::now(); const UInt64 & end_time = std::chrono::duration_cast(now.time_since_epoch()).count() + timeout; querySet.emplace(query, timeout, end_time); @@ -102,35 +102,47 @@ public: // Used when some task is done void appendDoneTasks(const std::shared_ptr & query) { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added to done tasks, query: {}", query->getInfo().query); done_tasks.push_back(query); + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "done tasks size: {}", done_tasks.size()); cond_var.notify_all(); } // Used when some task is cancelled void addToCancelledTasks(const std::shared_ptr & query) { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added to cancelled tasks, query: {}", query->getInfo().query); cancelled_tasks.push_back(query); + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancelled tasks size: {}", cancelled_tasks.size()); cond_var.notify_all(); } // Worker thread function void workerFunction() { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "workerFunction()"); std::unique_lock lock(m); while (!stop_thread) { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Iteration"); if (querySet.empty()) { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "minPriorityQueue.empty()"); // Wait until a new task is added or the thread is stopped cond_var.wait(lock, [this]() { return stop_thread || !querySet.empty(); }); } else { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "else"); + if (!cancelled_tasks.empty()) { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Something needs to be cancelled"); for (auto it = cancelled_tasks.begin(); it != cancelled_tasks.end();) { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "removing {} from cancelled tasks", (*it)->getInfo().query); + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancelling this query."); cancelTask(*it); removeQueryFromSet(*it); cancelled_tasks.erase(it); @@ -141,8 +153,10 @@ public: if (!done_tasks.empty()) { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Something is done"); for (auto it = done_tasks.begin(); it != done_tasks.end();) { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "removing {} from done tasks", (*it)->getInfo().query); removeQueryFromSet(*it); done_tasks.erase(it); } @@ -161,11 +175,13 @@ public: if ((end_time_ms <= now_ms && duration_milliseconds.count() != 0)) { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancel task because of the timeout, end_time_ms: {}, now_ms: {}", end_time_ms, now_ms); cancelTask(next_task.query); querySet.erase(next_task); } else { + LOG_TRACE(getLogger("CANCELLATION CHECKER"), "condvar, duration: {}", duration_milliseconds.count()); // Wait until the nearest endTime or until a new task is added that might have an earlier endTime or maybe some task cancelled if (duration_milliseconds.count()) { From 49c74324c494ab003fb2a747e2faba2502d756d1 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:46:25 +0200 Subject: [PATCH 042/502] Update CancellationChecker.h --- src/Interpreters/CancellationChecker.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index 4a870d9122e..af8ecb7cd0c 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include From ce8a308a70444bfffc20f019a4f5915f39f8ec4c Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:59:01 +0200 Subject: [PATCH 043/502] add mutexes + remove wrong entry --- src/Interpreters/CancellationChecker.h | 3 +++ src/Interpreters/ProcessList.cpp | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index af8ecb7cd0c..7a14a15f758 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -94,6 +94,7 @@ public: // Method to add a new task to the multiset void appendTask(const std::shared_ptr & query, const UInt64 & timeout) { + std::unique_lock lock(m); LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added. query: {}, timeout: {} milliseconds", query->getInfo().query, timeout); const auto & now = std::chrono::steady_clock::now(); const UInt64 & end_time = std::chrono::duration_cast(now.time_since_epoch()).count() + timeout; @@ -104,6 +105,7 @@ public: // Used when some task is done void appendDoneTasks(const std::shared_ptr & query) { + std::unique_lock lock(m); LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added to done tasks, query: {}", query->getInfo().query); done_tasks.push_back(query); LOG_TRACE(getLogger("CANCELLATION CHECKER"), "done tasks size: {}", done_tasks.size()); @@ -113,6 +115,7 @@ public: // Used when some task is cancelled void addToCancelledTasks(const std::shared_ptr & query) { + std::unique_lock lock(m); LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added to cancelled tasks, query: {}", query->getInfo().query); cancelled_tasks.push_back(query); LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancelled tasks size: {}", cancelled_tasks.size()); diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 326ccbe5ae9..fc12cd8731c 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -333,7 +333,6 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q } } - CancellationChecker::getInstance().appendDoneTasks(query); return res; } From 13a96ec29dededd7ff6c5a7adf4b314b2dfc31da Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 27 Sep 2024 16:37:22 +0200 Subject: [PATCH 044/502] Update TCPHandler.cpp --- src/Server/TCPHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 124a52ac59b..e563deb2262 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -2294,7 +2294,7 @@ void TCPHandler::decreaseCancellationStatus(const std::string & log_message) state.cancellation_status = CancellationStatus::FULLY_CANCELLED; } - if (state.cancellation_status == CancellationStatus::FULLY_CANCELLED) + if (state.cancellation_status == CancellationStatus::FULLY_CANCELLED && state.io.process_list_entry) CancellationChecker::getInstance().addToCancelledTasks(state.io.process_list_entry->getQueryStatus()); auto current_status = magic_enum::enum_name(state.cancellation_status); From fb91126fcfc1216ef0dd4ff9ee68e776eba3f805 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 3 Oct 2024 13:27:39 +0200 Subject: [PATCH 045/502] refactor --- src/Interpreters/CancellationChecker.cpp | 137 +++++++++++++++++ src/Interpreters/CancellationChecker.h | 179 +++-------------------- src/Server/TCPHandler.cpp | 4 - 3 files changed, 155 insertions(+), 165 deletions(-) create mode 100644 src/Interpreters/CancellationChecker.cpp diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp new file mode 100644 index 00000000000..b552427d4f4 --- /dev/null +++ b/src/Interpreters/CancellationChecker.cpp @@ -0,0 +1,137 @@ +#include +#include + +#include <__chrono/duration.h> +#include +#include + + +namespace DB +{ + +QueryToTrack::QueryToTrack( + std::shared_ptr query_, + UInt64 timeout_, + UInt64 endtime_) + : query(query_), timeout(timeout_), endtime(endtime_) +{ +} + +bool CompareEndTime::operator()(const QueryToTrack& a, const QueryToTrack& b) const +{ + if (a.endtime != b.endtime) + return a.endtime < b.endtime; + else + return a.query->getClientInfo().current_query_id < b.query->getClientInfo().current_query_id; +} + +CancellationChecker::CancellationChecker() : stop_thread(false) +{ +} + +CancellationChecker::~CancellationChecker() +{ + stop_thread = true; +} + +CancellationChecker& CancellationChecker::getInstance() +{ + static CancellationChecker instance; + return instance; +} + +void CancellationChecker::cancelTask(std::shared_ptr query) +{ + query->cancelQuery(/*kill=*/false); +} + +bool CancellationChecker::removeQueryFromSet(std::shared_ptr query) +{ + for (auto it = querySet.begin(); it != querySet.end();) + { + if (it->query == query) + { + LOG_TRACE(getLogger("CancellationChecker"), "Removing query {} from done tasks", query->getInfo().query); + it = querySet.erase(it); + return true; + } + else + ++it; + } + return false; +} + +void CancellationChecker::appendTask(const std::shared_ptr & query, const UInt64 & timeout) +{ + if (!timeout) + { + LOG_TRACE(getLogger("CancellationChecker"), "Did not add the task because the timeout is 0. Query: {}", query->getInfo().query); + return; + } + std::unique_lock lock(m); + LOG_TRACE(getLogger("CancellationChecker"), "Added to set. query: {}, timeout: {} milliseconds", query->getInfo().query, timeout); + const auto & now = std::chrono::steady_clock::now(); + const UInt64 & end_time = std::chrono::duration_cast(now.time_since_epoch()).count() + timeout; + querySet.emplace(query, timeout, end_time); + cond_var.notify_all(); +} + +void CancellationChecker::appendDoneTasks(const std::shared_ptr & query) +{ + std::unique_lock lock(m); + removeQueryFromSet(query); + cond_var.notify_all(); +} + +void CancellationChecker::workerFunction() +{ + LOG_TRACE(getLogger("CancellationChecker"), "Started worker function"); + std::unique_lock lock(m); + + while (!stop_thread) + { + size_t query_size = 0; + UInt64 end_time_ms = 0; + UInt64 duration = 0; + auto now = std::chrono::steady_clock::now(); + UInt64 now_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + std::chrono::steady_clock::duration duration_milliseconds = std::chrono::milliseconds(0); + + if (!querySet.empty()) + { + query_size = querySet.size(); + + const auto next_task = (*querySet.begin()); + + end_time_ms = next_task.endtime; + duration = next_task.timeout; + now = std::chrono::steady_clock::now(); + now_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + + // Convert UInt64 timeout to std::chrono::steady_clock::time_point + duration_milliseconds = std::chrono::milliseconds(next_task.timeout); + + if ((end_time_ms <= now_ms && duration_milliseconds.count() != 0)) + { + LOG_TRACE(getLogger("CancellationChecker"), "Cancelling the task because of the timeout: {}, query: {}", duration, next_task.query->getInfo().query); + cancelTask(next_task.query); + querySet.erase(next_task); + + continue; + } + } + + if (!duration_milliseconds.count()) + duration_milliseconds = std::chrono::years(1); // we put one year time to wait if we don't have any timeouts + + cond_var.wait_for(lock, duration_milliseconds, [this, end_time_ms, query_size]() + { + if (query_size) + return stop_thread || (!querySet.empty() && (*querySet.begin()).endtime < end_time_ms); + else + return stop_thread || !querySet.empty(); + }); + } +} + +} diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index 7a14a15f758..7030e6baa90 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -1,21 +1,19 @@ #pragma once -#include -#include -#include #include -#include -#include +#include +#include namespace DB { struct QueryToTrack { - QueryToTrack (std::shared_ptr query_, UInt64 timeout_, UInt64 endtime_): - query(query_), - timeout(timeout_), - endtime(endtime_) {} + QueryToTrack( + std::shared_ptr query_, + UInt64 timeout_, + UInt64 endtime_); + std::shared_ptr query; UInt64 timeout; UInt64 endtime; @@ -23,189 +21,48 @@ struct QueryToTrack struct CompareEndTime { - bool operator()(const QueryToTrack& a, const QueryToTrack& b) const - { - return a.endtime < b.endtime; - } + bool operator()(const QueryToTrack& a, const QueryToTrack& b) const; }; /* -A Singleton class that check if tasks are cancelled or timed out. +A Singleton class that checks if tasks are cancelled or timed out. Has a priority queue ordered by end time. Checker waits until the -first task in the list is done, then check if this task needs to be cancelled. +first task in the list is done, then checks if this task needs to be cancelled. If yes, sets a cancellation flag on this task, otherwise removes the task from the queue. */ class CancellationChecker { private: - // Private constructor for Singleton pattern - CancellationChecker() : stop_thread(false) - { - } - - ~CancellationChecker() - { - stop_thread = true; - } + CancellationChecker(); + ~CancellationChecker(); // Priority queue to manage tasks based on endTime std::multiset querySet; - std::vector> done_tasks; - std::vector> cancelled_tasks; - std::atomic stop_thread; std::mutex m; std::condition_variable cond_var; // Function to execute when a task's endTime is reached - void cancelTask(std::shared_ptr query) - { - query->cancelQuery(/*kill=*/false); - } - - bool removeQueryFromSet(std::shared_ptr query) - { - for (auto it = querySet.begin(); it != querySet.end();) - { - if (it->query == query) - { - it = querySet.erase(it); - return true; - } - else - ++it; - } - return false; - } + void cancelTask(std::shared_ptr query); + bool removeQueryFromSet(std::shared_ptr query); public: // Singleton instance retrieval - static CancellationChecker& getInstance() - { - static CancellationChecker instance; - return instance; - } + static CancellationChecker& getInstance(); // Deleted copy constructor and assignment operator CancellationChecker(const CancellationChecker&) = delete; CancellationChecker& operator=(const CancellationChecker&) = delete; // Method to add a new task to the multiset - void appendTask(const std::shared_ptr & query, const UInt64 & timeout) - { - std::unique_lock lock(m); - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added. query: {}, timeout: {} milliseconds", query->getInfo().query, timeout); - const auto & now = std::chrono::steady_clock::now(); - const UInt64 & end_time = std::chrono::duration_cast(now.time_since_epoch()).count() + timeout; - querySet.emplace(query, timeout, end_time); - cond_var.notify_all(); - } + void appendTask(const std::shared_ptr & query, const UInt64 & timeout); // Used when some task is done - void appendDoneTasks(const std::shared_ptr & query) - { - std::unique_lock lock(m); - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added to done tasks, query: {}", query->getInfo().query); - done_tasks.push_back(query); - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "done tasks size: {}", done_tasks.size()); - cond_var.notify_all(); - } - - // Used when some task is cancelled - void addToCancelledTasks(const std::shared_ptr & query) - { - std::unique_lock lock(m); - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "added to cancelled tasks, query: {}", query->getInfo().query); - cancelled_tasks.push_back(query); - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancelled tasks size: {}", cancelled_tasks.size()); - cond_var.notify_all(); - } + void appendDoneTasks(const std::shared_ptr & query); // Worker thread function - void workerFunction() - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "workerFunction()"); - std::unique_lock lock(m); - - while (!stop_thread) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Iteration"); - if (querySet.empty()) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "minPriorityQueue.empty()"); - // Wait until a new task is added or the thread is stopped - cond_var.wait(lock, [this]() { return stop_thread || !querySet.empty(); }); - } - else - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "else"); - - if (!cancelled_tasks.empty()) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Something needs to be cancelled"); - for (auto it = cancelled_tasks.begin(); it != cancelled_tasks.end();) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "removing {} from cancelled tasks", (*it)->getInfo().query); - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancelling this query."); - cancelTask(*it); - removeQueryFromSet(*it); - cancelled_tasks.erase(it); - } - if (querySet.empty()) - continue; - } - - if (!done_tasks.empty()) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "Something is done"); - for (auto it = done_tasks.begin(); it != done_tasks.end();) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "removing {} from done tasks", (*it)->getInfo().query); - removeQueryFromSet(*it); - done_tasks.erase(it); - } - if (querySet.empty()) - continue; - } - - const auto next_task = (*querySet.begin()); - - const UInt64 end_time_ms = next_task.endtime; - const auto now = std::chrono::steady_clock::now(); - const UInt64 now_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); - - // Convert UInt64 timeout to std::chrono::steady_clock::time_point - const std::chrono::steady_clock::duration duration_milliseconds = std::chrono::milliseconds(next_task.timeout); - - if ((end_time_ms <= now_ms && duration_milliseconds.count() != 0)) - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "cancel task because of the timeout, end_time_ms: {}, now_ms: {}", end_time_ms, now_ms); - cancelTask(next_task.query); - querySet.erase(next_task); - } - else - { - LOG_TRACE(getLogger("CANCELLATION CHECKER"), "condvar, duration: {}", duration_milliseconds.count()); - // Wait until the nearest endTime or until a new task is added that might have an earlier endTime or maybe some task cancelled - if (duration_milliseconds.count()) - { - cond_var.wait_for(lock, duration_milliseconds, [this, end_time_ms]() - { - return stop_thread || (!querySet.empty() && (*querySet.begin()).endtime < end_time_ms) || !done_tasks.empty() || !cancelled_tasks.empty(); - }); - } - else - { - cond_var.wait(lock, [this, end_time_ms]() - { - return stop_thread || (!querySet.empty() && (*querySet.begin()).endtime < end_time_ms) || !done_tasks.empty() || !cancelled_tasks.empty(); - }); - } - } - } - } - } + void workerFunction(); }; } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index e563deb2262..dadaf614f92 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -2294,9 +2293,6 @@ void TCPHandler::decreaseCancellationStatus(const std::string & log_message) state.cancellation_status = CancellationStatus::FULLY_CANCELLED; } - if (state.cancellation_status == CancellationStatus::FULLY_CANCELLED && state.io.process_list_entry) - CancellationChecker::getInstance().addToCancelledTasks(state.io.process_list_entry->getQueryStatus()); - auto current_status = magic_enum::enum_name(state.cancellation_status); LOG_INFO(log, "Change cancellation status from {} to {}. Log message: {}", prev_status, current_status, log_message); } From af9590a03cd0d42cb0056a9c54168875153685a7 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 3 Oct 2024 16:51:35 +0200 Subject: [PATCH 046/502] add cancellation reason --- src/Interpreters/CancellationChecker.cpp | 9 +++++---- src/Interpreters/CancellationChecker.h | 2 +- src/Interpreters/ProcessList.cpp | 13 ++++++++++++- src/Interpreters/ProcessList.h | 11 ++++++++++- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp index b552427d4f4..5b5f8bb52e3 100644 --- a/src/Interpreters/CancellationChecker.cpp +++ b/src/Interpreters/CancellationChecker.cpp @@ -1,4 +1,5 @@ #include +#include "Interpreters/ProcessList.h" #include #include <__chrono/duration.h> @@ -40,9 +41,9 @@ CancellationChecker& CancellationChecker::getInstance() return instance; } -void CancellationChecker::cancelTask(std::shared_ptr query) +void CancellationChecker::cancelTask(std::shared_ptr query, CancelReason reason) { - query->cancelQuery(/*kill=*/false); + query->cancelQuery(/*kill=*/false, /*reason=*/reason); } bool CancellationChecker::removeQueryFromSet(std::shared_ptr query) @@ -63,7 +64,7 @@ bool CancellationChecker::removeQueryFromSet(std::shared_ptr query) void CancellationChecker::appendTask(const std::shared_ptr & query, const UInt64 & timeout) { - if (!timeout) + if (timeout > 0) // Avoid cases when the timeout is less or equal zero { LOG_TRACE(getLogger("CancellationChecker"), "Did not add the task because the timeout is 0. Query: {}", query->getInfo().query); return; @@ -114,7 +115,7 @@ void CancellationChecker::workerFunction() if ((end_time_ms <= now_ms && duration_milliseconds.count() != 0)) { LOG_TRACE(getLogger("CancellationChecker"), "Cancelling the task because of the timeout: {}, query: {}", duration, next_task.query->getInfo().query); - cancelTask(next_task.query); + cancelTask(next_task.query, CancelReason::TIMEOUT); querySet.erase(next_task); continue; diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index 7030e6baa90..5491fd104d3 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -44,7 +44,7 @@ private: std::condition_variable cond_var; // Function to execute when a task's endTime is reached - void cancelTask(std::shared_ptr query); + void cancelTask(std::shared_ptr query, CancelReason reason); bool removeQueryFromSet(std::shared_ptr query); public: diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index fc12cd8731c..37c2be430c2 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -56,6 +56,7 @@ namespace ErrorCodes extern const int QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING; extern const int LOGICAL_ERROR; extern const int QUERY_WAS_CANCELLED; + extern const int TIMEOUT_EXCEEDED; } @@ -454,13 +455,15 @@ void QueryStatus::ExecutorHolder::remove() executor = nullptr; } -CancellationCode QueryStatus::cancelQuery(bool) +CancellationCode QueryStatus::cancelQuery(bool, CancelReason reason) { if (is_killed.load()) return CancellationCode::CancelSent; is_killed.store(true); + cancel_reason = reason; + std::vector executors_snapshot; { @@ -493,7 +496,11 @@ void QueryStatus::addPipelineExecutor(PipelineExecutor * e) /// addPipelineExecutor() from the cancelQuery() context, and this will /// lead to deadlock. if (is_killed.load()) + { + if (cancel_reason == CancelReason::TIMEOUT) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Query was timed out"); throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); + } std::lock_guard lock(executors_mutex); assert(!executors.contains(e)); @@ -519,7 +526,11 @@ void QueryStatus::removePipelineExecutor(PipelineExecutor * e) bool QueryStatus::checkTimeLimit() { if (is_killed.load()) + { + if (cancel_reason == CancelReason::TIMEOUT) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Query was timed out"); throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); + } return limits.checkTimeLimit(watch, overflow_mode); } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index accb73e12df..6dba3bb561d 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -42,6 +42,13 @@ class ThreadStatus; class ProcessListEntry; +enum CancelReason +{ + NOT_CANCELLED, + TIMEOUT, + MANUAL_CANCEL, +}; + /** Information of process list element. * To output in SHOW PROCESSLIST query. Does not contain any complex objects, that do something on copy or destructor. */ @@ -59,6 +66,7 @@ struct QueryStatusInfo Int64 peak_memory_usage; ClientInfo client_info; bool is_cancelled; + CancelReason cancel_reason; bool is_all_data_sent; /// Optional fields, filled by query @@ -105,6 +113,7 @@ protected: bool is_cancelling { false }; /// KILL was send to the query std::atomic is_killed { false }; + CancelReason cancel_reason { CancelReason::NOT_CANCELLED }; /// All data to the client already had been sent. /// Including EndOfStream or Exception. @@ -223,7 +232,7 @@ public: QueryStatusInfo getInfo(bool get_thread_list = false, bool get_profile_events = false, bool get_settings = false) const; - CancellationCode cancelQuery(bool kill); + CancellationCode cancelQuery(bool kill, CancelReason reason = CancelReason::MANUAL_CANCEL); bool isKilled() const { return is_killed; } From 95d299116662df9f5f65f57420d83f9311718d64 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 3 Oct 2024 18:25:03 +0200 Subject: [PATCH 047/502] small refactor --- src/Interpreters/CancellationChecker.cpp | 24 ++++++++++++------------ src/Interpreters/CancellationChecker.h | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp index 5b5f8bb52e3..e5b311ed45c 100644 --- a/src/Interpreters/CancellationChecker.cpp +++ b/src/Interpreters/CancellationChecker.cpp @@ -23,7 +23,7 @@ bool CompareEndTime::operator()(const QueryToTrack& a, const QueryToTrack& b) co if (a.endtime != b.endtime) return a.endtime < b.endtime; else - return a.query->getClientInfo().current_query_id < b.query->getClientInfo().current_query_id; + return a.query < b.query; } CancellationChecker::CancellationChecker() : stop_thread(false) @@ -48,23 +48,23 @@ void CancellationChecker::cancelTask(std::shared_ptr query, CancelR bool CancellationChecker::removeQueryFromSet(std::shared_ptr query) { - for (auto it = querySet.begin(); it != querySet.end();) + auto it = std::find_if(querySet.begin(), querySet.end(), [&](const QueryToTrack& task) { + return task.query == query; + }); + + if (it != querySet.end()) { - if (it->query == query) - { - LOG_TRACE(getLogger("CancellationChecker"), "Removing query {} from done tasks", query->getInfo().query); - it = querySet.erase(it); - return true; - } - else - ++it; + LOG_TRACE(getLogger("CancellationChecker"), "Removing query {} from done tasks", query->getInfo().query); + querySet.erase(it); + return true; } + return false; } -void CancellationChecker::appendTask(const std::shared_ptr & query, const UInt64 & timeout) +void CancellationChecker::appendTask(const std::shared_ptr & query, const Int64 & timeout) { - if (timeout > 0) // Avoid cases when the timeout is less or equal zero + if (timeout <= 0) // Avoid cases when the timeout is less or equal zero { LOG_TRACE(getLogger("CancellationChecker"), "Did not add the task because the timeout is 0. Query: {}", query->getInfo().query); return; diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index 5491fd104d3..252cc0e7420 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -56,7 +56,7 @@ public: CancellationChecker& operator=(const CancellationChecker&) = delete; // Method to add a new task to the multiset - void appendTask(const std::shared_ptr & query, const UInt64 & timeout); + void appendTask(const std::shared_ptr & query, const Int64 & timeout); // Used when some task is done void appendDoneTasks(const std::shared_ptr & query); From 568e25889b53d39cce30592a6f23b2446d69498c Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 3 Oct 2024 18:51:19 +0200 Subject: [PATCH 048/502] style check --- src/Interpreters/CancellationChecker.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp index e5b311ed45c..651efa73584 100644 --- a/src/Interpreters/CancellationChecker.cpp +++ b/src/Interpreters/CancellationChecker.cpp @@ -48,7 +48,8 @@ void CancellationChecker::cancelTask(std::shared_ptr query, CancelR bool CancellationChecker::removeQueryFromSet(std::shared_ptr query) { - auto it = std::find_if(querySet.begin(), querySet.end(), [&](const QueryToTrack& task) { + auto it = std::find_if(querySet.begin(), querySet.end(), [&](const QueryToTrack& task) + { return task.query == query; }); From 4ec7b10c735794469004ef8bafcb808acd2dbc2a Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 4 Oct 2024 20:46:28 +0200 Subject: [PATCH 049/502] add proper thread termination --- programs/server/Server.cpp | 2 ++ src/Interpreters/CancellationChecker.cpp | 13 ++++++++----- src/Interpreters/CancellationChecker.h | 3 ++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 6ec357a9927..2feba9dfc46 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -2318,6 +2318,8 @@ try if (current_connections) current_connections = waitServersToFinish(servers, servers_lock, server_settings.shutdown_wait_unfinished); + CancellationChecker::getInstance().terminateThread(); + if (current_connections) LOG_WARNING(log, "Closed connections. But {} remain." " Tip: To increase wait time add to config: 60", current_connections); diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp index 651efa73584..a7fbdf24497 100644 --- a/src/Interpreters/CancellationChecker.cpp +++ b/src/Interpreters/CancellationChecker.cpp @@ -30,17 +30,20 @@ CancellationChecker::CancellationChecker() : stop_thread(false) { } -CancellationChecker::~CancellationChecker() -{ - stop_thread = true; -} - CancellationChecker& CancellationChecker::getInstance() { static CancellationChecker instance; return instance; } +void CancellationChecker::terminateThread() +{ + LOG_TRACE(getLogger("CancellationChecker"), "Stopping CancellationChecker"); + stop_thread = true; + cond_var.notify_all(); +} + + void CancellationChecker::cancelTask(std::shared_ptr query, CancelReason reason) { query->cancelQuery(/*kill=*/false, /*reason=*/reason); diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index 252cc0e7420..2def3537253 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -34,7 +34,6 @@ class CancellationChecker { private: CancellationChecker(); - ~CancellationChecker(); // Priority queue to manage tasks based on endTime std::multiset querySet; @@ -55,6 +54,8 @@ public: CancellationChecker(const CancellationChecker&) = delete; CancellationChecker& operator=(const CancellationChecker&) = delete; + void terminateThread(); + // Method to add a new task to the multiset void appendTask(const std::shared_ptr & query, const Int64 & timeout); From 23ca36ab30777377d425d55ee6b4b87fefd6adb9 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Sat, 12 Oct 2024 14:40:55 +0800 Subject: [PATCH 050/502] add function toUnixTimestampEx and toUnixTimestamp64Second --- .../functions/date-time-functions.md | 40 +++++++++++- .../functions/type-conversion-functions.md | 63 +++++++++++++++---- src/Functions/FunctionsConversion.cpp | 26 ++++++++ src/Functions/toUnixTimestamp64.cpp | 13 ++++ .../01277_toUnixTimestamp64.reference | 12 ++-- .../0_stateless/01277_toUnixTimestamp64.sql | 15 +++-- .../03246_to_timestampex.reference | 9 +++ .../0_stateless/03246_to_timestampex.sql | 9 +++ 8 files changed, 162 insertions(+), 25 deletions(-) create mode 100644 src/Functions/toUnixTimestamp64.cpp create mode 100644 tests/queries/0_stateless/03246_to_timestampex.reference create mode 100644 tests/queries/0_stateless/03246_to_timestampex.sql diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 1d543f11cf3..57a92fa2358 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -195,7 +195,7 @@ makeDateTime64(year, month, day, hour, minute, second[, precision]) **Returned value** -- A date and time created from the supplied arguments. [DateTime64](../../sql-reference/data-types/datetime64.md). +- A date and time created from the supplied arguments. [DateTime64](../../sql-reference/data-types/datetime64.md). **Example** @@ -868,6 +868,44 @@ Behavior for * Functions `toStartOfDay`, `toStartOfHour`, `toStartOfFifteenMinutes`, `toStartOfTenMinutes`, `toStartOfFiveMinutes`, `toStartOfMinute`, `timeSlot` return `DateTime` if their argument is a `Date` or `DateTime`, and they return `DateTime64` if their argument is a `Date32` or `DateTime64`. ::: +## toUnixTimestampEx + +Like `toUnixTimestamp`, converts a string, a date or a date with time to the [Unix Timestamp](https://en.wikipedia.org/wiki/Unix_time) in `Int64` representation. + +If the function is called with a string, it accepts an optional timezone argument. + +**Syntax** + +``` sql +toUnixTimestampEx(date) +toUnixTimestampEx(str, [timezone]) +``` + +**Returned value** + +- Returns the unix timestamp. [UInt32](../data-types/int-uint.md). + +**Example** + +``` sql +SELECT + '1969-01-01 00:00:00' AS dt_str, + toUnixTimestamp(dt_str) AS from_str, + toUnixTimestampEx(dt_str) AS ex_str, + toUnixTimestamp(dt_str, 'Asia/Tokyo') AS from_str_tokyo, + toUnixTimestampEx(dt_str, 'Asia/Tokyo') AS ex_str_tokyo, + toUnixTimestamp(toDateTime(dt_str)) AS from_datetime, + toUnixTimestampEx(toDateTime64(dt_str, 0)) AS ex_datetime64 +``` + +Result: + +``` +┌─dt_str──────────────┬─from_str─┬────ex_str─┬─from_str_tokyo─┬─ex_str_tokyo─┬─from_datetime─┬─ex_datetime64─┐ +│ 1969-01-01 00:00:00 │ 0 │ -31564800 │ 0 │ -31568400 │ 0 │ -31564800 │ +└─────────────────────┴──────────┴───────────┴────────────────┴──────────────┴───────────────┴───────────────┘ +``` + ## toStartOfYear Rounds down a date or date with time to the first day of the year. Returns the date as a `Date` object. diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index b37bd057adc..e7b4f60afc7 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -122,7 +122,7 @@ Unsupported arguments: :::note If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md), overflow or underflow of the result occurs. -This is not considered an error. +This is not considered an error. For example: `SELECT toInt8(128) == -128;`. ::: @@ -370,7 +370,7 @@ Unsupported arguments: :::note If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md), overflow or underflow of the result occurs. -This is not considered an error. +This is not considered an error. For example: `SELECT toInt16(32768) == -32768;`. ::: @@ -618,7 +618,7 @@ Unsupported arguments: :::note If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md), the result over or under flows. -This is not considered an error. +This is not considered an error. For example: `SELECT toInt32(2147483648) == -2147483648;` ::: @@ -865,7 +865,7 @@ Unsupported types: :::note If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md), the result over or under flows. -This is not considered an error. +This is not considered an error. For example: `SELECT toInt64(9223372036854775808) == -9223372036854775808;` ::: @@ -1608,7 +1608,7 @@ Unsupported arguments: :::note If the input value cannot be represented within the bounds of [UInt8](../data-types/int-uint.md), overflow or underflow of the result occurs. -This is not considered an error. +This is not considered an error. For example: `SELECT toUInt8(256) == 0;`. ::: @@ -1856,7 +1856,7 @@ Unsupported arguments: :::note If the input value cannot be represented within the bounds of [UInt16](../data-types/int-uint.md), overflow or underflow of the result occurs. -This is not considered an error. +This is not considered an error. For example: `SELECT toUInt16(65536) == 0;`. ::: @@ -2104,7 +2104,7 @@ Unsupported arguments: :::note If the input value cannot be represented within the bounds of [UInt32](../data-types/int-uint.md), the result over or under flows. -This is not considered an error. +This is not considered an error. For example: `SELECT toUInt32(4294967296) == 0;` ::: @@ -2353,7 +2353,7 @@ Unsupported types: :::note If the input value cannot be represented within the bounds of [UInt64](../data-types/int-uint.md), the result over or under flows. -This is not considered an error. +This is not considered an error. For example: `SELECT toUInt64(18446744073709551616) == 0;` ::: @@ -3691,8 +3691,8 @@ toDateTime(expr[, time_zone ]) - `time_zone` — Time zone. [String](../data-types/string.md). :::note -If `expr` is a number, it is interpreted as the number of seconds since the beginning of the Unix Epoch (as Unix timestamp). -If `expr` is a [String](../data-types/string.md), it may be interpreted as a Unix timestamp or as a string representation of date / date with time. +If `expr` is a number, it is interpreted as the number of seconds since the beginning of the Unix Epoch (as Unix timestamp). +If `expr` is a [String](../data-types/string.md), it may be interpreted as a Unix timestamp or as a string representation of date / date with time. Thus, parsing of short numbers' string representations (up to 4 digits) is explicitly disabled due to ambiguity, e.g. a string `'1999'` may be both a year (an incomplete string representation of Date / DateTime) or a unix timestamp. Longer numeric strings are allowed. ::: @@ -5536,7 +5536,7 @@ Result: ## reinterpretAsUInt256 -Performs byte reinterpretation by treating the input value as a value of type UInt256. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless. +Performs byte reinterpretation by treating the input value as a value of type UInt256. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless. **Syntax** @@ -5612,7 +5612,7 @@ Result: ## reinterpretAsInt16 -Performs byte reinterpretation by treating the input value as a value of type Int16. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless. +Performs byte reinterpretation by treating the input value as a value of type Int16. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless. **Syntax** @@ -7170,6 +7170,45 @@ Result: └──────────────────────────────┘ ``` +## toUnixTimestamp64Second + +Converts a `DateTime64` to a `Int64` value with fixed second precision. The input value is scaled up or down appropriately depending on its precision. + +:::note +The output value is a timestamp in UTC, not in the timezone of `DateTime64`. +::: + +**Syntax** + +```sql +toUnixTimestamp64Second(value) +``` + +**Arguments** + +- `value` — DateTime64 value with any precision. [DateTime64](../data-types/datetime64.md). + +**Returned value** + +- `value` converted to the `Int64` data type. [Int64](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +WITH toDateTime64('2009-02-13 23:31:31.011', 3, 'UTC') AS dt64 +SELECT toUnixTimestamp64Second(dt64); +``` + +Result: + +```response +┌─toUnixTimestamp64Second(dt64)─┐ +│ 1234567891 │ +└───────────────────────────────┘ +``` + ## toUnixTimestamp64Micro Converts a `DateTime64` to a `Int64` value with fixed microsecond precision. The input value is scaled up or down appropriately depending on its precision. diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 615589a7d43..3f4e46eef0a 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -1150,6 +1150,9 @@ struct ConvertThroughParsing /// Function toUnixTimestamp has exactly the same implementation as toDateTime of String type. struct NameToUnixTimestamp { static constexpr auto name = "toUnixTimestamp"; }; +/// Function toUnixTimestampEx has exactly the same implementation as toDateTime of String type. +struct NameToUnixTimestampEx { static constexpr auto name = "toUnixTimestampEx"; }; + enum class BehaviourOnErrorFromString : uint8_t { ConvertDefaultBehaviorTag, @@ -1559,6 +1562,17 @@ struct ConvertImpl return ConvertImpl::template execute( arguments, result_type, input_rows_count, from_string_tag); } + else if constexpr (std::is_same_v + && std::is_same_v + && std::is_same_v) + { + /// Convert String to DateTime64 + ColumnPtr res = ConvertImpl::template execute( + arguments, result_type, input_rows_count, from_string_tag, 0); + /// Convert DateTime64 to Int64 + return ConvertImpl::template execute( + ColumnsWithTypeAndName{{res, result_type, arguments[0].name}}, result_type, input_rows_count, from_string_tag); + } else if constexpr ((std::is_same_v || std::is_same_v)) { switch (from_string_tag) @@ -1813,6 +1827,11 @@ struct ConvertImpl { vec_to[i] = static_cast(vec_from[i] * DATE_SECONDS_PER_DAY); } + else if constexpr (std::is_same_v + && (std::is_same_v || std::is_same_v)) + { + vec_to[i] = static_cast(vec_from[i] * DATE_SECONDS_PER_DAY); + } else { /// If From Data is Nan or Inf and we convert to integer type, throw exception @@ -2080,6 +2099,8 @@ public: if ((std::is_same_v && !arguments.empty() && (isDateTime64(arguments[0].type) || isDateTime(arguments[0].type))) // toUnixTimestamp(value[, timezone : String]) || std::is_same_v + // toUnixTimestampEx(value[, timezone : String]) + || std::is_same_v // toDate(value[, timezone : String]) || std::is_same_v // TODO: shall we allow timestamp argument for toDate? DateTime knows nothing about timezones and this argument is ignored below. // toDate32(value[, timezone : String]) @@ -2236,6 +2257,7 @@ private: if constexpr (IsDataTypeDecimal) { + if constexpr (std::is_same_v) { /// Account for optional timezone argument. @@ -2319,8 +2341,10 @@ private: } } else + { result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag); + } return true; }; @@ -2880,6 +2904,7 @@ using FunctionToIPv4 = FunctionConvert>; using FunctionToString = FunctionConvert; using FunctionToUnixTimestamp = FunctionConvert>; +using FunctionToUnixTimestampEx = FunctionConvert>; using FunctionToDecimal32 = FunctionConvert, NameToDecimal32, UnknownMonotonicity>; using FunctionToDecimal64 = FunctionConvert, NameToDecimal64, UnknownMonotonicity>; using FunctionToDecimal128 = FunctionConvert, NameToDecimal128, UnknownMonotonicity>; @@ -5420,6 +5445,7 @@ REGISTER_FUNCTION(Conversion) factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/toUnixTimestamp64.cpp b/src/Functions/toUnixTimestamp64.cpp new file mode 100644 index 00000000000..61249ed5609 --- /dev/null +++ b/src/Functions/toUnixTimestamp64.cpp @@ -0,0 +1,13 @@ +#include +#include + +namespace DB +{ + +REGISTER_FUNCTION(ToUnixTimestamp64Second) +{ + factory.registerFunction("toUnixTimestamp64Second", + [](ContextPtr){ return std::make_shared(0, "toUnixTimestamp64Second"); }); +} + +} diff --git a/tests/queries/0_stateless/01277_toUnixTimestamp64.reference b/tests/queries/0_stateless/01277_toUnixTimestamp64.reference index 7b66586b4d2..6b329f185a3 100644 --- a/tests/queries/0_stateless/01277_toUnixTimestamp64.reference +++ b/tests/queries/0_stateless/01277_toUnixTimestamp64.reference @@ -1,8 +1,8 @@ const column -2019-09-16 19:20:12.345 1568650812345 1568650812345000 1568650812345000000 -2019-09-16 19:20:12.345678 1568650812345 1568650812345678 1568650812345678000 -2019-09-16 19:20:12.345678910 1568650812345 1568650812345678 1568650812345678910 +2019-09-16 19:20:12.345 1568650812345 1568650812345000 1568650812345000000 1568650812 +2019-09-16 19:20:12.345678 1568650812345 1568650812345678 1568650812345678000 1568650812 +2019-09-16 19:20:12.345678910 1568650812345 1568650812345678 1568650812345678910 1568650812 non-const column -2019-09-16 19:20:12.345 1568650812345 1568650812345000 1568650812345000000 -2019-09-16 19:20:12.345678 1568650812345 1568650812345678 1568650812345678000 -2019-09-16 19:20:12.345678910 1568650812345 1568650812345678 1568650812345678910 +2019-09-16 19:20:12.345 1568650812345 1568650812345000 1568650812345000000 1568650812 +2019-09-16 19:20:12.345678 1568650812345 1568650812345678 1568650812345678000 1568650812 +2019-09-16 19:20:12.345678910 1568650812345 1568650812345678 1568650812345678910 1568650812 diff --git a/tests/queries/0_stateless/01277_toUnixTimestamp64.sql b/tests/queries/0_stateless/01277_toUnixTimestamp64.sql index 14ee57da5df..6f3e90bc92d 100644 --- a/tests/queries/0_stateless/01277_toUnixTimestamp64.sql +++ b/tests/queries/0_stateless/01277_toUnixTimestamp64.sql @@ -2,32 +2,35 @@ SELECT toUnixTimestamp64Milli(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} SELECT toUnixTimestamp64Micro(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} SELECT toUnixTimestamp64Nano(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} +SELECT toUnixTimestamp64Second(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} SELECT toUnixTimestamp64Milli('abc'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} SELECT toUnixTimestamp64Micro('abc'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} SELECT toUnixTimestamp64Nano('abc'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} +SELECT toUnixTimestamp64Second('abc'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} SELECT toUnixTimestamp64Milli('abc', 123); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} SELECT toUnixTimestamp64Micro('abc', 123); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} SELECT toUnixTimestamp64Nano('abc', 123); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} +SELECT toUnixTimestamp64Second('abc', 123); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} SELECT 'const column'; WITH toDateTime64('2019-09-16 19:20:12.345678910', 3, 'Asia/Istanbul') AS dt64 -SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); +SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64), toUnixTimestamp64Second(dt64); WITH toDateTime64('2019-09-16 19:20:12.345678910', 6, 'Asia/Istanbul') AS dt64 -SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); +SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64), toUnixTimestamp64Second(dt64); WITH toDateTime64('2019-09-16 19:20:12.345678910', 9, 'Asia/Istanbul') AS dt64 -SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); +SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64), toUnixTimestamp64Second(dt64); SELECT 'non-const column'; WITH toDateTime64('2019-09-16 19:20:12.345678910', 3, 'Asia/Istanbul') AS x -SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); +SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64), toUnixTimestamp64Second(dt64); WITH toDateTime64('2019-09-16 19:20:12.345678910', 6, 'Asia/Istanbul') AS x -SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); +SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64), toUnixTimestamp64Second(dt64); WITH toDateTime64('2019-09-16 19:20:12.345678910', 9, 'Asia/Istanbul') AS x -SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); +SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64), toUnixTimestamp64Second(dt64); diff --git a/tests/queries/0_stateless/03246_to_timestampex.reference b/tests/queries/0_stateless/03246_to_timestampex.reference new file mode 100644 index 00000000000..23b6c78b943 --- /dev/null +++ b/tests/queries/0_stateless/03246_to_timestampex.reference @@ -0,0 +1,9 @@ +1683676800 +1683676800 +1683676800 +1683676800 +-1293882467 +0 +0 +-28800 +-28800 diff --git a/tests/queries/0_stateless/03246_to_timestampex.sql b/tests/queries/0_stateless/03246_to_timestampex.sql new file mode 100644 index 00000000000..861a23c0bb1 --- /dev/null +++ b/tests/queries/0_stateless/03246_to_timestampex.sql @@ -0,0 +1,9 @@ +SELECT toUnixTimestampEx(makeDate(2023, 5, 10)); +SELECT toUnixTimestampEx(makeDate32(2023, 5, 10)); +SELECT toUnixTimestampEx(makeDate(2023, 5, 10), 'Pacific/Auckland'); +SELECT toUnixTimestampEx(makeDate32(2023, 5, 10), 'Pacific/Auckland'); +SELECT toUnixTimestampEx(toDateTime64('1928-12-31 12:12:12.123', 3, 'UTC')); +SELECT toUnixTimestampEx('1970-01-01 00:00:00', 'UTC'); +SELECT toUnixTimestampEx(materialize('1970-01-01 00:00:00'), 'UTC'); +SELECT toUnixTimestampEx('1970-01-01 00:00:00', 'Asia/Shanghai'); +SELECT toUnixTimestampEx(materialize('1970-01-01 00:00:00'), 'Asia/Shanghai'); From ae3ea224682e4da2086868adb9d26d05707c7d27 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Sat, 12 Oct 2024 14:49:11 +0800 Subject: [PATCH 051/502] clean code --- src/Functions/FunctionsConversion.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 3f4e46eef0a..9faa1c48d5a 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -2341,10 +2341,8 @@ private: } } else - { result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag); - } return true; }; From b3175447f66b32f839e54b9507f6714c79f3681f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 14 Oct 2024 19:44:21 +0200 Subject: [PATCH 052/502] review --- src/Interpreters/CancellationChecker.cpp | 32 ++++++++++--------- src/Interpreters/CancellationChecker.h | 9 ++++-- src/Interpreters/Context.cpp | 2 +- .../InterpreterKillQueryQuery.cpp | 4 +-- src/Interpreters/ProcessList.cpp | 2 +- src/Interpreters/ProcessList.h | 8 ++--- 6 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp index a7fbdf24497..8496bfdaa2e 100644 --- a/src/Interpreters/CancellationChecker.cpp +++ b/src/Interpreters/CancellationChecker.cpp @@ -13,17 +13,15 @@ namespace DB QueryToTrack::QueryToTrack( std::shared_ptr query_, UInt64 timeout_, - UInt64 endtime_) - : query(query_), timeout(timeout_), endtime(endtime_) + UInt64 endtime_, + OverflowMode overflow_mode_) + : query(query_), timeout(timeout_), endtime(endtime_), overflow_mode(overflow_mode_) { } bool CompareEndTime::operator()(const QueryToTrack& a, const QueryToTrack& b) const { - if (a.endtime != b.endtime) - return a.endtime < b.endtime; - else - return a.query < b.query; + return std::tie(a.endtime, a.query) < std::tie(b.endtime, b.query); } CancellationChecker::CancellationChecker() : stop_thread(false) @@ -43,16 +41,20 @@ void CancellationChecker::terminateThread() cond_var.notify_all(); } - -void CancellationChecker::cancelTask(std::shared_ptr query, CancelReason reason) +void CancellationChecker::cancelTask(QueryToTrack task, [[maybe_unused]]CancelReason reason) { - query->cancelQuery(/*kill=*/false, /*reason=*/reason); + if (task.query) + { + if (task.overflow_mode == OverflowMode::THROW) + task.query->cancelQuery(CancelReason::TIMEOUT); + else + task.query->checkTimeLimit(); + } } bool CancellationChecker::removeQueryFromSet(std::shared_ptr query) { - auto it = std::find_if(querySet.begin(), querySet.end(), [&](const QueryToTrack& task) - { + auto it = std::find_if(querySet.begin(), querySet.end(), [&](const QueryToTrack& task) { return task.query == query; }); @@ -66,7 +68,7 @@ bool CancellationChecker::removeQueryFromSet(std::shared_ptr query) return false; } -void CancellationChecker::appendTask(const std::shared_ptr & query, const Int64 & timeout) +void CancellationChecker::appendTask(const std::shared_ptr & query, const Int64 & timeout, OverflowMode overflow_mode) { if (timeout <= 0) // Avoid cases when the timeout is less or equal zero { @@ -77,7 +79,7 @@ void CancellationChecker::appendTask(const std::shared_ptr & query, LOG_TRACE(getLogger("CancellationChecker"), "Added to set. query: {}, timeout: {} milliseconds", query->getInfo().query, timeout); const auto & now = std::chrono::steady_clock::now(); const UInt64 & end_time = std::chrono::duration_cast(now.time_since_epoch()).count() + timeout; - querySet.emplace(query, timeout, end_time); + querySet.emplace(query, timeout, end_time, overflow_mode); cond_var.notify_all(); } @@ -118,8 +120,8 @@ void CancellationChecker::workerFunction() if ((end_time_ms <= now_ms && duration_milliseconds.count() != 0)) { - LOG_TRACE(getLogger("CancellationChecker"), "Cancelling the task because of the timeout: {}, query: {}", duration, next_task.query->getInfo().query); - cancelTask(next_task.query, CancelReason::TIMEOUT); + LOG_TRACE(getLogger("CancellationChecker"), "Cancelling the task because of the timeout: {} ms, query: {}", duration, next_task.query->getInfo().query); + cancelTask(next_task, CancelReason::TIMEOUT); querySet.erase(next_task); continue; diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index 2def3537253..e890fc9aec3 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -1,6 +1,7 @@ #pragma once #include +#include "QueryPipeline/SizeLimits.h" #include #include @@ -12,11 +13,13 @@ struct QueryToTrack QueryToTrack( std::shared_ptr query_, UInt64 timeout_, - UInt64 endtime_); + UInt64 endtime_, + OverflowMode overflow_mode_); std::shared_ptr query; UInt64 timeout; UInt64 endtime; + OverflowMode overflow_mode; }; struct CompareEndTime @@ -43,7 +46,7 @@ private: std::condition_variable cond_var; // Function to execute when a task's endTime is reached - void cancelTask(std::shared_ptr query, CancelReason reason); + void cancelTask(QueryToTrack task, CancelReason reason); bool removeQueryFromSet(std::shared_ptr query); public: @@ -57,7 +60,7 @@ public: void terminateThread(); // Method to add a new task to the multiset - void appendTask(const std::shared_ptr & query, const Int64 & timeout); + void appendTask(const std::shared_ptr & query, const Int64 & timeout, OverflowMode overflow_mode); // Used when some task is done void appendDoneTasks(const std::shared_ptr & query); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 47217231601..f323b297107 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2631,7 +2631,7 @@ bool Context::isBackgroundOperationContext() const void Context::killCurrentQuery() const { if (auto elem = getProcessListElement()) - elem->cancelQuery(true); + elem->cancelQuery(CancelReason::MANUAL_CANCEL); } bool Context::isCurrentQueryKilled() const diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index b390b29c03c..02a62a9a423 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -170,7 +170,7 @@ public: LOG_DEBUG(getLogger("KillQuery"), "Will kill query {} (synchronously)", curr_process.query_id); - auto code = process_list.sendCancelToQuery(curr_process.query_id, curr_process.user, true); + auto code = process_list.sendCancelToQuery(curr_process.query_id, curr_process.user); if (code != CancellationCode::QueryIsNotInitializedYet && code != CancellationCode::CancelSent) { @@ -237,7 +237,7 @@ BlockIO InterpreterKillQueryQuery::execute() { if (!query.test) LOG_DEBUG(getLogger("KillQuery"), "Will kill query {} (asynchronously)", query_desc.query_id); - auto code = (query.test) ? CancellationCode::Unknown : process_list.sendCancelToQuery(query_desc.query_id, query_desc.user, true); + auto code = (query.test) ? CancellationCode::Unknown : process_list.sendCancelToQuery(query_desc.query_id, query_desc.user); insertResultRow(query_desc.source_num, code, processes_block, header, res_columns); } diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 37c2be430c2..2981ff9421c 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -462,7 +462,7 @@ CancellationCode QueryStatus::cancelQuery(bool, CancelReason reason) is_killed.store(true); - cancel_reason = reason; + std::atomic_exchange(&cancel_reason, reason); std::vector executors_snapshot; diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 6dba3bb561d..200660e04c3 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -113,7 +113,7 @@ protected: bool is_cancelling { false }; /// KILL was send to the query std::atomic is_killed { false }; - CancelReason cancel_reason { CancelReason::NOT_CANCELLED }; + std::atomic cancel_reason { CancelReason::NOT_CANCELLED }; /// All data to the client already had been sent. /// Including EndOfStream or Exception. @@ -232,7 +232,7 @@ public: QueryStatusInfo getInfo(bool get_thread_list = false, bool get_profile_events = false, bool get_settings = false) const; - CancellationCode cancelQuery(bool kill, CancelReason reason = CancelReason::MANUAL_CANCEL); + CancellationCode cancelQuery(CancelReason reason); bool isKilled() const { return is_killed; } @@ -501,8 +501,8 @@ public: void decrementWaiters(); /// Try call cancel() for input and output streams of query with specified id and user - CancellationCode sendCancelToQuery(const String & current_query_id, const String & current_user, bool kill = false); - CancellationCode sendCancelToQuery(QueryStatusPtr elem, bool kill = false); + CancellationCode sendCancelToQuery(const String & current_query_id, const String & current_user); + CancellationCode sendCancelToQuery(QueryStatusPtr elem); void killAllQueries(); }; From c6caea5978ddcde2c94ed3c4fdb50af86d92e302 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 14 Oct 2024 19:57:09 +0200 Subject: [PATCH 053/502] style fix --- src/Interpreters/CancellationChecker.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp index 8496bfdaa2e..47ac118533b 100644 --- a/src/Interpreters/CancellationChecker.cpp +++ b/src/Interpreters/CancellationChecker.cpp @@ -54,7 +54,8 @@ void CancellationChecker::cancelTask(QueryToTrack task, [[maybe_unused]]CancelRe bool CancellationChecker::removeQueryFromSet(std::shared_ptr query) { - auto it = std::find_if(querySet.begin(), querySet.end(), [&](const QueryToTrack& task) { + auto it = std::find_if(querySet.begin(), querySet.end(), [&](const QueryToTrack& task) + { return task.query == query; }); From bbe701d3415fb7cc190a959e11f074d5b57b2e0b Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Tue, 15 Oct 2024 12:39:21 +0800 Subject: [PATCH 054/502] fix Check docs spelling error --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 616ad4a800c..aa0c2746391 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -3143,3 +3143,4 @@ DistributedCachePoolBehaviourOnLimit SharedJoin ShareSet unacked +toUnixTimestampEx From 73748075a29f384d7dc62599000589c71362954e Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:52:17 +0200 Subject: [PATCH 055/502] fix build --- src/Interpreters/ProcessList.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 2981ff9421c..8a5a46e5775 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -302,7 +302,7 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q processes.end(), query); - CancellationChecker::getInstance().appendTask(query, query_context->getSettingsRef()[Setting::max_execution_time].totalMilliseconds()); + CancellationChecker::getInstance().appendTask(query, query_context->getSettingsRef()[Setting::max_execution_time].totalMilliseconds(), query_context->getSettingsRef()[Setting::timeout_overflow_mode]); increaseQueryKindAmount(query_kind); @@ -455,7 +455,7 @@ void QueryStatus::ExecutorHolder::remove() executor = nullptr; } -CancellationCode QueryStatus::cancelQuery(bool, CancelReason reason) +CancellationCode QueryStatus::cancelQuery(CancelReason reason) { if (is_killed.load()) return CancellationCode::CancelSent; @@ -590,7 +590,7 @@ QueryStatusPtr ProcessList::tryGetProcessListElement(const String & current_quer } -CancellationCode ProcessList::sendCancelToQuery(const String & current_query_id, const String & current_user, bool kill) +CancellationCode ProcessList::sendCancelToQuery(const String & current_query_id, const String & current_user) { QueryStatusPtr elem; @@ -622,11 +622,11 @@ CancellationCode ProcessList::sendCancelToQuery(const String & current_query_id, cancelled_cv.notify_all(); }); - return elem->cancelQuery(kill); + return elem->cancelQuery(CancelReason::MANUAL_CANCEL); } -CancellationCode ProcessList::sendCancelToQuery(QueryStatusPtr elem, bool kill) +CancellationCode ProcessList::sendCancelToQuery(QueryStatusPtr elem) { /// Cancelling the query should be done without the lock. /// So here we first set is_cancelling, and later reset it. @@ -644,7 +644,7 @@ CancellationCode ProcessList::sendCancelToQuery(QueryStatusPtr elem, bool kill) cancelled_cv.notify_all(); }); - return elem->cancelQuery(kill); + return elem->cancelQuery(CancelReason::MANUAL_CANCEL); } @@ -670,7 +670,7 @@ void ProcessList::killAllQueries() } for (auto & cancelled_process : cancelled_processes) - cancelled_process->cancelQuery(true); + cancelled_process->cancelQuery(CancelReason::MANUAL_CANCEL); } From 15f7061109ce5c070f183e4a4c86a0f47c80c46b Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Wed, 16 Oct 2024 09:00:29 +0800 Subject: [PATCH 056/502] fix new functions doc check --- .../02415_all_new_functions_must_be_documented.reference | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index 7c541f272c8..072df78ee79 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -850,6 +850,8 @@ toUnixTimestamp toUnixTimestamp64Micro toUnixTimestamp64Milli toUnixTimestamp64Nano +toUnixTimestamp64Second +toUnixTimestampEx toValidUTF8 toWeek toYYYYMM From 9dc861f0abd1bc2a1cc0394866ee10a7a957623c Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Wed, 16 Oct 2024 15:03:06 +0800 Subject: [PATCH 057/502] fix tidy check fail --- src/Functions/FunctionsConversion.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 9faa1c48d5a..6afce88e054 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -1830,7 +1830,7 @@ struct ConvertImpl else if constexpr (std::is_same_v && (std::is_same_v || std::is_same_v)) { - vec_to[i] = static_cast(vec_from[i] * DATE_SECONDS_PER_DAY); + vec_to[i] = vec_from[i] * DATE_SECONDS_PER_DAY; } else { From 57798a19c0ad4a9c22e2c13d28dcf1f4256995e2 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 17 Oct 2024 19:37:33 +0200 Subject: [PATCH 058/502] add SCOPE_EXIT --- programs/server/Server.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 2feba9dfc46..5f3ab278060 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1101,10 +1101,6 @@ try PlacementInfo::PlacementInfo::instance().initialize(config()); - auto cancellation_task_holder = global_context->getSchedulePool().createTask("CancellationChecker", []{ CancellationChecker::getInstance().workerFunction(); }); - auto cancellation_task = std::make_unique(std::move(cancellation_task_holder)); - (*cancellation_task)->activateAndSchedule(); - zkutil::validateZooKeeperConfig(config()); bool has_zookeeper = zkutil::hasZooKeeperConfig(config()); @@ -1249,6 +1245,14 @@ try setOOMScore(oom_score, log); #endif + auto cancellation_task_holder = global_context->getSchedulePool().createTask("CancellationChecker", []{ CancellationChecker::getInstance().workerFunction(); }); + auto cancellation_task = std::make_unique(std::move(cancellation_task_holder)); + (*cancellation_task)->activateAndSchedule(); + + SCOPE_EXIT({ + CancellationChecker::getInstance().terminateThread(); + }); + global_context->setRemoteHostFilter(config()); global_context->setHTTPHeaderFilter(config()); @@ -2318,8 +2322,6 @@ try if (current_connections) current_connections = waitServersToFinish(servers, servers_lock, server_settings.shutdown_wait_unfinished); - CancellationChecker::getInstance().terminateThread(); - if (current_connections) LOG_WARNING(log, "Closed connections. But {} remain." " Tip: To increase wait time add to config: 60", current_connections); From 4220b9002cc814db1db52f1f3921493dcae4fc45 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 28 Oct 2024 00:35:40 +0800 Subject: [PATCH 059/502] Support reverse sorting key --- src/Interpreters/MutationsInterpreter.cpp | 10 ++- src/Parsers/ASTFunction.cpp | 8 +++ src/Parsers/ExpressionElementParsers.cpp | 23 +++++++ src/Parsers/ExpressionElementParsers.h | 8 +++ src/Parsers/ExpressionListParsers.cpp | 6 ++ src/Parsers/ExpressionListParsers.h | 6 ++ src/Parsers/ParserCreateQuery.cpp | 27 +++++++- .../Optimizations/optimizeReadInOrder.cpp | 10 +-- .../QueryPlan/ReadFromMergeTree.cpp | 17 ++++- src/Storages/KeyDescription.cpp | 19 +++++- src/Storages/KeyDescription.h | 3 + src/Storages/MergeTree/MergeTask.cpp | 8 ++- src/Storages/MergeTree/MergeTreeData.cpp | 38 ++++++++++- src/Storages/MergeTree/MergeTreeData.h | 4 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 25 +++++-- .../MergeTree/MergeTreeDataWriter.cpp | 16 ++++- src/Storages/MergeTree/MergeTreeSettings.cpp | 1 + src/Storages/StorageInMemoryMetadata.cpp | 7 ++ src/Storages/StorageInMemoryMetadata.h | 3 + .../03257_reverse_sorting_key.reference | 68 +++++++++++++++++++ .../0_stateless/03257_reverse_sorting_key.sql | 39 +++++++++++ 21 files changed, 321 insertions(+), 25 deletions(-) create mode 100644 tests/queries/0_stateless/03257_reverse_sorting_key.reference create mode 100644 tests/queries/0_stateless/03257_reverse_sorting_key.sql diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 0f25d5ac21c..2f8bb8c16ac 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -1436,6 +1436,7 @@ size_t MutationsInterpreter::evaluateCommandsSize() std::optional MutationsInterpreter::getStorageSortDescriptionIfPossible(const Block & header) const { Names sort_columns = metadata_snapshot->getSortingKeyColumns(); + std::vector reverse_flags = metadata_snapshot->getSortingKeyReverseFlags(); SortDescription sort_description; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); @@ -1443,9 +1444,16 @@ std::optional MutationsInterpreter::getStorageSortDescriptionIf for (size_t i = 0; i < sort_columns_size; ++i) { if (header.has(sort_columns[i])) - sort_description.emplace_back(sort_columns[i], 1, 1); + { + if (reverse_flags[i]) + sort_description.emplace_back(sort_columns[i], -1, 1); + else + sort_description.emplace_back(sort_columns[i], 1, 1); + } else + { return {}; + } } return sort_description; diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 53d44e2f325..7c26e4dde30 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -284,6 +284,14 @@ static bool formatNamedArgWithHiddenValue(IAST * arg, const IAST::FormatSettings void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { + /// Internal function for reversing MergeTree primary keys + if (name == "__descendingKey") + { + arguments->formatImpl(settings, state, frame); + settings.ostr << (settings.hilite ? hilite_keyword : "") << " DESC" << (settings.hilite ? hilite_none : ""); + return; + } + frame.expression_list_prepend_whitespace = false; if (kind == Kind::CODEC || kind == Kind::STATISTICS || kind == Kind::BACKUP_NAME) frame.allow_operators = false; diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 31efcb16f02..7fef956179e 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -2162,6 +2162,29 @@ bool ParserWithOptionalAlias::parseImpl(Pos & pos, ASTPtr & node, Expected & exp return true; } +bool ParserStorageOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserExpression elem_p; + ParserKeyword ascending(Keyword::ASCENDING); + ParserKeyword descending(Keyword::DESCENDING); + ParserKeyword asc(Keyword::ASC); + ParserKeyword desc(Keyword::DESC); + + ASTPtr expr_elem; + if (!elem_p.parse(pos, expr_elem, expected)) + return false; + + int direction = 1; + + if (descending.ignore(pos, expected) || desc.ignore(pos, expected)) + direction = -1; + else + ascending.ignore(pos, expected) || asc.ignore(pos, expected); + + node = direction == 1 ? std::move(expr_elem) : makeASTFunction("__descendingKey", std::move(expr_elem)); + return true; +} + bool ParserOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { diff --git a/src/Parsers/ExpressionElementParsers.h b/src/Parsers/ExpressionElementParsers.h index 903111f32db..467813b7aa0 100644 --- a/src/Parsers/ExpressionElementParsers.h +++ b/src/Parsers/ExpressionElementParsers.h @@ -432,6 +432,14 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +/** Element of storage ORDER BY expression - same as expression element, but in addition, ASC[ENDING] | DESC[ENDING] could be specified + */ +class ParserStorageOrderByElement : public IParserBase +{ +protected: + const char * getName() const override { return "element of storage ORDER BY expression"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; /** Element of ORDER BY expression - same as expression element, but in addition, ASC[ENDING] | DESC[ENDING] could be specified * and optionally, NULLS LAST|FIRST diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 6d4a2e57d10..5396340bc36 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -325,6 +325,12 @@ bool ParserNotEmptyExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected return nested_parser.parse(pos, node, expected) && !node->children.empty(); } +bool ParserStorageOrderByExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + return ParserList(std::make_unique(), std::make_unique(TokenType::Comma), false) + .parse(pos, node, expected); +} + bool ParserOrderByExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { return ParserList(std::make_unique(), std::make_unique(TokenType::Comma), false) diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index 6ab38416f32..0fc5875156d 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -249,6 +249,12 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +class ParserStorageOrderByExpressionList : public IParserBase +{ +protected: + const char * getName() const override { return "storage order by expression"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; class ParserOrderByExpressionList : public IParserBase { diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 76d399558be..b3b26baec6a 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -505,9 +505,13 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_sample_by(Keyword::SAMPLE_BY); ParserKeyword s_ttl(Keyword::TTL); ParserKeyword s_settings(Keyword::SETTINGS); + ParserToken s_lparen(TokenType::OpeningRoundBracket); + ParserToken s_rparen(TokenType::ClosingRoundBracket); ParserIdentifierWithOptionalParameters ident_with_optional_params_p; ParserExpression expression_p; + ParserStorageOrderByExpressionList order_list_p; + ParserStorageOrderByElement order_elem_p; ParserSetQuery settings_p(/* parse_only_internals_ = */ true); ParserTTLExpressionList parser_ttl_list; ParserStringLiteral string_literal_parser; @@ -556,11 +560,32 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!order_by && s_order_by.ignore(pos, expected)) { - if (expression_p.parse(pos, order_by, expected)) + auto old_pos = pos; + if (pos->type == TokenType::BareWord && std::string_view(pos->begin, pos->size()) == "tuple") + ++pos; + + if (s_lparen.ignore(pos, expected)) + { + auto tuple_function = std::make_shared(); + tuple_function->name = "tuple"; + if (order_list_p.parse(pos, order_by, expected)) + tuple_function->arguments = std::move(order_by); + else + tuple_function->arguments = std::make_shared(); + tuple_function->children.push_back(tuple_function->arguments); + order_by = std::move(tuple_function); + s_rparen.check(pos, expected); + storage_like = true; + continue; + } + + pos = old_pos; + if (order_elem_p.parse(pos, order_by, expected)) { storage_like = true; continue; } + return false; } diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index e64a88de62e..25564c55255 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -360,8 +360,8 @@ InputOrderInfoPtr buildInputOrderFromSortDescription( } /// This is a result direction we will read from MergeTree - /// 1 - in order, - /// -1 - in reverse order, + /// 1 - in same order of keys, + /// -1 - in reverse order of keys, /// 0 - usual read, don't apply optimization /// /// So far, 0 means any direction is possible. It is ok for constant prefix. @@ -372,6 +372,7 @@ InputOrderInfoPtr buildInputOrderFromSortDescription( while (next_description_column < description.size() && next_sort_key < sorting_key.column_names.size()) { const auto & sorting_key_column = sorting_key.column_names[next_sort_key]; + int reverse_indicator = sorting_key.reverse_flags[next_sort_key] ? -1 : 1; const auto & sort_column_description = description[next_description_column]; /// If required order depend on collation, it cannot be matched with primary key order. @@ -405,8 +406,7 @@ InputOrderInfoPtr buildInputOrderFromSortDescription( if (sort_column_description.column_name != sorting_key_column) break; - current_direction = sort_column_description.direction; - + current_direction = sort_column_description.direction * reverse_indicator; //std::cerr << "====== (no dag) Found direct match" << std::endl; @@ -433,7 +433,7 @@ InputOrderInfoPtr buildInputOrderFromSortDescription( /// 'SELECT x, y FROM table WHERE x = 42 ORDER BY x + 1, y + 1' /// Here, 'x + 1' would be a fixed point. But it is reasonable to read-in-order. - current_direction = sort_column_description.direction; + current_direction = sort_column_description.direction * reverse_indicator; if (match.monotonicity) { current_direction *= match.monotonicity->direction; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 3186df6a6b3..9bc259dd292 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -221,10 +221,15 @@ static bool checkAllPartsOnRemoteFS(const RangesInDataParts & parts) /// build sort description for output stream static SortDescription getSortDescriptionForOutputHeader( - const Header & output_header, const Names & sorting_key_columns, const int sort_direction, InputOrderInfoPtr input_order_info, PrewhereInfoPtr prewhere_info, bool enable_vertical_final) + const Header & output_header, + const Names & sorting_key_columns, + const int sort_direction, + InputOrderInfoPtr input_order_info, + PrewhereInfoPtr prewhere_info, + bool enable_vertical_final) { /// Updating sort description can be done after PREWHERE actions are applied to the header. - /// Aftert PREWHERE actions are applied, column names in header can differ from storage column names due to aliases + /// After PREWHERE actions are applied, column names in header can differ from storage column names due to aliases /// To mitigate it, we're trying to build original header and use it to deduce sorting description /// TODO: this approach is fragile, it'd be more robust to update sorting description for the whole plan during plan optimization Block original_header = output_header.cloneEmpty(); @@ -1386,6 +1391,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( continue; Names sort_columns = storage_snapshot->metadata->getSortingKeyColumns(); + std::vector reverse_flags = storage_snapshot->metadata->getSortingKeyReverseFlags(); SortDescription sort_description; sort_description.compile_sort_description = settings[Setting::compile_sort_description]; sort_description.min_count_to_compile_sort_description = settings[Setting::min_count_to_compile_sort_description]; @@ -1396,7 +1402,12 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( Names partition_key_columns = storage_snapshot->metadata->getPartitionKey().column_names; for (size_t i = 0; i < sort_columns_size; ++i) - sort_description.emplace_back(sort_columns[i], 1, 1); + { + if (reverse_flags[i]) + sort_description.emplace_back(sort_columns[i], -1, 1); + else + sort_description.emplace_back(sort_columns[i], 1, 1); + } for (auto & pipe : pipes) addMergingFinal( diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index 7e43966556e..1b7549f83ca 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -27,6 +27,7 @@ KeyDescription::KeyDescription(const KeyDescription & other) , expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr) , sample_block(other.sample_block) , column_names(other.column_names) + , reverse_flags(other.reverse_flags) , data_types(other.data_types) , additional_column(other.additional_column) { @@ -57,6 +58,7 @@ KeyDescription & KeyDescription::operator=(const KeyDescription & other) sample_block = other.sample_block; column_names = other.column_names; + reverse_flags = other.reverse_flags; data_types = other.data_types; /// additional_column is constant property It should never be lost. @@ -132,11 +134,26 @@ KeyDescription KeyDescription::getSortingKeyFromAST( } const auto & children = result.expression_list_ast->children; + ASTPtr expr = std::make_shared(); for (const auto & child : children) + { + if (auto * func = child->as()) + { + if (func->name == "__descendingKey") + { + auto & key = func->arguments->children.front(); + result.column_names.emplace_back(key->getColumnName()); + result.reverse_flags.emplace_back(true); + expr->children.push_back(key->clone()); + continue; + } + } result.column_names.emplace_back(child->getColumnName()); + result.reverse_flags.emplace_back(false); + expr->children.push_back(child->clone()); + } { - auto expr = result.expression_list_ast->clone(); auto syntax_result = TreeRewriter(context).analyze(expr, columns.getAllPhysical()); /// In expression we also need to store source columns result.expression = ExpressionAnalyzer(expr, syntax_result, context).getActions(false); diff --git a/src/Storages/KeyDescription.h b/src/Storages/KeyDescription.h index 527a36124aa..4fc4b79b807 100644 --- a/src/Storages/KeyDescription.h +++ b/src/Storages/KeyDescription.h @@ -27,6 +27,9 @@ struct KeyDescription /// Column names in key definition, example: x, toStartOfMonth(date), a * b. Names column_names; + /// Indicator of key column being sorted reversely, example: x DESC, y -> {1, 0}. + std::vector reverse_flags; + /// Types from sample block ordered in columns order. DataTypes data_types; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index e3ace824115..02365009bcd 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1680,6 +1680,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const /// Merge { Names sort_columns = global_ctx->metadata_snapshot->getSortingKeyColumns(); + std::vector reverse_flags = global_ctx->metadata_snapshot->getSortingKeyReverseFlags(); sort_description.compile_sort_description = global_ctx->data->getContext()->getSettingsRef()[Setting::compile_sort_description]; sort_description.min_count_to_compile_sort_description = global_ctx->data->getContext()->getSettingsRef()[Setting::min_count_to_compile_sort_description]; @@ -1689,7 +1690,12 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const Names partition_key_columns = global_ctx->metadata_snapshot->getPartitionKey().column_names; for (size_t i = 0; i < sort_columns_size; ++i) - sort_description.emplace_back(sort_columns[i], 1, 1); + { + if (reverse_flags[i]) + sort_description.emplace_back(sort_columns[i], -1, 1); + else + sort_description.emplace_back(sort_columns[i], 1, 1); + } const bool is_vertical_merge = (global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical); /// If merge is vertical we cannot calculate it diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 8611681a976..a13abb19cca 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -181,6 +181,7 @@ namespace Setting namespace MergeTreeSetting { + extern const MergeTreeSettingsBool allow_experimental_reverse_key; extern const MergeTreeSettingsBool allow_nullable_key; extern const MergeTreeSettingsBool allow_remote_fs_zero_copy_replication; extern const MergeTreeSettingsBool allow_suspicious_indices; @@ -450,6 +451,7 @@ MergeTreeData::MergeTreeData( bool sanity_checks = mode <= LoadingStrictnessLevel::CREATE; allow_nullable_key = !sanity_checks || (*settings)[MergeTreeSetting::allow_nullable_key]; + allow_reverse_key = !sanity_checks || (*settings)[MergeTreeSetting::allow_experimental_reverse_key]; /// Check sanity of MergeTreeSettings. Only when table is created. if (sanity_checks) @@ -651,12 +653,28 @@ void MergeTreeData::checkProperties( const StorageInMemoryMetadata & old_metadata, bool attach, bool allow_empty_sorting_key, + bool allow_reverse_sorting_key, bool allow_nullable_key_, ContextPtr local_context) const { if (!new_metadata.sorting_key.definition_ast && !allow_empty_sorting_key) throw Exception(ErrorCodes::BAD_ARGUMENTS, "ORDER BY cannot be empty"); + if (!allow_reverse_sorting_key) + { + size_t num_sorting_keys = new_metadata.sorting_key.column_names.size(); + for (size_t i = 0; i < num_sorting_keys; ++i) + { + if (new_metadata.sorting_key.reverse_flags[i]) + { + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Sorting key {} is reversed, but merge tree setting `allow_experimental_reverse_key` is disabled", + new_metadata.sorting_key.column_names[i]); + } + } + } + KeyDescription new_sorting_key = new_metadata.sorting_key; KeyDescription new_primary_key = new_metadata.primary_key; @@ -785,7 +803,14 @@ void MergeTreeData::checkProperties( /// We cannot alter a projection so far. So here we do not try to find a projection in old metadata. bool is_aggregate = projection.type == ProjectionDescription::Type::Aggregate; - checkProperties(*projection.metadata, *projection.metadata, attach, is_aggregate, true /* allow_nullable_key */, local_context); + checkProperties( + *projection.metadata, + *projection.metadata, + attach, + is_aggregate, + allow_reverse_key, + true /* allow_nullable_key */, + local_context); projections_names.insert(projection.name); } } @@ -805,7 +830,14 @@ void MergeTreeData::setProperties( bool attach, ContextPtr local_context) { - checkProperties(new_metadata, old_metadata, attach, false, allow_nullable_key, local_context); + checkProperties( + new_metadata, + old_metadata, + attach, + false, + allow_reverse_key, + allow_nullable_key, + local_context); setInMemoryMetadata(new_metadata); setVirtuals(createVirtuals(new_metadata)); } @@ -3632,7 +3664,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context } checkColumnFilenamesForCollision(new_metadata, /*throw_on_error=*/ true); - checkProperties(new_metadata, old_metadata, false, false, allow_nullable_key, local_context); + checkProperties(new_metadata, old_metadata, false, false, allow_reverse_key, allow_nullable_key, local_context); checkTTLExpressions(new_metadata, old_metadata); if (!columns_to_check_conversion.empty()) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 7a9730e8627..b7b65714af1 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1330,6 +1330,7 @@ protected: const StorageInMemoryMetadata & old_metadata, bool attach, bool allow_empty_sorting_key, + bool allow_reverse_sorting_key, bool allow_nullable_key_, ContextPtr local_context) const; @@ -1710,7 +1711,8 @@ private: virtual void startBackgroundMovesIfNeeded() = 0; - bool allow_nullable_key{}; + bool allow_nullable_key = false; + bool allow_reverse_key = false; void addPartContributionToDataVolume(const DataPartPtr & part); void removePartContributionToDataVolume(const DataPartPtr & part); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index d7305045a56..4d847e768c6 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1083,14 +1083,21 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( const auto & primary_key = metadata_snapshot->getPrimaryKey(); auto index_columns = std::make_shared(); + std::vector reverse_flags; const auto & key_indices = key_condition.getKeyIndices(); DataTypes key_types; for (size_t i : key_indices) { if (i < index->size()) + { index_columns->emplace_back(index->at(i), primary_key.data_types[i], primary_key.column_names[i]); + reverse_flags.push_back(primary_key.reverse_flags[i]); + } else + { index_columns->emplace_back(); /// The column of the primary key was not loaded in memory - we'll skip it. + reverse_flags.push_back(false); + } key_types.emplace_back(primary_key.data_types[i]); } @@ -1138,28 +1145,32 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( { for (size_t i = 0; i < used_key_size; ++i) { + auto & left = reverse_flags[i] ? index_right[i] : index_left[i]; + auto & right = reverse_flags[i] ? index_left[i] : index_right[i]; if ((*index_columns)[i].column) - create_field_ref(range.begin, i, index_left[i]); + create_field_ref(range.begin, i, left); else - index_left[i] = NEGATIVE_INFINITY; + left = NEGATIVE_INFINITY; - index_right[i] = POSITIVE_INFINITY; + right = POSITIVE_INFINITY; } } else { for (size_t i = 0; i < used_key_size; ++i) { + auto & left = reverse_flags[i] ? index_right[i] : index_left[i]; + auto & right = reverse_flags[i] ? index_left[i] : index_right[i]; if ((*index_columns)[i].column) { - create_field_ref(range.begin, i, index_left[i]); - create_field_ref(range.end, i, index_right[i]); + create_field_ref(range.begin, i, left); + create_field_ref(range.end, i, right); } else { /// If the PK column was not loaded in memory - exclude it from the analysis. - index_left[i] = NEGATIVE_INFINITY; - index_right[i] = POSITIVE_INFINITY; + left = NEGATIVE_INFINITY; + right = POSITIVE_INFINITY; } } } diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 67fef759ed4..56e868836d8 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -517,12 +517,18 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( data.getSortingKeyAndSkipIndicesExpression(metadata_snapshot, indices)->execute(block); Names sort_columns = metadata_snapshot->getSortingKeyColumns(); + std::vector reverse_flags = metadata_snapshot->getSortingKeyReverseFlags(); SortDescription sort_description; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); for (size_t i = 0; i < sort_columns_size; ++i) - sort_description.emplace_back(sort_columns[i], 1, 1); + { + if (reverse_flags[i]) + sort_description.emplace_back(sort_columns[i], -1, 1); + else + sort_description.emplace_back(sort_columns[i], 1, 1); + } ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterBlocks); @@ -785,12 +791,18 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( data.getSortingKeyAndSkipIndicesExpression(metadata_snapshot, {})->execute(block); Names sort_columns = metadata_snapshot->getSortingKeyColumns(); + std::vector reverse_flags = metadata_snapshot->getSortingKeyReverseFlags(); SortDescription sort_description; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); for (size_t i = 0; i < sort_columns_size; ++i) - sort_description.emplace_back(sort_columns[i], 1, 1); + { + if (reverse_flags[i]) + sort_description.emplace_back(sort_columns[i], -1, 1); + else + sort_description.emplace_back(sort_columns[i], 1, 1); + } ProfileEvents::increment(ProfileEvents::MergeTreeDataProjectionWriterBlocks); diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 8c6aafe48f2..930c3d5f586 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -194,6 +194,7 @@ namespace ErrorCodes DECLARE(String, storage_policy, "default", "Name of storage disk policy", 0) \ DECLARE(String, disk, "", "Name of storage disk. Can be specified instead of storage policy.", 0) \ DECLARE(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \ + DECLARE(Bool, allow_experimental_reverse_key, false, "Allow descending sorting key in MergeTree tables (experimental feature).", 0) \ DECLARE(Bool, remove_empty_parts, true, "Remove empty parts after they were pruned by TTL, mutation, or collapsing merge algorithm.", 0) \ DECLARE(Bool, assign_part_uuids, false, "Generate UUIDs for parts. Before enabling check that all replicas support new format.", 0) \ DECLARE(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited. This setting is the default that can be overridden by the query-level setting with the same name.", 0) \ diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 4c74c8f56d1..d9cfdf68999 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -513,6 +513,13 @@ Names StorageInMemoryMetadata::getSortingKeyColumns() const return {}; } +std::vector StorageInMemoryMetadata::getSortingKeyReverseFlags() const +{ + if (hasSortingKey()) + return sorting_key.reverse_flags; + return {}; +} + const KeyDescription & StorageInMemoryMetadata::getSamplingKey() const { return sampling_key; diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index 64ae499ec6e..d9ce3abcf49 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -222,6 +222,9 @@ struct StorageInMemoryMetadata /// Returns columns names in sorting key specified by user in ORDER BY /// expression. For example: 'a', 'x * y', 'toStartOfMonth(date)', etc. Names getSortingKeyColumns() const; + /// Returns reverse indicators of columns in sorting key specified by user in ORDER BY + /// expression. For example: ('a' DESC, 'x * y', 'toStartOfMonth(date)' DESC) -> {1, 0, 1}. + std::vector getSortingKeyReverseFlags() const; /// Returns column names that need to be read for FINAL to work. Names getColumnsRequiredForFinal() const { return getColumnsRequiredForSortingKey(); } diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key.reference b/tests/queries/0_stateless/03257_reverse_sorting_key.reference new file mode 100644 index 00000000000..5371ee21f44 --- /dev/null +++ b/tests/queries/0_stateless/03257_reverse_sorting_key.reference @@ -0,0 +1,68 @@ +3 +8 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + MergeSortingTransform + LimitsCheckingTransform + PartialSortingTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 +99 +98 +97 +96 +95 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + (Expression) + ExpressionTransform + (ReadFromMergeTree) + ReverseTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InReverseOrder) 0 → 1 +0 +1 +2 +3 +4 +3 1003 +6 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + FinishSortingTransform + PartialSortingTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 +0 1090 +0 1080 +0 1070 +0 1060 +0 1050 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + FinishSortingTransform + PartialSortingTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 +0 1000 +0 1010 +0 1020 +0 1030 +0 1040 diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key.sql b/tests/queries/0_stateless/03257_reverse_sorting_key.sql new file mode 100644 index 00000000000..caef4ccccb0 --- /dev/null +++ b/tests/queries/0_stateless/03257_reverse_sorting_key.sql @@ -0,0 +1,39 @@ +drop table if exists x1; + +drop table if exists x2; + +create table x1 (i Nullable(int)) engine MergeTree order by i desc settings allow_nullable_key = 1, index_granularity = 2, allow_experimental_reverse_key = 1; + +insert into x1 select * from numbers(100); + +select * from x1 where i = 3; + +select count() from x1 where i between 3 and 10; + +explain pipeline select * from x1 order by i desc limit 5; + +select * from x1 order by i desc limit 5; + +explain pipeline select * from x1 order by i limit 5; + +select * from x1 order by i limit 5; + +create table x2 (i Nullable(int), j Nullable(int)) engine MergeTree order by (i, j desc) settings allow_nullable_key = 1, index_granularity = 2, allow_experimental_reverse_key = 1; + +insert into x2 select number % 10, number + 1000 from numbers(100); + +select * from x2 where j = 1003; + +select count() from x2 where i between 3 and 10 and j between 1003 and 1008; + +explain pipeline select * from x2 order by i, j desc limit 5; + +select * from x2 order by i, j desc limit 5; + +explain pipeline select * from x2 order by i, j limit 5; + +select * from x2 order by i, j limit 5; + +drop table x1; + +drop table x2; From a5c2c8ba4f0c0fb5792ebaacf13d1739b2f16733 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 28 Oct 2024 09:21:22 +0800 Subject: [PATCH 060/502] Fix parsing --- src/Parsers/ASTFunction.cpp | 2 +- src/Parsers/ParserCreateQuery.cpp | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 7c26e4dde30..ae4ba0cc336 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -284,7 +284,7 @@ static bool formatNamedArgWithHiddenValue(IAST * arg, const IAST::FormatSettings void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - /// Internal function for reversing MergeTree primary keys + /// Internal function for reversing MergeTree sorting keys if (name == "__descendingKey") { arguments->formatImpl(settings, state, frame); diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index b3b26baec6a..0b84cf6548c 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -560,7 +560,20 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!order_by && s_order_by.ignore(pos, expected)) { - auto old_pos = pos; + if (expression_p.parse(pos, order_by, expected)) + { + storage_like = true; + continue; + } + + /// Check possible ASC|DESC suffix for single key + if (order_elem_p.parse(pos, order_by, expected)) + { + storage_like = true; + continue; + } + + /// Check possible ASC|DESC suffix for a list of keys if (pos->type == TokenType::BareWord && std::string_view(pos->begin, pos->size()) == "tuple") ++pos; @@ -579,13 +592,6 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) continue; } - pos = old_pos; - if (order_elem_p.parse(pos, order_by, expected)) - { - storage_like = true; - continue; - } - return false; } From e83fd1eb576e9ed7b4b402231d77ff013527407d Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 28 Oct 2024 09:23:31 +0800 Subject: [PATCH 061/502] Add doc --- .../settings/merge-tree-settings.md | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 45c4cdf9458..aa0fd4b4bb1 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -1081,7 +1081,7 @@ Default value: 0 bytes. Note that if both `min_free_disk_bytes_to_perform_insert` and `min_free_disk_ratio_to_perform_insert` are specified, ClickHouse will count on the value that will allow to perform inserts on a bigger amount of free memory. -## min_free_disk_ratio_to_perform_insert +## min_free_disk_ratio_to_perform_insert The minimum free to total disk space ratio to perform an `INSERT`. Must be a floating point value between 0 and 1. Note that this setting: - takes into account the `keep_free_space_bytes` setting. @@ -1095,3 +1095,28 @@ Possible values: Default value: 0.0 Note that if both `min_free_disk_ratio_to_perform_insert` and `min_free_disk_bytes_to_perform_insert` are specified, ClickHouse will count on the value that will allow to perform inserts on a bigger amount of free memory. + +## allow_experimental_reverse_key + +Enables support for descending sort order in MergeTree sorting keys. This setting is particularly useful for time series analysis and Top-N queries, allowing data to be stored in reverse chronological order to optimize query performance. + +With `allow_experimental_reverse_key` enabled, you can define descending sort orders within the `ORDER BY` clause of a MergeTree table. This enables the use of more efficient `ReadInOrder` optimizations instead of `ReadInReverseOrder` for descending queries. + +**Example** + +```sql +CREATE TABLE example +( + time DateTime, + key Int32, + value String +) ENGINE = MergeTree +ORDER BY (time DESC, key) -- Descending order on 'time' field +SETTINGS allow_experimental_reverse_key = 1; + +SELECT * FROM example WHERE key = 'xxx' ORDER BY time DESC LIMIT 10; +``` + +By using `ORDER BY time DESC` in the query, `ReadInOrder` is applied. + +**Default Value:** false From f2046602083db07ecf4b35685937470202017ab6 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 28 Oct 2024 09:44:11 +0800 Subject: [PATCH 062/502] Also normalize for min_max projection --- src/Storages/ProjectionsDescription.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Storages/ProjectionsDescription.cpp b/src/Storages/ProjectionsDescription.cpp index 9654b4ef37a..cf9ff121966 100644 --- a/src/Storages/ProjectionsDescription.cpp +++ b/src/Storages/ProjectionsDescription.cpp @@ -205,8 +205,15 @@ ProjectionDescription ProjectionDescription::getMinMaxCountProjection( } if (!primary_key_asts.empty()) { - select_expression_list->children.push_back(makeASTFunction("min", primary_key_asts.front()->clone())); - select_expression_list->children.push_back(makeASTFunction("max", primary_key_asts.front()->clone())); + ASTPtr first_key = primary_key_asts.front(); + if (auto * func = first_key->as()) + { + if (func->name == "__descendingKey") + first_key = func->arguments->children.front(); + } + + select_expression_list->children.push_back(makeASTFunction("min", first_key->clone())); + select_expression_list->children.push_back(makeASTFunction("max", first_key->clone())); } select_expression_list->children.push_back(makeASTFunction("count")); select_query->setExpression(ASTProjectionSelectQuery::Expression::SELECT, std::move(select_expression_list)); From 58fc2987ca973a8727b7faf9e1ddde6a766f5a58 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 28 Oct 2024 10:07:12 +0800 Subject: [PATCH 063/502] Different implementation --- src/Interpreters/ActionsVisitor.cpp | 7 +++++++ src/Interpreters/ExpressionAnalyzer.cpp | 8 +++++++- src/Parsers/ParserCreateQuery.cpp | 6 ------ src/Planner/PlannerActionsVisitor.cpp | 5 +++++ src/Storages/KeyDescription.cpp | 7 ++----- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 65c3fe8cfcf..f91d2a6e7d8 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -882,6 +882,13 @@ void checkFunctionHasEmptyNullsAction(const ASTFunction & node) void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & data) { + /// Special function __descendingKey + if (node.name == "__descendingKey") + { + visit(node.arguments->children.front(), data); + return; + } + auto column_name = ast->getColumnName(); if (data.hasColumn(column_name)) return; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 4e5cf7d2549..8b2f24e7579 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1788,8 +1788,14 @@ ActionsDAG ExpressionAnalyzer::getActionsDAG(bool add_aliases, bool remove_unuse else asts = ASTs(1, query); - for (const auto & ast : asts) + for (auto ast : asts) { + if (auto * func = ast->as()) + { + if (func->name == "__descendingKey") + ast = func->arguments->children.front(); + } + std::string name = ast->getColumnName(); std::string alias; if (add_aliases) diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 0b84cf6548c..a434f6631dd 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -560,12 +560,6 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!order_by && s_order_by.ignore(pos, expected)) { - if (expression_p.parse(pos, order_by, expected)) - { - storage_like = true; - continue; - } - /// Check possible ASC|DESC suffix for single key if (order_elem_p.parse(pos, order_by, expected)) { diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index aea304e0ecc..008532c0c77 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -923,6 +923,11 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitFunction(const QueryTreeNodePtr & node) { const auto & function_node = node->as(); + + /// Special function __descendingKey + if (function_node.getFunctionName() == "__descendingKey") + return visitFunction(function_node.getChildren().front()); + if (function_node.getFunctionName() == "indexHint") return visitIndexHintFunction(node); diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index 1b7549f83ca..aa9ee7313a4 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -134,26 +134,23 @@ KeyDescription KeyDescription::getSortingKeyFromAST( } const auto & children = result.expression_list_ast->children; - ASTPtr expr = std::make_shared(); for (const auto & child : children) { if (auto * func = child->as()) { if (func->name == "__descendingKey") { - auto & key = func->arguments->children.front(); - result.column_names.emplace_back(key->getColumnName()); + result.column_names.emplace_back(func->arguments->children.front()->getColumnName()); result.reverse_flags.emplace_back(true); - expr->children.push_back(key->clone()); continue; } } result.column_names.emplace_back(child->getColumnName()); result.reverse_flags.emplace_back(false); - expr->children.push_back(child->clone()); } { + auto expr = result.expression_list_ast->clone(); auto syntax_result = TreeRewriter(context).analyze(expr, columns.getAllPhysical()); /// In expression we also need to store source columns result.expression = ExpressionAnalyzer(expr, syntax_result, context).getActions(false); From fb8300e1510d2f9f9f8b9619d080ba355b7e7ba8 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 28 Oct 2024 16:00:34 +0800 Subject: [PATCH 064/502] Fix test --- tests/queries/0_stateless/03257_reverse_sorting_key.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key.sql b/tests/queries/0_stateless/03257_reverse_sorting_key.sql index caef4ccccb0..ab177197446 100644 --- a/tests/queries/0_stateless/03257_reverse_sorting_key.sql +++ b/tests/queries/0_stateless/03257_reverse_sorting_key.sql @@ -1,3 +1,5 @@ +set optimize_read_in_order = 1; + drop table if exists x1; drop table if exists x2; From b9a2f683ba051ab0ad46bde9979e0aa30961e5ba Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 29 Oct 2024 10:38:59 +0800 Subject: [PATCH 065/502] Reimplemented with more tests --- src/Interpreters/ActionsVisitor.cpp | 7 -- src/Interpreters/ExpressionAnalyzer.cpp | 8 +-- src/Parsers/ParserCreateQuery.cpp | 66 +++++++++++-------- src/Parsers/ParserCreateQuery.h | 6 ++ src/Parsers/ParserProjectionSelectQuery.cpp | 4 +- src/Planner/PlannerActionsVisitor.cpp | 4 -- src/Storages/KeyDescription.cpp | 24 +++++-- src/Storages/KeyDescription.h | 5 +- .../ReplicatedMergeTreeTableMetadata.cpp | 12 ++-- src/Storages/ProjectionsDescription.cpp | 11 +--- src/Storages/StorageKeeperMap.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 4 +- ...57_reverse_sorting_key_zookeeper.reference | 2 + .../03257_reverse_sorting_key_zookeeper.sql | 16 +++++ 14 files changed, 99 insertions(+), 72 deletions(-) create mode 100644 tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.reference create mode 100644 tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index f91d2a6e7d8..65c3fe8cfcf 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -882,13 +882,6 @@ void checkFunctionHasEmptyNullsAction(const ASTFunction & node) void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & data) { - /// Special function __descendingKey - if (node.name == "__descendingKey") - { - visit(node.arguments->children.front(), data); - return; - } - auto column_name = ast->getColumnName(); if (data.hasColumn(column_name)) return; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 8b2f24e7579..4e5cf7d2549 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1788,14 +1788,8 @@ ActionsDAG ExpressionAnalyzer::getActionsDAG(bool add_aliases, bool remove_unuse else asts = ASTs(1, query); - for (auto ast : asts) + for (const auto & ast : asts) { - if (auto * func = ast->as()) - { - if (func->name == "__descendingKey") - ast = func->arguments->children.front(); - } - std::string name = ast->getColumnName(); std::string alias; if (add_aliases) diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index a434f6631dd..56f661eb46e 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -494,6 +494,44 @@ bool ParserTablePropertiesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, E return true; } +bool ParserStorageOrderByClause::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserStorageOrderByExpressionList order_list_p; + ParserStorageOrderByElement order_elem_p; + ParserToken s_lparen(TokenType::OpeningRoundBracket); + ParserToken s_rparen(TokenType::ClosingRoundBracket); + + ASTPtr order_by; + + /// Check possible ASC|DESC suffix for single key + if (order_elem_p.parse(pos, order_by, expected)) + { + node = order_by; + return true; + } + + /// Check possible ASC|DESC suffix for a list of keys + if (pos->type == TokenType::BareWord && std::string_view(pos->begin, pos->size()) == "tuple") + ++pos; + + if (s_lparen.ignore(pos, expected)) + { + auto tuple_function = std::make_shared(); + tuple_function->name = "tuple"; + if (order_list_p.parse(pos, order_by, expected)) + tuple_function->arguments = std::move(order_by); + else + tuple_function->arguments = std::make_shared(); + tuple_function->children.push_back(tuple_function->arguments); + order_by = std::move(tuple_function); + s_rparen.check(pos, expected); + + node = order_by; + return true; + } + + return false; +} bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { @@ -505,13 +543,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_sample_by(Keyword::SAMPLE_BY); ParserKeyword s_ttl(Keyword::TTL); ParserKeyword s_settings(Keyword::SETTINGS); - ParserToken s_lparen(TokenType::OpeningRoundBracket); - ParserToken s_rparen(TokenType::ClosingRoundBracket); ParserIdentifierWithOptionalParameters ident_with_optional_params_p; ParserExpression expression_p; - ParserStorageOrderByExpressionList order_list_p; - ParserStorageOrderByElement order_elem_p; + ParserStorageOrderByClause order_by_p; ParserSetQuery settings_p(/* parse_only_internals_ = */ true); ParserTTLExpressionList parser_ttl_list; ParserStringLiteral string_literal_parser; @@ -560,32 +595,11 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!order_by && s_order_by.ignore(pos, expected)) { - /// Check possible ASC|DESC suffix for single key - if (order_elem_p.parse(pos, order_by, expected)) + if (order_by_p.parse(pos, order_by, expected)) { storage_like = true; continue; } - - /// Check possible ASC|DESC suffix for a list of keys - if (pos->type == TokenType::BareWord && std::string_view(pos->begin, pos->size()) == "tuple") - ++pos; - - if (s_lparen.ignore(pos, expected)) - { - auto tuple_function = std::make_shared(); - tuple_function->name = "tuple"; - if (order_list_p.parse(pos, order_by, expected)) - tuple_function->arguments = std::move(order_by); - else - tuple_function->arguments = std::make_shared(); - tuple_function->children.push_back(tuple_function->arguments); - order_by = std::move(tuple_function); - s_rparen.check(pos, expected); - storage_like = true; - continue; - } - return false; } diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 82da2e7ea0b..76e20ca107b 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -88,6 +88,12 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +class ParserStorageOrderByClause : public IParserBase +{ +protected: + const char * getName() const override { return "storage order by clause"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; template class IParserColumnDeclaration : public IParserBase diff --git a/src/Parsers/ParserProjectionSelectQuery.cpp b/src/Parsers/ParserProjectionSelectQuery.cpp index 9ddd536f847..cd29fcfddb3 100644 --- a/src/Parsers/ParserProjectionSelectQuery.cpp +++ b/src/Parsers/ParserProjectionSelectQuery.cpp @@ -22,7 +22,7 @@ bool ParserProjectionSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & ParserNotEmptyExpressionList exp_list_for_with_clause(false); ParserNotEmptyExpressionList exp_list_for_select_clause(true); /// Allows aliases without AS keyword. - ParserExpression order_expression_p; + ParserStorageOrderByExpressionList order_list_p; ASTPtr with_expression_list; ASTPtr select_expression_list; @@ -59,7 +59,7 @@ bool ParserProjectionSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & if (s_order_by.ignore(pos, expected)) { ASTPtr expr_list; - if (!ParserList(std::make_unique(), std::make_unique(TokenType::Comma)).parse(pos, expr_list, expected)) + if (!order_list_p.parse(pos, expr_list, expected)) return false; if (expr_list->children.size() == 1) diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 008532c0c77..09e5db2ce88 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -924,10 +924,6 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi { const auto & function_node = node->as(); - /// Special function __descendingKey - if (function_node.getFunctionName() == "__descendingKey") - return visitFunction(function_node.getChildren().front()); - if (function_node.getFunctionName() == "indexHint") return visitIndexHintFunction(node); diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index aa9ee7313a4..ca00ad8c936 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include @@ -24,6 +24,7 @@ namespace ErrorCodes KeyDescription::KeyDescription(const KeyDescription & other) : definition_ast(other.definition_ast ? other.definition_ast->clone() : nullptr) + , original_expression_list_ast(other.original_expression_list_ast ? other.original_expression_list_ast->clone() : nullptr) , expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr) , sample_block(other.sample_block) , column_names(other.column_names) @@ -45,12 +46,16 @@ KeyDescription & KeyDescription::operator=(const KeyDescription & other) else definition_ast.reset(); + if (other.original_expression_list_ast) + original_expression_list_ast = other.original_expression_list_ast->clone(); + else + original_expression_list_ast.reset(); + if (other.expression_list_ast) expression_list_ast = other.expression_list_ast->clone(); else expression_list_ast.reset(); - if (other.expression) expression = other.expression->clone(); else @@ -124,29 +129,33 @@ KeyDescription KeyDescription::getSortingKeyFromAST( { KeyDescription result; result.definition_ast = definition_ast; - result.expression_list_ast = extractKeyExpressionList(definition_ast); + result.original_expression_list_ast = extractKeyExpressionList(definition_ast); if (additional_column) { result.additional_column = additional_column; ASTPtr column_identifier = std::make_shared(*additional_column); - result.expression_list_ast->children.push_back(column_identifier); + result.original_expression_list_ast->children.push_back(column_identifier); } - const auto & children = result.expression_list_ast->children; + result.expression_list_ast = std::make_shared(); + const auto & children = result.original_expression_list_ast->children; for (const auto & child : children) { if (auto * func = child->as()) { if (func->name == "__descendingKey") { - result.column_names.emplace_back(func->arguments->children.front()->getColumnName()); + auto & real_key = func->arguments->children.front(); + result.column_names.emplace_back(real_key->getColumnName()); result.reverse_flags.emplace_back(true); + result.expression_list_ast->children.push_back(real_key->clone()); continue; } } result.column_names.emplace_back(child->getColumnName()); result.reverse_flags.emplace_back(false); + result.expression_list_ast->children.push_back(child->clone()); } { @@ -173,6 +182,7 @@ KeyDescription KeyDescription::getSortingKeyFromAST( KeyDescription KeyDescription::buildEmptyKey() { KeyDescription result; + result.original_expression_list_ast = std::make_shared(); result.expression_list_ast = std::make_shared(); result.expression = std::make_shared(ActionsDAG(), ExpressionActionsSettings{}); return result; @@ -184,7 +194,7 @@ KeyDescription KeyDescription::parse(const String & str, const ColumnsDescriptio if (str.empty()) return result; - ParserExpression parser; + ParserStorageOrderByClause parser; ASTPtr ast = parseQuery(parser, "(" + str + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); FunctionNameNormalizer::visit(ast.get()); diff --git a/src/Storages/KeyDescription.h b/src/Storages/KeyDescription.h index 4fc4b79b807..d3f8b3fad2b 100644 --- a/src/Storages/KeyDescription.h +++ b/src/Storages/KeyDescription.h @@ -14,7 +14,10 @@ struct KeyDescription /// primary key in merge tree can be part of sorting key) ASTPtr definition_ast; - /// ASTExpressionList with key fields, example: (x, toStartOfMonth(date))). + /// Original user defined ASTExpressionList with key fields, example: (x DESC, toStartOfMonth(date))). + ASTPtr original_expression_list_ast; + + /// Same as above but without special function __descendingKey, a.k.a, ASC|DESC suffix. ASTPtr expression_list_ast; /// Expression from expression_list_ast created by ExpressionAnalyzer. Useful, diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 10185115da4..9e44a9479d9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -78,14 +78,14 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr /// - When we have only ORDER BY, than store it in "primary key:" row of /metadata /// - When we have both, than store PRIMARY KEY in "primary key:" row and ORDER BY in "sorting key:" row of /metadata - primary_key = formattedASTNormalized(metadata_snapshot->getPrimaryKey().expression_list_ast); + primary_key = formattedASTNormalized(metadata_snapshot->getPrimaryKey().original_expression_list_ast); if (metadata_snapshot->isPrimaryKeyDefined()) { - /// We don't use preparsed AST `sorting_key.expression_list_ast` because + /// We don't use preparsed AST `sorting_key.original_expression_list_ast` because /// it contain version column for VersionedCollapsingMergeTree, which /// is not stored in ZooKeeper for compatibility reasons. So the best /// compatible way is just to convert definition_ast to list and - /// serialize it. In all other places key.expression_list_ast should be + /// serialize it. In all other places key.original_expression_list_ast should be /// used. sorting_key = formattedASTNormalized(extractKeyExpressionList(metadata_snapshot->getSortingKey().definition_ast)); } @@ -93,7 +93,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr data_format_version = data.format_version; if (data.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - partition_key = formattedASTNormalized(metadata_snapshot->getPartitionKey().expression_list_ast); + partition_key = formattedASTNormalized(metadata_snapshot->getPartitionKey().original_expression_list_ast); ttl_table = formattedASTNormalized(metadata_snapshot->getTableTTLs().definition_ast); @@ -301,7 +301,7 @@ void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const Replicat /// NOTE: You can make a less strict check of match expressions so that tables do not break from small changes /// in formatAST code. - String parsed_zk_primary_key = formattedAST(KeyDescription::parse(from_zk.primary_key, columns, context).expression_list_ast); + String parsed_zk_primary_key = formattedAST(KeyDescription::parse(from_zk.primary_key, columns, context).original_expression_list_ast); if (primary_key != parsed_zk_primary_key) throw Exception(ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in primary key. " "Stored in ZooKeeper: {}, parsed from ZooKeeper: {}, local: {}", @@ -313,7 +313,7 @@ void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const Replicat "Stored in ZooKeeper: {}, local: {}", DB::toString(from_zk.data_format_version.toUnderType()), DB::toString(data_format_version.toUnderType())); - String parsed_zk_partition_key = formattedAST(KeyDescription::parse(from_zk.partition_key, columns, context).expression_list_ast); + String parsed_zk_partition_key = formattedAST(KeyDescription::parse(from_zk.partition_key, columns, context).original_expression_list_ast); if (partition_key != parsed_zk_partition_key) throw Exception(ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in partition key expression. " diff --git a/src/Storages/ProjectionsDescription.cpp b/src/Storages/ProjectionsDescription.cpp index cf9ff121966..9654b4ef37a 100644 --- a/src/Storages/ProjectionsDescription.cpp +++ b/src/Storages/ProjectionsDescription.cpp @@ -205,15 +205,8 @@ ProjectionDescription ProjectionDescription::getMinMaxCountProjection( } if (!primary_key_asts.empty()) { - ASTPtr first_key = primary_key_asts.front(); - if (auto * func = first_key->as()) - { - if (func->name == "__descendingKey") - first_key = func->arguments->children.front(); - } - - select_expression_list->children.push_back(makeASTFunction("min", first_key->clone())); - select_expression_list->children.push_back(makeASTFunction("max", first_key->clone())); + select_expression_list->children.push_back(makeASTFunction("min", primary_key_asts.front()->clone())); + select_expression_list->children.push_back(makeASTFunction("max", primary_key_asts.front()->clone())); } select_expression_list->children.push_back(makeASTFunction("count")); select_query->setExpression(ASTProjectionSelectQuery::Expression::SELECT, std::move(select_expression_list)); diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 316eced1ed6..de950a939ea 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -375,7 +375,7 @@ StorageKeeperMap::StorageKeeperMap( WriteBufferFromOwnString out; out << "KeeperMap metadata format version: 1\n" << "columns: " << metadata.columns.toString() - << "primary key: " << formattedAST(metadata.getPrimaryKey().expression_list_ast) << "\n"; + << "primary key: " << formattedAST(metadata.getPrimaryKey().original_expression_list_ast) << "\n"; metadata_string = out.str(); if (zk_root_path.empty()) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 850623157a1..406e01aebb3 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -6254,9 +6254,9 @@ void StorageReplicatedMergeTree::alter( if (ast_to_str(future_metadata.sorting_key.definition_ast) != ast_to_str(current_metadata->sorting_key.definition_ast)) { /// We serialize definition_ast as list, because code which apply ALTER (setTableStructure) expect serialized non empty expression - /// list here and we cannot change this representation for compatibility. Also we have preparsed AST `sorting_key.expression_list_ast` + /// list here and we cannot change this representation for compatibility. Also we have preparsed AST `sorting_key.original_expression_list_ast` /// in KeyDescription, but it contain version column for VersionedCollapsingMergeTree, which shouldn't be defined as a part of key definition AST. - /// So the best compatible way is just to convert definition_ast to list and serialize it. In all other places key.expression_list_ast should be used. + /// So the best compatible way is just to convert definition_ast to list and serialize it. In all other places key.original_expression_list_ast should be used. future_metadata_in_zk.sorting_key = serializeAST(*extractKeyExpressionList(future_metadata.sorting_key.definition_ast)); } diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.reference b/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.reference new file mode 100644 index 00000000000..2f834350bf3 --- /dev/null +++ b/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.reference @@ -0,0 +1,2 @@ +metadata format version: 1\ndate column: \nsampling expression: \nindex granularity: 2\nmode: 0\nsign column: \nprimary key: i DESC\ndata format version: 1\npartition key: \ngranularity bytes: 10000\nmerge parameters format version: 2\n +metadata format version: 1\ndate column: \nsampling expression: \nindex granularity: 2\nmode: 0\nsign column: \nprimary key: i, j DESC\ndata format version: 1\npartition key: \ngranularity bytes: 10000\nmerge parameters format version: 2\n diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql b/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql new file mode 100644 index 00000000000..aa7a0325f25 --- /dev/null +++ b/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql @@ -0,0 +1,16 @@ +-- Tags: zookeeper + +drop table if exists x1; +drop table if exists x2; + +create table x1 (i Nullable(int)) engine ReplicatedMergeTree('/clickhouse/tables/{database}/x1', 'r1') order by i desc settings allow_nullable_key = 1, index_granularity = 2, index_granularity_bytes = 10000, allow_experimental_reverse_key = 1; + +create table x2 (i Nullable(int), j Nullable(int)) engine ReplicatedMergeTree('/clickhouse/tables/{database}/x2', 'r1') order by (i, j desc) settings allow_nullable_key = 1, index_granularity = 2, index_granularity_bytes = 10000, allow_experimental_reverse_key = 1; + +set allow_unrestricted_reads_from_keeper = 'true'; + +select value from system.zookeeper where path = '/clickhouse/tables/' || currentDatabase() || '/x1' and name = 'metadata'; +select value from system.zookeeper where path = '/clickhouse/tables/' || currentDatabase() || '/x2' and name = 'metadata'; + +drop table x1; +drop table x2; From c801184c4fa72cc0d9c83b07d8d442de8ae92778 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 29 Oct 2024 10:40:43 +0800 Subject: [PATCH 066/502] Stabilize test --- tests/queries/0_stateless/03257_reverse_sorting_key.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key.sql b/tests/queries/0_stateless/03257_reverse_sorting_key.sql index ab177197446..e3fabbb9620 100644 --- a/tests/queries/0_stateless/03257_reverse_sorting_key.sql +++ b/tests/queries/0_stateless/03257_reverse_sorting_key.sql @@ -8,6 +8,8 @@ create table x1 (i Nullable(int)) engine MergeTree order by i desc settings allo insert into x1 select * from numbers(100); +optimize table x1 final; + select * from x1 where i = 3; select count() from x1 where i between 3 and 10; @@ -24,6 +26,8 @@ create table x2 (i Nullable(int), j Nullable(int)) engine MergeTree order by (i, insert into x2 select number % 10, number + 1000 from numbers(100); +optimize table x2 final; + select * from x2 where j = 1003; select count() from x2 where i between 3 and 10 and j between 1003 and 1008; From a2ce825491130621e72a2df1b689cee06c053467 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 29 Oct 2024 14:55:15 +0800 Subject: [PATCH 067/502] Add another setting to fix test --- tests/queries/0_stateless/03257_reverse_sorting_key.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key.sql b/tests/queries/0_stateless/03257_reverse_sorting_key.sql index e3fabbb9620..454dace2476 100644 --- a/tests/queries/0_stateless/03257_reverse_sorting_key.sql +++ b/tests/queries/0_stateless/03257_reverse_sorting_key.sql @@ -1,4 +1,5 @@ set optimize_read_in_order = 1; +set read_in_order_two_level_merge_threshold=100; drop table if exists x1; From 8ff12a939010dd7adfa8526289b9e651de0eebc9 Mon Sep 17 00:00:00 2001 From: Kirill <71129570+kirillgarbar@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:05:46 +0300 Subject: [PATCH 068/502] Fix build --- src/Disks/ObjectStorages/DiskObjectStorage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 07cc21d48db..9b3169317e7 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -419,7 +419,7 @@ void DiskObjectStorage::removeSharedRecursiveWithLimit( if (check_limit_reached()) remove(); - if (metadata_storage->isFile(path_to_remove)) + if (metadata_storage->existsFile(path_to_remove)) { chassert(path_to_remove.starts_with(path)); local_paths.emplace_back(path_to_remove); From 6c7305e447bbe56c8548e5f1e40d274906e5ca3b Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:27:10 +0100 Subject: [PATCH 069/502] Small refactorings. --- src/Interpreters/CancellationChecker.cpp | 1 + src/Interpreters/Context.cpp | 2 +- src/Interpreters/ProcessList.cpp | 42 ++++++++++--------- src/Interpreters/ProcessList.h | 8 ++-- src/Processors/Executors/PipelineExecutor.cpp | 4 +- .../Transforms/LimitsCheckingTransform.cpp | 2 +- src/QueryPipeline/ExecutionSpeedLimits.cpp | 6 +-- src/QueryPipeline/ReadProgressCallback.cpp | 2 +- 8 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp index 47ac118533b..c32823b71a4 100644 --- a/src/Interpreters/CancellationChecker.cpp +++ b/src/Interpreters/CancellationChecker.cpp @@ -1,5 +1,6 @@ #include #include "Interpreters/ProcessList.h" +#include "QueryPipeline/SizeLimits.h" #include #include <__chrono/duration.h> diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index dccfb57f580..788f4c14588 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2675,7 +2675,7 @@ bool Context::isBackgroundOperationContext() const void Context::killCurrentQuery() const { if (auto elem = getProcessListElement()) - elem->cancelQuery(CancelReason::MANUAL_CANCEL); + elem->cancelQuery(CancelReason::CANCELLED_BY_USER); } bool Context::isCurrentQueryKilled() const diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 0f467b43ed9..898856c4f8b 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -490,17 +490,26 @@ CancellationCode QueryStatus::cancelQuery(CancelReason reason) return CancellationCode::CancelSent; } -void QueryStatus::addPipelineExecutor(PipelineExecutor * e) +void QueryStatus::throwProperExceptionIfNeeded(const UInt64 & max_execution_time, const UInt64 & elapsed_ns) +{ + if (is_killed.load()) + { + String additional_error_part; + if (!elapsed_ns) + additional_error_part = fmt::format("elapsed {} ms,", static_cast(elapsed_ns) / 1000000000ULL); + + if (cancel_reason == CancelReason::TIMEOUT) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout exceeded: {} maximum: {} ms", additional_error_part, max_execution_time / 1000.0); + throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); + } +} + +void QueryStatus::addPipelineExecutor(PipelineExecutor * e, UInt64 max_exec_time) { /// In case of asynchronous distributed queries it is possible to call /// addPipelineExecutor() from the cancelQuery() context, and this will /// lead to deadlock. - if (is_killed.load()) - { - if (cancel_reason == CancelReason::TIMEOUT) - throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Query was timed out"); - throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); - } + throwProperExceptionIfNeeded(max_exec_time); std::lock_guard lock(executors_mutex); assert(!executors.contains(e)); @@ -525,14 +534,10 @@ void QueryStatus::removePipelineExecutor(PipelineExecutor * e) bool QueryStatus::checkTimeLimit() { - if (is_killed.load()) - { - if (cancel_reason == CancelReason::TIMEOUT) - throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Query was timed out"); - throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); - } + auto elapsed_ns = watch.elapsed(); + throwProperExceptionIfNeeded(limits.max_execution_time.totalMilliseconds(), elapsed_ns); - return limits.checkTimeLimit(watch, overflow_mode); + return limits.checkTimeLimit(elapsed_ns, overflow_mode); } bool QueryStatus::checkTimeLimitSoft() @@ -540,10 +545,9 @@ bool QueryStatus::checkTimeLimitSoft() if (is_killed.load()) return false; - return limits.checkTimeLimit(watch, OverflowMode::BREAK); + return limits.checkTimeLimit(watch.elapsedNanoseconds(), OverflowMode::BREAK); } - void QueryStatus::setUserProcessList(ProcessListForUser * user_process_list_) { user_process_list = user_process_list_; @@ -622,7 +626,7 @@ CancellationCode ProcessList::sendCancelToQuery(const String & current_query_id, cancelled_cv.notify_all(); }); - return elem->cancelQuery(CancelReason::MANUAL_CANCEL); + return elem->cancelQuery(CancelReason::CANCELLED_BY_USER); } @@ -644,7 +648,7 @@ CancellationCode ProcessList::sendCancelToQuery(QueryStatusPtr elem) cancelled_cv.notify_all(); }); - return elem->cancelQuery(CancelReason::MANUAL_CANCEL); + return elem->cancelQuery(CancelReason::CANCELLED_BY_USER); } @@ -670,7 +674,7 @@ void ProcessList::killAllQueries() } for (auto & cancelled_process : cancelled_processes) - cancelled_process->cancelQuery(CancelReason::MANUAL_CANCEL); + cancelled_process->cancelQuery(CancelReason::CANCELLED_BY_USER); } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 200660e04c3..c0f791ca5b6 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -44,9 +44,9 @@ class ProcessListEntry; enum CancelReason { - NOT_CANCELLED, + UNDEFINED, TIMEOUT, - MANUAL_CANCEL, + CANCELLED_BY_USER, }; /** Information of process list element. @@ -113,7 +113,7 @@ protected: bool is_cancelling { false }; /// KILL was send to the query std::atomic is_killed { false }; - std::atomic cancel_reason { CancelReason::NOT_CANCELLED }; + std::atomic cancel_reason { CancelReason::UNDEFINED }; /// All data to the client already had been sent. /// Including EndOfStream or Exception. @@ -234,6 +234,8 @@ public: CancellationCode cancelQuery(CancelReason reason); + void throwProperExceptionIfNeeded(UInt64 max_execution_time); + bool isKilled() const { return is_killed; } /// Returns an entry in the ProcessList associated with this QueryStatus. The function can return nullptr. diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index e8e1d5a941b..7fbcc3c9d3f 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -32,6 +32,7 @@ namespace Setting { extern const SettingsBool log_processors_profiles; extern const SettingsBool opentelemetry_trace_processors; + extern const SettingsSeconds max_execution_time; } namespace ErrorCodes @@ -67,7 +68,8 @@ PipelineExecutor::PipelineExecutor(std::shared_ptr & processors, Que { // Add the pipeline to the QueryStatus at the end to avoid issues if other things throw // as that would leave the executor "linked" - process_list_element->addPipelineExecutor(this); + UInt64 max_exec_time = process_list_element->getContext()->getSettingsRef()[Setting::max_execution_time].totalMilliseconds(); + process_list_element->addPipelineExecutor(this, max_exec_time); } } diff --git a/src/Processors/Transforms/LimitsCheckingTransform.cpp b/src/Processors/Transforms/LimitsCheckingTransform.cpp index 02d2fef808c..7e7cf1f86bb 100644 --- a/src/Processors/Transforms/LimitsCheckingTransform.cpp +++ b/src/Processors/Transforms/LimitsCheckingTransform.cpp @@ -32,7 +32,7 @@ void LimitsCheckingTransform::transform(Chunk & chunk) info.started = true; } - if (!limits.speed_limits.checkTimeLimit(info.total_stopwatch, limits.timeout_overflow_mode)) + if (!limits.speed_limits.checkTimeLimit(info.total_stopwatch.elapsed(), limits.timeout_overflow_mode)) { stopReading(); return; diff --git a/src/QueryPipeline/ExecutionSpeedLimits.cpp b/src/QueryPipeline/ExecutionSpeedLimits.cpp index 05fd394db77..9b018a2628b 100644 --- a/src/QueryPipeline/ExecutionSpeedLimits.cpp +++ b/src/QueryPipeline/ExecutionSpeedLimits.cpp @@ -117,17 +117,15 @@ static bool handleOverflowMode(OverflowMode mode, int code, FormatStringHelper static_cast(max_execution_time.totalMicroseconds()) * 1000) return handleOverflowMode( overflow_mode, ErrorCodes::TIMEOUT_EXCEEDED, - "Timeout exceeded: elapsed {} seconds, maximum: {}", + "Timeout exceeded: elapsed {} seconds, maximum: {} seconds", static_cast(elapsed_ns) / 1000000000ULL, max_execution_time.totalMicroseconds() / 1000000.0); } diff --git a/src/QueryPipeline/ReadProgressCallback.cpp b/src/QueryPipeline/ReadProgressCallback.cpp index e90fc24d882..a0ababd8813 100644 --- a/src/QueryPipeline/ReadProgressCallback.cpp +++ b/src/QueryPipeline/ReadProgressCallback.cpp @@ -47,7 +47,7 @@ bool ReadProgressCallback::onProgress(uint64_t read_rows, uint64_t read_bytes, c { for (const auto & limits : storage_limits) { - if (!limits.local_limits.speed_limits.checkTimeLimit(total_stopwatch, limits.local_limits.timeout_overflow_mode)) + if (!limits.local_limits.speed_limits.checkTimeLimit(total_stopwatch.elapsed(), limits.local_limits.timeout_overflow_mode)) return false; } From efe8df1b463feafaef6010834cc928feae86c3ab Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:51:38 +0100 Subject: [PATCH 070/502] Fix checkTimeLimit definition. --- src/QueryPipeline/ExecutionSpeedLimits.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/QueryPipeline/ExecutionSpeedLimits.h b/src/QueryPipeline/ExecutionSpeedLimits.h index 0def483123a..33660b1983c 100644 --- a/src/QueryPipeline/ExecutionSpeedLimits.h +++ b/src/QueryPipeline/ExecutionSpeedLimits.h @@ -29,7 +29,7 @@ public: void throttle(size_t read_rows, size_t read_bytes, size_t total_rows_to_read, UInt64 total_elapsed_microseconds, OverflowMode timeout_overflow_mode) const; - bool checkTimeLimit(const Stopwatch & stopwatch, OverflowMode overflow_mode) const; + bool checkTimeLimit(const UInt64 & elapsed_ns, OverflowMode overflow_mode) const; }; } From 75a658c143f98ba0eef74a3322d6907b69c8196c Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 31 Oct 2024 13:32:43 +0100 Subject: [PATCH 071/502] Fix throwProperExceptionIfNeeded definition. --- src/Interpreters/ProcessList.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index c0f791ca5b6..9e79d7e1885 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -234,7 +234,7 @@ public: CancellationCode cancelQuery(CancelReason reason); - void throwProperExceptionIfNeeded(UInt64 max_execution_time); + void throwProperExceptionIfNeeded(const UInt64 & max_execution_time, const UInt64 & elapsed_ns = 0); bool isKilled() const { return is_killed; } From c58643e04baa4540e05a86f6857c46f6e96646c1 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 31 Oct 2024 16:19:41 +0100 Subject: [PATCH 072/502] Revert some changes. --- src/Interpreters/ProcessList.cpp | 3 ++- src/Processors/Executors/PipelineExecutor.cpp | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 898856c4f8b..3c04e3d6bba 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -504,11 +504,12 @@ void QueryStatus::throwProperExceptionIfNeeded(const UInt64 & max_execution_time } } -void QueryStatus::addPipelineExecutor(PipelineExecutor * e, UInt64 max_exec_time) +void QueryStatus::addPipelineExecutor(PipelineExecutor * e) { /// In case of asynchronous distributed queries it is possible to call /// addPipelineExecutor() from the cancelQuery() context, and this will /// lead to deadlock. + UInt64 max_exec_time = getContext()->getSettingsRef()[Setting::max_execution_time].totalMilliseconds(); throwProperExceptionIfNeeded(max_exec_time); std::lock_guard lock(executors_mutex); diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 7fbcc3c9d3f..94804822d1f 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -68,8 +68,7 @@ PipelineExecutor::PipelineExecutor(std::shared_ptr & processors, Que { // Add the pipeline to the QueryStatus at the end to avoid issues if other things throw // as that would leave the executor "linked" - UInt64 max_exec_time = process_list_element->getContext()->getSettingsRef()[Setting::max_execution_time].totalMilliseconds(); - process_list_element->addPipelineExecutor(this, max_exec_time); + process_list_element->addPipelineExecutor(this); } } From bc70d60854a2aa2a37477cee32a26fd6ceaaa2ce Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 31 Oct 2024 18:57:48 +0100 Subject: [PATCH 073/502] Remove unused value of now_ms. --- src/Interpreters/CancellationChecker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp index c32823b71a4..4554bea301b 100644 --- a/src/Interpreters/CancellationChecker.cpp +++ b/src/Interpreters/CancellationChecker.cpp @@ -103,7 +103,7 @@ void CancellationChecker::workerFunction() UInt64 end_time_ms = 0; UInt64 duration = 0; auto now = std::chrono::steady_clock::now(); - UInt64 now_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + UInt64 now_ms; std::chrono::steady_clock::duration duration_milliseconds = std::chrono::milliseconds(0); if (!querySet.empty()) From 47053b119b414e7e18655f138d984f2a204f1758 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 1 Nov 2024 20:07:00 +0100 Subject: [PATCH 074/502] Try to fix the case when bgschpool size = 1. --- programs/server/Server.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index a48147f0033..c577700f1d3 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1373,13 +1373,16 @@ try setOOMScore(oom_score, log); #endif - auto cancellation_task_holder = global_context->getSchedulePool().createTask("CancellationChecker", []{ CancellationChecker::getInstance().workerFunction(); }); - auto cancellation_task = std::make_unique(std::move(cancellation_task_holder)); - (*cancellation_task)->activateAndSchedule(); + if (server_settings[ServerSetting::background_schedule_pool_size] > 1) + { + auto cancellation_task_holder = global_context->getSchedulePool().createTask("CancellationChecker", []{ CancellationChecker::getInstance().workerFunction(); }); + auto cancellation_task = std::make_unique(std::move(cancellation_task_holder)); + (*cancellation_task)->activateAndSchedule(); - SCOPE_EXIT({ - CancellationChecker::getInstance().terminateThread(); - }); + SCOPE_EXIT({ + CancellationChecker::getInstance().terminateThread(); + }); + } global_context->setRemoteHostFilter(config()); global_context->setHTTPHeaderFilter(config()); From 40104a9cb5bd34996c6620672d94f9f49d03cedc Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 1 Nov 2024 20:16:57 +0100 Subject: [PATCH 075/502] Change function definition after a merge w/ upstream. --- src/Interpreters/ProcessList.cpp | 2 +- src/Interpreters/ProcessList.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 3bde1628f30..f85a05979b4 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -456,7 +456,7 @@ void QueryStatus::ExecutorHolder::remove() executor = nullptr; } -CancellationCode QueryStatus::cancelQuery(bool /* kill */, std::exception_ptr exception) +CancellationCode QueryStatus::cancelQuery(CancelReason reason, std::exception_ptr exception) { if (is_killed.exchange(true)) return CancellationCode::CancelSent; diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 8cba05cff6e..bc986a274d1 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -243,7 +243,7 @@ public: /// Cancels the current query. /// Optional argument `exception` allows to set an exception which checkTimeLimit() will throw instead of "QUERY_WAS_CANCELLED". - CancellationCode cancelQuery(bool kill, std::exception_ptr exception = nullptr); + CancellationCode cancelQuery(CancelReason reason, std::exception_ptr exception = nullptr); bool isKilled() const { return is_killed; } From e62b985f8c2caa5de287ad783abd8fdfbfbf4401 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 1 Nov 2024 20:32:34 +0100 Subject: [PATCH 076/502] Fix build. --- src/Backups/BackupCoordinationStageSync.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Backups/BackupCoordinationStageSync.cpp b/src/Backups/BackupCoordinationStageSync.cpp index 9a05f9490c2..dea70ff24d2 100644 --- a/src/Backups/BackupCoordinationStageSync.cpp +++ b/src/Backups/BackupCoordinationStageSync.cpp @@ -641,7 +641,7 @@ void BackupCoordinationStageSync::cancelQueryIfError() exception = state.hosts.at(*state.host_with_error).exception; } - process_list_element->cancelQuery(false, exception); + process_list_element->cancelQuery(CancelReason::CANCELLED_BY_USER, exception); state_changed.notify_all(); } @@ -692,7 +692,7 @@ void BackupCoordinationStageSync::cancelQueryIfDisconnectedTooLong() state.cancelled = true; } - process_list_element->cancelQuery(false, exception); + process_list_element->cancelQuery(CancelReason::TIMEOUT, exception); state_changed.notify_all(); } From 33821eedfb06ba0adabecf14206d177a4e6b51e6 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 4 Nov 2024 11:26:53 +0000 Subject: [PATCH 077/502] Formatting --- .../ObjectStorage/DataLakes/IStorageDataLake.h | 2 +- .../ObjectStorage/DataLakes/IcebergMetadata.cpp | 7 ++----- .../ObjectStorage/StorageObjectStorageSource.cpp | 10 +++++++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h index 46df4df59e0..e1d046d4095 100644 --- a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h +++ b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h @@ -101,7 +101,7 @@ public: } ConfigurationPtr configuration = base_configuration->clone(); - configuration->setPaths(metadata->getDataFiles()); + configuration->setPaths(metadata->getDataFileInfos()); std::string sample_path; return Storage::resolveSchemaFromData(object_storage_, configuration, format_settings_, sample_path, local_context); } diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index c95735e01ad..41a9b83ef15 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -515,11 +515,8 @@ DataLakeMetadataPtr IcebergMetadata::create( auto snapshots = object->get("snapshots").extract(); String manifest_list_file; - const auto path = snapshot->getValue("manifest-list"); - manifest_list_file = std::filesystem::path(configuration->getPath()) / "metadata" / std::filesystem::path(path).filename(); - break; - } - } + const auto path = object->getValue("manifest-list"); + manifest_list_file = std::filesystem::path(configuration->getPath()) / "metadata" / std::filesystem::path(path).filename(); Int32 format_version = object->getValue("format-version"); diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 33514833fd0..02c4a1da432 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1,23 +1,27 @@ #include "StorageObjectStorageSource.h" #include #include +#include #include #include -#include #include #include #include +#include +#include #include #include #include +#include #include #include #include #include #include #include -#include -#include +#include "Disks/IO/CachedOnDiskReadBufferFromFile.h" +#include "Interpreters/Cache/FileCache.h" +#include "Interpreters/Cache/FileCacheKey.h" namespace fs = std::filesystem; From f8437fcdac27c95c92d1a0331dffc209fcd21e99 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 31 Oct 2024 17:19:31 +0000 Subject: [PATCH 078/502] Try to add ASTStorageOrderByElement. --- src/Interpreters/MutationsInterpreter.cpp | 2 +- src/Parsers/ASTFunction.cpp | 8 -- src/Parsers/ASTOrderByElement.cpp | 14 ++++ src/Parsers/ASTOrderByElement.h | 19 +++++ src/Parsers/ExpressionElementParsers.cpp | 12 ++- src/Parsers/ExpressionElementParsers.h | 5 ++ src/Parsers/ExpressionListParsers.cpp | 2 +- src/Parsers/ExpressionListParsers.h | 5 ++ src/Parsers/ParserCreateQuery.cpp | 40 +++++----- src/Parsers/ParserCreateQuery.h | 5 ++ src/Parsers/ParserProjectionSelectQuery.cpp | 2 +- .../Optimizations/optimizeReadInOrder.cpp | 2 +- .../QueryPlan/ReadFromMergeTree.cpp | 2 +- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 2 +- src/Storages/KeyDescription.cpp | 76 +++++++++++-------- src/Storages/KeyDescription.h | 9 +-- src/Storages/MergeTree/MergeTask.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- .../MergeTree/MergeTreeDataWriter.cpp | 4 +- .../ReplicatedMergeTreeTableMetadata.cpp | 23 +++--- src/Storages/StorageKeeperMap.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 4 +- src/Storages/TTLDescription.cpp | 2 +- .../0_stateless/03257_reverse_sorting_key.sql | 8 +- 25 files changed, 163 insertions(+), 91 deletions(-) diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 2f8bb8c16ac..50b5143f6b2 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -1445,7 +1445,7 @@ std::optional MutationsInterpreter::getStorageSortDescriptionIf { if (header.has(sort_columns[i])) { - if (reverse_flags[i]) + if (!reverse_flags.empty() && reverse_flags[i]) sort_description.emplace_back(sort_columns[i], -1, 1); else sort_description.emplace_back(sort_columns[i], 1, 1); diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index ae4ba0cc336..53d44e2f325 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -284,14 +284,6 @@ static bool formatNamedArgWithHiddenValue(IAST * arg, const IAST::FormatSettings void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - /// Internal function for reversing MergeTree sorting keys - if (name == "__descendingKey") - { - arguments->formatImpl(settings, state, frame); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " DESC" << (settings.hilite ? hilite_none : ""); - return; - } - frame.expression_list_prepend_whitespace = false; if (kind == Kind::CODEC || kind == Kind::STATISTICS || kind == Kind::BACKUP_NAME) frame.allow_operators = false; diff --git a/src/Parsers/ASTOrderByElement.cpp b/src/Parsers/ASTOrderByElement.cpp index 09193a8b5e1..06b1a58a930 100644 --- a/src/Parsers/ASTOrderByElement.cpp +++ b/src/Parsers/ASTOrderByElement.cpp @@ -57,4 +57,18 @@ void ASTOrderByElement::formatImpl(const FormatSettings & settings, FormatState } } +void ASTStorageOrderByElement::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const +{ + hash_state.update(direction); + IAST::updateTreeHashImpl(hash_state, ignore_aliases); +} + +void ASTStorageOrderByElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +{ + children.front()->formatImpl(settings, state, frame); + + if (direction == -1) + settings.ostr << (settings.hilite ? hilite_keyword : "") << " DESC" << (settings.hilite ? hilite_none : ""); +} + } diff --git a/src/Parsers/ASTOrderByElement.h b/src/Parsers/ASTOrderByElement.h index 6edf84d7bde..836c896196c 100644 --- a/src/Parsers/ASTOrderByElement.h +++ b/src/Parsers/ASTOrderByElement.h @@ -82,4 +82,23 @@ private: std::unordered_map positions; }; +class ASTStorageOrderByElement : public IAST +{ +public: + int direction = 1; /// 1 for ASC, -1 for DESC + + ASTPtr clone() const override + { + auto clone = std::make_shared(*this); + clone->cloneChildren(); + return clone; + } + + String getID(char) const override { return "StorageOrderByElement"; } + void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; + +protected: + void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; +}; + } diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 7fef956179e..f5c7e76129b 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -2174,6 +2174,12 @@ bool ParserStorageOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & if (!elem_p.parse(pos, expr_elem, expected)) return false; + if (!allow_order) + { + node = std::move(expr_elem); + return true; + } + int direction = 1; if (descending.ignore(pos, expected) || desc.ignore(pos, expected)) @@ -2181,7 +2187,11 @@ bool ParserStorageOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & else ascending.ignore(pos, expected) || asc.ignore(pos, expected); - node = direction == 1 ? std::move(expr_elem) : makeASTFunction("__descendingKey", std::move(expr_elem)); + auto storage_elem = std::make_shared(); + storage_elem->children.push_back(std::move(expr_elem)); + storage_elem->direction = direction; + + node = std::move(storage_elem); return true; } diff --git a/src/Parsers/ExpressionElementParsers.h b/src/Parsers/ExpressionElementParsers.h index 467813b7aa0..a8e49f7e560 100644 --- a/src/Parsers/ExpressionElementParsers.h +++ b/src/Parsers/ExpressionElementParsers.h @@ -436,7 +436,12 @@ protected: */ class ParserStorageOrderByElement : public IParserBase { +public: + explicit ParserStorageOrderByElement(bool allow_order_) : allow_order(allow_order_) {} + protected: + bool allow_order; + const char * getName() const override { return "element of storage ORDER BY expression"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 5396340bc36..ac2381c3483 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -327,7 +327,7 @@ bool ParserNotEmptyExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected bool ParserStorageOrderByExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - return ParserList(std::make_unique(), std::make_unique(TokenType::Comma), false) + return ParserList(std::make_unique(allow_order), std::make_unique(TokenType::Comma), false) .parse(pos, node, expected); } diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index 0fc5875156d..8e2304f3e47 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -251,7 +251,12 @@ protected: class ParserStorageOrderByExpressionList : public IParserBase { +public: + explicit ParserStorageOrderByExpressionList(bool allow_order_) : allow_order(allow_order_) {} + protected: + bool allow_order; + const char * getName() const override { return "storage order by expression"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 56f661eb46e..5a852fc4017 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -26,6 +26,7 @@ #include #include #include +#include namespace DB @@ -496,8 +497,8 @@ bool ParserTablePropertiesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, E bool ParserStorageOrderByClause::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - ParserStorageOrderByExpressionList order_list_p; - ParserStorageOrderByElement order_elem_p; + ParserStorageOrderByExpressionList order_list_p(allow_order); + ParserStorageOrderByElement order_elem_p(allow_order); ParserToken s_lparen(TokenType::OpeningRoundBracket); ParserToken s_rparen(TokenType::ClosingRoundBracket); @@ -506,6 +507,11 @@ bool ParserStorageOrderByClause::parseImpl(Pos & pos, ASTPtr & node, Expected & /// Check possible ASC|DESC suffix for single key if (order_elem_p.parse(pos, order_by, expected)) { + /// This is needed because 'order by (x, y)' is parsed as tuple. + /// We can remove ASTStorageOrderByElement if no ASC|DESC suffix was specified. + if (const auto * elem = order_by->as(); elem && elem->direction > 0) + order_by = elem->children.front(); + node = order_by; return true; } @@ -514,23 +520,21 @@ bool ParserStorageOrderByClause::parseImpl(Pos & pos, ASTPtr & node, Expected & if (pos->type == TokenType::BareWord && std::string_view(pos->begin, pos->size()) == "tuple") ++pos; - if (s_lparen.ignore(pos, expected)) - { - auto tuple_function = std::make_shared(); - tuple_function->name = "tuple"; - if (order_list_p.parse(pos, order_by, expected)) - tuple_function->arguments = std::move(order_by); - else - tuple_function->arguments = std::make_shared(); - tuple_function->children.push_back(tuple_function->arguments); - order_by = std::move(tuple_function); - s_rparen.check(pos, expected); + if (!s_lparen.ignore(pos, expected)) + return false; - node = order_by; - return true; - } + if (!order_list_p.parse(pos, order_by, expected)) + order_by = std::make_shared(); - return false; + if (!s_rparen.ignore(pos, expected)) + return false; + + auto tuple_function = std::make_shared(); + tuple_function->name = "tuple"; + tuple_function->arguments = std::move(order_by); + + node = std::move(tuple_function); + return true; } bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) @@ -546,7 +550,7 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserIdentifierWithOptionalParameters ident_with_optional_params_p; ParserExpression expression_p; - ParserStorageOrderByClause order_by_p; + ParserStorageOrderByClause order_by_p(/*allow_order_*/ true); ParserSetQuery settings_p(/* parse_only_internals_ = */ true); ParserTTLExpressionList parser_ttl_list; ParserStringLiteral string_literal_parser; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 76e20ca107b..5ab42d72f4a 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -90,7 +90,12 @@ protected: class ParserStorageOrderByClause : public IParserBase { +public: + explicit ParserStorageOrderByClause(bool allow_order_) : allow_order(allow_order_) {} + protected: + bool allow_order; + const char * getName() const override { return "storage order by clause"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; diff --git a/src/Parsers/ParserProjectionSelectQuery.cpp b/src/Parsers/ParserProjectionSelectQuery.cpp index cd29fcfddb3..39456876b1c 100644 --- a/src/Parsers/ParserProjectionSelectQuery.cpp +++ b/src/Parsers/ParserProjectionSelectQuery.cpp @@ -22,7 +22,7 @@ bool ParserProjectionSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & ParserNotEmptyExpressionList exp_list_for_with_clause(false); ParserNotEmptyExpressionList exp_list_for_select_clause(true); /// Allows aliases without AS keyword. - ParserStorageOrderByExpressionList order_list_p; + ParserStorageOrderByExpressionList order_list_p(/*allow_order_*/ false); ASTPtr with_expression_list; ASTPtr select_expression_list; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index 25564c55255..cd37e24f068 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -372,7 +372,7 @@ InputOrderInfoPtr buildInputOrderFromSortDescription( while (next_description_column < description.size() && next_sort_key < sorting_key.column_names.size()) { const auto & sorting_key_column = sorting_key.column_names[next_sort_key]; - int reverse_indicator = sorting_key.reverse_flags[next_sort_key] ? -1 : 1; + int reverse_indicator = (!sorting_key.reverse_flags.empty() && sorting_key.reverse_flags[next_sort_key]) ? -1 : 1; const auto & sort_column_description = description[next_description_column]; /// If required order depend on collation, it cannot be matched with primary key order. diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 9bc259dd292..145f1327de9 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1403,7 +1403,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( for (size_t i = 0; i < sort_columns_size; ++i) { - if (reverse_flags[i]) + if (!reverse_flags.empty() && reverse_flags[i]) sort_description.emplace_back(sort_columns[i], -1, 1); else sort_description.emplace_back(sort_columns[i], 1, 1); diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index d2f3423400f..dc9a4e4ab93 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -411,7 +411,7 @@ ReadFromSystemNumbersStep::ReadFromSystemNumbersStep( context_) , column_names{column_names_} , storage{std::move(storage_)} - , key_expression{KeyDescription::parse(column_names[0], storage_snapshot->metadata->columns, context).expression} + , key_expression{KeyDescription::parse(column_names[0], storage_snapshot->metadata->columns, context, false).expression} , max_block_size{max_block_size_} , num_streams{num_streams_} , limit_length_and_offset(InterpreterSelectQuery::getLimitLengthAndOffset(query_info.query->as(), context)) diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index ca00ad8c936..e78b82ea29f 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -24,7 +25,6 @@ namespace ErrorCodes KeyDescription::KeyDescription(const KeyDescription & other) : definition_ast(other.definition_ast ? other.definition_ast->clone() : nullptr) - , original_expression_list_ast(other.original_expression_list_ast ? other.original_expression_list_ast->clone() : nullptr) , expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr) , sample_block(other.sample_block) , column_names(other.column_names) @@ -46,11 +46,6 @@ KeyDescription & KeyDescription::operator=(const KeyDescription & other) else definition_ast.reset(); - if (other.original_expression_list_ast) - original_expression_list_ast = other.original_expression_list_ast->clone(); - else - original_expression_list_ast.reset(); - if (other.expression_list_ast) expression_list_ast = other.expression_list_ast->clone(); else @@ -129,34 +124,38 @@ KeyDescription KeyDescription::getSortingKeyFromAST( { KeyDescription result; result.definition_ast = definition_ast; - result.original_expression_list_ast = extractKeyExpressionList(definition_ast); + auto key_expression_list = extractKeyExpressionList(definition_ast); + + result.expression_list_ast = std::make_shared(); + for (const auto & child : key_expression_list->children) + { + auto real_key = child; + if (auto * elem = child->as()) + { + real_key = elem->children.front(); + result.reverse_flags.emplace_back(elem->direction < 0); + } + + result.expression_list_ast->children.push_back(real_key); + result.column_names.emplace_back(real_key->getColumnName()); + } if (additional_column) { result.additional_column = additional_column; ASTPtr column_identifier = std::make_shared(*additional_column); - result.original_expression_list_ast->children.push_back(column_identifier); + result.column_names.emplace_back(column_identifier->getColumnName()); + result.expression_list_ast->children.push_back(column_identifier); + + if (!result.reverse_flags.empty()) + result.reverse_flags.emplace_back(false); } - result.expression_list_ast = std::make_shared(); - const auto & children = result.original_expression_list_ast->children; - for (const auto & child : children) - { - if (auto * func = child->as()) - { - if (func->name == "__descendingKey") - { - auto & real_key = func->arguments->children.front(); - result.column_names.emplace_back(real_key->getColumnName()); - result.reverse_flags.emplace_back(true); - result.expression_list_ast->children.push_back(real_key->clone()); - continue; - } - } - result.column_names.emplace_back(child->getColumnName()); - result.reverse_flags.emplace_back(false); - result.expression_list_ast->children.push_back(child->clone()); - } + if (!result.reverse_flags.empty() && result.reverse_flags.size() != result.expression_list_ast->children.size()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "The size of reverse_flags ({}) does not match the size of KeyDescription {}", + result.reverse_flags.size(), result.expression_list_ast->children.size()); { auto expr = result.expression_list_ast->clone(); @@ -179,22 +178,39 @@ KeyDescription KeyDescription::getSortingKeyFromAST( return result; } +ASTPtr KeyDescription::getOriginalExpressionList() const +{ + if (!expression_list_ast || reverse_flags.empty()) + return expression_list_ast; + + auto expr_list = std::make_shared(); + size_t size = expression_list_ast->children.size(); + for (size_t i = 0; i < size; ++i) + { + auto column_ast = std::make_shared(); + column_ast->children.push_back(expression_list_ast->children[i]); + column_ast->direction = (!reverse_flags.empty() && reverse_flags[i]) ? -1 : 1; + expr_list->children.push_back(std::move(column_ast)); + } + + return expr_list; +} + KeyDescription KeyDescription::buildEmptyKey() { KeyDescription result; - result.original_expression_list_ast = std::make_shared(); result.expression_list_ast = std::make_shared(); result.expression = std::make_shared(ActionsDAG(), ExpressionActionsSettings{}); return result; } -KeyDescription KeyDescription::parse(const String & str, const ColumnsDescription & columns, ContextPtr context) +KeyDescription KeyDescription::parse(const String & str, const ColumnsDescription & columns, ContextPtr context, bool allow_order) { KeyDescription result; if (str.empty()) return result; - ParserStorageOrderByClause parser; + ParserStorageOrderByClause parser(allow_order); ASTPtr ast = parseQuery(parser, "(" + str + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); FunctionNameNormalizer::visit(ast.get()); diff --git a/src/Storages/KeyDescription.h b/src/Storages/KeyDescription.h index d3f8b3fad2b..a90e8f6e9a4 100644 --- a/src/Storages/KeyDescription.h +++ b/src/Storages/KeyDescription.h @@ -14,10 +14,7 @@ struct KeyDescription /// primary key in merge tree can be part of sorting key) ASTPtr definition_ast; - /// Original user defined ASTExpressionList with key fields, example: (x DESC, toStartOfMonth(date))). - ASTPtr original_expression_list_ast; - - /// Same as above but without special function __descendingKey, a.k.a, ASC|DESC suffix. + /// ASTExpressionList with key fields, example: (x DESC, toStartOfMonth(date))). ASTPtr expression_list_ast; /// Expression from expression_list_ast created by ExpressionAnalyzer. Useful, @@ -73,6 +70,8 @@ struct KeyDescription const ColumnsDescription & columns, ContextPtr context); + ASTPtr getOriginalExpressionList() const; + KeyDescription() = default; /// We need custom copy constructors because we don't want @@ -84,7 +83,7 @@ struct KeyDescription static bool moduloToModuloLegacyRecursive(ASTPtr node_expr); /// Parse description from string - static KeyDescription parse(const String & str, const ColumnsDescription & columns, ContextPtr context); + static KeyDescription parse(const String & str, const ColumnsDescription & columns, ContextPtr context, bool allow_order); }; } diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 02365009bcd..fce9e7ec6ca 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1691,7 +1691,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const for (size_t i = 0; i < sort_columns_size; ++i) { - if (reverse_flags[i]) + if (!reverse_flags.empty() && reverse_flags[i]) sort_description.emplace_back(sort_columns[i], -1, 1); else sort_description.emplace_back(sort_columns[i], 1, 1); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index a13abb19cca..870f915bd46 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -665,7 +665,7 @@ void MergeTreeData::checkProperties( size_t num_sorting_keys = new_metadata.sorting_key.column_names.size(); for (size_t i = 0; i < num_sorting_keys; ++i) { - if (new_metadata.sorting_key.reverse_flags[i]) + if (!new_metadata.sorting_key.reverse_flags.empty() && new_metadata.sorting_key.reverse_flags[i]) { throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 4d847e768c6..ff1993cd548 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1091,7 +1091,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( if (i < index->size()) { index_columns->emplace_back(index->at(i), primary_key.data_types[i], primary_key.column_names[i]); - reverse_flags.push_back(primary_key.reverse_flags[i]); + reverse_flags.push_back(!primary_key.reverse_flags.empty() && primary_key.reverse_flags[i]); } else { diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 56e868836d8..8c004d1ca48 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -524,7 +524,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( for (size_t i = 0; i < sort_columns_size; ++i) { - if (reverse_flags[i]) + if (!reverse_flags.empty() && reverse_flags[i]) sort_description.emplace_back(sort_columns[i], -1, 1); else sort_description.emplace_back(sort_columns[i], 1, 1); @@ -798,7 +798,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( for (size_t i = 0; i < sort_columns_size; ++i) { - if (reverse_flags[i]) + if (!reverse_flags.empty() && reverse_flags[i]) sort_description.emplace_back(sort_columns[i], -1, 1); else sort_description.emplace_back(sort_columns[i], 1, 1); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 9e44a9479d9..f341ab4816b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -77,23 +77,26 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr /// So rules in zookeeper metadata is following: /// - When we have only ORDER BY, than store it in "primary key:" row of /metadata /// - When we have both, than store PRIMARY KEY in "primary key:" row and ORDER BY in "sorting key:" row of /metadata - - primary_key = formattedASTNormalized(metadata_snapshot->getPrimaryKey().original_expression_list_ast); if (metadata_snapshot->isPrimaryKeyDefined()) { - /// We don't use preparsed AST `sorting_key.original_expression_list_ast` because + primary_key = formattedASTNormalized(metadata_snapshot->getPrimaryKey().expression_list_ast); + /// We don't use preparsed AST `sorting_key.expression_list_ast` because /// it contain version column for VersionedCollapsingMergeTree, which /// is not stored in ZooKeeper for compatibility reasons. So the best /// compatible way is just to convert definition_ast to list and - /// serialize it. In all other places key.original_expression_list_ast should be + /// serialize it. In all other places key.expression_list_ast should be /// used. sorting_key = formattedASTNormalized(extractKeyExpressionList(metadata_snapshot->getSortingKey().definition_ast)); } + else + { + primary_key = formattedASTNormalized(metadata_snapshot->getPrimaryKey().getOriginalExpressionList()); + } data_format_version = data.format_version; if (data.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - partition_key = formattedASTNormalized(metadata_snapshot->getPartitionKey().original_expression_list_ast); + partition_key = formattedASTNormalized(metadata_snapshot->getPartitionKey().expression_list_ast); ttl_table = formattedASTNormalized(metadata_snapshot->getTableTTLs().definition_ast); @@ -301,7 +304,7 @@ void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const Replicat /// NOTE: You can make a less strict check of match expressions so that tables do not break from small changes /// in formatAST code. - String parsed_zk_primary_key = formattedAST(KeyDescription::parse(from_zk.primary_key, columns, context).original_expression_list_ast); + String parsed_zk_primary_key = formattedAST(KeyDescription::parse(from_zk.primary_key, columns, context, true).getOriginalExpressionList()); if (primary_key != parsed_zk_primary_key) throw Exception(ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in primary key. " "Stored in ZooKeeper: {}, parsed from ZooKeeper: {}, local: {}", @@ -313,7 +316,7 @@ void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const Replicat "Stored in ZooKeeper: {}, local: {}", DB::toString(from_zk.data_format_version.toUnderType()), DB::toString(data_format_version.toUnderType())); - String parsed_zk_partition_key = formattedAST(KeyDescription::parse(from_zk.partition_key, columns, context).original_expression_list_ast); + String parsed_zk_partition_key = formattedAST(KeyDescription::parse(from_zk.partition_key, columns, context, false).expression_list_ast); if (partition_key != parsed_zk_partition_key) throw Exception(ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in partition key expression. " @@ -326,7 +329,7 @@ void ReplicatedMergeTreeTableMetadata::checkEquals(const ReplicatedMergeTreeTabl checkImmutableFieldsEquals(from_zk, columns, context); - String parsed_zk_sampling_expression = formattedAST(KeyDescription::parse(from_zk.sampling_expression, columns, context).definition_ast); + String parsed_zk_sampling_expression = formattedAST(KeyDescription::parse(from_zk.sampling_expression, columns, context, false).definition_ast); if (sampling_expression != parsed_zk_sampling_expression) { throw Exception(ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in sample expression. " @@ -334,7 +337,7 @@ void ReplicatedMergeTreeTableMetadata::checkEquals(const ReplicatedMergeTreeTabl from_zk.sampling_expression, parsed_zk_sampling_expression, sampling_expression); } - String parsed_zk_sorting_key = formattedAST(extractKeyExpressionList(KeyDescription::parse(from_zk.sorting_key, columns, context).definition_ast)); + String parsed_zk_sorting_key = formattedAST(extractKeyExpressionList(KeyDescription::parse(from_zk.sorting_key, columns, context, true).definition_ast)); if (sorting_key != parsed_zk_sorting_key) { throw Exception(ErrorCodes::METADATA_MISMATCH, @@ -343,7 +346,7 @@ void ReplicatedMergeTreeTableMetadata::checkEquals(const ReplicatedMergeTreeTabl from_zk.sorting_key, parsed_zk_sorting_key, sorting_key); } - auto parsed_primary_key = KeyDescription::parse(primary_key, columns, context); + auto parsed_primary_key = KeyDescription::parse(primary_key, columns, context, true); String parsed_zk_ttl_table = formattedAST(TTLTableDescription::parse(from_zk.ttl_table, columns, context, parsed_primary_key).definition_ast); if (ttl_table != parsed_zk_ttl_table) { diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index de950a939ea..316eced1ed6 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -375,7 +375,7 @@ StorageKeeperMap::StorageKeeperMap( WriteBufferFromOwnString out; out << "KeeperMap metadata format version: 1\n" << "columns: " << metadata.columns.toString() - << "primary key: " << formattedAST(metadata.getPrimaryKey().original_expression_list_ast) << "\n"; + << "primary key: " << formattedAST(metadata.getPrimaryKey().expression_list_ast) << "\n"; metadata_string = out.str(); if (zk_root_path.empty()) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 406e01aebb3..850623157a1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -6254,9 +6254,9 @@ void StorageReplicatedMergeTree::alter( if (ast_to_str(future_metadata.sorting_key.definition_ast) != ast_to_str(current_metadata->sorting_key.definition_ast)) { /// We serialize definition_ast as list, because code which apply ALTER (setTableStructure) expect serialized non empty expression - /// list here and we cannot change this representation for compatibility. Also we have preparsed AST `sorting_key.original_expression_list_ast` + /// list here and we cannot change this representation for compatibility. Also we have preparsed AST `sorting_key.expression_list_ast` /// in KeyDescription, but it contain version column for VersionedCollapsingMergeTree, which shouldn't be defined as a part of key definition AST. - /// So the best compatible way is just to convert definition_ast to list and serialize it. In all other places key.original_expression_list_ast should be used. + /// So the best compatible way is just to convert definition_ast to list and serialize it. In all other places key.expression_list_ast should be used. future_metadata_in_zk.sorting_key = serializeAST(*extractKeyExpressionList(future_metadata.sorting_key.definition_ast)); } diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 4845984cc88..ebf3d12f2b6 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -275,7 +275,7 @@ TTLDescription TTLDescription::getTTLFromAST( for (size_t i = 0; i < ttl_element->group_by_key.size(); ++i) { if (ttl_element->group_by_key[i]->getColumnName() != pk_columns[i]) - throw Exception(ErrorCodes::BAD_TTL_EXPRESSION, "TTL Expression GROUP BY key should be a prefix of primary key"); + throw Exception(ErrorCodes::BAD_TTL_EXPRESSION, "TTL Expression GROUP BY key should be a prefix of primary key {} {}", ttl_element->group_by_key[i]->getColumnName(), pk_columns[i]); used_primary_key_columns_set.insert(pk_columns[i]); } diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key.sql b/tests/queries/0_stateless/03257_reverse_sorting_key.sql index 454dace2476..221d6e70927 100644 --- a/tests/queries/0_stateless/03257_reverse_sorting_key.sql +++ b/tests/queries/0_stateless/03257_reverse_sorting_key.sql @@ -15,11 +15,11 @@ select * from x1 where i = 3; select count() from x1 where i between 3 and 10; -explain pipeline select * from x1 order by i desc limit 5; +explain pipeline select * from x1 order by i desc limit 5 settings max_threads=1; select * from x1 order by i desc limit 5; -explain pipeline select * from x1 order by i limit 5; +explain pipeline select * from x1 order by i limit 5 settings max_threads=1; select * from x1 order by i limit 5; @@ -33,11 +33,11 @@ select * from x2 where j = 1003; select count() from x2 where i between 3 and 10 and j between 1003 and 1008; -explain pipeline select * from x2 order by i, j desc limit 5; +explain pipeline select * from x2 order by i, j desc limit 5 settings max_threads=1; select * from x2 order by i, j desc limit 5; -explain pipeline select * from x2 order by i, j limit 5; +explain pipeline select * from x2 order by i, j limit 5 settings max_threads=1; select * from x2 order by i, j limit 5; From c2bcf151c754c8e50380b17b07055e1ca67c4346 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 4 Nov 2024 13:45:53 +0000 Subject: [PATCH 079/502] Update tests tegs. --- tests/queries/0_stateless/03257_reverse_sorting_key.sql | 2 ++ .../queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key.sql b/tests/queries/0_stateless/03257_reverse_sorting_key.sql index 221d6e70927..6aaeb2bacd6 100644 --- a/tests/queries/0_stateless/03257_reverse_sorting_key.sql +++ b/tests/queries/0_stateless/03257_reverse_sorting_key.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-merge-tree-settings + set optimize_read_in_order = 1; set read_in_order_two_level_merge_threshold=100; diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql b/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql index aa7a0325f25..fd903d3a4de 100644 --- a/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql +++ b/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql @@ -1,4 +1,4 @@ --- Tags: zookeeper +-- Tags: zookeeper, no-random-merge-tree-settings drop table if exists x1; drop table if exists x2; From 9c6a75c1dde1ed60da51fb98a66306d752ce7242 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 6 Nov 2024 17:08:34 +0100 Subject: [PATCH 080/502] Support read-only part of Iceberg REST Catalog API --- src/CMakeLists.txt | 2 + src/Common/ErrorCodes.cpp | 1 + src/Core/SettingsEnums.cpp | 3 + src/Core/SettingsEnums.h | 6 + src/Databases/Iceberg/DatabaseIceberg.cpp | 225 ++++++++++++++ src/Databases/Iceberg/DatabaseIceberg.h | 56 ++++ .../Iceberg/DatabaseIcebergSettings.cpp | 82 +++++ .../Iceberg/DatabaseIcebergSettings.h | 39 +++ src/Databases/Iceberg/ICatalog.cpp | 31 ++ src/Databases/Iceberg/ICatalog.h | 67 ++++ src/Databases/Iceberg/RestCatalog.cpp | 286 ++++++++++++++++++ src/Databases/Iceberg/RestCatalog.h | 66 ++++ src/Databases/registerDatabases.cpp | 8 + .../DataLakes/IcebergMetadata.cpp | 11 +- .../ObjectStorage/DataLakes/IcebergMetadata.h | 5 + 15 files changed, 880 insertions(+), 8 deletions(-) create mode 100644 src/Databases/Iceberg/DatabaseIceberg.cpp create mode 100644 src/Databases/Iceberg/DatabaseIceberg.h create mode 100644 src/Databases/Iceberg/DatabaseIcebergSettings.cpp create mode 100644 src/Databases/Iceberg/DatabaseIcebergSettings.h create mode 100644 src/Databases/Iceberg/ICatalog.cpp create mode 100644 src/Databases/Iceberg/ICatalog.h create mode 100644 src/Databases/Iceberg/RestCatalog.cpp create mode 100644 src/Databases/Iceberg/RestCatalog.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3627d760d4c..b9a6c49a312 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -166,6 +166,8 @@ if (TARGET ch_contrib::hdfs) add_headers_and_sources(dbms Disks/ObjectStorages/HDFS) endif() +add_headers_and_sources(dbms Databases/Iceberg) + add_headers_and_sources(dbms Disks/ObjectStorages/Cached) add_headers_and_sources(dbms Disks/ObjectStorages/Local) add_headers_and_sources(dbms Disks/ObjectStorages/Web) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 3f4a75fae3c..89523a5694f 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -611,6 +611,7 @@ M(730, REFRESH_FAILED) \ M(731, QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE) \ M(733, TABLE_IS_BEING_RESTARTED) \ + M(734, ICEBERG_CATALOG_ERROR) \ \ M(900, DISTRIBUTED_CACHE_ERROR) \ M(901, CANNOT_USE_DISTRIBUTED_CACHE) \ diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 89e9cb295c3..69126285eb8 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -284,4 +284,7 @@ IMPLEMENT_SETTING_ENUM( {"StochasticSimple", MergeSelectorAlgorithm::STOCHASTIC_SIMPLE}, {"Trivial", MergeSelectorAlgorithm::TRIVIAL}}) +IMPLEMENT_SETTING_ENUM(DatabaseIcebergCatalogType, ErrorCodes::BAD_ARGUMENTS, + {{"rest", DatabaseIcebergCatalogType::REST}}) + } diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 35bdb8a7f65..d45b81bd8ee 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -361,4 +361,10 @@ DECLARE_SETTING_ENUM(GroupArrayActionWhenLimitReached) DECLARE_SETTING_ENUM(MergeSelectorAlgorithm) +enum class DatabaseIcebergCatalogType : uint8_t +{ + REST, +}; + +DECLARE_SETTING_ENUM(DatabaseIcebergCatalogType) } diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp new file mode 100644 index 00000000000..b81780071e9 --- /dev/null +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -0,0 +1,225 @@ +#include + +#if USE_AVRO +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include + + +namespace DB +{ +namespace DatabaseIcebergSetting +{ + extern const DatabaseIcebergSettingsString storage_endpoint; + extern const DatabaseIcebergSettingsDatabaseIcebergCatalogType catalog_type; +} + +namespace +{ + std::pair parseTableName(const std::string & name) + { + auto pos = name.rfind('.'); + if (pos == std::string::npos) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Table cannot have empty namespace: {}", name); + + auto table_name = name.substr(pos + 1); + auto namespace_name = name.substr(0, name.size() - table_name.size() - 1); + return {namespace_name, table_name}; + } +} + +DatabaseIceberg::DatabaseIceberg( + const std::string & database_name_, + const std::string & url_, + const DatabaseIcebergSettings & settings_, + ASTPtr database_engine_definition_) + : IDatabase(database_name_) + , url(url_) + , settings(settings_) + , database_engine_definition(database_engine_definition_) + , log(getLogger("DatabaseIceberg(" + database_name_ + ")")) +{ +} + +std::unique_ptr DatabaseIceberg::getCatalog(ContextPtr context_) const +{ + switch (settings[DatabaseIcebergSetting::catalog_type].value) + { + case DB::DatabaseIcebergCatalogType::REST: + { + return std::make_unique(getDatabaseName(), url, context_); + } + } +} + +bool DatabaseIceberg::empty() const +{ + return getCatalog(Context::getGlobalContextInstance())->existsCatalog(); +} + +bool DatabaseIceberg::isTableExist(const String & name, ContextPtr context_) const +{ + auto [namespace_name, table_name] = parseTableName(name); + return getCatalog(context_)->existsTable(namespace_name, table_name); +} + +StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_) const +{ + auto catalog = getCatalog(context_); + auto table_metadata = Iceberg::ICatalog::TableMetadata().withLocation(); + auto [namespace_name, table_name] = parseTableName(name); + if (!catalog->tryGetTableMetadata(namespace_name, table_name, table_metadata)) + return nullptr; + + auto configuration = std::make_shared(); + ASTStorage * storage = database_engine_definition->as(); + ASTs args = storage->engine->arguments->children; + + auto table_endpoint = std::filesystem::path(settings[DatabaseIcebergSetting::storage_endpoint].value) / table_metadata.getPath(); + args[0] = std::make_shared(table_endpoint.string()); + + LOG_TEST(log, "Using endpoint: {}", table_endpoint.string()); + + StorageObjectStorage::Configuration::initialize(*configuration, args, context_, false); + return std::make_shared( + configuration, + configuration->createObjectStorage(context_, /* is_readonly */ false), + context_, + StorageID(getDatabaseName(), name), + /* columns */ColumnsDescription{}, + /* constraints */ConstraintsDescription{}, + /* comment */"", + getFormatSettings(context_), + LoadingStrictnessLevel::CREATE); +} + +DatabaseTablesIteratorPtr DatabaseIceberg::getTablesIterator( + ContextPtr context_, + const FilterByNameFunction & /* filter_by_table_name */, + bool /* skip_not_loaded */) const +{ + Tables tables; + auto catalog = getCatalog(context_); + for (const auto & table_name : catalog->getTables()) + { + DataTypePtr type = std::make_shared(); + auto columns = ColumnsDescription{NamesAndTypesList({NameAndTypePair(std::string("a"), type)})}; + auto storage = std::make_shared(StorageID(getDatabaseName(), table_name), columns, ConstraintsDescription{}, ""); + tables.emplace(table_name, storage); + } + + return std::make_unique(tables, getDatabaseName()); +} + +ASTPtr DatabaseIceberg::getCreateDatabaseQuery() const +{ + const auto & create_query = std::make_shared(); + create_query->setDatabase(getDatabaseName()); + create_query->set(create_query->storage, database_engine_definition); + return create_query; +} + +ASTPtr DatabaseIceberg::getCreateTableQueryImpl( + const String & name, + ContextPtr context_, + bool /* throw_on_error */) const +{ + auto catalog = getCatalog(context_); + auto table_metadata = Iceberg::ICatalog::TableMetadata().withLocation().withSchema(); + auto [namespace_name, table_name] = parseTableName(name); + catalog->getTableMetadata(namespace_name, table_name, table_metadata); + + auto create_table_query = std::make_shared(); + auto table_storage_define = database_engine_definition->clone(); + + auto * storage = table_storage_define->as(); + storage->engine->kind = ASTFunction::Kind::TABLE_ENGINE; + storage->settings = {}; + + create_table_query->set(create_table_query->storage, table_storage_define); + + auto columns_declare_list = std::make_shared(); + auto columns_expression_list = std::make_shared(); + + columns_declare_list->set(columns_declare_list->columns, columns_expression_list); + create_table_query->set(create_table_query->columns_list, columns_declare_list); + + /// init create query. + create_table_query->setTable(name); + create_table_query->setDatabase(getDatabaseName()); + + for (const auto & column_type_and_name : table_metadata.getSchema()) + { + const auto column_declaration = std::make_shared(); + column_declaration->name = column_type_and_name.name; + column_declaration->type = makeASTDataType(column_type_and_name.type->getName()); + columns_expression_list->children.emplace_back(column_declaration); + } + + auto storage_engine_arguments = storage->engine->arguments; + if (storage_engine_arguments->children.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected number of arguments: {}", storage_engine_arguments->children.size()); + + auto table_endpoint = std::filesystem::path(settings[DatabaseIcebergSetting::storage_endpoint].value) / table_metadata.getPath(); + storage_engine_arguments->children[0] = std::make_shared(table_endpoint.string()); + + return create_table_query; +} + +void registerDatabaseIceberg(DatabaseFactory & factory) +{ + auto create_fn = [](const DatabaseFactory::Arguments & args) + { + const auto * database_engine_define = args.create_query.storage; + const auto & database_engine_name = args.engine_name; + + const ASTFunction * function_define = database_engine_define->engine; + if (!function_define->arguments) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine `{}` must have arguments", database_engine_name); + + ASTs & engine_args = function_define->arguments->children; + if (engine_args.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine `{}` must have arguments", database_engine_name); + + const size_t max_args_num = 3; + if (engine_args.size() != max_args_num) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine must have {} arguments", max_args_num); + + for (auto & engine_arg : engine_args) + engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.context); + + const auto url = engine_args[0]->as()->value.safeGet(); + + DatabaseIcebergSettings database_settings; + if (database_engine_define->settings) + database_settings.loadFromQuery(*database_engine_define); + + return std::make_shared( + args.database_name, + url, + database_settings, + database_engine_define->clone()); + }; + factory.registerDatabase("Iceberg", create_fn, { .supports_arguments = true, .supports_settings = true }); +} + +} + +#endif diff --git a/src/Databases/Iceberg/DatabaseIceberg.h b/src/Databases/Iceberg/DatabaseIceberg.h new file mode 100644 index 00000000000..1356f5aee47 --- /dev/null +++ b/src/Databases/Iceberg/DatabaseIceberg.h @@ -0,0 +1,56 @@ +#pragma once +#include "config.h" + +#if USE_AVRO +#include +#include +#include + +namespace DB +{ + +class DatabaseIceberg final : public IDatabase, WithContext +{ +public: + explicit DatabaseIceberg( + const std::string & database_name_, + const std::string & url_, + const DatabaseIcebergSettings & settings_, + ASTPtr database_engine_definition_); + + String getEngineName() const override { return "Iceberg"; } + String getMetadataPath() const override { return ""; } + + bool canContainMergeTreeTables() const override { return false; } + bool canContainDistributedTables() const override { return false; } + bool shouldBeEmptyOnDetach() const override { return false; } + + bool empty() const override; + + bool isTableExist(const String & name, ContextPtr context) const override; + StoragePtr tryGetTable(const String & name, ContextPtr context) const override; + + DatabaseTablesIteratorPtr getTablesIterator( + ContextPtr context, + const FilterByNameFunction & filter_by_table_name, + bool skip_not_loaded) const override; + + void shutdown() override {} + + ASTPtr getCreateDatabaseQuery() const override; + +protected: + ASTPtr getCreateTableQueryImpl(const String & table_name, ContextPtr context, bool throw_on_error) const override; + +private: + const std::string url; + const DatabaseIcebergSettings settings; + const ASTPtr database_engine_definition; + const LoggerPtr log; + + std::unique_ptr getCatalog(ContextPtr context_) const; + +}; + +} +#endif diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp new file mode 100644 index 00000000000..d0a93edb579 --- /dev/null +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp @@ -0,0 +1,82 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_SETTING; +} + +#define DATABASE_ICEBERG_RELATED_SETTINGS(DECLARE, ALIAS) \ + DECLARE(DatabaseIcebergCatalogType, catalog_type, DatabaseIcebergCatalogType::REST, "Catalog type", 0) \ + DECLARE(String, storage_endpoint, "", "Object storage endpoint", 0) \ + +#define LIST_OF_DATABASE_ICEBERG_SETTINGS(M, ALIAS) \ + DATABASE_ICEBERG_RELATED_SETTINGS(M, ALIAS) + +DECLARE_SETTINGS_TRAITS(DatabaseIcebergSettingsTraits, LIST_OF_DATABASE_ICEBERG_SETTINGS) +IMPLEMENT_SETTINGS_TRAITS(DatabaseIcebergSettingsTraits, LIST_OF_DATABASE_ICEBERG_SETTINGS) + +struct DatabaseIcebergSettingsImpl : public BaseSettings +{ +}; + +#define INITIALIZE_SETTING_EXTERN(TYPE, NAME, DEFAULT, DESCRIPTION, FLAGS) \ + DatabaseIcebergSettings##TYPE NAME = &DatabaseIcebergSettingsImpl ::NAME; + +namespace DatabaseIcebergSetting +{ +LIST_OF_DATABASE_ICEBERG_SETTINGS(INITIALIZE_SETTING_EXTERN, SKIP_ALIAS) +} + +#undef INITIALIZE_SETTING_EXTERN + +DatabaseIcebergSettings::DatabaseIcebergSettings() : impl(std::make_unique()) +{ +} + +DatabaseIcebergSettings::DatabaseIcebergSettings(const DatabaseIcebergSettings & settings) + : impl(std::make_unique(*settings.impl)) +{ +} + +DatabaseIcebergSettings::DatabaseIcebergSettings(DatabaseIcebergSettings && settings) noexcept + : impl(std::make_unique(std::move(*settings.impl))) +{ +} + +DatabaseIcebergSettings::~DatabaseIcebergSettings() = default; + +DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(DatabaseIcebergSettings, IMPLEMENT_SETTING_SUBSCRIPT_OPERATOR) + + +void DatabaseIcebergSettings::applyChanges(const SettingsChanges & changes) +{ + impl->applyChanges(changes); +} + +void DatabaseIcebergSettings::loadFromQuery(const ASTStorage & storage_def) +{ + if (storage_def.settings) + { + try + { + impl->applyChanges(storage_def.settings->changes); + } + catch (Exception & e) + { + if (e.code() == ErrorCodes::UNKNOWN_SETTING) + e.addMessage("for database engine " + storage_def.engine->name); + throw; + } + } +} + +} diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.h b/src/Databases/Iceberg/DatabaseIcebergSettings.h new file mode 100644 index 00000000000..d6908f6f3b7 --- /dev/null +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ +class ASTStorage; +struct DatabaseIcebergSettingsImpl; +class SettingsChanges; + +/// List of available types supported in DatabaseIcebergSettings object +#define DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(CLASS_NAME, M) \ + M(CLASS_NAME, String) \ + M(CLASS_NAME, UInt64) \ + M(CLASS_NAME, DatabaseIcebergCatalogType) + +DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(DatabaseIcebergSettings, DECLARE_SETTING_TRAIT) + +struct DatabaseIcebergSettings +{ + DatabaseIcebergSettings(); + DatabaseIcebergSettings(const DatabaseIcebergSettings & settings); + DatabaseIcebergSettings(DatabaseIcebergSettings && settings) noexcept; + ~DatabaseIcebergSettings(); + + DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(DatabaseIcebergSettings, DECLARE_SETTING_SUBSCRIPT_OPERATOR) + + void loadFromQuery(const ASTStorage & storage_def); + + void applyChanges(const SettingsChanges & changes); + +private: + std::unique_ptr impl; +}; +} diff --git a/src/Databases/Iceberg/ICatalog.cpp b/src/Databases/Iceberg/ICatalog.cpp new file mode 100644 index 00000000000..6ddeeeac58d --- /dev/null +++ b/src/Databases/Iceberg/ICatalog.cpp @@ -0,0 +1,31 @@ +#include +#include + +namespace DB::ErrorCodes +{ + extern const int NOT_IMPLEMENTED; + extern const int LOGICAL_ERROR; +} + +namespace Iceberg +{ + +std::string ICatalog::TableMetadata::getPath() const +{ + if (!with_location) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Data location was not requested"); + + if (location.starts_with("s3://")) + return location.substr(std::strlen("s3://")); + else + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unsupported location type: {}", location); +} + +const DB::NamesAndTypesList & ICatalog::TableMetadata::getSchema() const +{ + if (!with_schema) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Data location was not requested"); + return schema; +} + +} diff --git a/src/Databases/Iceberg/ICatalog.h b/src/Databases/Iceberg/ICatalog.h new file mode 100644 index 00000000000..6a0412391a4 --- /dev/null +++ b/src/Databases/Iceberg/ICatalog.h @@ -0,0 +1,67 @@ +#pragma once +#include +#include + +namespace Iceberg +{ + +class ICatalog +{ +public: + using Namespace = std::string; + using Namespaces = std::vector; + using Table = std::string; + using Tables = std::vector; + + class TableMetadata; + + explicit ICatalog(const std::string & catalog_name_) : catalog_name(catalog_name_) {} + + virtual ~ICatalog() = default; + + virtual bool existsCatalog() const = 0; + + virtual Tables getTables() const = 0; + + virtual bool existsTable( + const std::string & namespace_naem, + const std::string & table_name) const = 0; + + virtual void getTableMetadata( + const std::string & namespace_name, + const std::string & table_name, + TableMetadata & result) const = 0; + + virtual bool tryGetTableMetadata( + const std::string & namespace_name, + const std::string & table_name, + TableMetadata & result) const = 0; + +protected: + const std::string catalog_name; +}; + +class ICatalog::TableMetadata +{ +friend class RestCatalog; + +public: + TableMetadata() = default; + + std::string getPath() const; + + const DB::NamesAndTypesList & getSchema() const; + + TableMetadata & withLocation() { with_location = true; return *this; } + TableMetadata & withSchema() { with_schema = true; return *this; } + +private: + /// starts with s3://, file://, etc + std::string location; + /// column names and types + DB::NamesAndTypesList schema; + + bool with_location = false; + bool with_schema = false; +}; +} diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp new file mode 100644 index 00000000000..e767a4cffaf --- /dev/null +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -0,0 +1,286 @@ +#include + +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace DB::ErrorCodes +{ + extern const int ICEBERG_CATALOG_ERROR; + extern const int LOGICAL_ERROR; +} + +namespace Iceberg +{ + +static constexpr auto namespaces_endpoint = "namespaces"; + +RestCatalog::RestCatalog( + const std::string & catalog_name_, + const std::string & base_url_, + DB::ContextPtr context_) + : ICatalog(catalog_name_) + , DB::WithContext(context_) + , base_url(base_url_) + , log(getLogger("RestCatalog(" + catalog_name_ + ")")) +{ +} + +bool RestCatalog::existsCatalog() const +{ + try + { + createReadBuffer(namespaces_endpoint)->eof(); + return true; + } + catch (...) + { + DB::tryLogCurrentException(log); + return false; + } +} + +DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer(const std::string & endpoint, const Poco::URI::QueryParameters & params) const +{ + const auto & context = getContext(); + const auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); + + Poco::URI url(base_url / endpoint); + if (!params.empty()) + url.setQueryParameters(params); + + return DB::BuilderRWBufferFromHTTP(url) + .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) + .withSettings(getContext()->getReadSettings()) + .withTimeouts(timeouts) + .withHostFilter(&getContext()->getRemoteHostFilter()) + .withSkipNotFound(true) + .create(credentials); +} + +RestCatalog::Tables RestCatalog::getTables() const +{ + Namespaces namespaces; + getNamespacesRecursive("", namespaces); + + Tables tables; + for (const auto & current_namespace : namespaces) + { + auto tables_in_namespace = getTables(current_namespace); + std::move(tables_in_namespace.begin(), tables_in_namespace.end(), std::back_inserter(tables)); + } + return tables; +} + +void RestCatalog::getNamespacesRecursive(const Namespace & base_namespace, Namespaces & result) const +{ + auto namespaces = getNamespaces(base_namespace); + result.reserve(result.size() + namespaces.size()); + result.insert(result.end(), namespaces.begin(), namespaces.end()); + + for (const auto & current_namespace : namespaces) + { + chassert(current_namespace.starts_with(base_namespace)); + getNamespacesRecursive(current_namespace, result); + } +} + +Poco::URI::QueryParameters RestCatalog::createParentNamespaceParams(const std::string & base_namespace) const +{ + std::vector parts; + splitInto<'.'>(parts, base_namespace); + std::string parent_param; + for (const auto & part : parts) + { + if (!parent_param.empty()) + parent_param += static_cast(0x1F); + parent_param += part; + } + return {{"parent", parent_param}}; +} + +RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_namespace) const +{ + Poco::URI::QueryParameters params; + if (!base_namespace.empty()) + params = createParentNamespaceParams(base_namespace); + + try + { + auto buf = createReadBuffer(namespaces_endpoint, params); + auto namespaces = parseNamespaces(*buf, base_namespace); + LOG_TEST(log, "Loaded {} namespaces", namespaces.size()); + return namespaces; + } + catch (const DB::HTTPException & e) + { + std::string message = fmt::format( + "Received error while fetching list of namespaces from iceberg catalog `{}`. ", + catalog_name); + + if (e.code() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND) + message += "Namespace provided in the `parent` query parameter is not found. "; + + message += fmt::format( + "Code: {}, status: {}, message: {}", + e.code(), e.getHTTPStatus(), e.displayText()); + + throw DB::Exception(DB::ErrorCodes::ICEBERG_CATALOG_ERROR, "{}", message); + } +} + +RestCatalog::Namespaces RestCatalog::parseNamespaces(DB::ReadBuffer & buf, const std::string & base_namespace) const +{ + if (buf.eof()) + return {}; + + String json_str; + readJSONObjectPossiblyInvalid(json_str, buf); + + LOG_TEST(log, "Received response: {}", json_str); + + Poco::JSON::Parser parser; + Poco::Dynamic::Var json = parser.parse(json_str); + const Poco::JSON::Object::Ptr & object = json.extract(); + + auto namespaces_object = object->get("namespaces").extract(); + if (!namespaces_object) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot parse result"); + + Namespaces namespaces; + for (size_t i = 0; i < namespaces_object->size(); ++i) + { + auto current_namespace_array = namespaces_object->get(static_cast(i)).extract(); + if (current_namespace_array->size() == 0) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Expected namespace array to be non-empty"); + + const int current_namespace_idx = static_cast(current_namespace_array->size()) - 1; + const auto current_namespace = current_namespace_array->get(current_namespace_idx).extract(); + const auto full_namespace = base_namespace.empty() ? current_namespace : base_namespace + "." + current_namespace; + namespaces.push_back(full_namespace); + } + + return namespaces; +} + +RestCatalog::Tables RestCatalog::getTables(const Namespace & base_namespace) const +{ + const auto endpoint = std::string(namespaces_endpoint) + "/" + base_namespace + "/tables"; + auto buf = createReadBuffer(endpoint); + return parseTables(*buf, base_namespace); +} + +RestCatalog::Tables RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & base_namespace) const +{ + if (buf.eof()) + return {}; + + String json_str; + readJSONObjectPossiblyInvalid(json_str, buf); + + Poco::JSON::Parser parser; + Poco::Dynamic::Var json = parser.parse(json_str); + const Poco::JSON::Object::Ptr & object = json.extract(); + + auto identifiers_object = object->get("identifiers").extract(); + if (!identifiers_object) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot parse result"); + + Tables tables; + for (size_t i = 0; i < identifiers_object->size(); ++i) + { + auto current_table_json = identifiers_object->get(static_cast(i)).extract(); + auto table_name = current_table_json->get("name").extract(); + tables.push_back(base_namespace + "." + table_name); + } + return tables; +} + +bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name) const +{ + TableMetadata table_metadata; + return tryGetTableMetadata(namespace_name, table_name, table_metadata); +} + +bool RestCatalog::tryGetTableMetadata( + const std::string & namespace_name, + const std::string & table_name, + TableMetadata & result) const +{ + try + { + return getTableMetadataImpl(namespace_name, table_name, result); + } + catch (...) + { + DB::tryLogCurrentException(log); + return false; + } +} + +void RestCatalog::getTableMetadata( + const std::string & namespace_name, + const std::string & table_name, + TableMetadata & result) const +{ + if (!getTableMetadataImpl(namespace_name, table_name, result)) + throw DB::Exception(DB::ErrorCodes::ICEBERG_CATALOG_ERROR, "No response from iceberg catalog"); +} + +bool RestCatalog::getTableMetadataImpl( + const std::string & namespace_name, + const std::string & table_name, + TableMetadata & result) const +{ + LOG_TEST(log, "Checking table {} in namespace {}", table_name, namespace_name); + + const auto endpoint = std::string(namespaces_endpoint) + "/" + namespace_name + "/tables/" + table_name; + auto buf = createReadBuffer(endpoint); + + if (buf->eof()) + { + LOG_TEST(log, "Table doesn't exist (endpoint: {})", endpoint); + return false; + } + + String json_str; + readJSONObjectPossiblyInvalid(json_str, *buf); + + LOG_TEST(log, "Received metadata for table {}: {}", table_name, json_str); + + Poco::JSON::Parser parser; + Poco::Dynamic::Var json = parser.parse(json_str); + const Poco::JSON::Object::Ptr & object = json.extract(); + + auto metadata_object = object->get("metadata").extract(); + if (!metadata_object) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot parse result"); + + if (result.with_location) + { + result.location = metadata_object->get("location").extract(); + + LOG_TEST(log, "Location for table {}: {}", table_name, result.location); + } + + if (result.with_schema) + { + int format_version = metadata_object->getValue("format-version"); + result.schema = DB::IcebergMetadata::parseTableSchema(metadata_object, format_version, true).first; + } + return true; +} + +} diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h new file mode 100644 index 00000000000..3b2f46b2856 --- /dev/null +++ b/src/Databases/Iceberg/RestCatalog.h @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include + +namespace DB +{ +class ReadBuffer; +} + +namespace Iceberg +{ + +class RestCatalog final : public ICatalog, private DB::WithContext +{ +public: + explicit RestCatalog( + const std::string & catalog_name_, + const std::string & base_url_, + DB::ContextPtr context_); + + ~RestCatalog() override = default; + + bool existsCatalog() const override; + + Tables getTables() const override; + + bool existsTable(const std::string & namespace_name, const std::string & table_name) const override; + + void getTableMetadata( + const std::string & namespace_name, + const std::string & table_name, + TableMetadata & result) const override; + + bool tryGetTableMetadata( + const std::string & namespace_name, + const std::string & table_name, + TableMetadata & result) const override; + +private: + const std::filesystem::path base_url; + Poco::Net::HTTPBasicCredentials credentials{}; + LoggerPtr log; + + DB::ReadWriteBufferFromHTTPPtr createReadBuffer(const std::string & endpoint, const Poco::URI::QueryParameters & params = {}) const; + + Poco::URI::QueryParameters createParentNamespaceParams(const std::string & base_namespace) const; + + void getNamespacesRecursive(const Namespace & base_namespace, Namespaces & result) const; + + Namespaces getNamespaces(const Namespace & base_namespace) const; + + Namespaces parseNamespaces(DB::ReadBuffer & buf, const std::string & base_namespace) const; + + Tables getTables(const Namespace & base_namespace) const; + + Tables parseTables(DB::ReadBuffer & buf, const std::string & base_namespace) const; + + bool getTableMetadataImpl( + const std::string & namespace_name, + const std::string & table_name, + TableMetadata & result) const; +}; + +} diff --git a/src/Databases/registerDatabases.cpp b/src/Databases/registerDatabases.cpp index 4f7c229bdf4..8daba254891 100644 --- a/src/Databases/registerDatabases.cpp +++ b/src/Databases/registerDatabases.cpp @@ -36,6 +36,10 @@ void registerDatabaseS3(DatabaseFactory & factory); void registerDatabaseHDFS(DatabaseFactory & factory); #endif +#if USE_AVRO +void registerDatabaseIceberg(DatabaseFactory & factory); +#endif + void registerDatabases() { auto & factory = DatabaseFactory::instance(); @@ -68,5 +72,9 @@ void registerDatabases() #if USE_HDFS registerDatabaseHDFS(factory); #endif + +#if USE_AVRO + registerDatabaseIceberg(factory); +#endif } } diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index f0a80a41d4e..3be43daec09 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -70,9 +70,6 @@ IcebergMetadata::IcebergMetadata( { } -namespace -{ - enum class ManifestEntryStatus : uint8_t { EXISTING = 0, @@ -248,7 +245,7 @@ DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & t } -std::pair parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, int format_version, bool ignore_schema_evolution) +std::pair IcebergMetadata::parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, int format_version, bool ignore_schema_evolution) { Poco::JSON::Object::Ptr schema; Int32 current_schema_id; @@ -313,9 +310,9 @@ std::pair parseTableSchema(const Poco::JSON::Object::P for (size_t i = 0; i != fields->size(); ++i) { auto field = fields->getObject(static_cast(i)); - auto name = field->getValue("name"); + auto column_name = field->getValue("name"); bool required = field->getValue("required"); - names_and_types.push_back({name, getFieldType(field, "type", required)}); + names_and_types.push_back({column_name, getFieldType(field, "type", required)}); } return {std::move(names_and_types), current_schema_id}; @@ -380,8 +377,6 @@ std::pair getMetadataFileAndVersion( return *std::max_element(metadata_files_with_versions.begin(), metadata_files_with_versions.end()); } -} - DataLakeMetadataPtr IcebergMetadata::create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context) { diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index eb5cac591f2..b28de471b40 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -96,6 +96,11 @@ public: static DataLakeMetadataPtr create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context); + static std::pair parseTableSchema( + const Poco::JSON::Object::Ptr & metadata_object, + int format_version, + bool ignore_schema_evolution); + private: size_t getVersion() const { return metadata_version; } From aac75cdb71ea39074b4d12ec161927a19792f766 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 6 Nov 2024 17:26:36 +0100 Subject: [PATCH 081/502] Update 03257_reverse_sorting_key_zookeeper.sql --- .../queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql b/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql index fd903d3a4de..108d348fb76 100644 --- a/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql +++ b/tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql @@ -1,4 +1,4 @@ --- Tags: zookeeper, no-random-merge-tree-settings +-- Tags: zookeeper, no-random-merge-tree-settings, no-replicated-database drop table if exists x1; drop table if exists x2; From fb28c1645361fec0405797ac547a9c5a4bdcfea8 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 6 Nov 2024 20:42:50 +0100 Subject: [PATCH 082/502] Unfinished test --- .../docker_compose_iceberg_rest_catalog.yml | 27 ++++ .../compose/docker_compose_minio.yml | 4 + tests/integration/helpers/cluster.py | 16 +++ .../test_database_iceberg/__init__.py | 0 .../integration/test_database_iceberg/test.py | 118 ++++++++++++++++++ 5 files changed, 165 insertions(+) create mode 100644 tests/integration/compose/docker_compose_iceberg_rest_catalog.yml create mode 100644 tests/integration/test_database_iceberg/__init__.py create mode 100644 tests/integration/test_database_iceberg/test.py diff --git a/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml new file mode 100644 index 00000000000..7c786b84fc1 --- /dev/null +++ b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml @@ -0,0 +1,27 @@ +services: + rest: + image: tabulario/iceberg-rest + container_name: iceberg-rest + ports: + - 8181:8181 + environment: + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=minio123 + - AWS_REGION=us-east-1 + - CATALOG_WAREHOUSE=s3://warehouse/ + - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO + - CATALOG_S3_ENDPOINT=http://minio:9001 + + spark-iceberg: + image: tabulario/spark-iceberg + container_name: spark-iceberg + depends_on: + - rest + environment: + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=minio123 + - AWS_REGION=us-east-1 + ports: + - 8080:8080 + - 10000:10000 + - 10001:10001 diff --git a/tests/integration/compose/docker_compose_minio.yml b/tests/integration/compose/docker_compose_minio.yml index 7fbe3796a0c..b0dad3d1ab8 100644 --- a/tests/integration/compose/docker_compose_minio.yml +++ b/tests/integration/compose/docker_compose_minio.yml @@ -14,6 +14,10 @@ services: depends_on: - proxy1 - proxy2 + networks: + default: + aliases: + - warehouse.minio # HTTP proxies for Minio. proxy1: diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 6751f205fb8..56659c1eb88 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -440,6 +440,7 @@ class ClickHouseCluster: zookeeper_keyfile=None, zookeeper_certfile=None, with_spark=False, + with_iceberg_catalog=False, ): for param in list(os.environ.keys()): logging.debug("ENV %40s %s" % (param, os.environ[param])) @@ -1462,6 +1463,18 @@ class ClickHouseCluster: ) return self.base_minio_cmd + def setup_iceberg_catalog_cmd(self, instance, env_variables, docker_compose_yml_dir): + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_iceberg_rest_catalog.yml")] + ) + self.base_iceberg_catalog_cmd = self.compose_cmd( + "--env-file", + instance.env_file, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_iceberg_rest_catalog.yml"), + ) + return self.base_minio_cmd + def setup_azurite_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_azurite = True env_variables["AZURITE_PORT"] = str(self.azurite_port) @@ -1917,6 +1930,9 @@ class ClickHouseCluster: cmds.append( self.setup_minio_cmd(instance, env_variables, docker_compose_yml_dir) ) + cmds.append( + self.setup_iceberg_catalog_cmd(instance, env_variables, docker_compose_yml_dir) + ) if with_azurite and not self.with_azurite: cmds.append( diff --git a/tests/integration/test_database_iceberg/__init__.py b/tests/integration/test_database_iceberg/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py new file mode 100644 index 00000000000..dc1e75ad40d --- /dev/null +++ b/tests/integration/test_database_iceberg/test.py @@ -0,0 +1,118 @@ +import glob +import json +import logging +import os +import time +import uuid +import requests +import pytest + +from helpers.cluster import ClickHouseCluster, ClickHouseInstance, is_arm +from helpers.s3_tools import ( + get_file_contents, + list_s3_objects, + prepare_s3_bucket, +) + +BASE_URL = "http://rest:8181/v1" +BASE_URL_LOCAL = "http://localhost:8181/v1" + + +def create_namespace(name): + payload = { + "namespace": [name], + "properties": {"owner": "clickhouse", "description": "test namespace"}, + } + + headers = {"Content-Type": "application/json"} + response = requests.post( + f"{BASE_URL_LOCAL}/namespaces", headers=headers, data=json.dumps(payload) + ) + if response.status_code == 200: + print(f"Namespace '{name}' created successfully.") + else: + raise Exception( + f"Failed to create namespace. Status code: {response.status_code}, Response: {response.text}" + ) + + +def list_namespaces(): + response = requests.get(f"{BASE_URL_LOCAL}/namespaces") + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to list namespaces: {response.status_code}") + + +def create_table(name, namespace): + payload = { + "name": name, + "schema": { + "type": "struct", + "fields": [ + {"id": 1, "name": "name", "type": "String", "required": True}, + {"id": 2, "name": "age", "type": "Int", "required": False}, + ], + }, + } + + headers = {"Content-Type": "application/json"} + response = requests.post( + f"{BASE_URL_LOCAL}/namespaces/{namespace}/tables", + headers=headers, + data=json.dumps(payload), + ) + if response.status_code == 200: + print(f"Table '{name}' created successfully.") + else: + raise Exception( + f"Failed to create a table. Status code: {response.status_code}, Response: {response.text}" + ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster = ClickHouseCluster(__file__, with_spark=True) + cluster.add_instance( + "node1", + main_configs=[], + user_configs=[], + with_minio=True, + stay_alive=True, + ) + + logging.info("Starting cluster...") + cluster.start() + + cluster.minio_client.make_bucket("warehouse") + prepare_s3_bucket(cluster) + + yield cluster + + finally: + cluster.shutdown() + + +def test_simple(started_cluster): + # TODO: properly wait for container + time.sleep(10) + + namespace = "kssenii.test.namespace" + root_namespace = "kssenii" + + create_namespace(namespace) + assert root_namespace in list_namespaces()["namespaces"][0][0] + + node = started_cluster.instances["node1"] + node.query( + f""" +CREATE DATABASE demo ENGINE = Iceberg('{BASE_URL}', 'minio', 'minio123') +SETTINGS catalog_type = 'rest', storage_endpoint = 'http://{started_cluster.minio_ip}:{started_cluster.minio_port}/' + """ + ) + + table_name = "testtable" + create_table(table_name, "kssenii") + + assert namespace in node.query("USE demo; SHOW TABLES") From 1c3fcfa355e73957436490a3007019d29c201627 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 6 Nov 2024 20:57:43 +0100 Subject: [PATCH 083/502] Fix style check --- src/Databases/Iceberg/DatabaseIceberg.cpp | 7 ++++++- src/Databases/Iceberg/RestCatalog.h | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index b81780071e9..a84e4fe1cbe 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -31,13 +31,18 @@ namespace DatabaseIcebergSetting extern const DatabaseIcebergSettingsDatabaseIcebergCatalogType catalog_type; } +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + namespace { std::pair parseTableName(const std::string & name) { auto pos = name.rfind('.'); if (pos == std::string::npos) - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Table cannot have empty namespace: {}", name); + throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Table cannot have empty namespace: {}", name); auto table_name = name.substr(pos + 1); auto namespace_name = name.substr(0, name.size() - table_name.size() - 1); diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index 3b2f46b2856..b33d5accda4 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -1,3 +1,4 @@ +#pragma once #include #include #include From ca3dfe5d8ee416cc0487aca50e87def0e4fb0de3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 6 Nov 2024 20:05:08 +0000 Subject: [PATCH 084/502] Automatic style fix --- tests/integration/helpers/cluster.py | 15 ++++++++++++--- tests/integration/test_database_iceberg/test.py | 9 +++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 56659c1eb88..78e9c581629 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1463,9 +1463,16 @@ class ClickHouseCluster: ) return self.base_minio_cmd - def setup_iceberg_catalog_cmd(self, instance, env_variables, docker_compose_yml_dir): + def setup_iceberg_catalog_cmd( + self, instance, env_variables, docker_compose_yml_dir + ): self.base_cmd.extend( - ["--file", p.join(docker_compose_yml_dir, "docker_compose_iceberg_rest_catalog.yml")] + [ + "--file", + p.join( + docker_compose_yml_dir, "docker_compose_iceberg_rest_catalog.yml" + ), + ] ) self.base_iceberg_catalog_cmd = self.compose_cmd( "--env-file", @@ -1931,7 +1938,9 @@ class ClickHouseCluster: self.setup_minio_cmd(instance, env_variables, docker_compose_yml_dir) ) cmds.append( - self.setup_iceberg_catalog_cmd(instance, env_variables, docker_compose_yml_dir) + self.setup_iceberg_catalog_cmd( + instance, env_variables, docker_compose_yml_dir + ) ) if with_azurite and not self.with_azurite: diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index dc1e75ad40d..85b3fef6eb8 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -4,15 +4,12 @@ import logging import os import time import uuid -import requests + import pytest +import requests from helpers.cluster import ClickHouseCluster, ClickHouseInstance, is_arm -from helpers.s3_tools import ( - get_file_contents, - list_s3_objects, - prepare_s3_bucket, -) +from helpers.s3_tools import get_file_contents, list_s3_objects, prepare_s3_bucket BASE_URL = "http://rest:8181/v1" BASE_URL_LOCAL = "http://localhost:8181/v1" From ba6760436445a48d64aba02c6ca1333edcddb410 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 7 Nov 2024 16:45:09 +0000 Subject: [PATCH 085/502] Add dynamic schemas --- .../DataLakes/DataLakeConfiguration.h | 39 ++++++++++++++++--- .../ObjectStorage/StorageObjectStorage.cpp | 22 ++++++++++- .../ObjectStorage/StorageObjectStorage.h | 15 ++++--- .../StorageObjectStorageSource.cpp | 10 ++--- 4 files changed, 69 insertions(+), 17 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h index 3da556cd7b0..b15356b36fe 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h @@ -15,12 +15,20 @@ #include "Storages/ColumnsDescription.h" #include +#include #include +#include + namespace DB { +namespace ErrorCodes +{ +extern const int FORMAT_VERSION_TOO_OLD; +} + template concept StorageConfiguration = std::derived_from; @@ -38,12 +46,19 @@ public: { BaseStorageConfiguration::update(object_storage, local_context); auto new_metadata = DataLakeMetadata::create(object_storage, weak_from_this(), local_context); - if (current_metadata && *current_metadata == *new_metadata) - return; - current_metadata = std::move(new_metadata); - BaseStorageConfiguration::setPaths(current_metadata->getDataFiles()); - BaseStorageConfiguration::setPartitionColumns(current_metadata->getPartitionColumns()); + if (!current_metadata || (*current_metadata != *new_metadata)) + { + throw Exception( + ErrorCodes::FORMAT_VERSION_TOO_OLD, + "Storage thinks that actual metadata version is {}, but actual metadata version is {}", + (dynamic_cast(current_metadata.get()) != nullptr) + ? std::to_string(dynamic_cast(current_metadata.get())->getVersion()) + : "Absent", + (dynamic_cast(new_metadata.get()) != nullptr) + ? std::to_string(dynamic_cast(new_metadata.get())->getVersion()) + : "Absent"); + } } std::optional tryGetTableStructureFromMetadata() const override @@ -72,6 +87,20 @@ public: return current_metadata->getSchemaTransformer(data_path); } + ColumnsDescription updateAndGetCurrentSchema(ObjectStoragePtr object_storage, ContextPtr context) override + { + BaseStorageConfiguration::update(object_storage, context); + auto new_metadata = DataLakeMetadata::create(object_storage, weak_from_this(), context); + + if (!current_metadata || (*current_metadata != *new_metadata)) + { + current_metadata = std::move(new_metadata); + BaseStorageConfiguration::setPaths(current_metadata->getDataFiles()); + BaseStorageConfiguration::setPartitionColumns(current_metadata->getPartitionColumns()); + } + return ColumnsDescription{current_metadata->getTableSchema()}; + } + private: DataLakeMetadataPtr current_metadata; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 4278f04d376..38a4bb3d1cb 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -91,6 +91,7 @@ StorageObjectStorage::StorageObjectStorage( { try { + configuration->updateAndGetCurrentSchema(object_storage, context); configuration->update(object_storage, context); } catch (...) @@ -148,6 +149,24 @@ void StorageObjectStorage::Configuration::update(ObjectStoragePtr object_storage IObjectStorage::ApplyNewSettingsOptions options{.allow_client_change = !isStaticConfiguration()}; object_storage_ptr->applyNewSettings(context->getConfigRef(), getTypeName() + ".", context, options); } + +bool StorageObjectStorage::hasExternalDynamicMetadata() const +{ + return configuration->isDataLakeConfiguration(); +} + +void StorageObjectStorage::updateExternalDynamicMetadata(ContextPtr context_ptr) +{ + if (configuration->isDataLakeConfiguration()) + { + StorageInMemoryMetadata metadata; + metadata.setColumns(configuration->updateAndGetCurrentSchema(object_storage, context_ptr)); + setInMemoryMetadata(metadata); + return; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method updateExternalDynamicMetadata is not supported by storage {}", getName()); +} + namespace { class ReadFromObjectStorageStep : public SourceStepWithFilter @@ -363,7 +382,7 @@ SinkToStoragePtr StorageObjectStorage::write( auto paths = configuration->getPaths(); if (auto new_key = checkAndGetNewFileOnInsertIfNeeded(*object_storage, *configuration, settings, paths.front(), paths.size())) { - paths.emplace_back(*new_key); + paths.push_back(*new_key); } configuration->setPaths(paths); @@ -434,6 +453,7 @@ ColumnsDescription StorageObjectStorage::resolveSchemaFromData( { if (configuration->isDataLakeConfiguration()) { + configuration->updateAndGetCurrentSchema(object_storage, context); configuration->update(object_storage, context); auto table_structure = configuration->tryGetTableStructureFromMetadata(); if (table_structure) diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 798711c7f3e..e543e4e9653 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -1,6 +1,4 @@ #pragma once -#include -#include #include #include #include @@ -9,12 +7,10 @@ #include #include #include - -#include - #include "Interpreters/ActionsDAG.h" #include "Storages/ColumnsDescription.h" +#include namespace DB { @@ -129,6 +125,10 @@ public: std::string & sample_path, const ContextPtr & context); + bool hasExternalDynamicMetadata() const override; + + void updateExternalDynamicMetadata(ContextPtr) override; + protected: String getPathSample(StorageInMemoryMetadata metadata, ContextPtr context); @@ -215,6 +215,11 @@ public: virtual std::shared_ptr getSchemaTransformer(const String&) const { return {}; } + virtual ColumnsDescription updateAndGetCurrentSchema(ObjectStoragePtr, ContextPtr) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method updateAndGetCurrentSchema is not supported by storage {}", getEngineName()); + } + virtual ReadFromFormatInfo prepareReadingFromFormat( ObjectStoragePtr object_storage, const Strings & requested_columns, diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 9855bab0232..cdd8d9f39b8 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -35,7 +35,8 @@ namespace CurrentMetrics extern const Metric StorageObjectStorageThreads; extern const Metric StorageObjectStorageThreadsActive; extern const Metric StorageObjectStorageThreadsScheduled; - } +} + namespace DB { namespace Setting @@ -682,10 +683,7 @@ StorageObjectStorage::ObjectInfoPtr StorageObjectStorageSource::GlobIterator::ne return {}; } - for (const auto & relative_metadata : *result) - { - new_batch.emplace_back(std::make_shared(relative_metadata->relative_path, relative_metadata->metadata)); - } + new_batch = std::move(result.value()); for (auto it = new_batch.begin(); it != new_batch.end();) { if (!recursive && !re2::RE2::FullMatch((*it)->getPath(), *matcher)) @@ -755,7 +753,7 @@ StorageObjectStorageSource::KeysIterator::KeysIterator( /// TODO: should we add metadata if we anyway fetch it if file_progress_callback is passed? for (auto && key : keys) { - auto object_info = std::make_shared(key, std::nullopt); + auto object_info = std::make_shared(key); read_keys_->emplace_back(object_info); } } From 34da26df32eca5d950312cf4271b568e02302db2 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 7 Nov 2024 18:35:32 +0000 Subject: [PATCH 086/502] Resolve issues --- .../DataLakes/DataLakeConfiguration.h | 9 +-- .../DataLakes/IcebergMetadata.cpp | 77 +++++++++---------- .../ObjectStorage/DataLakes/IcebergMetadata.h | 26 ++++--- .../ObjectStorage/StorageObjectStorage.h | 5 ++ 4 files changed, 56 insertions(+), 61 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h index b15356b36fe..af8bd88a006 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h @@ -47,17 +47,12 @@ public: BaseStorageConfiguration::update(object_storage, local_context); auto new_metadata = DataLakeMetadata::create(object_storage, weak_from_this(), local_context); + //Metadata must have been updated to this moment in updateAndGetCurrentSchema method and should be the same here if (!current_metadata || (*current_metadata != *new_metadata)) { throw Exception( ErrorCodes::FORMAT_VERSION_TOO_OLD, - "Storage thinks that actual metadata version is {}, but actual metadata version is {}", - (dynamic_cast(current_metadata.get()) != nullptr) - ? std::to_string(dynamic_cast(current_metadata.get())->getVersion()) - : "Absent", - (dynamic_cast(new_metadata.get()) != nullptr) - ? std::to_string(dynamic_cast(new_metadata.get())->getVersion()) - : "Absent"); + "Metadata is not consinsent with the one which was used to infer table schema. Please, retry the query."); } } diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 4bae09e45a7..cc9835ea25c 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -119,6 +119,27 @@ bool operator==(const Poco::JSON::Object & first, const Poco::JSON::Object & sec } return first_string_stream.str() == second_string_stream.str(); } + +Int32 parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor) +{ + Int32 format_version = metadata_object->getValue("format-version"); + if (format_version == 2) + { + auto fields = metadata_object->get("schemas").extract(); + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + schema_processor.addIcebergTableSchema(field); + } + return metadata_object->getValue("current-schema-id"); + } + else + { + auto schema = metadata_object->getObject("schema"); + schema_processor.addIcebergTableSchema(schema); + return schema->getValue("schema-id"); + } +} } DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name) @@ -242,27 +263,6 @@ bool IcebergSchemaProcessor::allowPrimitiveTypeConversion(const String & old_typ return allowed_type_conversion; } -Int32 parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor) -{ - Int32 format_version = metadata_object->getValue("format-version"); - if (format_version == 2) - { - auto fields = metadata_object->get("schemas").extract(); - for (size_t i = 0; i != fields->size(); ++i) - { - auto field = fields->getObject(static_cast(i)); - schema_processor.addIcebergTableSchema(field); - } - return metadata_object->getValue("current-schema-id"); - } - else - { - auto schema = metadata_object->getObject("schema"); - schema_processor.addIcebergTableSchema(schema); - return schema->getValue("schema-id"); - } -} - std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) { @@ -372,31 +372,28 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformatio current_new_id = -1; }); Poco::JSON::Object::Ptr old_schema, new_schema; - if (transform_dags_by_ids.contains({old_id, new_id})) + auto required_transform_dag_it = transform_dags_by_ids.find({old_id, new_id}); + if (required_transform_dag_it != transform_dags_by_ids.end()) { - return transform_dags_by_ids.at({old_id, new_id}); - } - try - { - old_schema = iceberg_table_schemas_by_ids.at(old_id); - } - catch (std::exception &) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", old_id); + return required_transform_dag_it->second; } + if (old_id == new_id) { return nullptr; } - try + + auto old_schema_it = iceberg_table_schemas_by_ids.find(old_id); + if (old_schema_it == iceberg_table_schemas_by_ids.end()) { - new_schema = iceberg_table_schemas_by_ids.at(new_id); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", old_id); } - catch (std::exception &) + auto new_schema_it = iceberg_table_schemas_by_ids.find(new_id); + if (new_schema_it == iceberg_table_schemas_by_ids.end()) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", new_id); } - return transform_dags_by_ids[{old_id, new_id}] = getSchemaTransformationDag(old_schema, new_schema); + return transform_dags_by_ids[{old_id, new_id}] = getSchemaTransformationDag(old_schema_it->second, new_schema_it->second); } void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schema_ptr) @@ -425,14 +422,10 @@ void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schem std::shared_ptr IcebergSchemaProcessor::getClickhouseTableSchemaById(Int32 id) { - try - { - return clickhouse_table_schemas_by_ids.at(id); - } - catch (std::exception &) - { + auto it = clickhouse_table_schemas_by_ids.find(id); + if (it == clickhouse_table_schemas_by_ids.end()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with id {} is unknown", id); - } + return it->second; } MutableColumns parseAvro(avro::DataFileReaderBase & file_reader, const Block & header, const FormatSettings & settings) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 7f952ea666c..a8e15a28182 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -1,7 +1,8 @@ #pragma once -#include #include "config.h" +#include +#include #if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format. @@ -80,16 +81,20 @@ public: std::shared_ptr getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id); private: - std::map iceberg_table_schemas_by_ids; - std::map> clickhouse_table_schemas_by_ids; - std::map, std::shared_ptr> transform_dags_by_ids; + std::unordered_map + iceberg_table_schemas_by_ids; + std::unordered_map> + clickhouse_table_schemas_by_ids; + std::map, std::shared_ptr> + transform_dags_by_ids; - NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr & schema); - DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type); - DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & type_key, bool required); - DataTypePtr getSimpleType(const String & type_name); + NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr &schema); + DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr &type); + DataTypePtr getFieldType(const Poco::JSON::Object::Ptr &field, + const String &type_key, bool required); + DataTypePtr getSimpleType(const String &type_name); std::shared_ptr getSchemaTransformationDag( - [[maybe_unused]] const Poco::JSON::Object::Ptr & old_schema, [[maybe_unused]] const Poco::JSON::Object::Ptr & new_schema); + const Poco::JSON::Object::Ptr & old_schema, [const Poco::JSON::Object::Ptr & new_schema); bool allowPrimitiveTypeConversion(const String & old_type, const String & new_type); @@ -97,9 +102,6 @@ private: Int32 current_old_id = -1; Int32 current_new_id = -1; - - // std::pair - // getRemappingForStructField(const Poco::JSON::Array::Ptr & old_node, const Poco::JSON::Array::Ptr & new_node, const Node * input_node); }; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index e543e4e9653..bd994b051e6 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -18,6 +18,11 @@ class ReadBufferIterator; class SchemaCache; class NamedCollection; +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + /** * A general class containing implementation for external table engines From 362b20bb7bb95e5b8d98b23eacd5af40ddadc568 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 7 Nov 2024 18:42:54 +0000 Subject: [PATCH 087/502] Removed compilation bug --- .../ObjectStorage/DataLakes/IcebergMetadata.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index a8e15a28182..3022622d7f6 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -93,15 +93,15 @@ private: DataTypePtr getFieldType(const Poco::JSON::Object::Ptr &field, const String &type_key, bool required); DataTypePtr getSimpleType(const String &type_name); - std::shared_ptr getSchemaTransformationDag( - const Poco::JSON::Object::Ptr & old_schema, [const Poco::JSON::Object::Ptr & new_schema); + std::shared_ptr + getSchemaTransformationDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema); - bool allowPrimitiveTypeConversion(const String & old_type, const String & new_type); + bool allowPrimitiveTypeConversion(const String & old_type, const String & new_type); - const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); + const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); - Int32 current_old_id = -1; - Int32 current_new_id = -1; + Int32 current_old_id = -1; + Int32 current_new_id = -1; }; From 3ec06cd7c7340e66f354dd8bc25180f1d970427b Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 8 Nov 2024 19:20:15 +0100 Subject: [PATCH 088/502] Make the max_execution_time bigger due to new logs --- .../0_stateless/01290_max_execution_speed_distributed.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql index d0dc554f425..864467c0942 100644 --- a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql +++ b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql @@ -1,6 +1,7 @@ -- Tags: distributed SET max_execution_speed = 1000000; +SET max_execution_time = 1000; SET timeout_before_checking_execution_speed = 0; SET max_block_size = 100; From af76992121bf6b88747242b7a34d562b2654accf Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 8 Nov 2024 19:35:12 +0100 Subject: [PATCH 089/502] Empty commit. From b6e2db14ad20026ba51fc39b1732bf581326e81a Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 8 Nov 2024 18:55:47 +0000 Subject: [PATCH 090/502] Resolve some problems --- .../DataLakes/DataLakeConfiguration.h | 19 ++++++++--- .../DataLakes/DeltaLakeMetadata.cpp | 5 +-- .../DataLakes/IDataLakeMetadata.h | 1 + .../DataLakes/IcebergMetadata.cpp | 1 + .../ObjectStorage/DataLakes/IcebergMetadata.h | 32 +++++++++---------- .../ObjectStorage/StorageObjectStorage.cpp | 26 +++++++-------- .../ObjectStorage/StorageObjectStorage.h | 2 ++ .../registerStorageObjectStorage.cpp | 2 ++ .../integration/test_storage_iceberg/test.py | 17 ++++++++-- 9 files changed, 65 insertions(+), 40 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h index af8bd88a006..be1304e3340 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h @@ -47,12 +47,20 @@ public: BaseStorageConfiguration::update(object_storage, local_context); auto new_metadata = DataLakeMetadata::create(object_storage, weak_from_this(), local_context); - //Metadata must have been updated to this moment in updateAndGetCurrentSchema method and should be the same here if (!current_metadata || (*current_metadata != *new_metadata)) { - throw Exception( - ErrorCodes::FORMAT_VERSION_TOO_OLD, - "Metadata is not consinsent with the one which was used to infer table schema. Please, retry the query."); + if (hasExternalDynamicMetadata()) + { + throw Exception( + ErrorCodes::FORMAT_VERSION_TOO_OLD, + "Metadata is not consinsent with the one which was used to infer table schema. Please, retry the query."); + } + else + { + current_metadata = std::move(new_metadata); + BaseStorageConfiguration::setPaths(current_metadata->getDataFiles()); + BaseStorageConfiguration::setPartitionColumns(current_metadata->getPartitionColumns()); + } } } @@ -82,6 +90,8 @@ public: return current_metadata->getSchemaTransformer(data_path); } + bool hasExternalDynamicMetadata() override { return current_metadata && current_metadata->supportsExternalMetadataChange(); } + ColumnsDescription updateAndGetCurrentSchema(ObjectStoragePtr object_storage, ContextPtr context) override { BaseStorageConfiguration::update(object_storage, context); @@ -99,6 +109,7 @@ public: private: DataLakeMetadataPtr current_metadata; + ReadFromFormatInfo prepareReadingFromFormat( ObjectStoragePtr object_storage, const Strings & requested_columns, diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp index dc5a32c582c..ef0adc15186 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp @@ -691,10 +691,7 @@ DeltaLakeMetadata::DeltaLakeMetadata(ObjectStoragePtr object_storage_, Configura { auto impl = DeltaLakeMetadataImpl(object_storage_, configuration_, context_); auto result = impl.processMetadataFiles(); - for (const auto & data_file_name : result.data_files) - { - data_files.emplace_back(data_file_name); - } + data_files = result.data_files; schema = result.schema; partition_columns = result.partition_columns; diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h index db44be94021..de6324369c3 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h @@ -18,6 +18,7 @@ public: virtual const std::unordered_map & getColumnNameToPhysicalNameMapping() const = 0; virtual std::shared_ptr getInitialSchemaByPath(const String &) const { return {}; } virtual std::shared_ptr getSchemaTransformer(const String &) const { return {}; } + virtual bool supportsExternalMetadataChange() const { return false; } }; using DataLakeMetadataPtr = std::unique_ptr; diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index cc9835ea25c..475d158818e 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -142,6 +142,7 @@ Int32 parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, IcebergS } } + DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name) { if (type_name == "boolean") diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 3022622d7f6..26e26831cc8 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -81,27 +81,23 @@ public: std::shared_ptr getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id); private: - std::unordered_map - iceberg_table_schemas_by_ids; - std::unordered_map> - clickhouse_table_schemas_by_ids; - std::map, std::shared_ptr> - transform_dags_by_ids; + std::unordered_map iceberg_table_schemas_by_ids; + std::unordered_map> clickhouse_table_schemas_by_ids; + std::map, std::shared_ptr> transform_dags_by_ids; - NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr &schema); - DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr &type); - DataTypePtr getFieldType(const Poco::JSON::Object::Ptr &field, - const String &type_key, bool required); - DataTypePtr getSimpleType(const String &type_name); - std::shared_ptr - getSchemaTransformationDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema); + NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr & schema); + DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type); + DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & type_key, bool required); + DataTypePtr getSimpleType(const String & type_name); + std::shared_ptr + getSchemaTransformationDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema); - bool allowPrimitiveTypeConversion(const String & old_type, const String & new_type); + bool allowPrimitiveTypeConversion(const String & old_type, const String & new_type); - const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); + const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); - Int32 current_old_id = -1; - Int32 current_new_id = -1; + Int32 current_old_id = -1; + Int32 current_new_id = -1; }; @@ -206,6 +202,8 @@ public: return {}; } + bool supportsExternalMetadataChange() const override { return true; } + private: mutable std::unordered_map> initial_schemas; mutable std::unordered_map> schema_transformers; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 38a4bb3d1cb..5764f3fac43 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -25,6 +25,7 @@ #include "Databases/LoadingStrictnessLevel.h" #include "Storages/ColumnsDescription.h" +#include namespace DB { @@ -91,8 +92,10 @@ StorageObjectStorage::StorageObjectStorage( { try { - configuration->updateAndGetCurrentSchema(object_storage, context); - configuration->update(object_storage, context); + if (configuration->hasExternalDynamicMetadata()) + configuration->updateAndGetCurrentSchema(object_storage, context); + else + configuration->update(object_storage, context); } catch (...) { @@ -152,19 +155,14 @@ void StorageObjectStorage::Configuration::update(ObjectStoragePtr object_storage bool StorageObjectStorage::hasExternalDynamicMetadata() const { - return configuration->isDataLakeConfiguration(); + return configuration->hasExternalDynamicMetadata(); } void StorageObjectStorage::updateExternalDynamicMetadata(ContextPtr context_ptr) { - if (configuration->isDataLakeConfiguration()) - { - StorageInMemoryMetadata metadata; - metadata.setColumns(configuration->updateAndGetCurrentSchema(object_storage, context_ptr)); - setInMemoryMetadata(metadata); - return; - } - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method updateExternalDynamicMetadata is not supported by storage {}", getName()); + StorageInMemoryMetadata metadata; + metadata.setColumns(configuration->updateAndGetCurrentSchema(object_storage, context_ptr)); + setInMemoryMetadata(metadata); } namespace @@ -453,8 +451,10 @@ ColumnsDescription StorageObjectStorage::resolveSchemaFromData( { if (configuration->isDataLakeConfiguration()) { - configuration->updateAndGetCurrentSchema(object_storage, context); - configuration->update(object_storage, context); + if (configuration->hasExternalDynamicMetadata()) + configuration->updateAndGetCurrentSchema(object_storage, context); + else + configuration->update(object_storage, context); auto table_structure = configuration->tryGetTableStructureFromMetadata(); if (table_structure) { diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index bd994b051e6..3e7bf72625a 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -216,6 +216,8 @@ public: virtual bool isDataLakeConfiguration() const { return false; } + virtual bool hasExternalDynamicMetadata() { return false; } + virtual std::shared_ptr getInitialSchemaByPath(const String&) const { return {}; } virtual std::shared_ptr getSchemaTransformer(const String&) const { return {}; } diff --git a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp index e94f1860176..0c8a88bfd18 100644 --- a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include "Common/logger_useful.h" namespace DB { diff --git a/tests/integration/test_storage_iceberg/test.py b/tests/integration/test_storage_iceberg/test.py index b93fda4df4a..1ef27da76a7 100644 --- a/tests/integration/test_storage_iceberg/test.py +++ b/tests/integration/test_storage_iceberg/test.py @@ -1302,11 +1302,18 @@ def test_filesystem_cache(started_cluster, storage_type): instance.query("SYSTEM FLUSH LOGS") - count = int( + written_to_cache_first_select = int( instance.query( f"SELECT ProfileEvents['CachedReadBufferCacheWriteBytes'] FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" ) ) + + read_from_cache_first_select = int( + instance.query( + f"SELECT ProfileEvents['CachedReadBufferReadFromCacheBytes'] FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ) + ) + assert 0 < int( instance.query( f"SELECT ProfileEvents['S3GetObject'] FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" @@ -1321,11 +1328,17 @@ def test_filesystem_cache(started_cluster, storage_type): instance.query("SYSTEM FLUSH LOGS") - assert count == int( + read_from_cache_second_select = int( instance.query( f"SELECT ProfileEvents['CachedReadBufferReadFromCacheBytes'] FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" ) ) + + assert ( + read_from_cache_second_select + == read_from_cache_first_select + written_to_cache_first_select + ) + assert 0 == int( instance.query( f"SELECT ProfileEvents['S3GetObject'] FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" From b2c77df001e18784ad559dd6a4656785a3c2f3ba Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 11 Nov 2024 20:27:34 +0000 Subject: [PATCH 091/502] Support ALTER from Object to JSON --- src/DataTypes/ObjectUtils.cpp | 2 +- src/DataTypes/ObjectUtils.h | 2 +- src/Formats/FormatSettings.h | 2 +- src/Functions/FunctionsConversion.cpp | 1 + .../ClusterProxy/executeQuery.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- src/Storages/AlterCommands.cpp | 3 +- src/Storages/MergeTree/MergeTreeData.cpp | 9 +- .../StorageFromMergeTreeDataPart.cpp | 2 +- src/Storages/StorageDistributed.cpp | 2 +- src/Storages/StorageMemory.cpp | 2 +- .../03270_object_to_json_alter.reference | 132 ++++++++++++++++++ .../03270_object_to_json_alter.sql | 56 ++++++++ ...3271_ghdata_object_to_json_alter.reference | 12 ++ .../03271_ghdata_object_to_json_alter.sh | 30 ++++ ...object_to_json_alter_shared_data.reference | 12 ++ ...ghdata_object_to_json_alter_shared_data.sh | 30 ++++ 17 files changed, 288 insertions(+), 13 deletions(-) create mode 100644 tests/queries/0_stateless/03270_object_to_json_alter.reference create mode 100644 tests/queries/0_stateless/03270_object_to_json_alter.sql create mode 100644 tests/queries/0_stateless/03271_ghdata_object_to_json_alter.reference create mode 100755 tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh create mode 100644 tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.reference create mode 100755 tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 4f3c22aa8a3..f8a1ff7002f 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -521,7 +521,7 @@ DataTypePtr createConcreteEmptyDynamicColumn(const DataTypePtr & type_in_storage throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, "Type {} unexpectedly has dynamic columns", type_in_storage->getName()); } -bool hasDynamicSubcolumns(const ColumnsDescription & columns) +bool hasDynamicSubcolumnsDeprecated(const ColumnsDescription & columns) { return std::any_of(columns.begin(), columns.end(), [](const auto & column) diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index d4109b971a4..830f0e4c95e 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -63,7 +63,7 @@ DataTypePtr createConcreteEmptyDynamicColumn(const DataTypePtr & type_in_storage void extendObjectColumns(NamesAndTypesList & columns_list, const ColumnsDescription & object_columns, bool with_subcolumns); /// Checks whether @columns contain any column with dynamic subcolumns. -bool hasDynamicSubcolumns(const ColumnsDescription & columns); +bool hasDynamicSubcolumnsDeprecated(const ColumnsDescription & columns); /// Updates types of objects in @object_columns inplace /// according to types in new_columns. diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 315c2d1bc32..ed4357b6cfd 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -220,7 +220,7 @@ struct FormatSettings bool escape_forward_slashes = true; bool read_named_tuples_as_objects = false; bool use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects = false; - bool write_named_tuples_as_objects = false; + bool write_named_tuples_as_objects = true; bool skip_null_value_in_named_tuples = false; bool defaults_for_missing_elements_in_named_tuple = false; bool ignore_unknown_keys_in_named_tuple = false; diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index ee04916e7b4..c7f0a490e09 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -4103,6 +4103,7 @@ private: ColumnStringHelpers::WriteHelper write_helper(assert_cast(*json_string), input_rows_count); auto & write_buffer = write_helper.getWriteBuffer(); FormatSettings format_settings = context ? getFormatSettings(context) : FormatSettings{}; + format_settings.json.quote_64bit_integers = false; auto serialization = arguments[0].type->getDefaultSerialization(); for (size_t i = 0; i < input_rows_count; ++i) { diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index e88fdeb0379..dc1a8e5a1c3 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -755,7 +755,7 @@ void executeQueryWithParallelReplicasCustomKey( } ColumnsDescriptionByShardNum columns_object; - if (hasDynamicSubcolumns(columns)) + if (hasDynamicSubcolumnsDeprecated(columns)) columns_object = getExtendedObjectsOfRemoteTables(*query_info.cluster, storage_id, columns, context); ClusterProxy::SelectStreamFactory select_stream_factory diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index f6586f8bfc2..41b8871b726 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1875,7 +1875,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, validateVirtualColumns(*res); - if (!res->supportsDynamicSubcolumnsDeprecated() && hasDynamicSubcolumns(res->getInMemoryMetadataPtr()->getColumns()) && mode <= LoadingStrictnessLevel::CREATE) + if (!res->supportsDynamicSubcolumnsDeprecated() && hasDynamicSubcolumnsDeprecated(res->getInMemoryMetadataPtr()->getColumns()) && mode <= LoadingStrictnessLevel::CREATE) { throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot create table with column of type Object, " diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index fbca222b1e7..9f41c845510 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1450,9 +1450,8 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const const auto old_data_type = all_columns.getColumn(options, column_name).type; bool new_type_has_deprecated_object = command.data_type->hasDynamicSubcolumnsDeprecated(); - bool old_type_has_deprecated_object = old_data_type->hasDynamicSubcolumnsDeprecated(); - if (new_type_has_deprecated_object || old_type_has_deprecated_object) + if (new_type_has_deprecated_object) throw Exception( ErrorCodes::BAD_ARGUMENTS, "The change of data type {} of column {} to {} is not allowed. It has known bugs", diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index b2f35d0a309..773f4d95b1e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4094,7 +4094,7 @@ void MergeTreeData::checkPartDynamicColumns(MutableDataPartPtr & part, DataParts const auto & columns = metadata_snapshot->getColumns(); auto virtuals = getVirtualsPtr(); - if (!hasDynamicSubcolumns(columns)) + if (!hasDynamicSubcolumnsDeprecated(columns)) return; const auto & part_columns = part->getColumns(); @@ -8728,8 +8728,11 @@ void MergeTreeData::resetObjectColumnsFromActiveParts(const DataPartsLock & /*lo { auto metadata_snapshot = getInMemoryMetadataPtr(); const auto & columns = metadata_snapshot->getColumns(); - if (!hasDynamicSubcolumns(columns)) + if (!hasDynamicSubcolumnsDeprecated(columns)) + { + object_columns = ColumnsDescription{}; return; + } auto range = getDataPartsStateRange(DataPartState::Active); object_columns = getConcreteObjectColumns(range, columns); @@ -8739,7 +8742,7 @@ void MergeTreeData::updateObjectColumns(const DataPartPtr & part, const DataPart { auto metadata_snapshot = getInMemoryMetadataPtr(); const auto & columns = metadata_snapshot->getColumns(); - if (!hasDynamicSubcolumns(columns)) + if (!hasDynamicSubcolumnsDeprecated(columns)) return; DB::updateObjectColumns(object_columns, columns, part->getColumns()); diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.cpp b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.cpp index 3df03b55cd4..9e59c086e79 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.cpp @@ -50,7 +50,7 @@ StorageSnapshotPtr StorageFromMergeTreeDataPart::getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr /*query_context*/) const { const auto & storage_columns = metadata_snapshot->getColumns(); - if (!hasDynamicSubcolumns(storage_columns)) + if (!hasDynamicSubcolumnsDeprecated(storage_columns)) return std::make_shared(*this, metadata_snapshot); auto data_parts = storage.getDataPartsVectorForInternalUsage(); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 4f5a95ab508..337800338d1 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -686,7 +686,7 @@ std::optional StorageDistributed::getOptimizedQueryP static bool requiresObjectColumns(const ColumnsDescription & all_columns, ASTPtr query) { - if (!hasDynamicSubcolumns(all_columns)) + if (!hasDynamicSubcolumnsDeprecated(all_columns)) return false; if (!query) diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 71447889d86..e352b16546e 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -180,7 +180,7 @@ StorageSnapshotPtr StorageMemory::getStorageSnapshot(const StorageMetadataPtr & /// rows and bytes counters into the MultiVersion-ed struct, then everything would be consistent. snapshot_data->rows_approx = total_size_rows.load(std::memory_order_relaxed); - if (!hasDynamicSubcolumns(metadata_snapshot->getColumns())) + if (!hasDynamicSubcolumnsDeprecated(metadata_snapshot->getColumns())) return std::make_shared(*this, metadata_snapshot, ColumnsDescription{}, std::move(snapshot_data)); auto object_columns = getConcreteObjectColumns( diff --git a/tests/queries/0_stateless/03270_object_to_json_alter.reference b/tests/queries/0_stateless/03270_object_to_json_alter.reference new file mode 100644 index 00000000000..a04884fb374 --- /dev/null +++ b/tests/queries/0_stateless/03270_object_to_json_alter.reference @@ -0,0 +1,132 @@ +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 diff --git a/tests/queries/0_stateless/03270_object_to_json_alter.sql b/tests/queries/0_stateless/03270_object_to_json_alter.sql new file mode 100644 index 00000000000..61db4d0d33a --- /dev/null +++ b/tests/queries/0_stateless/03270_object_to_json_alter.sql @@ -0,0 +1,56 @@ +set allow_experimental_object_type = 1; +set allow_experimental_json_type = 1; +set max_block_size = 100; +set max_insert_block_size = 100; +set min_insert_block_size_rows = 100; +set output_format_json_quote_64bit_integers = 0; + +drop table if exists test; + +create table test (json Object('json')) engine=Memory; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON; +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; + +create table test (json Object('json')) engine=Memory; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON(max_dynamic_paths=10); +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; + +create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=10000000, min_bytes_for_wide_part=100000000; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON; +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; + +create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=10000000, min_bytes_for_wide_part=100000000; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON(max_dynamic_paths=10); +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; + +create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON(); +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; + +create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON(max_dynamic_paths=10); +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; diff --git a/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.reference b/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.reference new file mode 100644 index 00000000000..ca2fb7e8ff9 --- /dev/null +++ b/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.reference @@ -0,0 +1,12 @@ +5000 +leonardomso/33-js-concepts 3 +ytdl-org/youtube-dl 3 +Bogdanp/neko 2 +bminossi/AllVideoPocsFromHackerOne 2 +disclose/diodata 2 +Commit 182 +chipeo345 119 +phanwi346 114 +Nicholas Piggin 95 +direwolf-github 49 +2 diff --git a/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh b/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh new file mode 100755 index 00000000000..02d39e5bf61 --- /dev/null +++ b/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-s3-storage, long +# ^ no-s3-storage: too memory hungry + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata (data Object('json')) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'" --allow_experimental_object_type + +cat $CUR_DIR/data_json/ghdata_sample.json | ${CLICKHOUSE_CLIENT} \ + --max_memory_usage 10G --query "INSERT INTO ghdata FORMAT JSONAsObject" + +${CLICKHOUSE_CLIENT} -q "ALTER TABLE ghdata MODIFY column data JSON SETTINGS mutations_sync=1" --allow_experimental_json_type 1 + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ghdata WHERE NOT ignore(*)" + +${CLICKHOUSE_CLIENT} -q \ +"SELECT data.repo.name, count() AS stars FROM ghdata \ + WHERE data.type = 'WatchEvent' GROUP BY data.repo.name ORDER BY stars DESC, data.repo.name LIMIT 5" --allow_suspicious_types_in_group_by=1 --allow_suspicious_types_in_order_by=1 + +${CLICKHOUSE_CLIENT} --enable_analyzer=1 -q \ +"SELECT data.payload.commits[].author.name AS name, count() AS c FROM ghdata \ + ARRAY JOIN data.payload.commits[].author.name \ + GROUP BY name ORDER BY c DESC, name LIMIT 5" --allow_suspicious_types_in_group_by=1 --allow_suspicious_types_in_order_by=1 + +${CLICKHOUSE_CLIENT} -q "SELECT max(data.payload.pull_request.assignees[].size0) FROM ghdata" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata" diff --git a/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.reference b/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.reference new file mode 100644 index 00000000000..ca2fb7e8ff9 --- /dev/null +++ b/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.reference @@ -0,0 +1,12 @@ +5000 +leonardomso/33-js-concepts 3 +ytdl-org/youtube-dl 3 +Bogdanp/neko 2 +bminossi/AllVideoPocsFromHackerOne 2 +disclose/diodata 2 +Commit 182 +chipeo345 119 +phanwi346 114 +Nicholas Piggin 95 +direwolf-github 49 +2 diff --git a/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh b/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh new file mode 100755 index 00000000000..62723feacb6 --- /dev/null +++ b/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-s3-storage, long +# ^ no-s3-storage: too memory hungry + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata (data String) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'" + +cat $CUR_DIR/data_json/ghdata_sample.json | ${CLICKHOUSE_CLIENT} \ + --max_memory_usage 10G --query "INSERT INTO ghdata FORMAT JSONAsString" + +${CLICKHOUSE_CLIENT} -q "ALTER TABLE ghdata MODIFY column data JSON(max_dynamic_paths=8) SETTINGS mutations_sync=1" --allow_experimental_json_type 1 + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ghdata WHERE NOT ignore(*)" + +${CLICKHOUSE_CLIENT} -q \ +"SELECT data.repo.name, count() AS stars FROM ghdata \ + WHERE data.type = 'WatchEvent' GROUP BY data.repo.name ORDER BY stars DESC, data.repo.name LIMIT 5" --allow_suspicious_types_in_group_by=1 --allow_suspicious_types_in_order_by=1 + +${CLICKHOUSE_CLIENT} --enable_analyzer=1 -q \ +"SELECT data.payload.commits[].author.name AS name, count() AS c FROM ghdata \ + ARRAY JOIN data.payload.commits[].author.name \ + GROUP BY name ORDER BY c DESC, name LIMIT 5" --allow_suspicious_types_in_group_by=1 --allow_suspicious_types_in_order_by=1 + +${CLICKHOUSE_CLIENT} -q "SELECT max(data.payload.pull_request.assignees[].size0) FROM ghdata" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata" From 1fe395bcd6451379f19265419964a318482881e8 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 11 Nov 2024 20:34:43 +0000 Subject: [PATCH 092/502] Fix test --- .../03272_ghdata_object_to_json_alter_shared_data.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh b/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh index 62723feacb6..79dea6554fc 100755 --- a/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh +++ b/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh @@ -7,10 +7,10 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata" -${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata (data String) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata (data Object('json')) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'" --allow_experimental_object_type=1 cat $CUR_DIR/data_json/ghdata_sample.json | ${CLICKHOUSE_CLIENT} \ - --max_memory_usage 10G --query "INSERT INTO ghdata FORMAT JSONAsString" + --max_memory_usage 10G --query "INSERT INTO ghdata FORMAT JSONAsObject" ${CLICKHOUSE_CLIENT} -q "ALTER TABLE ghdata MODIFY column data JSON(max_dynamic_paths=8) SETTINGS mutations_sync=1" --allow_experimental_json_type 1 From 6ac3631d4b086e74488616d9965d7b754f9e50eb Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 11 Nov 2024 20:36:16 +0000 Subject: [PATCH 093/502] Update tests --- tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh | 3 +-- .../03272_ghdata_object_to_json_alter_shared_data.sh | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh b/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh index 02d39e5bf61..fc217780aa3 100755 --- a/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh +++ b/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-s3-storage, long -# ^ no-s3-storage: too memory hungry +# Tags: no-fasttest, long CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh b/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh index 79dea6554fc..64a364fb2e9 100755 --- a/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh +++ b/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-s3-storage, long -# ^ no-s3-storage: too memory hungry +# Tags: no-fasttest, long CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 725344359d3449b531471ed07f3a564b42b8852a Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 12 Nov 2024 13:23:00 +0000 Subject: [PATCH 094/502] Update tests --- tests/queries/0_stateless/02725_object_column_alter.sql | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/02725_object_column_alter.sql b/tests/queries/0_stateless/02725_object_column_alter.sql index 0e41070742a..9625e7c9777 100644 --- a/tests/queries/0_stateless/02725_object_column_alter.sql +++ b/tests/queries/0_stateless/02725_object_column_alter.sql @@ -3,13 +3,7 @@ SET allow_experimental_object_type=1; DROP TABLE IF EXISTS t_to; -DROP TABLE IF EXISTS t_from; - CREATE TABLE t_to (id UInt64, value Nullable(String)) ENGINE MergeTree() ORDER BY id; -CREATE TABLE t_from (id UInt64, value Object('json')) ENGINE MergeTree() ORDER BY id; - ALTER TABLE t_to MODIFY COLUMN value Object('json'); -- { serverError BAD_ARGUMENTS } -ALTER TABLE t_from MODIFY COLUMN value Nullable(String); -- { serverError BAD_ARGUMENTS } - DROP TABLE t_to; -DROP TABLE t_from; + From e565f5f1c7cea4e4a6406b24827feef47c058e6c Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 12 Nov 2024 16:41:39 +0100 Subject: [PATCH 095/502] Add a test --- .../docker_compose_iceberg_rest_catalog.yml | 54 +++++++-- tests/integration/helpers/cluster.py | 10 +- .../integration/test_database_iceberg/test.py | 103 ++++++++++++++++-- 3 files changed, 144 insertions(+), 23 deletions(-) diff --git a/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml index 7c786b84fc1..0a4be37335f 100644 --- a/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml +++ b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml @@ -1,7 +1,21 @@ services: + spark-iceberg: + image: tabulario/spark-iceberg + container_name: spark-iceberg + build: spark/ + depends_on: + - rest + - minio + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + ports: + - 8080:8080 + - 10000:10000 + - 10001:10001 rest: image: tabulario/iceberg-rest - container_name: iceberg-rest ports: - 8181:8181 environment: @@ -10,18 +24,36 @@ services: - AWS_REGION=us-east-1 - CATALOG_WAREHOUSE=s3://warehouse/ - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - - CATALOG_S3_ENDPOINT=http://minio:9001 - - spark-iceberg: - image: tabulario/spark-iceberg - container_name: spark-iceberg + - CATALOG_S3_ENDPOINT=http://minio:9000 + minio: + image: minio/minio + container_name: minio + environment: + - MINIO_ROOT_USER=minio + - MINIO_ROOT_PASSWORD=minio123 + - MINIO_DOMAIN=minio + networks: + default: + aliases: + - warehouse.minio + ports: + - 9001:9001 + - 9000:9000 + command: ["server", "/data", "--console-address", ":9001"] + mc: depends_on: - - rest + - minio + image: minio/mc + container_name: mc environment: - AWS_ACCESS_KEY_ID=minio - AWS_SECRET_ACCESS_KEY=minio123 - AWS_REGION=us-east-1 - ports: - - 8080:8080 - - 10000:10000 - - 10001:10001 + entrypoint: > + /bin/sh -c " + until (/usr/bin/mc config host add minio http://minio:9000 minio minio123) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc rm -r --force minio/warehouse; + /usr/bin/mc mb minio/warehouse --ignore-existing; + /usr/bin/mc policy set public minio/warehouse; + tail -f /dev/null + " diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 78e9c581629..517d0e36865 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -440,7 +440,6 @@ class ClickHouseCluster: zookeeper_keyfile=None, zookeeper_certfile=None, with_spark=False, - with_iceberg_catalog=False, ): for param in list(os.environ.keys()): logging.debug("ENV %40s %s" % (param, os.environ[param])) @@ -569,6 +568,7 @@ class ClickHouseCluster: self.resolver_logs_dir = os.path.join(self.instances_dir, "resolver") self.spark_session = None + self.with_iceberg_catalog = False self.with_azurite = False self.azurite_container = "azurite-container" @@ -1480,7 +1480,8 @@ class ClickHouseCluster: "--file", p.join(docker_compose_yml_dir, "docker_compose_iceberg_rest_catalog.yml"), ) - return self.base_minio_cmd + return self.base_iceberg_catalog_cmd + #return self.base_minio_cmd def setup_azurite_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_azurite = True @@ -1648,6 +1649,7 @@ class ClickHouseCluster: with_hive=False, with_coredns=False, with_prometheus=False, + with_iceberg_catalog=False, handle_prometheus_remote_write=False, handle_prometheus_remote_read=False, use_old_analyzer=None, @@ -1750,6 +1752,7 @@ class ClickHouseCluster: with_coredns=with_coredns, with_cassandra=with_cassandra, with_ldap=with_ldap, + with_iceberg_catalog=with_iceberg_catalog, use_old_analyzer=use_old_analyzer, server_bin_path=self.server_bin_path, odbc_bridge_bin_path=self.odbc_bridge_bin_path, @@ -1937,6 +1940,8 @@ class ClickHouseCluster: cmds.append( self.setup_minio_cmd(instance, env_variables, docker_compose_yml_dir) ) + + if with_iceberg_catalog and not self.with_iceberg_catalog: cmds.append( self.setup_iceberg_catalog_cmd( instance, env_variables, docker_compose_yml_dir @@ -3383,6 +3388,7 @@ class ClickHouseInstance: with_coredns, with_cassandra, with_ldap, + with_iceberg_catalog, use_old_analyzer, server_bin_path, odbc_bridge_bin_path, diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 85b3fef6eb8..cc72b50a93c 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -7,12 +7,31 @@ import uuid import pytest import requests +from minio import Minio +import urllib3 + +from pyiceberg.catalog import load_catalog +from pyiceberg.schema import Schema +from pyiceberg.types import ( + TimestampType, + FloatType, + DoubleType, + StringType, + NestedField, + StructType, +) +from pyiceberg.partitioning import PartitionSpec, PartitionField +from pyiceberg.transforms import DayTransform +from pyiceberg.table.sorting import SortOrder, SortField +from pyiceberg.transforms import IdentityTransform + from helpers.cluster import ClickHouseCluster, ClickHouseInstance, is_arm from helpers.s3_tools import get_file_contents, list_s3_objects, prepare_s3_bucket BASE_URL = "http://rest:8181/v1" BASE_URL_LOCAL = "http://localhost:8181/v1" +BASE_URL_LOCAL_RAW = "http://localhost:8181" def create_namespace(name): @@ -44,6 +63,7 @@ def list_namespaces(): def create_table(name, namespace): payload = { "name": name, + "location": "s3://warehouse/", "schema": { "type": "struct", "fields": [ @@ -70,20 +90,19 @@ def create_table(name, namespace): @pytest.fixture(scope="module") def started_cluster(): try: - cluster = ClickHouseCluster(__file__, with_spark=True) + cluster = ClickHouseCluster(__file__) cluster.add_instance( "node1", main_configs=[], user_configs=[], - with_minio=True, stay_alive=True, + with_iceberg_catalog=True, ) logging.info("Starting cluster...") cluster.start() - cluster.minio_client.make_bucket("warehouse") - prepare_s3_bucket(cluster) + # prepare_s3_bucket(cluster) yield cluster @@ -95,10 +114,10 @@ def test_simple(started_cluster): # TODO: properly wait for container time.sleep(10) - namespace = "kssenii.test.namespace" - root_namespace = "kssenii" + namespace_1 = "clickhouse.test.A" + root_namespace = "clickhouse" - create_namespace(namespace) + create_namespace(namespace_1) assert root_namespace in list_namespaces()["namespaces"][0][0] node = started_cluster.instances["node1"] @@ -109,7 +128,71 @@ SETTINGS catalog_type = 'rest', storage_endpoint = 'http://{started_cluster.mini """ ) - table_name = "testtable" - create_table(table_name, "kssenii") + catalog_name = "demo" - assert namespace in node.query("USE demo; SHOW TABLES") + catalog = load_catalog( + "demo", + **{ + "uri": BASE_URL_LOCAL_RAW, + "type": "rest", + "s3.endpoint": f"http://minio:9000", + "s3.access-key-id": "minio", + "s3.secret-access-key": "minio123", + }, + ) + namespace_2 = "clickhouse.test.B" + catalog.create_namespace(namespace_2) + + assert [(root_namespace,)] == catalog.list_namespaces() + + tables = catalog.list_tables(namespace_2) + assert len(tables) == 0 + + schema = Schema( + NestedField( + field_id=1, name="datetime", field_type=TimestampType(), required=True + ), + NestedField(field_id=2, name="symbol", field_type=StringType(), required=True), + NestedField(field_id=3, name="bid", field_type=FloatType(), required=False), + NestedField(field_id=4, name="ask", field_type=DoubleType(), required=False), + NestedField( + field_id=5, + name="details", + field_type=StructType( + NestedField( + field_id=4, + name="created_by", + field_type=StringType(), + required=False, + ), + ), + required=False, + ), + ) + + partition_spec = PartitionSpec( + PartitionField( + source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day" + ) + ) + + sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform())) + + for table in ["tableA", "tableB"]: + catalog.create_table( + identifier=f"{namespace_2}.{table}", + schema=schema, + location=f"s3://warehouse", + partition_spec=partition_spec, + sort_order=sort_order, + ) + + def check(): + assert f"{namespace_2}.tableA\n{namespace_2}.tableB\n" == node.query("SELECT name FROM system.tables WHERE database = 'demo' ORDER BY name") + + expected = "CREATE TABLE demo.`clickhouse.test.B.tableA`\\n(\\n `datetime` DateTime64(6),\\n `symbol` String,\\n `bid` Nullable(Float32),\\n `ask` Nullable(Float64),\\n `details` Tuple(created_by Nullable(String))\\n)\\nENGINE = Iceberg(\\'http://None:9001/warehouse\\', \\'minio\\', \\'[HIDDEN]\\')\n"; + assert expected == node.query(f"SHOW CREATE TABLE demo.`{namespace_2}.tableA`") + + check() + node.restart_clickhouse() + check() From f8255fb4ae94888c828640d828e1f5c526239ab0 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 12 Nov 2024 18:17:10 +0100 Subject: [PATCH 096/502] One more test --- .../integration/test_database_iceberg/test.py | 245 ++++++++++-------- 1 file changed, 130 insertions(+), 115 deletions(-) diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index cc72b50a93c..a1f4ad6ad06 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -10,6 +10,7 @@ import requests from minio import Minio import urllib3 +from helpers.test_tools import TSV, csv_compare from pyiceberg.catalog import load_catalog from pyiceberg.schema import Schema from pyiceberg.types import ( @@ -33,23 +34,33 @@ BASE_URL = "http://rest:8181/v1" BASE_URL_LOCAL = "http://localhost:8181/v1" BASE_URL_LOCAL_RAW = "http://localhost:8181" +CATALOG_NAME = "demo" -def create_namespace(name): - payload = { - "namespace": [name], - "properties": {"owner": "clickhouse", "description": "test namespace"}, - } - - headers = {"Content-Type": "application/json"} - response = requests.post( - f"{BASE_URL_LOCAL}/namespaces", headers=headers, data=json.dumps(payload) +DEFAULT_SCHEMA = Schema( + NestedField(field_id=1, name="datetime", field_type=TimestampType(), required=True), + NestedField(field_id=2, name="symbol", field_type=StringType(), required=True), + NestedField(field_id=3, name="bid", field_type=FloatType(), required=False), + NestedField(field_id=4, name="ask", field_type=DoubleType(), required=False), + NestedField( + field_id=5, + name="details", + field_type=StructType( + NestedField( + field_id=4, + name="created_by", + field_type=StringType(), + required=False, + ), + ), + required=False, + ), +) +DEFAULT_PARTITION_SPEC = PartitionSpec( + PartitionField( + source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day" ) - if response.status_code == 200: - print(f"Namespace '{name}' created successfully.") - else: - raise Exception( - f"Failed to create namespace. Status code: {response.status_code}, Response: {response.text}" - ) +) +DEFAULT_SORT_ORDER = SortOrder(SortField(source_id=2, transform=IdentityTransform())) def list_namespaces(): @@ -60,31 +71,44 @@ def list_namespaces(): raise Exception(f"Failed to list namespaces: {response.status_code}") -def create_table(name, namespace): - payload = { - "name": name, - "location": "s3://warehouse/", - "schema": { - "type": "struct", - "fields": [ - {"id": 1, "name": "name", "type": "String", "required": True}, - {"id": 2, "name": "age", "type": "Int", "required": False}, - ], +def load_catalog_impl(): + return load_catalog( + CATALOG_NAME, + **{ + "uri": BASE_URL_LOCAL_RAW, + "type": "rest", + "s3.endpoint": f"http://minio:9000", + "s3.access-key-id": "minio", + "s3.secret-access-key": "minio123", }, - } - - headers = {"Content-Type": "application/json"} - response = requests.post( - f"{BASE_URL_LOCAL}/namespaces/{namespace}/tables", - headers=headers, - data=json.dumps(payload), ) - if response.status_code == 200: - print(f"Table '{name}' created successfully.") - else: - raise Exception( - f"Failed to create a table. Status code: {response.status_code}, Response: {response.text}" - ) + + +def create_table( + catalog, + namespace, + table, + schema=DEFAULT_SCHEMA, + partition_spec=DEFAULT_PARTITION_SPEC, + sort_order=DEFAULT_SORT_ORDER, +): + catalog.create_table( + identifier=f"{namespace}.{table}", + schema=schema, + location=f"s3://warehouse", + partition_spec=partition_spec, + sort_order=sort_order, + ) + + +def create_clickhouse_iceberg_database(started_cluster, node, name): + node.query( + f""" +DROP DATABASE IF EXISTS {name}; +CREATE DATABASE {name} ENGINE = Iceberg('{BASE_URL}', 'minio', 'minio123') +SETTINGS catalog_type = 'rest', storage_endpoint = 'http://{started_cluster.minio_ip}:{started_cluster.minio_port}/' + """ + ) @pytest.fixture(scope="module") @@ -102,7 +126,8 @@ def started_cluster(): logging.info("Starting cluster...") cluster.start() - # prepare_s3_bucket(cluster) + # TODO: properly wait for container + time.sleep(10) yield cluster @@ -111,88 +136,78 @@ def started_cluster(): def test_simple(started_cluster): - # TODO: properly wait for container - time.sleep(10) - - namespace_1 = "clickhouse.test.A" - root_namespace = "clickhouse" - - create_namespace(namespace_1) - assert root_namespace in list_namespaces()["namespaces"][0][0] - node = started_cluster.instances["node1"] - node.query( - f""" -CREATE DATABASE demo ENGINE = Iceberg('{BASE_URL}', 'minio', 'minio123') -SETTINGS catalog_type = 'rest', storage_endpoint = 'http://{started_cluster.minio_ip}:{started_cluster.minio_port}/' - """ - ) - catalog_name = "demo" + root_namespace = "clickhouse" + namespace_1 = "clickhouse.testA.A" + namespace_2 = "clickhouse.testB.B" + namespace_1_tables = ["tableA", "tableB"] + namespace_2_tables = ["tableC", "tableD"] - catalog = load_catalog( - "demo", - **{ - "uri": BASE_URL_LOCAL_RAW, - "type": "rest", - "s3.endpoint": f"http://minio:9000", - "s3.access-key-id": "minio", - "s3.secret-access-key": "minio123", - }, - ) - namespace_2 = "clickhouse.test.B" - catalog.create_namespace(namespace_2) + catalog = load_catalog_impl() + for namespace in [namespace_1, namespace_2]: + catalog.create_namespace(namespace) + + assert root_namespace in list_namespaces()["namespaces"][0][0] assert [(root_namespace,)] == catalog.list_namespaces() - tables = catalog.list_tables(namespace_2) - assert len(tables) == 0 + for namespace in [namespace_1, namespace_2]: + assert len(catalog.list_tables(namespace)) == 0 - schema = Schema( - NestedField( - field_id=1, name="datetime", field_type=TimestampType(), required=True - ), - NestedField(field_id=2, name="symbol", field_type=StringType(), required=True), - NestedField(field_id=3, name="bid", field_type=FloatType(), required=False), - NestedField(field_id=4, name="ask", field_type=DoubleType(), required=False), - NestedField( - field_id=5, - name="details", - field_type=StructType( - NestedField( - field_id=4, - name="created_by", - field_type=StringType(), - required=False, - ), - ), - required=False, - ), + create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) + + tables_list = "" + for table in namespace_1_tables: + create_table(catalog, namespace_1, table) + if len(tables_list) > 0: + tables_list += "\n" + tables_list += f"{namespace_1}.{table}" + + for table in namespace_2_tables: + create_table(catalog, namespace_2, table) + if len(tables_list) > 0: + tables_list += "\n" + tables_list += f"{namespace_2}.{table}" + + assert ( + tables_list + == node.query( + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' ORDER BY name" + ).strip() ) - - partition_spec = PartitionSpec( - PartitionField( - source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day" - ) - ) - - sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform())) - - for table in ["tableA", "tableB"]: - catalog.create_table( - identifier=f"{namespace_2}.{table}", - schema=schema, - location=f"s3://warehouse", - partition_spec=partition_spec, - sort_order=sort_order, - ) - - def check(): - assert f"{namespace_2}.tableA\n{namespace_2}.tableB\n" == node.query("SELECT name FROM system.tables WHERE database = 'demo' ORDER BY name") - - expected = "CREATE TABLE demo.`clickhouse.test.B.tableA`\\n(\\n `datetime` DateTime64(6),\\n `symbol` String,\\n `bid` Nullable(Float32),\\n `ask` Nullable(Float64),\\n `details` Tuple(created_by Nullable(String))\\n)\\nENGINE = Iceberg(\\'http://None:9001/warehouse\\', \\'minio\\', \\'[HIDDEN]\\')\n"; - assert expected == node.query(f"SHOW CREATE TABLE demo.`{namespace_2}.tableA`") - - check() node.restart_clickhouse() - check() + assert ( + tables_list + == node.query( + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' ORDER BY name" + ).strip() + ) + + expected = f"CREATE TABLE {CATALOG_NAME}.`{namespace_2}.tableC`\\n(\\n `datetime` DateTime64(6),\\n `symbol` String,\\n `bid` Nullable(Float32),\\n `ask` Nullable(Float64),\\n `details` Tuple(created_by Nullable(String))\\n)\\nENGINE = Iceberg(\\'http://None:9001/warehouse\\', \\'minio\\', \\'[HIDDEN]\\')\n" + assert expected == node.query( + f"SHOW CREATE TABLE {CATALOG_NAME}.`{namespace_2}.tableC`" + ) + + +def test_different_namespaces(started_cluster): + node = started_cluster.instances["node1"] + namespaces = ["A", "A.B.C", "A.B.C.D", "A.B.C.D.E", "A.B.C.D.E.F", "A.B.C.D.E.FF", "B", "B.C", "B.CC"] + tables = ["A", "B", "C", "D", "E", "F"] + catalog = load_catalog_impl() + + for namespace in namespaces: + #if namespace in catalog.list_namespaces()["namesoaces"]: + # catalog.drop_namespace(namespace) + catalog.create_namespace(namespace) + for table in tables: + create_table(catalog, namespace, table) + + create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) + + for namespace in namespaces: + for table in tables: + table_name = f"{namespace}.{table}" + assert int(node.query( + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}'" + )) From 19854ff78ae84b0f8c03a8e7df8406ed3a37371d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 12 Nov 2024 17:24:25 +0000 Subject: [PATCH 097/502] Automatic style fix --- tests/integration/helpers/cluster.py | 2 +- .../integration/test_database_iceberg/test.py | 41 +++++++++++-------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 517d0e36865..0d0cf585763 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1481,7 +1481,7 @@ class ClickHouseCluster: p.join(docker_compose_yml_dir, "docker_compose_iceberg_rest_catalog.yml"), ) return self.base_iceberg_catalog_cmd - #return self.base_minio_cmd + # return self.base_minio_cmd def setup_azurite_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_azurite = True diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index a1f4ad6ad06..43686694a60 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -7,28 +7,25 @@ import uuid import pytest import requests -from minio import Minio import urllib3 - -from helpers.test_tools import TSV, csv_compare +from minio import Minio from pyiceberg.catalog import load_catalog +from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema +from pyiceberg.table.sorting import SortField, SortOrder +from pyiceberg.transforms import DayTransform, IdentityTransform from pyiceberg.types import ( - TimestampType, - FloatType, DoubleType, - StringType, + FloatType, NestedField, + StringType, StructType, + TimestampType, ) -from pyiceberg.partitioning import PartitionSpec, PartitionField -from pyiceberg.transforms import DayTransform -from pyiceberg.table.sorting import SortOrder, SortField -from pyiceberg.transforms import IdentityTransform - from helpers.cluster import ClickHouseCluster, ClickHouseInstance, is_arm from helpers.s3_tools import get_file_contents, list_s3_objects, prepare_s3_bucket +from helpers.test_tools import TSV, csv_compare BASE_URL = "http://rest:8181/v1" BASE_URL_LOCAL = "http://localhost:8181/v1" @@ -192,12 +189,22 @@ def test_simple(started_cluster): def test_different_namespaces(started_cluster): node = started_cluster.instances["node1"] - namespaces = ["A", "A.B.C", "A.B.C.D", "A.B.C.D.E", "A.B.C.D.E.F", "A.B.C.D.E.FF", "B", "B.C", "B.CC"] + namespaces = [ + "A", + "A.B.C", + "A.B.C.D", + "A.B.C.D.E", + "A.B.C.D.E.F", + "A.B.C.D.E.FF", + "B", + "B.C", + "B.CC", + ] tables = ["A", "B", "C", "D", "E", "F"] catalog = load_catalog_impl() for namespace in namespaces: - #if namespace in catalog.list_namespaces()["namesoaces"]: + # if namespace in catalog.list_namespaces()["namesoaces"]: # catalog.drop_namespace(namespace) catalog.create_namespace(namespace) for table in tables: @@ -208,6 +215,8 @@ def test_different_namespaces(started_cluster): for namespace in namespaces: for table in tables: table_name = f"{namespace}.{table}" - assert int(node.query( - f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}'" - )) + assert int( + node.query( + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}'" + ) + ) From f8fbd0330c8a2a9c485bbf8a389d9b330ed76e3f Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 12 Nov 2024 19:04:22 +0100 Subject: [PATCH 098/502] Update requirements.txt --- docker/test/integration/runner/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/integration/runner/requirements.txt b/docker/test/integration/runner/requirements.txt index bb0c4d001e6..bccf15b8495 100644 --- a/docker/test/integration/runner/requirements.txt +++ b/docker/test/integration/runner/requirements.txt @@ -101,3 +101,4 @@ wadllib==1.3.6 websocket-client==1.8.0 wheel==0.38.1 zipp==1.0.0 +pyiceberg==0.7.1 From e7f708696f2b5501a90b67f95f72adeda362fa57 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 13 Nov 2024 13:26:16 +0000 Subject: [PATCH 099/502] Fix tests --- ...61_tuple_map_object_to_json_cast.reference | 24 +++++++++---------- ...ghdata_object_to_json_alter_shared_data.sh | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/03261_tuple_map_object_to_json_cast.reference b/tests/queries/0_stateless/03261_tuple_map_object_to_json_cast.reference index 0ae94e68663..bfc10723941 100644 --- a/tests/queries/0_stateless/03261_tuple_map_object_to_json_cast.reference +++ b/tests/queries/0_stateless/03261_tuple_map_object_to_json_cast.reference @@ -1,20 +1,20 @@ Map to JSON {"a":"0","b":"1970-01-01","c":[],"d":[{"e":"0"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a":"1","b":"1970-01-02","c":["0"],"d":[{"e":"1"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a":"2","b":"1970-01-03","c":["0","1"],"d":[{"e":"2"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a":"3","b":"1970-01-04","c":["0","1","2"],"d":[{"e":"3"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a":"4","b":"1970-01-05","c":["0","1","2","3"],"d":[{"e":"4"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"1","b":"1970-01-02","c":["0"],"d":[{"e":"1"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(Int64))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"2","b":"1970-01-03","c":["0","1"],"d":[{"e":"2"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(Int64))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"3","b":"1970-01-04","c":["0","1","2"],"d":[{"e":"3"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(Int64))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"4","b":"1970-01-05","c":["0","1","2","3"],"d":[{"e":"4"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(Int64))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} {"a0":"0","b0":"1970-01-01","c0":[],"d0":[{"e0":"0"}]} {'a0':'Int64','b0':'Date','c0':'Array(Nullable(String))','d0':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a1":"1","b1":"1970-01-02","c1":["0"],"d1":[{"e1":"1"}]} {'a1':'Int64','b1':'Date','c1':'Array(Nullable(String))','d1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a2":"2","b2":"1970-01-03","c2":["0","1"],"d2":[{"e2":"2"}]} {'a2':'Int64','b2':'Date','c2':'Array(Nullable(String))','d2':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a0":"3","b0":"1970-01-04","c0":["0","1","2"],"d0":[{"e0":"3"}]} {'a0':'Int64','b0':'Date','c0':'Array(Nullable(String))','d0':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a1":"4","b1":"1970-01-05","c1":["0","1","2","3"],"d1":[{"e1":"4"}]} {'a1':'Int64','b1':'Date','c1':'Array(Nullable(String))','d1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a1":"1","b1":"1970-01-02","c1":["0"],"d1":[{"e1":"1"}]} {'a1':'Int64','b1':'Date','c1':'Array(Nullable(Int64))','d1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a2":"2","b2":"1970-01-03","c2":["0","1"],"d2":[{"e2":"2"}]} {'a2':'Int64','b2':'Date','c2':'Array(Nullable(Int64))','d2':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a0":"3","b0":"1970-01-04","c0":["0","1","2"],"d0":[{"e0":"3"}]} {'a0':'Int64','b0':'Date','c0':'Array(Nullable(Int64))','d0':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a1":"4","b1":"1970-01-05","c1":["0","1","2","3"],"d1":[{"e1":"4"}]} {'a1':'Int64','b1':'Date','c1':'Array(Nullable(Int64))','d1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} Tuple to JSON {"a":"0","b":"1970-01-01","c":[],"d":[{"e":"0"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a":"1","b":"1970-01-02","c":["0"],"d":[{"e":"1"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a":"2","b":"1970-01-03","c":["0","1"],"d":[{"e":"2"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a":"3","b":"1970-01-04","c":["0","1","2"],"d":[{"e":"3"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} -{"a":"4","b":"1970-01-05","c":["0","1","2","3"],"d":[{"e":"4"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"1","b":"1970-01-02","c":["0"],"d":[{"e":"1"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(Int64))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"2","b":"1970-01-03","c":["0","1"],"d":[{"e":"2"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(Int64))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"3","b":"1970-01-04","c":["0","1","2"],"d":[{"e":"3"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(Int64))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} +{"a":"4","b":"1970-01-05","c":["0","1","2","3"],"d":[{"e":"4"}]} {'a':'Int64','b':'Date','c':'Array(Nullable(Int64))','d':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'} Object to JSON {"a":"0","b":"1970-01-01","c":[],"d":{"e":["0"]}} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d.e':'Array(Nullable(Int64))'} {"a":"1","b":"1970-01-02","c":["0"],"d":{"e":["1"]}} {'a':'Int64','b':'Date','c':'Array(Nullable(String))','d.e':'Array(Nullable(Int64))'} diff --git a/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh b/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh index 64a364fb2e9..fad2833b74e 100755 --- a/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh +++ b/tests/queries/0_stateless/03272_ghdata_object_to_json_alter_shared_data.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest, long +# Tags: no-fasttest, no-msan, no-tsan, no-asan, long CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 35ed7eb293d790f5224109851dba4bec3f81c32c Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 13 Nov 2024 13:28:38 +0000 Subject: [PATCH 100/502] Make small refactoring --- .../DataLakes/IcebergMetadata.cpp | 9 ++---- .../ObjectStorage/DataLakes/IcebergMetadata.h | 32 ++++++++++++------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 475d158818e..7e23b9df272 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -594,7 +594,6 @@ Strings IcebergMetadata::getDataFiles() const manifest_files.emplace_back(std::filesystem::path(configuration_ptr->getPath()) / "metadata" / filename); } - std::map files; LOG_TEST(log, "Collect data files"); for (const auto & manifest_file : manifest_files) { @@ -728,23 +727,21 @@ Strings IcebergMetadata::getDataFiles() const if (ManifestEntryStatus(status) == ManifestEntryStatus::DELETED) { LOG_TEST(log, "Processing delete file for path: {}", file_path); - chassert(files.contains(file_path) == 0); + chassert(schema_id_by_data_file.contains(file_path) == 0); } else { LOG_TEST(log, "Processing data file for path: {}", file_path); - files[file_path] = schema_object_id; + schema_id_by_data_file[file_path] = schema_object_id; } } schema_processor.addIcebergTableSchema(schema_object); } - for (const auto & [file_path, schema_object_id] : files) + for (const auto & [file_path, schema_object_id] : schema_id_by_data_file) { data_files.emplace_back(file_path); - initial_schemas[file_path] = schema_processor.getClickhouseTableSchemaById(schema_object_id); - schema_transformers[file_path] = schema_processor.getSchemaTransformationDagByIds(schema_object_id, current_schema_id); } return data_files; } diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 26e26831cc8..4b711aee351 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -1,8 +1,9 @@ #pragma once -#include "config.h" #include +#include #include +#include "config.h" #if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format. @@ -188,25 +189,22 @@ public: std::shared_ptr getInitialSchemaByPath(const String & data_path) const override { - auto it = initial_schemas.find(data_path); - if (it != initial_schemas.end()) - return it->second; - return {}; + auto version_if_outdated = getSchemaVersionByFileIfOutdated(data_path); + return version_if_outdated.has_value() ? schema_processor.getClickhouseTableSchemaById(version_if_outdated.value()) : nullptr; } std::shared_ptr getSchemaTransformer(const String & data_path) const override { - auto it = schema_transformers.find(data_path); - if (it != schema_transformers.end()) - return it->second; - return {}; + auto version_if_outdated = getSchemaVersionByFileIfOutdated(data_path); + return version_if_outdated.has_value() + ? schema_processor.getSchemaTransformationDagByIds(version_if_outdated.value(), current_schema_id) + : nullptr; } bool supportsExternalMetadataChange() const override { return true; } private: - mutable std::unordered_map> initial_schemas; - mutable std::unordered_map> schema_transformers; + mutable std::unordered_map schema_id_by_data_file; const ObjectStoragePtr object_storage; const ConfigurationObserverPtr configuration; @@ -220,6 +218,18 @@ private: mutable IcebergSchemaProcessor schema_processor; NamesAndTypesList schema; LoggerPtr log; + + std::optional getSchemaVersionByFileIfOutdated(String data_path) const + { + auto schema_id = schema_id_by_data_file.find(data_path); + if (schema_id == schema_id_by_data_file.end()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot find schema version for data file: {}", data_path); + } + if (schema_id->second == current_schema_id) + return std::nullopt; + return std::optional{schema_id->second}; + } }; } From 1e8ee2034a0aa2d23b5256e8cbb1d5c80ca07ab4 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 13 Nov 2024 15:26:32 +0100 Subject: [PATCH 101/502] Fix test run with runner --- docker/test/integration/runner/requirements.txt | 4 ++-- .../compose/docker_compose_iceberg_rest_catalog.yml | 7 +++---- tests/integration/test_database_iceberg/test.py | 5 +++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/test/integration/runner/requirements.txt b/docker/test/integration/runner/requirements.txt index bccf15b8495..9e3f88e1350 100644 --- a/docker/test/integration/runner/requirements.txt +++ b/docker/test/integration/runner/requirements.txt @@ -36,7 +36,7 @@ geomet==0.2.1.post1 grpcio-tools==1.60.0 grpcio==1.60.0 gssapi==1.8.3 -httplib2==0.20.2 +httplib2==0.22.0 idna==3.7 importlib-metadata==4.6.4 iniconfig==2.0.0 @@ -72,7 +72,7 @@ pyarrow==17.0.0 pycparser==2.22 pycryptodome==3.20.0 pymongo==3.11.0 -pyparsing==2.4.7 +pyparsing==3.1.0 pyspark==3.3.2 pyspnego==0.10.2 pytest-order==1.0.0 diff --git a/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml index 0a4be37335f..b529bf7d3ff 100644 --- a/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml +++ b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml @@ -17,7 +17,7 @@ services: rest: image: tabulario/iceberg-rest ports: - - 8181:8181 + - 8182:8181 environment: - AWS_ACCESS_KEY_ID=minio - AWS_SECRET_ACCESS_KEY=minio123 @@ -36,9 +36,8 @@ services: default: aliases: - warehouse.minio - ports: - - 9001:9001 - - 9000:9000 + expose: + - 9001 command: ["server", "/data", "--console-address", ":9001"] mc: depends_on: diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 43686694a60..05344b7467c 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -28,8 +28,8 @@ from helpers.s3_tools import get_file_contents, list_s3_objects, prepare_s3_buck from helpers.test_tools import TSV, csv_compare BASE_URL = "http://rest:8181/v1" -BASE_URL_LOCAL = "http://localhost:8181/v1" -BASE_URL_LOCAL_RAW = "http://localhost:8181" +BASE_URL_LOCAL = "http://localhost:8182/v1" +BASE_URL_LOCAL_RAW = "http://localhost:8182" CATALOG_NAME = "demo" @@ -125,6 +125,7 @@ def started_cluster(): # TODO: properly wait for container time.sleep(10) + #cluster.minio_client.make_bucket("warehouse") yield cluster From 770f7be49dd7b1d5d5053438bbc817628c52e7c2 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Wed, 13 Nov 2024 15:01:46 +0000 Subject: [PATCH 102/502] Format Settings: Better date_time_overflow_behavior Bring back the previous documentation for `date_time_overflow_behavior` from https://github.com/ClickHouse/ClickHouse/blob/d2b596409df73b7817a30b4d7a49159d1de33538/docs/en/operations/settings/settings.md?plain=1#L4506 --- src/Core/FormatFactorySettings.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h index a095bffc4c9..88cc4c6e9a6 100644 --- a/src/Core/FormatFactorySettings.h +++ b/src/Core/FormatFactorySettings.h @@ -1231,7 +1231,15 @@ Execute a pipeline for reading dictionary source in several threads. It's suppor Prefer more precise (but slower) float parsing algorithm )", 0) \ DECLARE(DateTimeOverflowBehavior, date_time_overflow_behavior, "ignore", R"( -Overflow mode for Date, Date32, DateTime, DateTime64 types. Possible values: 'ignore', 'throw', 'saturate'. +Defines the behavior when [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md), [DateTime64](../../sql-reference/data-types/datetime64.md) or integers are converted into Date, Date32, DateTime or DateTime64 but the value cannot be represented in the result type. + +Possible values: + +- `ignore` — Silently ignore overflows. The result is random. +- `throw` — Throw an exception in case of conversion overflow. +- `saturate` — Silently saturate the result. If the value is smaller than the smallest value that can be represented by the target type, the result is chosen as the smallest representable value. If the value is bigger than the largest value that can be represented by the target type, the result is chosen as the largest representable value. + +Default value: `ignore`. )", 0) \ DECLARE(Bool, validate_experimental_and_suspicious_types_inside_nested_types, true, R"( Validate usage of experimental and suspicious types inside nested types like Array/Map/Tuple From 6dfd4ad942e648ce215e5dc8ab32445b50c45fc3 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 13 Nov 2024 19:58:04 +0100 Subject: [PATCH 103/502] Better --- src/Core/SettingsEnums.cpp | 7 ++ src/Core/SettingsEnums.h | 11 ++ src/Databases/Iceberg/DatabaseIceberg.cpp | 63 ++++++++--- src/Databases/Iceberg/DatabaseIceberg.h | 3 +- .../Iceberg/DatabaseIcebergSettings.cpp | 1 + .../Iceberg/DatabaseIcebergSettings.h | 3 +- src/Databases/Iceberg/ICatalog.cpp | 2 +- src/Databases/Iceberg/ICatalog.h | 2 + src/Databases/Iceberg/RestCatalog.cpp | 47 ++++++-- src/Databases/Iceberg/RestCatalog.h | 9 +- .../docker_compose_iceberg_rest_catalog.yml | 7 +- .../integration/test_database_iceberg/test.py | 105 +++++++++++++++--- 12 files changed, 213 insertions(+), 47 deletions(-) diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 69126285eb8..19581fb2f93 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -287,4 +287,11 @@ IMPLEMENT_SETTING_ENUM( IMPLEMENT_SETTING_ENUM(DatabaseIcebergCatalogType, ErrorCodes::BAD_ARGUMENTS, {{"rest", DatabaseIcebergCatalogType::REST}}) +IMPLEMENT_SETTING_ENUM(DatabaseIcebergStorageType, ErrorCodes::BAD_ARGUMENTS, + {{"s3", DatabaseIcebergStorageType::S3}, + {"azure", DatabaseIcebergStorageType::Azure}, + {"hdfs", DatabaseIcebergStorageType::HDFS}, + {"local", DatabaseIcebergStorageType::Local}, + }) + } diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index d45b81bd8ee..004e2aec789 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -367,4 +367,15 @@ enum class DatabaseIcebergCatalogType : uint8_t }; DECLARE_SETTING_ENUM(DatabaseIcebergCatalogType) + +enum class DatabaseIcebergStorageType : uint8_t +{ + S3, + Azure, + Local, + HDFS, +}; + +DECLARE_SETTING_ENUM(DatabaseIcebergStorageType) + } diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index a84e4fe1cbe..7efde3a5671 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -29,6 +28,7 @@ namespace DatabaseIcebergSetting { extern const DatabaseIcebergSettingsString storage_endpoint; extern const DatabaseIcebergSettingsDatabaseIcebergCatalogType catalog_type; + extern const DatabaseIcebergSettingsDatabaseIcebergStorageType storage_type; } namespace ErrorCodes @@ -38,6 +38,10 @@ namespace ErrorCodes namespace { + /// Parse a string, containing at least one dot, into a two substrings: + /// A.B.C.D.E -> A.B.C.D and E, where + /// `A.B.C.D` is a table "namespace". + /// `E` is a table name. std::pair parseTableName(const std::string & name) { auto pos = name.rfind('.'); @@ -74,41 +78,73 @@ std::unique_ptr DatabaseIceberg::getCatalog(ContextPtr contex } } +std::shared_ptr DatabaseIceberg::getConfiguration() const +{ + switch (settings[DatabaseIcebergSetting::storage_type].value) + { + case DB::DatabaseIcebergStorageType::S3: + { + return std::make_shared(); + } + case DB::DatabaseIcebergStorageType::Azure: + { + return std::make_shared(); + } + case DB::DatabaseIcebergStorageType::HDFS: + { + return std::make_shared(); + } + case DB::DatabaseIcebergStorageType::Local: + { + return std::make_shared(); + } + } +} + bool DatabaseIceberg::empty() const { - return getCatalog(Context::getGlobalContextInstance())->existsCatalog(); + return getCatalog(Context::getGlobalContextInstance())->empty(); } bool DatabaseIceberg::isTableExist(const String & name, ContextPtr context_) const { - auto [namespace_name, table_name] = parseTableName(name); + const auto [namespace_name, table_name] = parseTableName(name); return getCatalog(context_)->existsTable(namespace_name, table_name); } StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_) const { auto catalog = getCatalog(context_); - auto table_metadata = Iceberg::ICatalog::TableMetadata().withLocation(); + auto table_metadata = Iceberg::ICatalog::TableMetadata().withLocation().withSchema(); auto [namespace_name, table_name] = parseTableName(name); + if (!catalog->tryGetTableMetadata(namespace_name, table_name, table_metadata)) return nullptr; - auto configuration = std::make_shared(); + /// Take database engine definition AST as base. ASTStorage * storage = database_engine_definition->as(); ASTs args = storage->engine->arguments->children; - auto table_endpoint = std::filesystem::path(settings[DatabaseIcebergSetting::storage_endpoint].value) / table_metadata.getPath(); + /// Replace Iceberg Catalog endpoint with storage path endpoint of requested table. + auto table_endpoint = std::filesystem::path(settings[DatabaseIcebergSetting::storage_endpoint].value) + / table_metadata.getPath() + / ""; + args[0] = std::make_shared(table_endpoint.string()); - LOG_TEST(log, "Using endpoint: {}", table_endpoint.string()); + LOG_TEST(log, "Using table endpoint: {}", table_endpoint.string()); + + const auto columns = ColumnsDescription(table_metadata.getSchema()); + const auto configuration = getConfiguration(); + /// with_table_structure = false: because there will be no table stucture in table definition AST. + StorageObjectStorage::Configuration::initialize(*configuration, args, context_, /* with_table_structure */false); - StorageObjectStorage::Configuration::initialize(*configuration, args, context_, false); return std::make_shared( configuration, configuration->createObjectStorage(context_, /* is_readonly */ false), context_, StorageID(getDatabaseName(), name), - /* columns */ColumnsDescription{}, + /* columns */columns, /* constraints */ConstraintsDescription{}, /* comment */"", getFormatSettings(context_), @@ -117,16 +153,17 @@ StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_ DatabaseTablesIteratorPtr DatabaseIceberg::getTablesIterator( ContextPtr context_, - const FilterByNameFunction & /* filter_by_table_name */, + const FilterByNameFunction & filter_by_table_name, bool /* skip_not_loaded */) const { Tables tables; auto catalog = getCatalog(context_); for (const auto & table_name : catalog->getTables()) { - DataTypePtr type = std::make_shared(); - auto columns = ColumnsDescription{NamesAndTypesList({NameAndTypePair(std::string("a"), type)})}; - auto storage = std::make_shared(StorageID(getDatabaseName(), table_name), columns, ConstraintsDescription{}, ""); + if (filter_by_table_name && !filter_by_table_name(table_name)) + continue; + + auto storage = tryGetTable(table_name, context_); tables.emplace(table_name, storage); } diff --git a/src/Databases/Iceberg/DatabaseIceberg.h b/src/Databases/Iceberg/DatabaseIceberg.h index 1356f5aee47..41ed17f58d0 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.h +++ b/src/Databases/Iceberg/DatabaseIceberg.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -49,7 +50,7 @@ private: const LoggerPtr log; std::unique_ptr getCatalog(ContextPtr context_) const; - + std::shared_ptr getConfiguration() const; }; } diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp index d0a93edb579..8383c372a52 100644 --- a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp @@ -16,6 +16,7 @@ namespace ErrorCodes #define DATABASE_ICEBERG_RELATED_SETTINGS(DECLARE, ALIAS) \ DECLARE(DatabaseIcebergCatalogType, catalog_type, DatabaseIcebergCatalogType::REST, "Catalog type", 0) \ + DECLARE(DatabaseIcebergStorageType, storage_type, DatabaseIcebergStorageType::S3, "Storage type: S3, Local, Azure, HDFS", 0) \ DECLARE(String, storage_endpoint, "", "Object storage endpoint", 0) \ #define LIST_OF_DATABASE_ICEBERG_SETTINGS(M, ALIAS) \ diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.h b/src/Databases/Iceberg/DatabaseIcebergSettings.h index d6908f6f3b7..5d9d120efed 100644 --- a/src/Databases/Iceberg/DatabaseIcebergSettings.h +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.h @@ -16,7 +16,8 @@ class SettingsChanges; #define DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(CLASS_NAME, M) \ M(CLASS_NAME, String) \ M(CLASS_NAME, UInt64) \ - M(CLASS_NAME, DatabaseIcebergCatalogType) + M(CLASS_NAME, DatabaseIcebergCatalogType) \ + M(CLASS_NAME, DatabaseIcebergStorageType) DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(DatabaseIcebergSettings, DECLARE_SETTING_TRAIT) diff --git a/src/Databases/Iceberg/ICatalog.cpp b/src/Databases/Iceberg/ICatalog.cpp index 6ddeeeac58d..a67d88fc555 100644 --- a/src/Databases/Iceberg/ICatalog.cpp +++ b/src/Databases/Iceberg/ICatalog.cpp @@ -18,7 +18,7 @@ std::string ICatalog::TableMetadata::getPath() const if (location.starts_with("s3://")) return location.substr(std::strlen("s3://")); else - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unsupported location type: {}", location); + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unexpected path format: {}", location); } const DB::NamesAndTypesList & ICatalog::TableMetadata::getSchema() const diff --git a/src/Databases/Iceberg/ICatalog.h b/src/Databases/Iceberg/ICatalog.h index 6a0412391a4..a9e8c24b01b 100644 --- a/src/Databases/Iceberg/ICatalog.h +++ b/src/Databases/Iceberg/ICatalog.h @@ -21,6 +21,8 @@ public: virtual bool existsCatalog() const = 0; + virtual bool empty() const = 0; + virtual Tables getTables() const = 0; virtual bool existsTable( diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index e767a4cffaf..86abec72c29 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -52,6 +52,30 @@ bool RestCatalog::existsCatalog() const } } +bool RestCatalog::empty() const +{ + try + { + bool found_table = false; + auto stop_condition = [&](const std::string & namespace_name) -> bool + { + const auto tables = getTables(namespace_name, /* limit */1); + found_table = !tables.empty(); + return found_table; + }; + + Namespaces namespaces; + getNamespacesRecursive("", namespaces, stop_condition); + + return found_table; + } + catch (...) + { + DB::tryLogCurrentException(log); + return true; + } +} + DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer(const std::string & endpoint, const Poco::URI::QueryParameters & params) const { const auto & context = getContext(); @@ -73,7 +97,7 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer(const std::string & RestCatalog::Tables RestCatalog::getTables() const { Namespaces namespaces; - getNamespacesRecursive("", namespaces); + getNamespacesRecursive("", namespaces, {}); Tables tables; for (const auto & current_namespace : namespaces) @@ -84,7 +108,7 @@ RestCatalog::Tables RestCatalog::getTables() const return tables; } -void RestCatalog::getNamespacesRecursive(const Namespace & base_namespace, Namespaces & result) const +void RestCatalog::getNamespacesRecursive(const Namespace & base_namespace, Namespaces & result, StopCondition stop_condition) const { auto namespaces = getNamespaces(base_namespace); result.reserve(result.size() + namespaces.size()); @@ -93,7 +117,11 @@ void RestCatalog::getNamespacesRecursive(const Namespace & base_namespace, Names for (const auto & current_namespace : namespaces) { chassert(current_namespace.starts_with(base_namespace)); - getNamespacesRecursive(current_namespace, result); + + if (stop_condition && stop_condition(current_namespace)) + break; + + getNamespacesRecursive(current_namespace, result, stop_condition); } } @@ -175,14 +203,14 @@ RestCatalog::Namespaces RestCatalog::parseNamespaces(DB::ReadBuffer & buf, const return namespaces; } -RestCatalog::Tables RestCatalog::getTables(const Namespace & base_namespace) const +RestCatalog::Tables RestCatalog::getTables(const Namespace & base_namespace, size_t limit) const { const auto endpoint = std::string(namespaces_endpoint) + "/" + base_namespace + "/tables"; auto buf = createReadBuffer(endpoint); - return parseTables(*buf, base_namespace); + return parseTables(*buf, base_namespace, limit); } -RestCatalog::Tables RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & base_namespace) const +RestCatalog::Tables RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit) const { if (buf.eof()) return {}; @@ -201,9 +229,12 @@ RestCatalog::Tables RestCatalog::parseTables(DB::ReadBuffer & buf, const std::st Tables tables; for (size_t i = 0; i < identifiers_object->size(); ++i) { - auto current_table_json = identifiers_object->get(static_cast(i)).extract(); - auto table_name = current_table_json->get("name").extract(); + const auto current_table_json = identifiers_object->get(static_cast(i)).extract(); + const auto table_name = current_table_json->get("name").extract(); + tables.push_back(base_namespace + "." + table_name); + if (limit && tables.size() >= limit) + break; } return tables; } diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index b33d5accda4..e83aa8eabe9 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -25,6 +25,8 @@ public: bool existsCatalog() const override; + bool empty() const override; + Tables getTables() const override; bool existsTable(const std::string & namespace_name, const std::string & table_name) const override; @@ -48,15 +50,16 @@ private: Poco::URI::QueryParameters createParentNamespaceParams(const std::string & base_namespace) const; - void getNamespacesRecursive(const Namespace & base_namespace, Namespaces & result) const; + using StopCondition = std::function; + void getNamespacesRecursive(const Namespace & base_namespace, Namespaces & result, StopCondition stop_condition) const; Namespaces getNamespaces(const Namespace & base_namespace) const; Namespaces parseNamespaces(DB::ReadBuffer & buf, const std::string & base_namespace) const; - Tables getTables(const Namespace & base_namespace) const; + Tables getTables(const Namespace & base_namespace, size_t limit = 0) const; - Tables parseTables(DB::ReadBuffer & buf, const std::string & base_namespace) const; + Tables parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit) const; bool getTableMetadataImpl( const std::string & namespace_name, diff --git a/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml index b529bf7d3ff..860d9e29230 100644 --- a/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml +++ b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml @@ -22,7 +22,7 @@ services: - AWS_ACCESS_KEY_ID=minio - AWS_SECRET_ACCESS_KEY=minio123 - AWS_REGION=us-east-1 - - CATALOG_WAREHOUSE=s3://warehouse/ + - CATALOG_WAREHOUSE=s3://iceberg_data/ - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - CATALOG_S3_ENDPOINT=http://minio:9000 minio: @@ -36,8 +36,9 @@ services: default: aliases: - warehouse.minio - expose: - - 9001 + ports: + - 9001:9001 + - 9002:9000 command: ["server", "/data", "--console-address", ":9001"] mc: depends_on: diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 05344b7467c..4341b9cb30a 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -22,7 +22,9 @@ from pyiceberg.types import ( StructType, TimestampType, ) - +import pyarrow as pa +import random +from datetime import datetime, timedelta from helpers.cluster import ClickHouseCluster, ClickHouseInstance, is_arm from helpers.s3_tools import get_file_contents, list_s3_objects, prepare_s3_bucket from helpers.test_tools import TSV, csv_compare @@ -34,9 +36,11 @@ BASE_URL_LOCAL_RAW = "http://localhost:8182" CATALOG_NAME = "demo" DEFAULT_SCHEMA = Schema( - NestedField(field_id=1, name="datetime", field_type=TimestampType(), required=True), - NestedField(field_id=2, name="symbol", field_type=StringType(), required=True), - NestedField(field_id=3, name="bid", field_type=FloatType(), required=False), + NestedField( + field_id=1, name="datetime", field_type=TimestampType(), required=False + ), + NestedField(field_id=2, name="symbol", field_type=StringType(), required=False), + NestedField(field_id=3, name="bid", field_type=DoubleType(), required=False), NestedField(field_id=4, name="ask", field_type=DoubleType(), required=False), NestedField( field_id=5, @@ -52,11 +56,15 @@ DEFAULT_SCHEMA = Schema( required=False, ), ) + +DEFAULT_CREATE_TABLE = "CREATE TABLE {}.`{}.{}`\\n(\\n `datetime` Nullable(DateTime64(6)),\\n `symbol` Nullable(String),\\n `bid` Nullable(Float64),\\n `ask` Nullable(Float64),\\n `details` Tuple(created_by Nullable(String))\\n)\\nENGINE = Iceberg(\\'http://minio:9000/warehouse/data\\', \\'minio\\', \\'[HIDDEN]\\')\n" + DEFAULT_PARTITION_SPEC = PartitionSpec( PartitionField( source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day" ) ) + DEFAULT_SORT_ORDER = SortOrder(SortField(source_id=2, transform=IdentityTransform())) @@ -68,13 +76,13 @@ def list_namespaces(): raise Exception(f"Failed to list namespaces: {response.status_code}") -def load_catalog_impl(): +def load_catalog_impl(started_cluster): return load_catalog( CATALOG_NAME, **{ "uri": BASE_URL_LOCAL_RAW, "type": "rest", - "s3.endpoint": f"http://minio:9000", + "s3.endpoint": f"http://localhost:9002", "s3.access-key-id": "minio", "s3.secret-access-key": "minio123", }, @@ -89,25 +97,51 @@ def create_table( partition_spec=DEFAULT_PARTITION_SPEC, sort_order=DEFAULT_SORT_ORDER, ): - catalog.create_table( + return catalog.create_table( identifier=f"{namespace}.{table}", schema=schema, - location=f"s3://warehouse", + location=f"s3://warehouse/data", partition_spec=partition_spec, sort_order=sort_order, ) +def generate_record(): + return { + "datetime": datetime.now(), + "symbol": str("kek"), + "bid": round(random.uniform(100, 200), 2), + "ask": round(random.uniform(200, 300), 2), + "details": {"created_by": "Alice Smith"}, + } + + def create_clickhouse_iceberg_database(started_cluster, node, name): node.query( f""" DROP DATABASE IF EXISTS {name}; CREATE DATABASE {name} ENGINE = Iceberg('{BASE_URL}', 'minio', 'minio123') -SETTINGS catalog_type = 'rest', storage_endpoint = 'http://{started_cluster.minio_ip}:{started_cluster.minio_port}/' +SETTINGS catalog_type = 'rest', storage_endpoint = 'http://minio:9000/' """ ) +def print_objects(): + minio_client = Minio( + f"localhost:9002", + access_key="minio", + secret_key="minio123", + secure=False, + http_client=urllib3.PoolManager(cert_reqs="CERT_NONE"), + ) + + objects = list(minio_client.list_objects("warehouse", "", recursive=True)) + names = [x.object_name for x in objects] + names.sort() + for name in names: + print(f"Found object: {name}") + + @pytest.fixture(scope="module") def started_cluster(): try: @@ -125,7 +159,6 @@ def started_cluster(): # TODO: properly wait for container time.sleep(10) - #cluster.minio_client.make_bucket("warehouse") yield cluster @@ -133,7 +166,7 @@ def started_cluster(): cluster.shutdown() -def test_simple(started_cluster): +def test_list_tables(started_cluster): node = started_cluster.instances["node1"] root_namespace = "clickhouse" @@ -142,7 +175,7 @@ def test_simple(started_cluster): namespace_1_tables = ["tableA", "tableB"] namespace_2_tables = ["tableC", "tableD"] - catalog = load_catalog_impl() + catalog = load_catalog_impl(started_cluster) for namespace in [namespace_1, namespace_2]: catalog.create_namespace(namespace) @@ -182,13 +215,13 @@ def test_simple(started_cluster): ).strip() ) - expected = f"CREATE TABLE {CATALOG_NAME}.`{namespace_2}.tableC`\\n(\\n `datetime` DateTime64(6),\\n `symbol` String,\\n `bid` Nullable(Float32),\\n `ask` Nullable(Float64),\\n `details` Tuple(created_by Nullable(String))\\n)\\nENGINE = Iceberg(\\'http://None:9001/warehouse\\', \\'minio\\', \\'[HIDDEN]\\')\n" + expected = DEFAULT_CREATE_TABLE.format(CATALOG_NAME, namespace_2, "tableC") assert expected == node.query( f"SHOW CREATE TABLE {CATALOG_NAME}.`{namespace_2}.tableC`" ) -def test_different_namespaces(started_cluster): +def test_many_namespaces(started_cluster): node = started_cluster.instances["node1"] namespaces = [ "A", @@ -202,11 +235,9 @@ def test_different_namespaces(started_cluster): "B.CC", ] tables = ["A", "B", "C", "D", "E", "F"] - catalog = load_catalog_impl() + catalog = load_catalog_impl(started_cluster) for namespace in namespaces: - # if namespace in catalog.list_namespaces()["namesoaces"]: - # catalog.drop_namespace(namespace) catalog.create_namespace(namespace) for table in tables: create_table(catalog, namespace, table) @@ -221,3 +252,43 @@ def test_different_namespaces(started_cluster): f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}'" ) ) + + +def test_select(started_cluster): + node = started_cluster.instances["node1"] + + test_ref = "test_list_tables" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + + namespace = f"{root_namespace}.A.B.C" + namespaces_to_create = [ + root_namespace, + f"{root_namespace}.A", + f"{root_namespace}.A.B", + f"{root_namespace}.A.B.C", + ] + + catalog = load_catalog_impl(started_cluster) + + for namespace in namespaces_to_create: + catalog.create_namespace(namespace) + assert len(catalog.list_tables(namespace)) == 0 + + table = create_table(catalog, namespace, table_name) + + num_rows = 10 + data = [generate_record() for _ in range(num_rows)] + df = pa.Table.from_pylist(data) + table.append(df) + + create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) + + expected = DEFAULT_CREATE_TABLE.format(CATALOG_NAME, namespace, table_name) + assert expected == node.query( + f"SHOW CREATE TABLE {CATALOG_NAME}.`{namespace}.{table_name}`" + ) + + assert num_rows == int( + node.query(f"SELECT count() FROM {CATALOG_NAME}.`{namespace}.{table_name}`") + ) From 58edfbe1136bb8c8dd5f0a29b3ecce0d2a3335b9 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 13 Nov 2024 20:12:52 +0100 Subject: [PATCH 104/502] Fix style check --- src/Databases/Iceberg/DatabaseIceberg.cpp | 2 +- tests/integration/test_database_iceberg/test.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 7efde3a5671..81c39554635 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -136,7 +136,7 @@ StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_ const auto columns = ColumnsDescription(table_metadata.getSchema()); const auto configuration = getConfiguration(); - /// with_table_structure = false: because there will be no table stucture in table definition AST. + /// with_table_structure = false: because there will be no table structure in table definition AST. StorageObjectStorage::Configuration::initialize(*configuration, args, context_, /* with_table_structure */false); return std::make_shared( diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 4341b9cb30a..37d2bbf0ea8 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -2,9 +2,12 @@ import glob import json import logging import os +import random import time import uuid +from datetime import datetime, timedelta +import pyarrow as pa import pytest import requests import urllib3 @@ -22,9 +25,7 @@ from pyiceberg.types import ( StructType, TimestampType, ) -import pyarrow as pa -import random -from datetime import datetime, timedelta + from helpers.cluster import ClickHouseCluster, ClickHouseInstance, is_arm from helpers.s3_tools import get_file_contents, list_s3_objects, prepare_s3_bucket from helpers.test_tools import TSV, csv_compare From 6c3003c6ce0c41542e410921afcebc6d71b0e51c Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 13 Nov 2024 20:30:33 +0100 Subject: [PATCH 105/502] Cleanup --- src/Databases/Iceberg/DatabaseIceberg.cpp | 38 +++++++++------ src/Databases/Iceberg/DatabaseIceberg.h | 10 +++- src/Databases/Iceberg/ICatalog.cpp | 4 +- src/Databases/Iceberg/ICatalog.h | 58 +++++++++++------------ src/Databases/Iceberg/RestCatalog.cpp | 27 ++++------- src/Databases/Iceberg/RestCatalog.h | 8 ++-- 6 files changed, 73 insertions(+), 72 deletions(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 81c39554635..311ae9bf2dc 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -101,6 +101,13 @@ std::shared_ptr DatabaseIceberg::getConfigu } } +std::string DatabaseIceberg::getStorageEndpointForTable(const Iceberg::TableMetadata & table_metadata) const +{ + return std::filesystem::path(settings[DatabaseIcebergSetting::storage_endpoint].value) + / table_metadata.getPath() + / ""; +} + bool DatabaseIceberg::empty() const { return getCatalog(Context::getGlobalContextInstance())->empty(); @@ -115,7 +122,7 @@ bool DatabaseIceberg::isTableExist(const String & name, ContextPtr context_) con StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_) const { auto catalog = getCatalog(context_); - auto table_metadata = Iceberg::ICatalog::TableMetadata().withLocation().withSchema(); + auto table_metadata = Iceberg::TableMetadata().withLocation().withSchema(); auto [namespace_name, table_name] = parseTableName(name); if (!catalog->tryGetTableMetadata(namespace_name, table_name, table_metadata)) @@ -126,17 +133,16 @@ StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_ ASTs args = storage->engine->arguments->children; /// Replace Iceberg Catalog endpoint with storage path endpoint of requested table. - auto table_endpoint = std::filesystem::path(settings[DatabaseIcebergSetting::storage_endpoint].value) - / table_metadata.getPath() - / ""; + auto table_endpoint = getStorageEndpointForTable(table_metadata); + args[0] = std::make_shared(table_endpoint); - args[0] = std::make_shared(table_endpoint.string()); - - LOG_TEST(log, "Using table endpoint: {}", table_endpoint.string()); + LOG_TEST(log, "Using table endpoint: {}", table_endpoint); const auto columns = ColumnsDescription(table_metadata.getSchema()); const auto configuration = getConfiguration(); - /// with_table_structure = false: because there will be no table structure in table definition AST. + + /// with_table_structure = false: because there will be + /// no table structure in table definition AST. StorageObjectStorage::Configuration::initialize(*configuration, args, context_, /* with_table_structure */false); return std::make_shared( @@ -184,8 +190,9 @@ ASTPtr DatabaseIceberg::getCreateTableQueryImpl( bool /* throw_on_error */) const { auto catalog = getCatalog(context_); - auto table_metadata = Iceberg::ICatalog::TableMetadata().withLocation().withSchema(); - auto [namespace_name, table_name] = parseTableName(name); + auto table_metadata = Iceberg::TableMetadata().withLocation().withSchema(); + + const auto [namespace_name, table_name] = parseTableName(name); catalog->getTableMetadata(namespace_name, table_name, table_metadata); auto create_table_query = std::make_shared(); @@ -203,7 +210,6 @@ ASTPtr DatabaseIceberg::getCreateTableQueryImpl( columns_declare_list->set(columns_declare_list->columns, columns_expression_list); create_table_query->set(create_table_query->columns_list, columns_declare_list); - /// init create query. create_table_query->setTable(name); create_table_query->setDatabase(getDatabaseName()); @@ -217,10 +223,14 @@ ASTPtr DatabaseIceberg::getCreateTableQueryImpl( auto storage_engine_arguments = storage->engine->arguments; if (storage_engine_arguments->children.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected number of arguments: {}", storage_engine_arguments->children.size()); + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Unexpected number of arguments: {}", + storage_engine_arguments->children.size()); + } - auto table_endpoint = std::filesystem::path(settings[DatabaseIcebergSetting::storage_endpoint].value) / table_metadata.getPath(); - storage_engine_arguments->children[0] = std::make_shared(table_endpoint.string()); + auto table_endpoint = getStorageEndpointForTable(table_metadata); + storage_engine_arguments->children[0] = std::make_shared(table_endpoint); return create_table_query; } diff --git a/src/Databases/Iceberg/DatabaseIceberg.h b/src/Databases/Iceberg/DatabaseIceberg.h index 41ed17f58d0..5a87e3179af 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.h +++ b/src/Databases/Iceberg/DatabaseIceberg.h @@ -10,6 +10,10 @@ namespace DB { +/// TODO: +/// - http basic auth for catalog +/// - tests with azure, hdfs, local + class DatabaseIceberg final : public IDatabase, WithContext { public: @@ -20,7 +24,6 @@ public: ASTPtr database_engine_definition_); String getEngineName() const override { return "Iceberg"; } - String getMetadataPath() const override { return ""; } bool canContainMergeTreeTables() const override { return false; } bool canContainDistributedTables() const override { return false; } @@ -44,13 +47,18 @@ protected: ASTPtr getCreateTableQueryImpl(const String & table_name, ContextPtr context, bool throw_on_error) const override; private: + /// Iceberg Catalog url. const std::string url; + /// SETTINGS from CREATE query. const DatabaseIcebergSettings settings; + /// Database engine definition taken from initial CREATE DATABASE query. const ASTPtr database_engine_definition; + const LoggerPtr log; std::unique_ptr getCatalog(ContextPtr context_) const; std::shared_ptr getConfiguration() const; + std::string getStorageEndpointForTable(const Iceberg::TableMetadata & table_metadata) const; }; } diff --git a/src/Databases/Iceberg/ICatalog.cpp b/src/Databases/Iceberg/ICatalog.cpp index a67d88fc555..d07d9613e1a 100644 --- a/src/Databases/Iceberg/ICatalog.cpp +++ b/src/Databases/Iceberg/ICatalog.cpp @@ -10,7 +10,7 @@ namespace DB::ErrorCodes namespace Iceberg { -std::string ICatalog::TableMetadata::getPath() const +std::string TableMetadata::getPath() const { if (!with_location) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Data location was not requested"); @@ -21,7 +21,7 @@ std::string ICatalog::TableMetadata::getPath() const throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unexpected path format: {}", location); } -const DB::NamesAndTypesList & ICatalog::TableMetadata::getSchema() const +const DB::NamesAndTypesList & TableMetadata::getSchema() const { if (!with_schema) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Data location was not requested"); diff --git a/src/Databases/Iceberg/ICatalog.h b/src/Databases/Iceberg/ICatalog.h index a9e8c24b01b..7bd2cf3010c 100644 --- a/src/Databases/Iceberg/ICatalog.h +++ b/src/Databases/Iceberg/ICatalog.h @@ -5,22 +5,41 @@ namespace Iceberg { +class TableMetadata +{ +friend class RestCatalog; + +public: + TableMetadata() = default; + + std::string getPath() const; + + const DB::NamesAndTypesList & getSchema() const; + + TableMetadata & withLocation() { with_location = true; return *this; } + TableMetadata & withSchema() { with_schema = true; return *this; } + +private: + /// starts with s3://, file://, etc + std::string location; + /// column names and types + DB::NamesAndTypesList schema; + + bool with_location = false; + bool with_schema = false; +}; + + class ICatalog { public: - using Namespace = std::string; - using Namespaces = std::vector; - using Table = std::string; - using Tables = std::vector
; - - class TableMetadata; + using Namespaces = std::vector; + using Tables = std::vector; explicit ICatalog(const std::string & catalog_name_) : catalog_name(catalog_name_) {} virtual ~ICatalog() = default; - virtual bool existsCatalog() const = 0; - virtual bool empty() const = 0; virtual Tables getTables() const = 0; @@ -43,27 +62,4 @@ protected: const std::string catalog_name; }; -class ICatalog::TableMetadata -{ -friend class RestCatalog; - -public: - TableMetadata() = default; - - std::string getPath() const; - - const DB::NamesAndTypesList & getSchema() const; - - TableMetadata & withLocation() { with_location = true; return *this; } - TableMetadata & withSchema() { with_schema = true; return *this; } - -private: - /// starts with s3://, file://, etc - std::string location; - /// column names and types - DB::NamesAndTypesList schema; - - bool with_location = false; - bool with_schema = false; -}; } diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index 86abec72c29..729ed641494 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -38,20 +38,6 @@ RestCatalog::RestCatalog( { } -bool RestCatalog::existsCatalog() const -{ - try - { - createReadBuffer(namespaces_endpoint)->eof(); - return true; - } - catch (...) - { - DB::tryLogCurrentException(log); - return false; - } -} - bool RestCatalog::empty() const { try @@ -108,7 +94,7 @@ RestCatalog::Tables RestCatalog::getTables() const return tables; } -void RestCatalog::getNamespacesRecursive(const Namespace & base_namespace, Namespaces & result, StopCondition stop_condition) const +void RestCatalog::getNamespacesRecursive(const std::string & base_namespace, Namespaces & result, StopCondition stop_condition) const { auto namespaces = getNamespaces(base_namespace); result.reserve(result.size() + namespaces.size()); @@ -194,16 +180,19 @@ RestCatalog::Namespaces RestCatalog::parseNamespaces(DB::ReadBuffer & buf, const if (current_namespace_array->size() == 0) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Expected namespace array to be non-empty"); - const int current_namespace_idx = static_cast(current_namespace_array->size()) - 1; - const auto current_namespace = current_namespace_array->get(current_namespace_idx).extract(); - const auto full_namespace = base_namespace.empty() ? current_namespace : base_namespace + "." + current_namespace; + const int idx = static_cast(current_namespace_array->size()) - 1; + const auto current_namespace = current_namespace_array->get(idx).extract(); + const auto full_namespace = base_namespace.empty() + ? current_namespace + : base_namespace + "." + current_namespace; + namespaces.push_back(full_namespace); } return namespaces; } -RestCatalog::Tables RestCatalog::getTables(const Namespace & base_namespace, size_t limit) const +RestCatalog::Tables RestCatalog::getTables(const std::string & base_namespace, size_t limit) const { const auto endpoint = std::string(namespaces_endpoint) + "/" + base_namespace + "/tables"; auto buf = createReadBuffer(endpoint); diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index e83aa8eabe9..12ff31ad8d1 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -23,8 +23,6 @@ public: ~RestCatalog() override = default; - bool existsCatalog() const override; - bool empty() const override; Tables getTables() const override; @@ -51,13 +49,13 @@ private: Poco::URI::QueryParameters createParentNamespaceParams(const std::string & base_namespace) const; using StopCondition = std::function; - void getNamespacesRecursive(const Namespace & base_namespace, Namespaces & result, StopCondition stop_condition) const; + void getNamespacesRecursive(const std::string & base_namespace, Namespaces & result, StopCondition stop_condition) const; - Namespaces getNamespaces(const Namespace & base_namespace) const; + Namespaces getNamespaces(const std::string & base_namespace) const; Namespaces parseNamespaces(DB::ReadBuffer & buf, const std::string & base_namespace) const; - Tables getTables(const Namespace & base_namespace, size_t limit = 0) const; + Tables getTables(const std::string & base_namespace, size_t limit = 0) const; Tables parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit) const; From b95eef773332af52d5a16e3b8b3da51e9815a317 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 14 Nov 2024 09:18:46 +0000 Subject: [PATCH 106/502] Add Setting for backward compatibility --- .../table-engines/integrations/iceberg.md | 2 + .../DataLakes/DataLakeConfiguration.h | 6 +- .../ObjectStorage/DataLakes/IcebergMetadata.h | 4 + .../ObjectStorage/StorageObjectStorage.cpp | 12 +- .../ObjectStorage/StorageObjectStorage.h | 6 +- .../StorageObjectStorageSettings.cpp | 104 +++++ .../StorageObjectStorageSettings.h | 73 ++++ .../registerStorageObjectStorage.cpp | 44 +- .../registerQueueStorage.cpp | 2 +- .../TableFunctionObjectStorage.h | 2 +- .../integration/test_storage_iceberg/test.py | 408 ++++++++++++++++-- 11 files changed, 604 insertions(+), 59 deletions(-) create mode 100644 src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp create mode 100644 src/Storages/ObjectStorage/StorageObjectStorageSettings.h diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md index b5933bfe977..ff21d1c6681 100644 --- a/docs/en/engines/table-engines/integrations/iceberg.md +++ b/docs/en/engines/table-engines/integrations/iceberg.md @@ -79,6 +79,8 @@ At the moment, with the help of CH, you can read iceberg tables, the schema of w Currently, it is not possible to change nested structures or the types of elements within arrays and maps. +To read a table where the schema has changed after its creation with dynamic schema inference, set allow_dynamic_metadata_for_data_lakes = true when creating the table. + ### Data cache {#data-cache} `Iceberg` table engine and table function support data caching same as `S3`, `AzureBlobStorage`, `HDFS` storages. See [here](../../../engines/table-engines/integrations/s3.md#data-cache). diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h index be1304e3340..ede70567da4 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h @@ -90,7 +90,11 @@ public: return current_metadata->getSchemaTransformer(data_path); } - bool hasExternalDynamicMetadata() override { return current_metadata && current_metadata->supportsExternalMetadataChange(); } + bool hasExternalDynamicMetadata() override + { + return StorageObjectStorage::Configuration::allow_dynamic_metadata_for_data_lakes && current_metadata + && current_metadata->supportsExternalMetadataChange(); + } ColumnsDescription updateAndGetCurrentSchema(ObjectStoragePtr object_storage, ContextPtr context) override { diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 4b711aee351..b6cfaeb852b 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -20,6 +20,10 @@ namespace DB { +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} /** * Iceberg supports the next data types (see https://iceberg.apache.org/spec/#schemas-and-data-types): diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 5764f3fac43..1f669044ab4 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -24,6 +24,7 @@ #include #include "Databases/LoadingStrictnessLevel.h" #include "Storages/ColumnsDescription.h" +#include "Storages/ObjectStorage/StorageObjectStorageSettings.h" #include @@ -43,6 +44,11 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +namespace StorageObjectStorageSetting +{ +extern const StorageObjectStorageSettingsBool allow_dynamic_metadata_for_data_lakes; +} + String StorageObjectStorage::getPathSample(StorageInMemoryMetadata metadata, ContextPtr context) { auto query_settings = configuration->getQuerySettings(context); @@ -533,7 +539,8 @@ void StorageObjectStorage::Configuration::initialize( Configuration & configuration, ASTs & engine_args, ContextPtr local_context, - bool with_table_structure) + bool with_table_structure, + std::unique_ptr settings) { if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, local_context)) configuration.fromNamedCollection(*named_collection, local_context); @@ -557,6 +564,9 @@ void StorageObjectStorage::Configuration::initialize( else FormatFactory::instance().checkFormatName(configuration.format); + if (settings) + configuration.allow_dynamic_metadata_for_data_lakes + = (*settings)[StorageObjectStorageSetting::allow_dynamic_metadata_for_data_lakes]; configuration.initialized = true; } diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 3e7bf72625a..c058c6e12ff 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include "Interpreters/ActionsDAG.h" @@ -167,7 +168,8 @@ public: Configuration & configuration, ASTs & engine_args, ContextPtr local_context, - bool with_table_structure); + bool with_table_structure, + std::unique_ptr settings); /// Storage type: s3, hdfs, azure, local. virtual ObjectStorageType getType() const = 0; @@ -252,6 +254,8 @@ protected: bool initialized = false; DataLakePartitionColumns partition_columns; + + bool allow_dynamic_metadata_for_data_lakes; }; } diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp new file mode 100644 index 00000000000..3bb08bc1419 --- /dev/null +++ b/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp @@ -0,0 +1,104 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +#define STORAGE_OBJECT_STORAGE_RELATED_SETTINGS(DECLARE, ALIAS) DECLARE(Bool, allow_dynamic_metadata_for_data_lakes, false, "", 0) + +#define LIST_OF_STORAGE_OBJECT_STORAGE_SETTINGS(M, ALIAS) \ + STORAGE_OBJECT_STORAGE_RELATED_SETTINGS(M, ALIAS) \ + LIST_OF_ALL_FORMAT_SETTINGS(M, ALIAS) + +DECLARE_SETTINGS_TRAITS(StorageObjectStorageSettingsTraits, LIST_OF_STORAGE_OBJECT_STORAGE_SETTINGS) +IMPLEMENT_SETTINGS_TRAITS(StorageObjectStorageSettingsTraits, LIST_OF_STORAGE_OBJECT_STORAGE_SETTINGS) + +struct StorageObjectStorageSettingsImpl : public BaseSettings +{ +}; + +#define INITIALIZE_SETTING_EXTERN(TYPE, NAME, DEFAULT, DESCRIPTION, FLAGS) \ + StorageObjectStorageSettings##TYPE NAME = &StorageObjectStorageSettingsImpl ::NAME; + +namespace StorageObjectStorageSetting +{ +LIST_OF_STORAGE_OBJECT_STORAGE_SETTINGS(INITIALIZE_SETTING_EXTERN, SKIP_ALIAS) +} + +#undef INITIALIZE_SETTING_EXTERN + +StorageObjectStorageSettings::StorageObjectStorageSettings() : impl(std::make_unique()) +{ +} + +StorageObjectStorageSettings::StorageObjectStorageSettings(const StorageObjectStorageSettings & settings) + : impl(std::make_unique(*settings.impl)) +{ +} + +StorageObjectStorageSettings::StorageObjectStorageSettings(StorageObjectStorageSettings && settings) noexcept + : impl(std::make_unique(std::move(*settings.impl))) +{ +} + + +void StorageObjectStorageSettings::dumpToSystemEngineSettingsColumns( + MutableColumnsAndConstraints & params, + const std::string & table_name, + const std::string & database_name, + const StorageObjectStorage & storage) const +{ + MutableColumns & res_columns = params.res_columns; + + /// We cannot use setting.isValueChanged(), because we do not store initial settings in storage. + /// Therefore check if the setting was changed via table metadata. + const auto & settings_changes = storage.getInMemoryMetadataPtr()->settings_changes->as()->changes; + auto is_changed = [&](const std::string & setting_name) -> bool + { + return settings_changes.end() + != std::find_if( + settings_changes.begin(), + settings_changes.end(), + [&](const SettingChange & change) { return change.name == setting_name; }); + }; + + for (const auto & change : impl->all()) + { + size_t i = 0; + res_columns[i++]->insert(database_name); + res_columns[i++]->insert(table_name); + res_columns[i++]->insert(change.getName()); + res_columns[i++]->insert(convertFieldToString(change.getValue())); + res_columns[i++]->insert(change.getTypeName()); + res_columns[i++]->insert(is_changed(change.getName())); + res_columns[i++]->insert(change.getDescription()); + res_columns[i++]->insert(false); + } +} + +StorageObjectStorageSettings::~StorageObjectStorageSettings() = default; + +STORAGE_OBJECT_STORAGE_SETTINGS_SUPPORTED_TYPES(StorageObjectStorageSettings, IMPLEMENT_SETTING_SUBSCRIPT_OPERATOR) + + +void StorageObjectStorageSettings::loadFromQuery(ASTStorage & storage_def) +{ + if (storage_def.settings) + { + impl->applyChanges(storage_def.settings->changes); + } +} + +Field StorageObjectStorageSettings::get(const std::string & name) +{ + return impl->get(name); +} + +} diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSettings.h b/src/Storages/ObjectStorage/StorageObjectStorageSettings.h new file mode 100644 index 00000000000..e58a7e3c184 --- /dev/null +++ b/src/Storages/ObjectStorage/StorageObjectStorageSettings.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ +class ASTStorage; +struct StorageObjectStorageSettingsImpl; +struct MutableColumnsAndConstraints; +class StorageObjectStorage; +class SettingsChanges; + +/// List of available types supported in StorageObjectStorageSettingsSettings object +#define STORAGE_OBJECT_STORAGE_SETTINGS_SUPPORTED_TYPES(CLASS_NAME, M) \ + M(CLASS_NAME, ArrowCompression) \ + M(CLASS_NAME, Bool) \ + M(CLASS_NAME, CapnProtoEnumComparingMode) \ + M(CLASS_NAME, Char) \ + M(CLASS_NAME, DateTimeInputFormat) \ + M(CLASS_NAME, DateTimeOutputFormat) \ + M(CLASS_NAME, DateTimeOverflowBehavior) \ + M(CLASS_NAME, Double) \ + M(CLASS_NAME, EscapingRule) \ + M(CLASS_NAME, Float) \ + M(CLASS_NAME, IdentifierQuotingRule) \ + M(CLASS_NAME, IdentifierQuotingStyle) \ + M(CLASS_NAME, Int64) \ + M(CLASS_NAME, IntervalOutputFormat) \ + M(CLASS_NAME, MsgPackUUIDRepresentation) \ + M(CLASS_NAME, ObjectStorageQueueAction) \ + M(CLASS_NAME, ObjectStorageQueueMode) \ + M(CLASS_NAME, ORCCompression) \ + M(CLASS_NAME, ParquetCompression) \ + M(CLASS_NAME, ParquetVersion) \ + M(CLASS_NAME, SchemaInferenceMode) \ + M(CLASS_NAME, String) \ + M(CLASS_NAME, UInt32) \ + M(CLASS_NAME, UInt64) \ + M(CLASS_NAME, UInt64Auto) \ + M(CLASS_NAME, URI) + +STORAGE_OBJECT_STORAGE_SETTINGS_SUPPORTED_TYPES(StorageObjectStorageSettings, DECLARE_SETTING_TRAIT) + +struct StorageObjectStorageSettings +{ + StorageObjectStorageSettings(); + StorageObjectStorageSettings(const StorageObjectStorageSettings & settings); + StorageObjectStorageSettings(StorageObjectStorageSettings && settings) noexcept; + ~StorageObjectStorageSettings(); + + STORAGE_OBJECT_STORAGE_SETTINGS_SUPPORTED_TYPES(StorageObjectStorageSettings, DECLARE_SETTING_SUBSCRIPT_OPERATOR) + + void dumpToSystemEngineSettingsColumns( + MutableColumnsAndConstraints & params, + const std::string & table_name, + const std::string & database_name, + const StorageObjectStorage & storage) const; + + void loadFromQuery(ASTStorage & storage_def); + + void applyChanges(const SettingsChanges & changes); + + Field get(const std::string & name); + +private: + std::unique_ptr impl; +}; + +} diff --git a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp index 0c8a88bfd18..c56038f441a 100644 --- a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp @@ -9,6 +9,7 @@ #include #include #include "Common/logger_useful.h" +#include "Storages/ObjectStorage/StorageObjectStorageSettings.h" namespace DB { @@ -31,7 +32,12 @@ createStorageObjectStorage(const StorageFactory::Arguments & args, StorageObject if (engine_args.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "External data source must have arguments"); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, context, false); + + auto queue_settings = std::make_unique(); + + queue_settings->loadFromQuery(*args.storage_def); + + StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, context, false, std::move(queue_settings)); // Use format settings from global server context + settings from // the SETTINGS clause of the create query. Settings from current @@ -165,12 +171,10 @@ void registerStorageIceberg(StorageFactory & factory) [&](const StorageFactory::Arguments & args) { auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - return createStorageObjectStorage(args, configuration, args.getLocalContext()); }, { - .supports_settings = false, + .supports_settings = true, .supports_schema_inference = true, .source_access_type = AccessType::S3, }); @@ -180,59 +184,51 @@ void registerStorageIceberg(StorageFactory & factory) [&](const StorageFactory::Arguments & args) { auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - return createStorageObjectStorage(args, configuration, args.getLocalContext()); }, { - .supports_settings = false, + .supports_settings = true, .supports_schema_inference = true, .source_access_type = AccessType::S3, }); -#endif -#if USE_AZURE_BLOB_STORAGE +# endif +# if USE_AZURE_BLOB_STORAGE factory.registerStorage( "IcebergAzure", [&](const StorageFactory::Arguments & args) { auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), true); - return createStorageObjectStorage(args, configuration, args.getLocalContext()); }, { - .supports_settings = false, + .supports_settings = true, .supports_schema_inference = true, .source_access_type = AccessType::AZURE, }); -#endif -#if USE_HDFS +# endif +# if USE_HDFS factory.registerStorage( "IcebergHDFS", [&](const StorageFactory::Arguments & args) { auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - return createStorageObjectStorage(args, configuration, args.getLocalContext()); }, { - .supports_settings = false, + .supports_settings = true, .supports_schema_inference = true, .source_access_type = AccessType::HDFS, }); -#endif +# endif factory.registerStorage( "IcebergLocal", [&](const StorageFactory::Arguments & args) { auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - return createStorageObjectStorage(args, configuration, args.getLocalContext()); }, { - .supports_settings = false, + .supports_settings = true, .supports_schema_inference = true, .source_access_type = AccessType::FILE, }); @@ -250,8 +246,6 @@ void registerStorageDeltaLake(StorageFactory & factory) [&](const StorageFactory::Arguments & args) { auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - return createStorageObjectStorage(args, configuration, args.getLocalContext()); }, { @@ -259,7 +253,7 @@ void registerStorageDeltaLake(StorageFactory & factory) .supports_schema_inference = true, .source_access_type = AccessType::S3, }); -#endif +# endif UNUSED(factory); } #endif @@ -272,8 +266,6 @@ void registerStorageHudi(StorageFactory & factory) [&](const StorageFactory::Arguments & args) { auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getLocalContext(), false); - return createStorageObjectStorage(args, configuration, args.getLocalContext()); }, { diff --git a/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp b/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp index b8e1480862b..f637a73ee7f 100644 --- a/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp +++ b/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp @@ -32,7 +32,7 @@ StoragePtr createQueueStorage(const StorageFactory::Arguments & args) throw Exception(ErrorCodes::BAD_ARGUMENTS, "External data source must have arguments"); auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getContext(), false); + StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getContext(), false, nullptr); // Use format settings from global server context + settings from // the SETTINGS clause of the create query. Settings from current diff --git a/src/TableFunctions/TableFunctionObjectStorage.h b/src/TableFunctions/TableFunctionObjectStorage.h index 19cd637bd80..08a4624ca0e 100644 --- a/src/TableFunctions/TableFunctionObjectStorage.h +++ b/src/TableFunctions/TableFunctionObjectStorage.h @@ -130,7 +130,7 @@ public: virtual void parseArgumentsImpl(ASTs & args, const ContextPtr & context) { - StorageObjectStorage::Configuration::initialize(*getConfiguration(), args, context, true); + StorageObjectStorage::Configuration::initialize(*getConfiguration(), args, context, true, nullptr); } static void updateStructureAndFormatArgumentsIfNeeded( diff --git a/tests/integration/test_storage_iceberg/test.py b/tests/integration/test_storage_iceberg/test.py index 1ef27da76a7..1c807b8c45e 100644 --- a/tests/integration/test_storage_iceberg/test.py +++ b/tests/integration/test_storage_iceberg/test.py @@ -182,8 +182,15 @@ def get_creation_expression( cluster, format="Parquet", table_function=False, + allow_dynamic_metadata_for_datalakes=False, **kwargs, ): + allow_dynamic_metadata_for_datalakes_suffix = ( + " SETTINGS allow_dynamic_metadata_for_datalakes = 1" + if allow_dynamic_metadata_for_datalakes + else "" + ) + if storage_type == "s3": if "bucket" in kwargs: bucket = kwargs["bucket"] @@ -193,49 +200,59 @@ def get_creation_expression( if table_function: return f"icebergS3(s3, filename = 'iceberg_data/default/{table_name}/', format={format}, url = 'http://minio1:9001/{bucket}/')" else: - return f""" + return ( + f""" DROP TABLE IF EXISTS {table_name}; CREATE TABLE {table_name} ENGINE=IcebergS3(s3, filename = 'iceberg_data/default/{table_name}/', format={format}, url = 'http://minio1:9001/{bucket}/')""" + + allow_dynamic_metadata_for_datalakes_suffix + ) elif storage_type == "azure": if table_function: return f""" icebergAzure(azure, container = '{cluster.azure_container_name}', storage_account_url = '{cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"]}', blob_path = '/iceberg_data/default/{table_name}/', format={format}) """ else: - return f""" + return ( + f""" DROP TABLE IF EXISTS {table_name}; CREATE TABLE {table_name} ENGINE=IcebergAzure(azure, container = {cluster.azure_container_name}, storage_account_url = '{cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"]}', blob_path = '/iceberg_data/default/{table_name}/', format={format})""" + + allow_dynamic_metadata_for_datalakes_suffix + ) elif storage_type == "hdfs": if table_function: return f""" icebergHDFS(hdfs, filename= 'iceberg_data/default/{table_name}/', format={format}, url = 'hdfs://hdfs1:9000/') """ else: - return f""" + return ( + f""" DROP TABLE IF EXISTS {table_name}; CREATE TABLE {table_name} - ENGINE=IcebergHDFS(hdfs, filename = 'iceberg_data/default/{table_name}/', format={format}, url = 'hdfs://hdfs1:9000/');""" + ENGINE=IcebergHDFS(hdfs, filename = 'iceberg_data/default/{table_name}/', format={format}, url = 'hdfs://hdfs1:9000/')""" + + allow_dynamic_metadata_for_datalakes_suffix + ) elif storage_type == "local": if table_function: return f""" icebergLocal(local, path = '/iceberg_data/default/{table_name}/', format={format}) """ else: - return f""" + return ( + f""" DROP TABLE IF EXISTS {table_name}; CREATE TABLE {table_name} - ENGINE=IcebergLocal(local, path = '/iceberg_data/default/{table_name}/', format={format});""" + ENGINE=IcebergLocal(local, path = '/iceberg_data/default/{table_name}/', format={format})""" + + allow_dynamic_metadata_for_datalakes_suffix + ) else: raise Exception(f"Unknown iceberg storage type: {storage_type}") -def check_schema_and_data( - instance, table_function_expression, expected_schema, expected_data -): - schema = instance.query(f"DESC {table_function_expression}") - data = instance.query(f"SELECT * FROM {table_function_expression} ORDER BY ALL") +def check_schema_and_data(instance, table_expression, expected_schema, expected_data): + schema = instance.query(f"DESC {table_expression}") + data = instance.query(f"SELECT * FROM {table_expression} ORDER BY ALL") schema = list( map( lambda x: x.split("\t")[:2], @@ -586,7 +603,10 @@ def test_delete_files(started_cluster, format_version, storage_type): @pytest.mark.parametrize("format_version", ["1", "2"]) @pytest.mark.parametrize("storage_type", ["s3", "azure", "hdfs", "local"]) -def test_evolved_schema(started_cluster, format_version, storage_type): +@pytest.mark.parametrize("is_table_function", [False, True]) +def test_evolved_schema_simple( + started_cluster, format_version, storage_type, is_table_function +): if is_arm() and storage_type == "hdfs": pytest.skip("Disabled test IcebergHDFS for aarch64") instance = started_cluster.instances["node1"] @@ -629,13 +649,24 @@ def test_evolved_schema(started_cluster, format_version, storage_type): """ ) - table_function = get_creation_expression( - storage_type, TABLE_NAME, started_cluster, table_function=True + table_creation_expression = get_creation_expression( + storage_type, + TABLE_NAME, + started_cluster, + table_function=is_table_function, + allow_dynamic_metadata_for_datalakes=True, ) + table_select_expression = ( + TABLE_NAME if not is_table_function else table_creation_expression + ) + + if not is_table_function: + instance.query(table_creation_expression) + check_schema_and_data( instance, - table_function, + table_select_expression, [ ["a", "Int32"], ["b", "Nullable(Float32)"], @@ -653,7 +684,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["a", "Int32"], ["b", "Nullable(Float32)"], @@ -671,7 +702,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["a", "Int32"], ["b", "Nullable(Float64)"], @@ -689,7 +720,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["a", "Int32"], ["b", "Nullable(Float64)"], @@ -707,7 +738,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["d", "Array(Nullable(Int32))"], ["a", "Int32"], @@ -725,7 +756,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["d", "Array(Nullable(Int32))"], ["b", "Nullable(Float64)"], @@ -746,7 +777,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["d", "Array(Nullable(Int32))"], ["b", "Nullable(Float64)"], @@ -768,7 +799,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["d", "Array(Nullable(Int32))"], ["b", "Nullable(Float64)"], @@ -790,7 +821,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["d", "Array(Nullable(Int32))"], ["b", "Nullable(Float64)"], @@ -813,7 +844,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["d", "Array(Nullable(Int32))"], ["b", "Nullable(Float64)"], @@ -836,7 +867,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["d", "Array(Nullable(Int32))"], ["b", "Nullable(Float64)"], @@ -860,7 +891,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["d", "Array(Nullable(Int32))"], ["b", "Nullable(Float64)"], @@ -884,7 +915,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["d", "Array(Nullable(Int32))"], ["b", "Nullable(Float64)"], @@ -909,7 +940,7 @@ def test_evolved_schema(started_cluster, format_version, storage_type): check_schema_and_data( instance, - table_function, + table_select_expression, [ ["b", "Nullable(Float64)"], ["a", "Nullable(Int64)"], @@ -926,6 +957,327 @@ def test_evolved_schema(started_cluster, format_version, storage_type): ) +@pytest.mark.parametrize("format_version", ["1", "2"]) +@pytest.mark.parametrize("storage_type", ["s3", "azure", "hdfs", "local"]) +def test_not_evolved_schema(started_cluster, format_version, storage_type): + if is_arm() and storage_type == "hdfs": + pytest.skip("Disabled test IcebergHDFS for aarch64") + instance = started_cluster.instances["node1"] + spark = started_cluster.spark_session + TABLE_NAME = ( + "test_evolved_schema_simple_" + + format_version + + "_" + + storage_type + + "_" + + get_uuid_str() + ) + + def execute_spark_query(query: str): + spark.sql(query) + default_upload_directory( + started_cluster, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + return + + execute_spark_query( + f""" + DROP TABLE IF EXISTS {TABLE_NAME}; + """ + ) + + execute_spark_query( + f""" + CREATE TABLE IF NOT EXISTS {TABLE_NAME} ( + a int NOT NULL, + b float, + c decimal(9,2) NOT NULL, + d array + ) + USING iceberg + OPTIONS ('format-version'='{format_version}') + """ + ) + + instance.query( + get_creation_expression( + storage_type, + TABLE_NAME, + started_cluster, + table_function=False, + allow_dynamic_metadata_for_datalakes=False, + ) + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [], + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES (4, 3.0, 7.12, ARRAY(5, 6, 7)); + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [["4", "3", "7.12", "[5,6,7]"]], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN b TYPE double; + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [["4", "3", "7.12", "[5,6,7]"]], + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES (7, 5.0, 18.1, ARRAY(6, 7, 9)); + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [["4", "3", "7.12", "[5,6,7]"], ["7", "5", "18.1", "[6,7,9]"]], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN d FIRST; + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [["4", "3", "7.12", "[5,6,7]"], ["7", "5", "18.1", "[6,7,9]"]], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN b AFTER d; + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [["4", "3", "7.12", "[5,6,7]"], ["7", "5", "18.1", "[6,7,9]"]], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} + ADD COLUMNS ( + e string + ); + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [["4", "3", "7.12", "[5,6,7]"], ["7", "5", "18.1", "[6,7,9]"]], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN c TYPE decimal(12, 2); + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [["4", "3", "7.12", "[5,6,7]"], ["7", "5", "18.1", "[6,7,9]"]], + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES (ARRAY(5, 6, 7), 3, -30, 7.12, 'AAA'); + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [ + ["-30", "3", "7.12", "[5,6,7]"], + ["4", "3", "7.12", "[5,6,7]"], + ["7", "5", "18.1", "[6,7,9]"], + ], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN a TYPE BIGINT; + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [ + ["-30", "3", "7.12", "[5,6,7]"], + ["4", "3", "7.12", "[5,6,7]"], + ["7", "5", "18.1", "[6,7,9]"], + ], + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES (ARRAY(), 3.0, 12, -9.13, 'BBB'); + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [ + ["-30", "3", "7.12", "[5,6,7]"], + ["4", "3", "7.12", "[5,6,7]"], + ["7", "5", "18.1", "[6,7,9]"], + ["12", "3", "-9.13", "[]"], + ], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} ALTER COLUMN a DROP NOT NULL; + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [ + ["-30", "3", "7.12", "[5,6,7]"], + ["4", "3", "7.12", "[5,6,7]"], + ["7", "5", "18.1", "[6,7,9]"], + ["12", "3", "-9.13", "[]"], + ], + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES (NULL, 3.4, NULL, -9.13, NULL); + """ + ) + + check_schema_and_data( + instance, + TABLE_NAME, + [ + ["a", "Int32"], + ["b", "Nullable(Float32)"], + ["c", "Decimal(9, 2)"], + ["d", "Array(Nullable(Int32))"], + ], + [ + ["-30", "3", "7.12", "[5,6,7]"], + ["0", "3.4", "-9.13", "[]"], + ["4", "3", "7.12", "[5,6,7]"], + ["7", "5", "18.1", "[6,7,9]"], + ["12", "3", "-9.13", "[]"], + ], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} DROP COLUMN d; + """ + ) + + error = instance.query_and_get_error(f"SELECT * FROM {TABLE_NAME} ORDER BY ALL") + + assert "Not found column" in error + + @pytest.mark.parametrize("format_version", ["1", "2"]) @pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) def test_evolved_schema_complex(started_cluster, format_version, storage_type): From 2aee85a776683bf15751ee40df469e62c42c5b27 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 14 Nov 2024 09:19:14 +0000 Subject: [PATCH 107/502] Fix style --- src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index b6cfaeb852b..497f27c4b3a 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -13,9 +13,9 @@ #include #include -# include -# include -# include +#include +#include +#include namespace DB { From bf7329fa7428ae235a30b9df74aa65281f49ebe3 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 14 Nov 2024 09:31:31 +0000 Subject: [PATCH 108/502] Added hashe for pair --- .../ObjectStorage/DataLakes/IcebergMetadata.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 497f27c4b3a..6f1058e3cf3 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -17,12 +17,22 @@ #include #include +template <> +struct std::hash> +{ + size_t operator()(const std::pair & p) const + { + std::hash hash; + return hash(static_cast(p.first) << 32 | static_cast(p.second)); + } +}; + namespace DB { namespace ErrorCodes { -extern const int BAD_ARGUMENTS; + extern const int BAD_ARGUMENTS; } /** @@ -88,7 +98,7 @@ public: private: std::unordered_map iceberg_table_schemas_by_ids; std::unordered_map> clickhouse_table_schemas_by_ids; - std::map, std::shared_ptr> transform_dags_by_ids; + std::unordered_map, std::shared_ptr> transform_dags_by_ids; NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr & schema); DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type); From 326c91c02b7157c5f2834ff03f77a5288af60956 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 14 Nov 2024 13:56:09 +0100 Subject: [PATCH 109/502] Fix test --- src/Databases/Iceberg/DatabaseIceberg.h | 2 +- .../integration/test_database_iceberg/test.py | 49 ++++++++++++------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.h b/src/Databases/Iceberg/DatabaseIceberg.h index 5a87e3179af..f7b7544f80a 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.h +++ b/src/Databases/Iceberg/DatabaseIceberg.h @@ -11,7 +11,7 @@ namespace DB { /// TODO: -/// - http basic auth for catalog +/// - auth: oauth, bearer token? /// - tests with azure, hdfs, local class DatabaseIceberg final : public IDatabase, WithContext diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 37d2bbf0ea8..310880c282a 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -58,7 +58,7 @@ DEFAULT_SCHEMA = Schema( ), ) -DEFAULT_CREATE_TABLE = "CREATE TABLE {}.`{}.{}`\\n(\\n `datetime` Nullable(DateTime64(6)),\\n `symbol` Nullable(String),\\n `bid` Nullable(Float64),\\n `ask` Nullable(Float64),\\n `details` Tuple(created_by Nullable(String))\\n)\\nENGINE = Iceberg(\\'http://minio:9000/warehouse/data\\', \\'minio\\', \\'[HIDDEN]\\')\n" +DEFAULT_CREATE_TABLE = "CREATE TABLE {}.`{}.{}`\\n(\\n `datetime` Nullable(DateTime64(6)),\\n `symbol` Nullable(String),\\n `bid` Nullable(Float64),\\n `ask` Nullable(Float64),\\n `details` Tuple(created_by Nullable(String))\\n)\\nENGINE = Iceberg(\\'http://minio:9000/warehouse/data/\\', \\'minio\\', \\'[HIDDEN]\\')\n" DEFAULT_PARTITION_SPEC = PartitionSpec( PartitionField( @@ -170,9 +170,9 @@ def started_cluster(): def test_list_tables(started_cluster): node = started_cluster.instances["node1"] - root_namespace = "clickhouse" - namespace_1 = "clickhouse.testA.A" - namespace_2 = "clickhouse.testB.B" + root_namespace = f"clickhouse_{uuid.uuid4()}" + namespace_1 = f"{root_namespace}.testA.A" + namespace_2 = f"{root_namespace}.testB.B" namespace_1_tables = ["tableA", "tableB"] namespace_2_tables = ["tableC", "tableD"] @@ -181,8 +181,19 @@ def test_list_tables(started_cluster): for namespace in [namespace_1, namespace_2]: catalog.create_namespace(namespace) - assert root_namespace in list_namespaces()["namespaces"][0][0] - assert [(root_namespace,)] == catalog.list_namespaces() + found = False + for namespace_list in list_namespaces()["namespaces"]: + if root_namespace == namespace_list[0]: + found = True + break + assert found + + found = False + for namespace_list in catalog.list_namespaces(): + if root_namespace == namespace_list[0]: + found = True + break + assert found for namespace in [namespace_1, namespace_2]: assert len(catalog.list_tables(namespace)) == 0 @@ -205,14 +216,14 @@ def test_list_tables(started_cluster): assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" ).strip() ) node.restart_clickhouse() assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" ).strip() ) @@ -224,16 +235,18 @@ def test_list_tables(started_cluster): def test_many_namespaces(started_cluster): node = started_cluster.instances["node1"] + root_namespace_1 = f"A_{uuid.uuid4()}" + root_namespace_2 = f"B_{uuid.uuid4()}" namespaces = [ - "A", - "A.B.C", - "A.B.C.D", - "A.B.C.D.E", - "A.B.C.D.E.F", - "A.B.C.D.E.FF", - "B", - "B.C", - "B.CC", + f"{root_namespace_1}", + f"{root_namespace_1}.B.C", + f"{root_namespace_1}.B.C.D", + f"{root_namespace_1}.B.C.D.E", + f"{root_namespace_1}.B.C.D.E.F", + f"{root_namespace_1}.B.C.D.E.FF", + f"{root_namespace_2}", + f"{root_namespace_2}.C", + f"{root_namespace_2}.CC", ] tables = ["A", "B", "C", "D", "E", "F"] catalog = load_catalog_impl(started_cluster) @@ -258,7 +271,7 @@ def test_many_namespaces(started_cluster): def test_select(started_cluster): node = started_cluster.instances["node1"] - test_ref = "test_list_tables" + test_ref = f"test_list_tables_{uuid.uuid4()}" table_name = f"{test_ref}_table" root_namespace = f"{test_ref}_namespace" From 0b76015272d75fae705514393534307fdff14385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 6 Nov 2024 16:04:05 +0000 Subject: [PATCH 110/502] Extract common expressions in `WHERE` --- .../Passes/LogicalExpressionOptimizerPass.cpp | 289 +++++++++++++++++- .../Passes/LogicalExpressionOptimizerPass.h | 2 +- 2 files changed, 282 insertions(+), 9 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index e147ad68e40..24717164e4b 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -1,18 +1,16 @@ #include -#include - -#include -#include #include -#include +#include #include +#include +#include #include #include - #include -#include #include +#include +#include namespace DB { @@ -173,6 +171,16 @@ bool isTwoArgumentsFromDifferentSides(const FunctionNode & node_function, const (first_src->isEqual(rhs_join) && second_src->isEqual(lhs_join)); } +void extractNodesFromSetInto(QueryTreeNodePtrWithHashSet && set, QueryTreeNodes & into) +{ + for (auto it = set.begin(); it != set.end();) + { + auto it_to_extract = it++; + auto nh = set.extract(it_to_extract); + into.push_back(std::move(nh.value().node)); + } +} + /// Visitor that optimizes logical expressions _only_ in JOIN ON section class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext { @@ -385,7 +393,7 @@ private: } else if (and_arguments.size() == 1) { - /// Replace AND with a single argument with the argument itself + /// Replace AND with a single argument by the argument itself new_or_operands.emplace_back(and_arguments[0]); } } @@ -459,6 +467,26 @@ public: } } + void leaveImpl(QueryTreeNodePtr & node) + { + auto * query_node = node->as(); + if (!query_node) + return; + + auto & where_node = query_node->getWhere(); + if (!where_node) + return; + + auto * function_node = where_node->as(); + if (!function_node) + return; + + if (function_node->getFunctionName() == "or") + tryOptimizeCommonExpressionsInOr(where_node); + else if (function_node->getFunctionName() == "and") + tryOptimizeCommonExpressionsInAnd(where_node); + } + private: void tryOptimizeAndEqualsNotEqualsChain(QueryTreeNodePtr & node) { @@ -629,6 +657,251 @@ private: function_node.resolveAsFunction(and_function_resolver); } + // Returns the flattened AND/OR node if the passed-in node can be flattened. Doesn't modify the passed-in node. + std::shared_ptr getFlattenedAndOrOr(const FunctionNode & node) + { + const auto & function_name = node.getFunctionName(); + if (function_name != "or" && function_name != "and") + return nullptr; + + const auto & arguments = node.getArguments().getNodes(); + QueryTreeNodePtrWithHashMap arguments_to_replace; + + for (const auto & argument : arguments) + { + auto * maybe_function = argument->as(); + if (!maybe_function || maybe_function->getFunctionName() != function_name) + continue; + + auto maybe_flattened = getFlattenedAndOrOr(*maybe_function); + if (maybe_flattened) + arguments_to_replace.emplace(argument, std::move(maybe_flattened->getArguments().getNodes())); + else + arguments_to_replace.emplace(argument, maybe_function->getArguments().getNodes()); + } + + if (arguments_to_replace.empty()) + return nullptr; // nothing to flatten + + auto flattened = std::make_shared(function_name); + + auto & new_arguments = flattened->getArguments().getNodes(); + new_arguments.reserve(arguments.size()); + for (const auto & argument : arguments) + { + if (auto it = arguments_to_replace.find(argument); it != arguments_to_replace.end()) + new_arguments.insert( + new_arguments.end(), std::make_move_iterator(it->second.begin()), std::make_move_iterator(it->second.end())); + else + new_arguments.push_back(argument); + } + + auto function_resolver = FunctionFactory::instance().get(function_name, getContext()); + flattened->resolveAsFunction(function_resolver); + + return flattened; + } + + struct CommonExpressionExtractionResult + { + // new_node: if the new node is empty, then contain the new node, otherwise nullptr + // common_expressions: the extracted common expressions. The new expressions can be created + // as the conjunction of new_node and the nodes in common_expressions. + // Examples: + // Input: (A & B & C) | (A & D & E) + // Result: new_node = (B & C) | (D & E), common_expressions = {A} + // + // Input: (A & B) | (A & B) + // Result: new_node = nullptr, common_expressions = {A, B} + // + // This is a special case: A & B & C is a subset of A & B, thus the conjunction of extracted + // expressions is equivalent with the passed-in expression, we have to discard C. With C the + // new expression would be more restrictive. + // Input: (A & B) | (A & B & C) + // Result: new_node = nullptr, common_expressions = {A, B} + QueryTreeNodePtr new_node; + QueryTreeNodePtrWithHashSet common_expressions; + }; + + std::optional tryExtractCommonExpressions(const QueryTreeNodePtr & node) + { + auto * or_node = node->as(); + if (!or_node || or_node->getFunctionName() != "or") + return {}; // the optimization can only be done on or nodes + + auto flattened_or_node = getFlattenedAndOrOr(*or_node); + if (flattened_or_node) + or_node = flattened_or_node.get(); + + auto & or_argument_nodes = or_node->getArguments().getNodes(); + + chassert(or_argument_nodes.size() > 1); + + bool first_argument = true; + QueryTreeNodePtrWithHashSet common_exprs; + QueryTreeNodePtrWithHashMap flattened_ands; + + for (auto & maybe_and_node : or_argument_nodes) + { + auto * and_node = maybe_and_node->as(); + if (!and_node || and_node->getFunctionName() != "and") + return {}; // one of the nodes is not an "AND", thus there is no common expression to extract + + auto flattened_and_node = getFlattenedAndOrOr(*and_node); + if (flattened_and_node) + { + flattened_ands.emplace(maybe_and_node, flattened_and_node); + and_node = flattened_and_node.get(); + } + + if (first_argument) + { + auto & current_arguments = and_node->getArguments().getNodes(); + common_exprs.reserve(current_arguments.size()); + + for (auto & or_argument : current_arguments) + { + QueryTreeNodePtrWithHash ptr_with_hash{or_argument}; + common_exprs.insert(ptr_with_hash); + } + first_argument = false; + } + else + { + QueryTreeNodePtrWithHashSet new_common_expr_hashes; + + for (auto & or_argument : and_node->getArguments()) + { + if (auto ptr_with_hash = QueryTreeNodePtrWithHash{or_argument}; common_exprs.contains(ptr_with_hash)) + new_common_expr_hashes.insert(ptr_with_hash); + } + + common_exprs = std::move(new_common_expr_hashes); + + if (common_exprs.empty()) + return {}; // There are no common expressions + } + } + + chassert(!common_exprs.empty()); + + QueryTreeNodePtrWithHashSet new_or_arguments; + bool has_completely_extracted_and_expression = false; + + for (auto & or_argument : or_argument_nodes) + { + if (auto it = flattened_ands.find(or_argument); it != flattened_ands.end()) + or_argument = it->second; + + auto & and_node = or_argument->as(); + auto & and_arguments = and_node.getArguments().getNodes(); + and_arguments.erase( + std::remove_if( + and_arguments.begin(), + and_arguments.end(), + [&common_exprs](const QueryTreeNodePtr & ptr) + { + QueryTreeNodeWithHash ptr_with_hash{ptr}; + return common_exprs.contains(ptr_with_hash); + }), + and_arguments.end()); + + if (and_arguments.empty()) + { + has_completely_extracted_and_expression = true; + // As we will discard new_or_arguments, no need for further processing + break; + } + else if (and_arguments.size() == 1) + { + new_or_arguments.emplace(std::move(and_arguments.front())); + } + else + { + auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); + and_node.resolveAsFunction(and_function_resolver); + new_or_arguments.emplace(or_argument); + } + } + + // If all the arguments of the OR expression is eliminated or one argument is completely eliminated, there is no need for new node. + if (new_or_arguments.empty() || has_completely_extracted_and_expression) + return CommonExpressionExtractionResult{nullptr, std::move(common_exprs)}; + + // There are at least two arguments in the passed-in OR expression, thus we either completely eliminated at least one arguments, or there should be at least 2 remaining arguments. + // The complete elimination is handled above, so at this point we can be sure there are at least 2 arguments. + chassert(new_or_arguments.size() >= 2); + + auto new_or_node = std::make_shared("or"); + extractNodesFromSetInto(std::move(new_or_arguments), new_or_node->getArguments().getNodes()); + + auto or_function_resolver = FunctionFactory::instance().get("or", getContext()); + new_or_node->resolveAsFunction(or_function_resolver); + + return CommonExpressionExtractionResult{new_or_node, common_exprs}; + } + + void tryOptimizeCommonExpressionsInOr(QueryTreeNodePtr & node) + { + auto * root_node = node->as(); + chassert(root_node && root_node->getFunctionName() == "or"); + + if (auto maybe_result = tryExtractCommonExpressions(node); maybe_result.has_value()) + { + auto & result = *maybe_result; + auto & root_arguments = root_node->getArguments().getNodes(); + root_arguments.clear(); + if (result.new_node != nullptr) + root_arguments.push_back(std::move(result.new_node)); + extractNodesFromSetInto(std::move(result.common_expressions), root_arguments); + + if (root_arguments.size() == 1) + { + node = std::move(root_arguments[0]); + return; + } + + auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); + root_node->resolveAsFunction(and_function_resolver); + } + } + + void tryOptimizeCommonExpressionsInAnd(QueryTreeNodePtr & node) + { + auto * root_node = node->as(); + chassert(root_node && root_node->getFunctionName() == "and"); + + QueryTreeNodePtrWithHashSet new_top_level_arguments; + auto extracted_something = false; + auto & root_arguments = root_node->getArguments(); + + for (const auto & argument : root_node->getArguments()) + { + if (auto maybe_result = tryExtractCommonExpressions(argument); maybe_result.has_value()) + { + extracted_something = true; + auto & result = *maybe_result; + if (result.new_node != nullptr) + new_top_level_arguments.emplace(std::move(result.new_node)); + new_top_level_arguments.merge(std::move(result.common_expressions)); + } + else + { + new_top_level_arguments.emplace(argument); + } + } + + if (!extracted_something) + return; + + auto & root_argument_nodes = root_arguments.getNodes(); + root_argument_nodes.clear(); + extractNodesFromSetInto(std::move(new_top_level_arguments), root_argument_nodes); + + auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); + root_node->resolveAsFunction(and_function_resolver); + } + void tryReplaceOrEqualsChainWithIn(QueryTreeNodePtr & node) { auto & function_node = node->as(); diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h index 5f109993f3f..dace434984c 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h @@ -98,7 +98,7 @@ namespace DB * ------------------------------- * * 7. Remove redundant equality checks on boolean functions. - * - these requndant checks cause the primary index to not be used when if the query involves any primary key columns + * - these redundant checks cause the primary index to not be used when if the query involves any primary key columns * ------------------------------- * SELECT * FROM t1 WHERE a IN (n) = 1 * SELECT * FROM t1 WHERE a IN (n) = 0 From b77501713b88452c3461d9a1e77b1c923c315644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 6 Nov 2024 16:49:14 +0000 Subject: [PATCH 111/502] Add tests to `WHERE` --- ...2_common_expression_optimization.reference | 273 ++++++++++++++++++ .../03262_common_expression_optimization.sql | 58 ++++ 2 files changed, 331 insertions(+) create mode 100644 tests/queries/0_stateless/03262_common_expression_optimization.reference create mode 100644 tests/queries/0_stateless/03262_common_expression_optimization.sql diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference new file mode 100644 index 00000000000..5fa9f03a482 --- /dev/null +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -0,0 +1,273 @@ +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 3 + COLUMN id: 6, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 7, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 8, column_name: A, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 4 + COLUMN id: 6, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 8, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 9, nodes: 2 + COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 4 + COLUMN id: 6, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 8, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 9, nodes: 2 + COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 3 + COLUMN id: 6, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 7, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 3 + COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 3 + COLUMN id: 6, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 7, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 3 + COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 3 + FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: F, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 3 + FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: F, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 3 + COLUMN id: 8, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: D, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: B, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 3 + COLUMN id: 8, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: D, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: B, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: A, result_type: UInt8, source_id: 3 + FUNCTION id: 10, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + FUNCTION id: 12, function_name: sipHash64, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 13, nodes: 1 + COLUMN id: 14, column_name: C, result_type: UInt8, source_id: 3 + FUNCTION id: 15, function_name: sipHash64, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 16, nodes: 1 + COLUMN id: 17, column_name: D, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: A, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 10, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + FUNCTION id: 12, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 2 + COLUMN id: 14, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 15, column_name: D, result_type: UInt8, source_id: 3 + FUNCTION id: 16, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 17, nodes: 2 + COLUMN id: 18, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 19, column_name: F, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: A, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 10, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + FUNCTION id: 12, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 2 + COLUMN id: 9, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 14, column_name: D, result_type: UInt8, source_id: 3 + FUNCTION id: 15, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 16, nodes: 2 + COLUMN id: 17, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 18, column_name: F, result_type: UInt8, source_id: 3 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql new file mode 100644 index 00000000000..1fda7804f02 --- /dev/null +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -0,0 +1,58 @@ +DROP TABLE IF EXISTS x; +CREATE TABLE x (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) ENGINE = MergeTree ORDER BY x; +INSERT INTO x SELECT * FROM generateRandom('x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8') LIMIT 10000; + +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE A AND ((B AND C) OR (B AND C AND F)); + +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE A AND ((B AND C AND E) OR (B AND C AND F)); + +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)); + +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)); + +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))); + +-- Without AND as a root +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE ((B AND C) OR (B AND C AND F)); + +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE ((B AND C AND E) OR (B AND C AND F)); + +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE ((B AND (C AND E)) OR (B AND C AND F)); + +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE ((B AND C) OR (B AND D) OR (B AND E)); + +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE ((B AND C) OR ((B AND D) OR (B AND E))); + +-- Complex expression +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))); + +-- Flattening is only happening if something can be extracted +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE ((A AND B) OR ((C AND D) OR (E AND F))); + +EXPLAIN QUERY TREE SELECT count() +FROM x +WHERE ((A AND B) OR ((B AND D) OR (E AND F))); From 752527dd788305e83502976bfd14599473fafcc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 6 Nov 2024 17:00:19 +0000 Subject: [PATCH 112/502] Add test case with duplicate expressions --- ...2_common_expression_optimization.reference | 28 +++++++++++++++++++ .../03262_common_expression_optimization.sql | 6 ++++ 2 files changed, 34 insertions(+) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index 5fa9f03a482..2f17a4da831 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -271,3 +271,31 @@ QUERY id: 0 LIST id: 16, nodes: 2 COLUMN id: 17, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 18, column_name: F, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 3 + FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 3 + FUNCTION id: 8, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 9, nodes: 2 + COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 + FUNCTION id: 11, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 12, nodes: 2 + COLUMN id: 13, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 13, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 14, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 15, column_name: A, result_type: UInt8, source_id: 3 + COLUMN id: 16, column_name: B, result_type: UInt8, source_id: 3 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index 1fda7804f02..fe7a18cfe02 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -56,3 +56,9 @@ WHERE ((A AND B) OR ((C AND D) OR (E AND F))); EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))); + +-- Duplicates +EXPLAIN QUERY TREE +SELECT count() +FROM x +WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) From 3401c820050440fb592e40acf3f247a5738f3c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 6 Nov 2024 22:07:13 +0000 Subject: [PATCH 113/502] Optimize also `ON` expressions --- .../Passes/LogicalExpressionOptimizerPass.cpp | 506 +++++++++--------- ...2_common_expression_optimization.reference | 34 ++ .../03262_common_expression_optimization.sql | 12 +- 3 files changed, 302 insertions(+), 250 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 24717164e4b..f5f1049f4d8 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -181,6 +181,252 @@ void extractNodesFromSetInto(QueryTreeNodePtrWithHashSet && set, QueryTreeNodes } } +// Returns the flattened AND/OR node if the passed-in node can be flattened. Doesn't modify the passed-in node. +std::shared_ptr getFlattenedAndOrOr(const FunctionNode & node, const ContextPtr & context) +{ + const auto & function_name = node.getFunctionName(); + if (function_name != "or" && function_name != "and") + return nullptr; + + const auto & arguments = node.getArguments().getNodes(); + QueryTreeNodePtrWithHashMap arguments_to_replace; + + for (const auto & argument : arguments) + { + auto * maybe_function = argument->as(); + if (!maybe_function || maybe_function->getFunctionName() != function_name) + continue; + + auto maybe_flattened = getFlattenedAndOrOr(*maybe_function, context); + if (maybe_flattened) + arguments_to_replace.emplace(argument, std::move(maybe_flattened->getArguments().getNodes())); + else + arguments_to_replace.emplace(argument, maybe_function->getArguments().getNodes()); + } + + if (arguments_to_replace.empty()) + return nullptr; // nothing to flatten + + auto flattened = std::make_shared(function_name); + + auto & new_arguments = flattened->getArguments().getNodes(); + new_arguments.reserve(arguments.size()); + for (const auto & argument : arguments) + { + if (auto it = arguments_to_replace.find(argument); it != arguments_to_replace.end()) + new_arguments.insert( + new_arguments.end(), std::make_move_iterator(it->second.begin()), std::make_move_iterator(it->second.end())); + else + new_arguments.push_back(argument); + } + + auto function_resolver = FunctionFactory::instance().get(function_name, context); + flattened->resolveAsFunction(function_resolver); + + return flattened; +} + +struct CommonExpressionExtractionResult +{ + // new_node: if the new node is empty, then contain the new node, otherwise nullptr + // common_expressions: the extracted common expressions. The new expressions can be created + // as the conjunction of new_node and the nodes in common_expressions. + // Examples: + // Input: (A & B & C) | (A & D & E) + // Result: new_node = (B & C) | (D & E), common_expressions = {A} + // + // Input: (A & B) | (A & B) + // Result: new_node = nullptr, common_expressions = {A, B} + // + // This is a special case: A & B & C is a subset of A & B, thus the conjunction of extracted + // expressions is equivalent with the passed-in expression, we have to discard C. With C the + // new expression would be more restrictive. + // Input: (A & B) | (A & B & C) + // Result: new_node = nullptr, common_expressions = {A, B} + QueryTreeNodePtr new_node; + QueryTreeNodePtrWithHashSet common_expressions; +}; + +std::optional tryExtractCommonExpressions(const QueryTreeNodePtr & node, const ContextPtr & context) +{ + auto * or_node = node->as(); + if (!or_node || or_node->getFunctionName() != "or") + return {}; // the optimization can only be done on or nodes + + auto flattened_or_node = getFlattenedAndOrOr(*or_node, context); + if (flattened_or_node) + or_node = flattened_or_node.get(); + + auto & or_argument_nodes = or_node->getArguments().getNodes(); + + chassert(or_argument_nodes.size() > 1); + + bool first_argument = true; + QueryTreeNodePtrWithHashSet common_exprs; + QueryTreeNodePtrWithHashMap flattened_ands; + + for (auto & maybe_and_node : or_argument_nodes) + { + auto * and_node = maybe_and_node->as(); + if (!and_node || and_node->getFunctionName() != "and") + return {}; // one of the nodes is not an "AND", thus there is no common expression to extract + + auto flattened_and_node = getFlattenedAndOrOr(*and_node, context); + if (flattened_and_node) + { + flattened_ands.emplace(maybe_and_node, flattened_and_node); + and_node = flattened_and_node.get(); + } + + if (first_argument) + { + auto & current_arguments = and_node->getArguments().getNodes(); + common_exprs.reserve(current_arguments.size()); + + for (auto & or_argument : current_arguments) + { + QueryTreeNodePtrWithHash ptr_with_hash{or_argument}; + common_exprs.insert(ptr_with_hash); + } + first_argument = false; + } + else + { + QueryTreeNodePtrWithHashSet new_common_expr_hashes; + + for (auto & or_argument : and_node->getArguments()) + { + if (auto ptr_with_hash = QueryTreeNodePtrWithHash{or_argument}; common_exprs.contains(ptr_with_hash)) + new_common_expr_hashes.insert(ptr_with_hash); + } + + common_exprs = std::move(new_common_expr_hashes); + + if (common_exprs.empty()) + return {}; // There are no common expressions + } + } + + chassert(!common_exprs.empty()); + + QueryTreeNodePtrWithHashSet new_or_arguments; + bool has_completely_extracted_and_expression = false; + + for (auto & or_argument : or_argument_nodes) + { + if (auto it = flattened_ands.find(or_argument); it != flattened_ands.end()) + or_argument = it->second; + + auto & and_node = or_argument->as(); + auto & and_arguments = and_node.getArguments().getNodes(); + and_arguments.erase( + std::remove_if( + and_arguments.begin(), + and_arguments.end(), + [&common_exprs](const QueryTreeNodePtr & ptr) + { + QueryTreeNodeWithHash ptr_with_hash{ptr}; + return common_exprs.contains(ptr_with_hash); + }), + and_arguments.end()); + + if (and_arguments.empty()) + { + has_completely_extracted_and_expression = true; + // As we will discard new_or_arguments, no need for further processing + break; + } + else if (and_arguments.size() == 1) + { + new_or_arguments.emplace(std::move(and_arguments.front())); + } + else + { + auto and_function_resolver = FunctionFactory::instance().get("and", context); + and_node.resolveAsFunction(and_function_resolver); + new_or_arguments.emplace(or_argument); + } + } + + // If all the arguments of the OR expression is eliminated or one argument is completely eliminated, there is no need for new node. + if (new_or_arguments.empty() || has_completely_extracted_and_expression) + return CommonExpressionExtractionResult{nullptr, std::move(common_exprs)}; + + // There are at least two arguments in the passed-in OR expression, thus we either completely eliminated at least one arguments, or there should be at least 2 remaining arguments. + // The complete elimination is handled above, so at this point we can be sure there are at least 2 arguments. + chassert(new_or_arguments.size() >= 2); + + auto new_or_node = std::make_shared("or"); + extractNodesFromSetInto(std::move(new_or_arguments), new_or_node->getArguments().getNodes()); + + auto or_function_resolver = FunctionFactory::instance().get("or", context); + new_or_node->resolveAsFunction(or_function_resolver); + + return CommonExpressionExtractionResult{new_or_node, common_exprs}; +} + +void tryOptimizeCommonExpressionsInOr(QueryTreeNodePtr & node, const ContextPtr & context) +{ + auto * root_node = node->as(); + chassert(root_node && root_node->getFunctionName() == "or"); + + if (auto maybe_result = tryExtractCommonExpressions(node, context); maybe_result.has_value()) + { + auto & result = *maybe_result; + auto & root_arguments = root_node->getArguments().getNodes(); + root_arguments.clear(); + if (result.new_node != nullptr) + root_arguments.push_back(std::move(result.new_node)); + extractNodesFromSetInto(std::move(result.common_expressions), root_arguments); + + if (root_arguments.size() == 1) + { + node = std::move(root_arguments[0]); + return; + } + + auto and_function_resolver = FunctionFactory::instance().get("and", context); + root_node->resolveAsFunction(and_function_resolver); + } +} + +void tryOptimizeCommonExpressionsInAnd(QueryTreeNodePtr & node, const ContextPtr & context) +{ + auto * root_node = node->as(); + chassert(root_node && root_node->getFunctionName() == "and"); + + QueryTreeNodePtrWithHashSet new_top_level_arguments; + auto extracted_something = false; + auto & root_arguments = root_node->getArguments(); + + for (const auto & argument : root_node->getArguments()) + { + if (auto maybe_result = tryExtractCommonExpressions(argument, context); maybe_result.has_value()) + { + extracted_something = true; + auto & result = *maybe_result; + if (result.new_node != nullptr) + new_top_level_arguments.emplace(std::move(result.new_node)); + new_top_level_arguments.merge(std::move(result.common_expressions)); + } + else + { + new_top_level_arguments.emplace(argument); + } + } + + if (!extracted_something) + return; + + auto & root_argument_nodes = root_arguments.getNodes(); + root_argument_nodes.clear(); + extractNodesFromSetInto(std::move(new_top_level_arguments), root_argument_nodes); + + auto and_function_resolver = FunctionFactory::instance().get("and", context); + root_node->resolveAsFunction(and_function_resolver); +} + + /// Visitor that optimizes logical expressions _only_ in JOIN ON section class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext { @@ -231,11 +477,18 @@ public: void leaveImpl(QueryTreeNodePtr & node) { - if (!need_rerun_resolve) + auto * function_node = node->as(); + + if (!function_node) return; - if (auto * function_node = node->as()) + if (need_rerun_resolve) rerunFunctionResolve(function_node, getContext()); + + if (function_node->getFunctionName() == "or") + tryOptimizeCommonExpressionsInOr(node, getContext()); + else if (function_node->getFunctionName() == "and") + tryOptimizeCommonExpressionsInAnd(node, getContext()); } private: @@ -482,9 +735,9 @@ public: return; if (function_node->getFunctionName() == "or") - tryOptimizeCommonExpressionsInOr(where_node); + tryOptimizeCommonExpressionsInOr(where_node, getContext()); else if (function_node->getFunctionName() == "and") - tryOptimizeCommonExpressionsInAnd(where_node); + tryOptimizeCommonExpressionsInAnd(where_node, getContext()); } private: @@ -657,251 +910,6 @@ private: function_node.resolveAsFunction(and_function_resolver); } - // Returns the flattened AND/OR node if the passed-in node can be flattened. Doesn't modify the passed-in node. - std::shared_ptr getFlattenedAndOrOr(const FunctionNode & node) - { - const auto & function_name = node.getFunctionName(); - if (function_name != "or" && function_name != "and") - return nullptr; - - const auto & arguments = node.getArguments().getNodes(); - QueryTreeNodePtrWithHashMap arguments_to_replace; - - for (const auto & argument : arguments) - { - auto * maybe_function = argument->as(); - if (!maybe_function || maybe_function->getFunctionName() != function_name) - continue; - - auto maybe_flattened = getFlattenedAndOrOr(*maybe_function); - if (maybe_flattened) - arguments_to_replace.emplace(argument, std::move(maybe_flattened->getArguments().getNodes())); - else - arguments_to_replace.emplace(argument, maybe_function->getArguments().getNodes()); - } - - if (arguments_to_replace.empty()) - return nullptr; // nothing to flatten - - auto flattened = std::make_shared(function_name); - - auto & new_arguments = flattened->getArguments().getNodes(); - new_arguments.reserve(arguments.size()); - for (const auto & argument : arguments) - { - if (auto it = arguments_to_replace.find(argument); it != arguments_to_replace.end()) - new_arguments.insert( - new_arguments.end(), std::make_move_iterator(it->second.begin()), std::make_move_iterator(it->second.end())); - else - new_arguments.push_back(argument); - } - - auto function_resolver = FunctionFactory::instance().get(function_name, getContext()); - flattened->resolveAsFunction(function_resolver); - - return flattened; - } - - struct CommonExpressionExtractionResult - { - // new_node: if the new node is empty, then contain the new node, otherwise nullptr - // common_expressions: the extracted common expressions. The new expressions can be created - // as the conjunction of new_node and the nodes in common_expressions. - // Examples: - // Input: (A & B & C) | (A & D & E) - // Result: new_node = (B & C) | (D & E), common_expressions = {A} - // - // Input: (A & B) | (A & B) - // Result: new_node = nullptr, common_expressions = {A, B} - // - // This is a special case: A & B & C is a subset of A & B, thus the conjunction of extracted - // expressions is equivalent with the passed-in expression, we have to discard C. With C the - // new expression would be more restrictive. - // Input: (A & B) | (A & B & C) - // Result: new_node = nullptr, common_expressions = {A, B} - QueryTreeNodePtr new_node; - QueryTreeNodePtrWithHashSet common_expressions; - }; - - std::optional tryExtractCommonExpressions(const QueryTreeNodePtr & node) - { - auto * or_node = node->as(); - if (!or_node || or_node->getFunctionName() != "or") - return {}; // the optimization can only be done on or nodes - - auto flattened_or_node = getFlattenedAndOrOr(*or_node); - if (flattened_or_node) - or_node = flattened_or_node.get(); - - auto & or_argument_nodes = or_node->getArguments().getNodes(); - - chassert(or_argument_nodes.size() > 1); - - bool first_argument = true; - QueryTreeNodePtrWithHashSet common_exprs; - QueryTreeNodePtrWithHashMap flattened_ands; - - for (auto & maybe_and_node : or_argument_nodes) - { - auto * and_node = maybe_and_node->as(); - if (!and_node || and_node->getFunctionName() != "and") - return {}; // one of the nodes is not an "AND", thus there is no common expression to extract - - auto flattened_and_node = getFlattenedAndOrOr(*and_node); - if (flattened_and_node) - { - flattened_ands.emplace(maybe_and_node, flattened_and_node); - and_node = flattened_and_node.get(); - } - - if (first_argument) - { - auto & current_arguments = and_node->getArguments().getNodes(); - common_exprs.reserve(current_arguments.size()); - - for (auto & or_argument : current_arguments) - { - QueryTreeNodePtrWithHash ptr_with_hash{or_argument}; - common_exprs.insert(ptr_with_hash); - } - first_argument = false; - } - else - { - QueryTreeNodePtrWithHashSet new_common_expr_hashes; - - for (auto & or_argument : and_node->getArguments()) - { - if (auto ptr_with_hash = QueryTreeNodePtrWithHash{or_argument}; common_exprs.contains(ptr_with_hash)) - new_common_expr_hashes.insert(ptr_with_hash); - } - - common_exprs = std::move(new_common_expr_hashes); - - if (common_exprs.empty()) - return {}; // There are no common expressions - } - } - - chassert(!common_exprs.empty()); - - QueryTreeNodePtrWithHashSet new_or_arguments; - bool has_completely_extracted_and_expression = false; - - for (auto & or_argument : or_argument_nodes) - { - if (auto it = flattened_ands.find(or_argument); it != flattened_ands.end()) - or_argument = it->second; - - auto & and_node = or_argument->as(); - auto & and_arguments = and_node.getArguments().getNodes(); - and_arguments.erase( - std::remove_if( - and_arguments.begin(), - and_arguments.end(), - [&common_exprs](const QueryTreeNodePtr & ptr) - { - QueryTreeNodeWithHash ptr_with_hash{ptr}; - return common_exprs.contains(ptr_with_hash); - }), - and_arguments.end()); - - if (and_arguments.empty()) - { - has_completely_extracted_and_expression = true; - // As we will discard new_or_arguments, no need for further processing - break; - } - else if (and_arguments.size() == 1) - { - new_or_arguments.emplace(std::move(and_arguments.front())); - } - else - { - auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); - and_node.resolveAsFunction(and_function_resolver); - new_or_arguments.emplace(or_argument); - } - } - - // If all the arguments of the OR expression is eliminated or one argument is completely eliminated, there is no need for new node. - if (new_or_arguments.empty() || has_completely_extracted_and_expression) - return CommonExpressionExtractionResult{nullptr, std::move(common_exprs)}; - - // There are at least two arguments in the passed-in OR expression, thus we either completely eliminated at least one arguments, or there should be at least 2 remaining arguments. - // The complete elimination is handled above, so at this point we can be sure there are at least 2 arguments. - chassert(new_or_arguments.size() >= 2); - - auto new_or_node = std::make_shared("or"); - extractNodesFromSetInto(std::move(new_or_arguments), new_or_node->getArguments().getNodes()); - - auto or_function_resolver = FunctionFactory::instance().get("or", getContext()); - new_or_node->resolveAsFunction(or_function_resolver); - - return CommonExpressionExtractionResult{new_or_node, common_exprs}; - } - - void tryOptimizeCommonExpressionsInOr(QueryTreeNodePtr & node) - { - auto * root_node = node->as(); - chassert(root_node && root_node->getFunctionName() == "or"); - - if (auto maybe_result = tryExtractCommonExpressions(node); maybe_result.has_value()) - { - auto & result = *maybe_result; - auto & root_arguments = root_node->getArguments().getNodes(); - root_arguments.clear(); - if (result.new_node != nullptr) - root_arguments.push_back(std::move(result.new_node)); - extractNodesFromSetInto(std::move(result.common_expressions), root_arguments); - - if (root_arguments.size() == 1) - { - node = std::move(root_arguments[0]); - return; - } - - auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); - root_node->resolveAsFunction(and_function_resolver); - } - } - - void tryOptimizeCommonExpressionsInAnd(QueryTreeNodePtr & node) - { - auto * root_node = node->as(); - chassert(root_node && root_node->getFunctionName() == "and"); - - QueryTreeNodePtrWithHashSet new_top_level_arguments; - auto extracted_something = false; - auto & root_arguments = root_node->getArguments(); - - for (const auto & argument : root_node->getArguments()) - { - if (auto maybe_result = tryExtractCommonExpressions(argument); maybe_result.has_value()) - { - extracted_something = true; - auto & result = *maybe_result; - if (result.new_node != nullptr) - new_top_level_arguments.emplace(std::move(result.new_node)); - new_top_level_arguments.merge(std::move(result.common_expressions)); - } - else - { - new_top_level_arguments.emplace(argument); - } - } - - if (!extracted_something) - return; - - auto & root_argument_nodes = root_arguments.getNodes(); - root_argument_nodes.clear(); - extractNodesFromSetInto(std::move(new_top_level_arguments), root_argument_nodes); - - auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); - root_node->resolveAsFunction(and_function_resolver); - } - void tryReplaceOrEqualsChainWithIn(QueryTreeNodePtr & node) { auto & function_node = node->as(); diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index 2f17a4da831..222772a92c0 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -299,3 +299,37 @@ QUERY id: 0 COLUMN id: 14, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 15, column_name: A, result_type: UInt8, source_id: 3 COLUMN id: 16, column_name: B, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + JOIN id: 3, strictness: ALL, kind: INNER + LEFT TABLE EXPRESSION + TABLE id: 4, alias: __table1, table_name: default.x + RIGHT TABLE EXPRESSION + TABLE id: 5, alias: __table2, table_name: default.y + JOIN EXPRESSION + FUNCTION id: 6, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + FUNCTION id: 8, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 9, nodes: 2 + FUNCTION id: 10, function_name: greater, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + COLUMN id: 12, column_name: B, result_type: UInt8, source_id: 5 + CONSTANT id: 13, constant_value: UInt64_1, constant_value_type: UInt8 + FUNCTION id: 14, function_name: greater, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 15, nodes: 2 + COLUMN id: 16, column_name: B, result_type: UInt8, source_id: 4 + CONSTANT id: 17, constant_value: UInt64_1, constant_value_type: UInt8 + FUNCTION id: 18, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 19, nodes: 2 + COLUMN id: 20, column_name: A, result_type: UInt8, source_id: 4 + COLUMN id: 21, column_name: A, result_type: UInt8, source_id: 5 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index fe7a18cfe02..9cfe8b31586 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-settings + DROP TABLE IF EXISTS x; CREATE TABLE x (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) ENGINE = MergeTree ORDER BY x; INSERT INTO x SELECT * FROM generateRandom('x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8') LIMIT 10000; @@ -61,4 +63,12 @@ WHERE ((A AND B) OR ((B AND D) OR (E AND F))); EXPLAIN QUERY TREE SELECT count() FROM x -WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) +WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)); + +CREATE TABLE y (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) ENGINE = MergeTree ORDER BY x; + +-- JOIN expressions +-- As the optimization code is shared between ON and WHERE, it is enough to test that the optimization is done also in ON +EXPLAIN QUERY TREE +SELECT count() +FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B > 1) OR ((x.A = y.A) AND y.B > 1); From b6bcd90cbd6f24399a696f9c937f2065e67f4b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 6 Nov 2024 22:46:38 +0000 Subject: [PATCH 114/502] Add header docs --- .../Passes/LogicalExpressionOptimizerPass.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h index dace434984c..dac6ccdc880 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h @@ -108,6 +108,22 @@ namespace DB * SELECT * FROM t1 WHERE a IN (n) * SELECT * FROM t1 WHERE NOT a IN (n) * ------------------------------- + * + * 8. Extract common expressions from AND expressions of a single OR expression only in WHERE and ON expressions. + * If possible, AND and OR expressions will be flattened during performing this. + * This might break some lazily evaluated expressions, but this optimization can be turned off by optimize_extract_common_expressions = 0. + * ------------------------------- + * SELECT * FROM t1 WHERE a AND ((b AND c) OR (b AND d) OR (b AND e)) + * SELECT * FROM t1 WHERE a AND ((b AND c) OR ((b AND d) OR (b AND e))) -- needs flattening + * SELECT * FROM t1 WHERE (a AND b AND c) OR (a AND b AND d) + * SELECT * FROM t1 WHERE (a AND b) OR (a AND b AND c) + * + * will be transformed into + * + * SELECT * FROM t1 WHERE a AND b AND (c OR d AND e) + * SELECT * FROM t1 WHERE a AND b AND (c OR d AND e) + * SELECT * FROM t1 WHERE a AND b AND (c OR d) + * SELECT * FROM t1 WHERE a AND b */ class LogicalExpressionOptimizerPass final : public IQueryTreePass From 3083a39268b37466e7c4e8a3667295b3410e2fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 6 Nov 2024 22:54:23 +0000 Subject: [PATCH 115/502] Add some queries to check optimization only does only equivalent transformations --- ...2_common_expression_optimization.reference | 306 +++++++++++++++++- .../03262_common_expression_optimization.sql | 96 +++--- 2 files changed, 352 insertions(+), 50 deletions(-) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index 222772a92c0..d9a490425ce 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -1,3 +1,23 @@ +-4890149726841642150 1 1 1 0 0 0 +6142641079032712283 1 1 1 0 0 0 +-558050477524086008 1 1 1 0 0 0 +670122215481070522 1 1 1 0 0 0 +-2818642828283588318 1 1 1 0 0 0 +-6769625230840672398 1 1 1 0 0 0 +2360327909276087486 1 1 1 0 0 0 +5422332327868912849 1 1 1 0 0 0 +-6185482623805481021 1 1 1 0 0 0 +-6276639858940903557 1 1 1 0 0 0 +-4890149726841642150 1 1 1 0 0 0 +6142641079032712283 1 1 1 0 0 0 +-558050477524086008 1 1 1 0 0 0 +670122215481070522 1 1 1 0 0 0 +-2818642828283588318 1 1 1 0 0 0 +-6769625230840672398 1 1 1 0 0 0 +2360327909276087486 1 1 1 0 0 0 +5422332327868912849 1 1 1 0 0 0 +-6185482623805481021 1 1 1 0 0 0 +-6276639858940903557 1 1 1 0 0 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -13,6 +33,26 @@ QUERY id: 0 COLUMN id: 6, column_name: B, result_type: UInt8, source_id: 3 COLUMN id: 7, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 8, column_name: A, result_type: UInt8, source_id: 3 +-1417696215412225994 1 1 1 0 0 1 +-7491222816216432223 1 1 1 0 0 1 +-7667054131998434193 1 1 1 0 0 1 +1150290670655138617 1 1 1 0 0 1 +-3651160188931679372 1 1 1 0 0 1 +7727375535152756381 1 1 1 0 0 1 +1248518317831711188 1 1 1 0 0 1 +-7224086688461492380 1 1 1 0 0 1 +9032006395921235708 1 1 1 0 0 1 +7044288989182144315 1 1 1 0 0 1 +-1417696215412225994 1 1 1 0 0 1 +-7491222816216432223 1 1 1 0 0 1 +-7667054131998434193 1 1 1 0 0 1 +1150290670655138617 1 1 1 0 0 1 +-3651160188931679372 1 1 1 0 0 1 +7727375535152756381 1 1 1 0 0 1 +1248518317831711188 1 1 1 0 0 1 +-7224086688461492380 1 1 1 0 0 1 +9032006395921235708 1 1 1 0 0 1 +7044288989182144315 1 1 1 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -33,6 +73,26 @@ QUERY id: 0 COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 +-1417696215412225994 1 1 1 0 0 1 +-7491222816216432223 1 1 1 0 0 1 +-7667054131998434193 1 1 1 0 0 1 +1150290670655138617 1 1 1 0 0 1 +-3651160188931679372 1 1 1 0 0 1 +7727375535152756381 1 1 1 0 0 1 +1248518317831711188 1 1 1 0 0 1 +-7224086688461492380 1 1 1 0 0 1 +9032006395921235708 1 1 1 0 0 1 +7044288989182144315 1 1 1 0 0 1 +-1417696215412225994 1 1 1 0 0 1 +-7491222816216432223 1 1 1 0 0 1 +-7667054131998434193 1 1 1 0 0 1 +1150290670655138617 1 1 1 0 0 1 +-3651160188931679372 1 1 1 0 0 1 +7727375535152756381 1 1 1 0 0 1 +1248518317831711188 1 1 1 0 0 1 +-7224086688461492380 1 1 1 0 0 1 +9032006395921235708 1 1 1 0 0 1 +7044288989182144315 1 1 1 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -53,6 +113,26 @@ QUERY id: 0 COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 +-7064227703501672459 1 1 0 0 1 0 +-5989383837337135212 1 1 0 0 1 0 +-2960971598243161531 1 1 0 0 1 0 +-2091785646518281290 1 1 0 0 1 0 +-2587761388937978501 1 1 0 0 1 0 +4189632917510416150 1 1 0 0 1 0 +329410740548269838 1 1 0 0 1 0 +-966942443397313698 1 1 0 0 1 0 +8454473642815416800 1 1 0 0 1 0 +8268045340657794461 1 1 0 0 1 0 +-7064227703501672459 1 1 0 0 1 0 +-5989383837337135212 1 1 0 0 1 0 +-2960971598243161531 1 1 0 0 1 0 +-2091785646518281290 1 1 0 0 1 0 +-2587761388937978501 1 1 0 0 1 0 +4189632917510416150 1 1 0 0 1 0 +329410740548269838 1 1 0 0 1 0 +-966942443397313698 1 1 0 0 1 0 +8454473642815416800 1 1 0 0 1 0 +8268045340657794461 1 1 0 0 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -73,6 +153,26 @@ QUERY id: 0 COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 +-7064227703501672459 1 1 0 0 1 0 +-5989383837337135212 1 1 0 0 1 0 +-2960971598243161531 1 1 0 0 1 0 +-2091785646518281290 1 1 0 0 1 0 +-2587761388937978501 1 1 0 0 1 0 +4189632917510416150 1 1 0 0 1 0 +329410740548269838 1 1 0 0 1 0 +-966942443397313698 1 1 0 0 1 0 +8454473642815416800 1 1 0 0 1 0 +8268045340657794461 1 1 0 0 1 0 +-7064227703501672459 1 1 0 0 1 0 +-5989383837337135212 1 1 0 0 1 0 +-2960971598243161531 1 1 0 0 1 0 +-2091785646518281290 1 1 0 0 1 0 +-2587761388937978501 1 1 0 0 1 0 +4189632917510416150 1 1 0 0 1 0 +329410740548269838 1 1 0 0 1 0 +-966942443397313698 1 1 0 0 1 0 +8454473642815416800 1 1 0 0 1 0 +8268045340657794461 1 1 0 0 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -93,6 +193,26 @@ QUERY id: 0 COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 +7046366705027993992 0 1 1 0 0 0 +5876398964123690873 0 1 1 0 0 0 +-4849840134697008981 0 1 1 0 0 0 +8219785106025342771 0 1 1 0 0 0 +7106055360641449065 0 1 1 0 0 0 +7782502397032484090 0 1 1 0 0 0 +1587022221789919998 0 1 1 0 0 0 +-5488510425784433218 0 1 1 0 0 0 +-3751993301879274716 0 1 1 0 0 0 +3995320070609749801 0 1 1 0 0 0 +7046366705027993992 0 1 1 0 0 0 +5876398964123690873 0 1 1 0 0 0 +-4849840134697008981 0 1 1 0 0 0 +8219785106025342771 0 1 1 0 0 0 +7106055360641449065 0 1 1 0 0 0 +7782502397032484090 0 1 1 0 0 0 +1587022221789919998 0 1 1 0 0 0 +-5488510425784433218 0 1 1 0 0 0 +-3751993301879274716 0 1 1 0 0 0 +3995320070609749801 0 1 1 0 0 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -107,6 +227,26 @@ QUERY id: 0 LIST id: 5, nodes: 2 COLUMN id: 6, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 +331058316081182921 0 1 1 0 0 1 +-2878539180531591201 0 1 1 0 0 1 +5634271855584736521 0 1 1 0 0 1 +602708131544673121 0 1 1 0 0 1 +-7538506014351088906 0 1 1 0 0 1 +-7767907992783882398 0 1 1 0 0 1 +3427035445395945973 0 1 1 0 0 1 +1130846464892565445 0 1 1 0 0 1 +-7229259894257180861 0 1 1 0 0 1 +-9156504923995237243 0 1 1 0 0 1 +331058316081182921 0 1 1 0 0 1 +-2878539180531591201 0 1 1 0 0 1 +5634271855584736521 0 1 1 0 0 1 +602708131544673121 0 1 1 0 0 1 +-7538506014351088906 0 1 1 0 0 1 +-7767907992783882398 0 1 1 0 0 1 +3427035445395945973 0 1 1 0 0 1 +1130846464892565445 0 1 1 0 0 1 +-7229259894257180861 0 1 1 0 0 1 +-9156504923995237243 0 1 1 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -126,6 +266,26 @@ QUERY id: 0 COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 10, column_name: B, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 +331058316081182921 0 1 1 0 0 1 +-2878539180531591201 0 1 1 0 0 1 +5634271855584736521 0 1 1 0 0 1 +602708131544673121 0 1 1 0 0 1 +-7538506014351088906 0 1 1 0 0 1 +-7767907992783882398 0 1 1 0 0 1 +3427035445395945973 0 1 1 0 0 1 +1130846464892565445 0 1 1 0 0 1 +-7229259894257180861 0 1 1 0 0 1 +-9156504923995237243 0 1 1 0 0 1 +331058316081182921 0 1 1 0 0 1 +-2878539180531591201 0 1 1 0 0 1 +5634271855584736521 0 1 1 0 0 1 +602708131544673121 0 1 1 0 0 1 +-7538506014351088906 0 1 1 0 0 1 +-7767907992783882398 0 1 1 0 0 1 +3427035445395945973 0 1 1 0 0 1 +1130846464892565445 0 1 1 0 0 1 +-7229259894257180861 0 1 1 0 0 1 +-9156504923995237243 0 1 1 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -145,6 +305,26 @@ QUERY id: 0 COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 10, column_name: B, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 +-8350064335402276481 0 1 0 0 1 0 +2766908592531260293 0 1 0 0 1 0 +-8500464668406226757 0 1 0 0 1 0 +-4469615932557529717 0 1 0 0 1 0 +-2676464305117434926 0 1 0 0 1 0 +-3455317119139662830 0 1 0 0 1 0 +5588544998641306169 0 1 0 0 1 0 +-9173414302980908256 0 1 0 0 1 0 +7212851845568196641 0 1 0 0 1 0 +-3704547074519116929 0 1 0 0 1 0 +-8350064335402276481 0 1 0 0 1 0 +2766908592531260293 0 1 0 0 1 0 +-8500464668406226757 0 1 0 0 1 0 +-4469615932557529717 0 1 0 0 1 0 +-2676464305117434926 0 1 0 0 1 0 +-3455317119139662830 0 1 0 0 1 0 +5588544998641306169 0 1 0 0 1 0 +-9173414302980908256 0 1 0 0 1 0 +7212851845568196641 0 1 0 0 1 0 +-3704547074519116929 0 1 0 0 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -164,6 +344,26 @@ QUERY id: 0 COLUMN id: 9, column_name: D, result_type: UInt8, source_id: 3 COLUMN id: 10, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: B, result_type: UInt8, source_id: 3 +-8350064335402276481 0 1 0 0 1 0 +2766908592531260293 0 1 0 0 1 0 +-8500464668406226757 0 1 0 0 1 0 +-4469615932557529717 0 1 0 0 1 0 +-2676464305117434926 0 1 0 0 1 0 +-3455317119139662830 0 1 0 0 1 0 +5588544998641306169 0 1 0 0 1 0 +-9173414302980908256 0 1 0 0 1 0 +7212851845568196641 0 1 0 0 1 0 +-3704547074519116929 0 1 0 0 1 0 +-8350064335402276481 0 1 0 0 1 0 +2766908592531260293 0 1 0 0 1 0 +-8500464668406226757 0 1 0 0 1 0 +-4469615932557529717 0 1 0 0 1 0 +-2676464305117434926 0 1 0 0 1 0 +-3455317119139662830 0 1 0 0 1 0 +5588544998641306169 0 1 0 0 1 0 +-9173414302980908256 0 1 0 0 1 0 +7212851845568196641 0 1 0 0 1 0 +-3704547074519116929 0 1 0 0 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -183,6 +383,26 @@ QUERY id: 0 COLUMN id: 9, column_name: D, result_type: UInt8, source_id: 3 COLUMN id: 10, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: B, result_type: UInt8, source_id: 3 +7296788055532769937 0 1 0 0 0 0 +-8085083078325036349 0 1 0 0 0 0 +8056746393406743698 0 1 0 0 0 0 +-5618901525230504315 0 1 0 0 0 0 +-6790762979674887120 0 1 0 0 0 0 +7609629462704427873 0 1 0 0 0 0 +7776384315538625189 0 1 0 0 0 0 +1381341868259371991 0 1 0 0 0 0 +2483552736214921240 0 1 0 0 0 0 +5286533408672438546 0 1 0 0 0 0 +7296788055532769937 0 1 0 0 0 0 +-8085083078325036349 0 1 0 0 0 0 +8056746393406743698 0 1 0 0 0 0 +-5618901525230504315 0 1 0 0 0 0 +-6790762979674887120 0 1 0 0 0 0 +7609629462704427873 0 1 0 0 0 0 +7776384315538625189 0 1 0 0 0 0 +1381341868259371991 0 1 0 0 0 0 +2483552736214921240 0 1 0 0 0 0 +5286533408672438546 0 1 0 0 0 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -211,6 +431,26 @@ QUERY id: 0 ARGUMENTS LIST id: 16, nodes: 1 COLUMN id: 17, column_name: D, result_type: UInt8, source_id: 3 +2589067852420946093 0 0 0 0 1 1 +566510304360553038 0 0 0 0 1 1 +4712156778925374545 0 0 0 0 1 1 +6513794688600930866 0 0 0 0 1 1 +8420009359281429633 0 0 0 0 1 1 +2151442296722776334 0 0 0 0 1 1 +-7155015417725275505 0 0 0 0 1 1 +8063264694611659226 0 0 0 0 1 1 +7596093258110041712 0 0 0 0 1 1 +-5463634588980750408 0 0 0 0 1 1 +2589067852420946093 0 0 0 0 1 1 +566510304360553038 0 0 0 0 1 1 +4712156778925374545 0 0 0 0 1 1 +6513794688600930866 0 0 0 0 1 1 +8420009359281429633 0 0 0 0 1 1 +2151442296722776334 0 0 0 0 1 1 +-7155015417725275505 0 0 0 0 1 1 +8063264694611659226 0 0 0 0 1 1 +7596093258110041712 0 0 0 0 1 1 +-5463634588980750408 0 0 0 0 1 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -241,6 +481,26 @@ QUERY id: 0 LIST id: 17, nodes: 2 COLUMN id: 18, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 19, column_name: F, result_type: UInt8, source_id: 3 +813655996633899418 0 0 0 0 1 1 +6513794688600930866 0 0 0 0 1 1 +2151442296722776334 0 0 0 0 1 1 +-2316167413850459561 0 0 0 0 1 1 +-5864912137959118127 0 0 0 0 1 1 +-5326152364639478952 0 0 0 0 1 1 +-5463634588980750408 0 0 0 0 1 1 +6704351461338887300 0 0 0 0 1 1 +-973750167321101279 0 0 0 0 1 1 +8063264694611659226 0 0 0 0 1 1 +813655996633899418 0 0 0 0 1 1 +6513794688600930866 0 0 0 0 1 1 +2151442296722776334 0 0 0 0 1 1 +-2316167413850459561 0 0 0 0 1 1 +-5864912137959118127 0 0 0 0 1 1 +-5326152364639478952 0 0 0 0 1 1 +-5463634588980750408 0 0 0 0 1 1 +6704351461338887300 0 0 0 0 1 1 +-973750167321101279 0 0 0 0 1 1 +8063264694611659226 0 0 0 0 1 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -271,6 +531,26 @@ QUERY id: 0 LIST id: 16, nodes: 2 COLUMN id: 17, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 18, column_name: F, result_type: UInt8, source_id: 3 +6015136002503835948 1 1 0 0 0 1 +-3883799183493487473 1 1 0 0 0 1 +-3002427897361111856 1 1 0 0 0 1 +-2523952504416578832 1 1 0 0 0 1 +-3187045566703922613 1 1 0 0 0 1 +4838814183373278674 1 1 0 0 0 1 +620980830413267931 1 1 0 0 0 1 +-5907701125400227703 1 1 0 0 0 1 +-1174991811984636509 1 1 0 0 0 1 +446104724568266173 1 1 0 0 0 1 +6015136002503835948 1 1 0 0 0 1 +-3883799183493487473 1 1 0 0 0 1 +-3002427897361111856 1 1 0 0 0 1 +-2523952504416578832 1 1 0 0 0 1 +-3187045566703922613 1 1 0 0 0 1 +4838814183373278674 1 1 0 0 0 1 +620980830413267931 1 1 0 0 0 1 +-5907701125400227703 1 1 0 0 0 1 +-1174991811984636509 1 1 0 0 0 1 +446104724568266173 1 1 0 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -299,6 +579,26 @@ QUERY id: 0 COLUMN id: 14, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 15, column_name: A, result_type: UInt8, source_id: 3 COLUMN id: 16, column_name: B, result_type: UInt8, source_id: 3 +-9220342792229035180 1 1 0 1 0 1 -9218747728814747926 1 1 0 0 0 1 +-9220342792229035180 1 1 0 1 0 1 -9217081065560798881 1 0 0 1 0 1 +-9220342792229035180 1 1 0 1 0 1 -9211856624818647543 1 1 1 0 0 0 +-9220342792229035180 1 1 0 1 0 1 -9190487214870855200 1 1 1 1 1 1 +-9220342792229035180 1 1 0 1 0 1 -9160211915115471089 1 0 0 0 1 1 +-9220342792229035180 1 1 0 1 0 1 -9105451307219421605 1 0 1 1 1 1 +-9220342792229035180 1 1 0 1 0 1 -9077371913032868764 1 1 0 0 1 1 +-9220342792229035180 1 1 0 1 0 1 -9077295854519266595 1 1 0 0 1 0 +-9220342792229035180 1 1 0 1 0 1 -9041842932373383548 1 1 0 0 0 1 +-9220342792229035180 1 1 0 1 0 1 -9037094316313599791 1 0 0 1 0 1 +-9220342792229035180 1 1 0 1 0 1 -9218747728814747926 1 1 0 0 0 1 +-9220342792229035180 1 1 0 1 0 1 -9217081065560798881 1 0 0 1 0 1 +-9220342792229035180 1 1 0 1 0 1 -9211856624818647543 1 1 1 0 0 0 +-9220342792229035180 1 1 0 1 0 1 -9190487214870855200 1 1 1 1 1 1 +-9220342792229035180 1 1 0 1 0 1 -9160211915115471089 1 0 0 0 1 1 +-9220342792229035180 1 1 0 1 0 1 -9105451307219421605 1 0 1 1 1 1 +-9220342792229035180 1 1 0 1 0 1 -9077371913032868764 1 1 0 0 1 1 +-9220342792229035180 1 1 0 1 0 1 -9077295854519266595 1 1 0 0 1 0 +-9220342792229035180 1 1 0 1 0 1 -9041842932373383548 1 1 0 0 0 1 +-9220342792229035180 1 1 0 1 0 1 -9037094316313599791 1 0 0 1 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -318,12 +618,12 @@ QUERY id: 0 FUNCTION id: 8, function_name: or, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 9, nodes: 2 - FUNCTION id: 10, function_name: greater, function_type: ordinary, result_type: UInt8 + FUNCTION id: 10, function_name: equals, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 11, nodes: 2 - COLUMN id: 12, column_name: B, result_type: UInt8, source_id: 5 + COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 5 CONSTANT id: 13, constant_value: UInt64_1, constant_value_type: UInt8 - FUNCTION id: 14, function_name: greater, function_type: ordinary, result_type: UInt8 + FUNCTION id: 14, function_name: equals, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 15, nodes: 2 COLUMN id: 16, column_name: B, result_type: UInt8, source_id: 4 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index 9cfe8b31586..14728fa1fc4 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -2,73 +2,75 @@ DROP TABLE IF EXISTS x; CREATE TABLE x (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) ENGINE = MergeTree ORDER BY x; -INSERT INTO x SELECT * FROM generateRandom('x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8') LIMIT 10000; +INSERT INTO x SELECT x, A%2 AS A, B%2 AS B, C%2 AS C, D%2 AS D, E%2 AS E, F%2 AS F FROM generateRandom('x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8', 42) LIMIT 2000; -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE A AND ((B AND C) OR (B AND C AND F)); -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE A AND ((B AND C AND E) OR (B AND C AND F)); +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND C AND F)); -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)); +SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)); -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)); +SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)); -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))); +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)); + +SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))); -- Without AND as a root -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE ((B AND C) OR (B AND C AND F)); +SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR (B AND C AND F)); -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE ((B AND C AND E) OR (B AND C AND F)); +SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C AND E) OR (B AND C AND F)); -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE ((B AND (C AND E)) OR (B AND C AND F)); +SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)); -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE ((B AND C) OR (B AND D) OR (B AND E)); +SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)); -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE ((B AND C) OR ((B AND D) OR (B AND E))); +SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))); -- Complex expression -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))); +SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))); -- Flattening is only happening if something can be extracted -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE ((A AND B) OR ((C AND D) OR (E AND F))); +SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))); -EXPLAIN QUERY TREE SELECT count() -FROM x -WHERE ((A AND B) OR ((B AND D) OR (E AND F))); +SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))); -- Duplicates -EXPLAIN QUERY TREE -SELECT count() -FROM x -WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)); +SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; +SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)); +DROP TABLE IF EXISTS y; CREATE TABLE y (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) ENGINE = MergeTree ORDER BY x; +INSERT INTO y SELECT x, A%2 AS A, B%2 AS B, C%2 AS C, D%2 AS D, E%2 AS E, F%2 AS F FROM generateRandom('x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8', 43) LIMIT 2000; -- JOIN expressions -- As the optimization code is shared between ON and WHERE, it is enough to test that the optimization is done also in ON -EXPLAIN QUERY TREE -SELECT count() -FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B > 1) OR ((x.A = y.A) AND y.B > 1); +SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, enable_analyzer = 0; +SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, enable_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1); From 1abd75e905006884035e401306261b1af55ea2f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 7 Nov 2024 09:44:41 +0000 Subject: [PATCH 116/502] Make the order of arguments deterministic --- .../Passes/LogicalExpressionOptimizerPass.cpp | 85 ++- ...2_common_expression_optimization.reference | 641 +++++++++--------- 2 files changed, 373 insertions(+), 353 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index f5f1049f4d8..a30b44fc4db 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -171,14 +171,11 @@ bool isTwoArgumentsFromDifferentSides(const FunctionNode & node_function, const (first_src->isEqual(rhs_join) && second_src->isEqual(lhs_join)); } -void extractNodesFromSetInto(QueryTreeNodePtrWithHashSet && set, QueryTreeNodes & into) +void insertIfNotPresentInSet(QueryTreeNodePtrWithHashSet& set, QueryTreeNodes &nodes, QueryTreeNodePtr node) { - for (auto it = set.begin(); it != set.end();) - { - auto it_to_extract = it++; - auto nh = set.extract(it_to_extract); - into.push_back(std::move(nh.value().node)); - } + const auto [_, inserted] = set.emplace(node); + if (inserted) + nodes.push_back(std::move(node)); } // Returns the flattened AND/OR node if the passed-in node can be flattened. Doesn't modify the passed-in node. @@ -230,7 +227,8 @@ struct CommonExpressionExtractionResult { // new_node: if the new node is empty, then contain the new node, otherwise nullptr // common_expressions: the extracted common expressions. The new expressions can be created - // as the conjunction of new_node and the nodes in common_expressions. + // as the conjunction of new_node and the nodes in common_expressions. It is guaranteed that + // the common expressions are deduplicated. // Examples: // Input: (A & B & C) | (A & D & E) // Result: new_node = (B & C) | (D & E), common_expressions = {A} @@ -244,7 +242,7 @@ struct CommonExpressionExtractionResult // Input: (A & B) | (A & B & C) // Result: new_node = nullptr, common_expressions = {A, B} QueryTreeNodePtr new_node; - QueryTreeNodePtrWithHashSet common_expressions; + QueryTreeNodes common_expressions; }; std::optional tryExtractCommonExpressions(const QueryTreeNodePtr & node, const ContextPtr & context) @@ -262,9 +260,15 @@ std::optional tryExtractCommonExpressions(cons chassert(or_argument_nodes.size() > 1); bool first_argument = true; - QueryTreeNodePtrWithHashSet common_exprs; + QueryTreeNodePtrWithHashSet common_exprs_set; + QueryTreeNodes common_exprs; QueryTreeNodePtrWithHashMap flattened_ands; + auto insert_possible_new_common_expr = [&common_exprs_set, &common_exprs](QueryTreeNodePtr node_to_insert) + { + insertIfNotPresentInSet(common_exprs_set, common_exprs, std::move(node_to_insert)); + }; + for (auto & maybe_and_node : or_argument_nodes) { auto * and_node = maybe_and_node->as(); @@ -284,23 +288,26 @@ std::optional tryExtractCommonExpressions(cons common_exprs.reserve(current_arguments.size()); for (auto & or_argument : current_arguments) - { - QueryTreeNodePtrWithHash ptr_with_hash{or_argument}; - common_exprs.insert(ptr_with_hash); - } + insert_possible_new_common_expr(or_argument); first_argument = false; } else { - QueryTreeNodePtrWithHashSet new_common_expr_hashes; + QueryTreeNodePtrWithHashSet new_common_exprs_set; + QueryTreeNodes new_common_exprs; + for (auto & or_argument : and_node->getArguments()) { - if (auto ptr_with_hash = QueryTreeNodePtrWithHash{or_argument}; common_exprs.contains(ptr_with_hash)) - new_common_expr_hashes.insert(ptr_with_hash); + if (auto ptr_with_hash = QueryTreeNodePtrWithHash{or_argument}; common_exprs_set.contains(ptr_with_hash)) + { + new_common_exprs.push_back(or_argument); + new_common_exprs_set.insert(std::move(ptr_with_hash)); + } } - common_exprs = std::move(new_common_expr_hashes); + common_exprs_set = std::move(new_common_exprs_set); + common_exprs = std::move(new_common_exprs); if (common_exprs.empty()) return {}; // There are no common expressions @@ -309,7 +316,8 @@ std::optional tryExtractCommonExpressions(cons chassert(!common_exprs.empty()); - QueryTreeNodePtrWithHashSet new_or_arguments; + QueryTreeNodePtrWithHashSet new_or_arguments_set; + QueryTreeNodes new_or_arguments; bool has_completely_extracted_and_expression = false; for (auto & or_argument : or_argument_nodes) @@ -323,10 +331,10 @@ std::optional tryExtractCommonExpressions(cons std::remove_if( and_arguments.begin(), and_arguments.end(), - [&common_exprs](const QueryTreeNodePtr & ptr) + [&common_exprs_set](const QueryTreeNodePtr & ptr) { QueryTreeNodeWithHash ptr_with_hash{ptr}; - return common_exprs.contains(ptr_with_hash); + return common_exprs_set.contains(ptr_with_hash); }), and_arguments.end()); @@ -338,13 +346,18 @@ std::optional tryExtractCommonExpressions(cons } else if (and_arguments.size() == 1) { - new_or_arguments.emplace(std::move(and_arguments.front())); + const auto [_, inserted] = new_or_arguments_set.insert(and_arguments.front()); + if (inserted) + new_or_arguments.push_back(std::move(and_arguments.front())); + } else { auto and_function_resolver = FunctionFactory::instance().get("and", context); and_node.resolveAsFunction(and_function_resolver); - new_or_arguments.emplace(or_argument); + const auto [_, inserted] = new_or_arguments_set.insert(or_argument); + if (inserted) + new_or_arguments.push_back(std::move(or_argument)); } } @@ -357,7 +370,7 @@ std::optional tryExtractCommonExpressions(cons chassert(new_or_arguments.size() >= 2); auto new_or_node = std::make_shared("or"); - extractNodesFromSetInto(std::move(new_or_arguments), new_or_node->getArguments().getNodes()); + new_or_node->getArguments().getNodes() = std::move(new_or_arguments); auto or_function_resolver = FunctionFactory::instance().get("or", context); new_or_node->resolveAsFunction(or_function_resolver); @@ -374,10 +387,9 @@ void tryOptimizeCommonExpressionsInOr(QueryTreeNodePtr & node, const ContextPtr { auto & result = *maybe_result; auto & root_arguments = root_node->getArguments().getNodes(); - root_arguments.clear(); + root_arguments = std::move(result.common_expressions); if (result.new_node != nullptr) root_arguments.push_back(std::move(result.new_node)); - extractNodesFromSetInto(std::move(result.common_expressions), root_arguments); if (root_arguments.size() == 1) { @@ -385,6 +397,8 @@ void tryOptimizeCommonExpressionsInOr(QueryTreeNodePtr & node, const ContextPtr return; } + // The OR expression must be replaced by and AND expression that will contain the common expressions + // and the new_node, if it is not nullptr. auto and_function_resolver = FunctionFactory::instance().get("and", context); root_node->resolveAsFunction(and_function_resolver); } @@ -395,7 +409,13 @@ void tryOptimizeCommonExpressionsInAnd(QueryTreeNodePtr & node, const ContextPtr auto * root_node = node->as(); chassert(root_node && root_node->getFunctionName() == "and"); - QueryTreeNodePtrWithHashSet new_top_level_arguments; + QueryTreeNodePtrWithHashSet new_top_level_arguments_set; + QueryTreeNodes new_top_level_arguments; + + auto insert_possible_new_top_level_arg = [&new_top_level_arguments_set, &new_top_level_arguments](QueryTreeNodePtr node_to_insert) + { + insertIfNotPresentInSet(new_top_level_arguments_set, new_top_level_arguments, std::move(node_to_insert)); + }; auto extracted_something = false; auto & root_arguments = root_node->getArguments(); @@ -406,21 +426,20 @@ void tryOptimizeCommonExpressionsInAnd(QueryTreeNodePtr & node, const ContextPtr extracted_something = true; auto & result = *maybe_result; if (result.new_node != nullptr) - new_top_level_arguments.emplace(std::move(result.new_node)); - new_top_level_arguments.merge(std::move(result.common_expressions)); + insert_possible_new_top_level_arg(std::move(result.new_node)); + for (auto& common_expr: result.common_expressions) + insert_possible_new_top_level_arg(std::move(common_expr)); } else { - new_top_level_arguments.emplace(argument); + insert_possible_new_top_level_arg(argument); } } if (!extracted_something) return; - auto & root_argument_nodes = root_arguments.getNodes(); - root_argument_nodes.clear(); - extractNodesFromSetInto(std::move(new_top_level_arguments), root_argument_nodes); + root_arguments.getNodes() = std::move(new_top_level_arguments); auto and_function_resolver = FunctionFactory::instance().get("and", context); root_node->resolveAsFunction(and_function_resolver); diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index d9a490425ce..a6c332f18de 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -18,6 +18,235 @@ 5422332327868912849 1 1 1 0 0 0 -6185482623805481021 1 1 1 0 0 0 -6276639858940903557 1 1 1 0 0 0 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 3 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 8, column_name: C, result_type: UInt8, source_id: 3 +-1417696215412225994 1 1 1 0 0 1 +-7491222816216432223 1 1 1 0 0 1 +-7667054131998434193 1 1 1 0 0 1 +1150290670655138617 1 1 1 0 0 1 +-3651160188931679372 1 1 1 0 0 1 +7727375535152756381 1 1 1 0 0 1 +1248518317831711188 1 1 1 0 0 1 +-7224086688461492380 1 1 1 0 0 1 +9032006395921235708 1 1 1 0 0 1 +7044288989182144315 1 1 1 0 0 1 +-1417696215412225994 1 1 1 0 0 1 +-7491222816216432223 1 1 1 0 0 1 +-7667054131998434193 1 1 1 0 0 1 +1150290670655138617 1 1 1 0 0 1 +-3651160188931679372 1 1 1 0 0 1 +7727375535152756381 1 1 1 0 0 1 +1248518317831711188 1 1 1 0 0 1 +-7224086688461492380 1 1 1 0 0 1 +9032006395921235708 1 1 1 0 0 1 +7044288989182144315 1 1 1 0 0 1 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 4 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + FUNCTION id: 7, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 3 +-1417696215412225994 1 1 1 0 0 1 +-7491222816216432223 1 1 1 0 0 1 +-7667054131998434193 1 1 1 0 0 1 +1150290670655138617 1 1 1 0 0 1 +-3651160188931679372 1 1 1 0 0 1 +7727375535152756381 1 1 1 0 0 1 +1248518317831711188 1 1 1 0 0 1 +-7224086688461492380 1 1 1 0 0 1 +9032006395921235708 1 1 1 0 0 1 +7044288989182144315 1 1 1 0 0 1 +-1417696215412225994 1 1 1 0 0 1 +-7491222816216432223 1 1 1 0 0 1 +-7667054131998434193 1 1 1 0 0 1 +1150290670655138617 1 1 1 0 0 1 +-3651160188931679372 1 1 1 0 0 1 +7727375535152756381 1 1 1 0 0 1 +1248518317831711188 1 1 1 0 0 1 +-7224086688461492380 1 1 1 0 0 1 +9032006395921235708 1 1 1 0 0 1 +7044288989182144315 1 1 1 0 0 1 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 4 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + FUNCTION id: 7, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 3 +-7064227703501672459 1 1 0 0 1 0 +-5989383837337135212 1 1 0 0 1 0 +-2960971598243161531 1 1 0 0 1 0 +-2091785646518281290 1 1 0 0 1 0 +-2587761388937978501 1 1 0 0 1 0 +4189632917510416150 1 1 0 0 1 0 +329410740548269838 1 1 0 0 1 0 +-966942443397313698 1 1 0 0 1 0 +8454473642815416800 1 1 0 0 1 0 +8268045340657794461 1 1 0 0 1 0 +-7064227703501672459 1 1 0 0 1 0 +-5989383837337135212 1 1 0 0 1 0 +-2960971598243161531 1 1 0 0 1 0 +-2091785646518281290 1 1 0 0 1 0 +-2587761388937978501 1 1 0 0 1 0 +4189632917510416150 1 1 0 0 1 0 +329410740548269838 1 1 0 0 1 0 +-966942443397313698 1 1 0 0 1 0 +8454473642815416800 1 1 0 0 1 0 +8268045340657794461 1 1 0 0 1 0 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 3 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + FUNCTION id: 7, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 3 + COLUMN id: 9, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 12, column_name: B, result_type: UInt8, source_id: 3 +-7064227703501672459 1 1 0 0 1 0 +-5989383837337135212 1 1 0 0 1 0 +-2960971598243161531 1 1 0 0 1 0 +-2091785646518281290 1 1 0 0 1 0 +-2587761388937978501 1 1 0 0 1 0 +4189632917510416150 1 1 0 0 1 0 +329410740548269838 1 1 0 0 1 0 +-966942443397313698 1 1 0 0 1 0 +8454473642815416800 1 1 0 0 1 0 +8268045340657794461 1 1 0 0 1 0 +-7064227703501672459 1 1 0 0 1 0 +-5989383837337135212 1 1 0 0 1 0 +-2960971598243161531 1 1 0 0 1 0 +-2091785646518281290 1 1 0 0 1 0 +-2587761388937978501 1 1 0 0 1 0 +4189632917510416150 1 1 0 0 1 0 +329410740548269838 1 1 0 0 1 0 +-966942443397313698 1 1 0 0 1 0 +8454473642815416800 1 1 0 0 1 0 +8268045340657794461 1 1 0 0 1 0 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 3 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + FUNCTION id: 7, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 3 + COLUMN id: 9, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 12, column_name: B, result_type: UInt8, source_id: 3 +7046366705027993992 0 1 1 0 0 0 +5876398964123690873 0 1 1 0 0 0 +-4849840134697008981 0 1 1 0 0 0 +8219785106025342771 0 1 1 0 0 0 +7106055360641449065 0 1 1 0 0 0 +7782502397032484090 0 1 1 0 0 0 +1587022221789919998 0 1 1 0 0 0 +-5488510425784433218 0 1 1 0 0 0 +-3751993301879274716 0 1 1 0 0 0 +3995320070609749801 0 1 1 0 0 0 +7046366705027993992 0 1 1 0 0 0 +5876398964123690873 0 1 1 0 0 0 +-4849840134697008981 0 1 1 0 0 0 +8219785106025342771 0 1 1 0 0 0 +7106055360641449065 0 1 1 0 0 0 +7782502397032484090 0 1 1 0 0 0 +1587022221789919998 0 1 1 0 0 0 +-5488510425784433218 0 1 1 0 0 0 +-3751993301879274716 0 1 1 0 0 0 +3995320070609749801 0 1 1 0 0 0 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 7, column_name: C, result_type: UInt8, source_id: 3 +331058316081182921 0 1 1 0 0 1 +-2878539180531591201 0 1 1 0 0 1 +5634271855584736521 0 1 1 0 0 1 +602708131544673121 0 1 1 0 0 1 +-7538506014351088906 0 1 1 0 0 1 +-7767907992783882398 0 1 1 0 0 1 +3427035445395945973 0 1 1 0 0 1 +1130846464892565445 0 1 1 0 0 1 +-7229259894257180861 0 1 1 0 0 1 +-9156504923995237243 0 1 1 0 0 1 +331058316081182921 0 1 1 0 0 1 +-2878539180531591201 0 1 1 0 0 1 +5634271855584736521 0 1 1 0 0 1 +602708131544673121 0 1 1 0 0 1 +-7538506014351088906 0 1 1 0 0 1 +-7767907992783882398 0 1 1 0 0 1 +3427035445395945973 0 1 1 0 0 1 +1130846464892565445 0 1 1 0 0 1 +-7229259894257180861 0 1 1 0 0 1 +-9156504923995237243 0 1 1 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -32,107 +261,31 @@ QUERY id: 0 LIST id: 5, nodes: 3 COLUMN id: 6, column_name: B, result_type: UInt8, source_id: 3 COLUMN id: 7, column_name: C, result_type: UInt8, source_id: 3 - COLUMN id: 8, column_name: A, result_type: UInt8, source_id: 3 --1417696215412225994 1 1 1 0 0 1 --7491222816216432223 1 1 1 0 0 1 --7667054131998434193 1 1 1 0 0 1 -1150290670655138617 1 1 1 0 0 1 --3651160188931679372 1 1 1 0 0 1 -7727375535152756381 1 1 1 0 0 1 -1248518317831711188 1 1 1 0 0 1 --7224086688461492380 1 1 1 0 0 1 -9032006395921235708 1 1 1 0 0 1 -7044288989182144315 1 1 1 0 0 1 --1417696215412225994 1 1 1 0 0 1 --7491222816216432223 1 1 1 0 0 1 --7667054131998434193 1 1 1 0 0 1 -1150290670655138617 1 1 1 0 0 1 --3651160188931679372 1 1 1 0 0 1 -7727375535152756381 1 1 1 0 0 1 -1248518317831711188 1 1 1 0 0 1 --7224086688461492380 1 1 1 0 0 1 -9032006395921235708 1 1 1 0 0 1 -7044288989182144315 1 1 1 0 0 1 -QUERY id: 0 - PROJECTION COLUMNS - count() UInt64 - PROJECTION - LIST id: 1, nodes: 1 - FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.x - WHERE - FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 5, nodes: 4 - COLUMN id: 6, column_name: C, result_type: UInt8, source_id: 3 - COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 FUNCTION id: 8, function_name: or, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 9, nodes: 2 - COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 - COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 - COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 --1417696215412225994 1 1 1 0 0 1 --7491222816216432223 1 1 1 0 0 1 --7667054131998434193 1 1 1 0 0 1 -1150290670655138617 1 1 1 0 0 1 --3651160188931679372 1 1 1 0 0 1 -7727375535152756381 1 1 1 0 0 1 -1248518317831711188 1 1 1 0 0 1 --7224086688461492380 1 1 1 0 0 1 -9032006395921235708 1 1 1 0 0 1 -7044288989182144315 1 1 1 0 0 1 --1417696215412225994 1 1 1 0 0 1 --7491222816216432223 1 1 1 0 0 1 --7667054131998434193 1 1 1 0 0 1 -1150290670655138617 1 1 1 0 0 1 --3651160188931679372 1 1 1 0 0 1 -7727375535152756381 1 1 1 0 0 1 -1248518317831711188 1 1 1 0 0 1 --7224086688461492380 1 1 1 0 0 1 -9032006395921235708 1 1 1 0 0 1 -7044288989182144315 1 1 1 0 0 1 -QUERY id: 0 - PROJECTION COLUMNS - count() UInt64 - PROJECTION - LIST id: 1, nodes: 1 - FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.x - WHERE - FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 5, nodes: 4 - COLUMN id: 6, column_name: C, result_type: UInt8, source_id: 3 - COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 - FUNCTION id: 8, function_name: or, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 9, nodes: 2 - COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 - COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 - COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 --7064227703501672459 1 1 0 0 1 0 --5989383837337135212 1 1 0 0 1 0 --2960971598243161531 1 1 0 0 1 0 --2091785646518281290 1 1 0 0 1 0 --2587761388937978501 1 1 0 0 1 0 -4189632917510416150 1 1 0 0 1 0 -329410740548269838 1 1 0 0 1 0 --966942443397313698 1 1 0 0 1 0 -8454473642815416800 1 1 0 0 1 0 -8268045340657794461 1 1 0 0 1 0 --7064227703501672459 1 1 0 0 1 0 --5989383837337135212 1 1 0 0 1 0 --2960971598243161531 1 1 0 0 1 0 --2091785646518281290 1 1 0 0 1 0 --2587761388937978501 1 1 0 0 1 0 -4189632917510416150 1 1 0 0 1 0 -329410740548269838 1 1 0 0 1 0 --966942443397313698 1 1 0 0 1 0 -8454473642815416800 1 1 0 0 1 0 -8268045340657794461 1 1 0 0 1 0 + COLUMN id: 10, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: F, result_type: UInt8, source_id: 3 +331058316081182921 0 1 1 0 0 1 +-2878539180531591201 0 1 1 0 0 1 +5634271855584736521 0 1 1 0 0 1 +602708131544673121 0 1 1 0 0 1 +-7538506014351088906 0 1 1 0 0 1 +-7767907992783882398 0 1 1 0 0 1 +3427035445395945973 0 1 1 0 0 1 +1130846464892565445 0 1 1 0 0 1 +-7229259894257180861 0 1 1 0 0 1 +-9156504923995237243 0 1 1 0 0 1 +331058316081182921 0 1 1 0 0 1 +-2878539180531591201 0 1 1 0 0 1 +5634271855584736521 0 1 1 0 0 1 +602708131544673121 0 1 1 0 0 1 +-7538506014351088906 0 1 1 0 0 1 +-7767907992783882398 0 1 1 0 0 1 +3427035445395945973 0 1 1 0 0 1 +1130846464892565445 0 1 1 0 0 1 +-7229259894257180861 0 1 1 0 0 1 +-9156504923995237243 0 1 1 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -146,33 +299,32 @@ QUERY id: 0 ARGUMENTS LIST id: 5, nodes: 3 COLUMN id: 6, column_name: B, result_type: UInt8, source_id: 3 - FUNCTION id: 7, function_name: or, function_type: ordinary, result_type: UInt8 + COLUMN id: 7, column_name: C, result_type: UInt8, source_id: 3 + FUNCTION id: 8, function_name: or, function_type: ordinary, result_type: UInt8 ARGUMENTS - LIST id: 8, nodes: 3 - COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 - COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 - COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 - COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 --7064227703501672459 1 1 0 0 1 0 --5989383837337135212 1 1 0 0 1 0 --2960971598243161531 1 1 0 0 1 0 --2091785646518281290 1 1 0 0 1 0 --2587761388937978501 1 1 0 0 1 0 -4189632917510416150 1 1 0 0 1 0 -329410740548269838 1 1 0 0 1 0 --966942443397313698 1 1 0 0 1 0 -8454473642815416800 1 1 0 0 1 0 -8268045340657794461 1 1 0 0 1 0 --7064227703501672459 1 1 0 0 1 0 --5989383837337135212 1 1 0 0 1 0 --2960971598243161531 1 1 0 0 1 0 --2091785646518281290 1 1 0 0 1 0 --2587761388937978501 1 1 0 0 1 0 -4189632917510416150 1 1 0 0 1 0 -329410740548269838 1 1 0 0 1 0 --966942443397313698 1 1 0 0 1 0 -8454473642815416800 1 1 0 0 1 0 -8268045340657794461 1 1 0 0 1 0 + LIST id: 9, nodes: 2 + COLUMN id: 10, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: F, result_type: UInt8, source_id: 3 +-8350064335402276481 0 1 0 0 1 0 +2766908592531260293 0 1 0 0 1 0 +-8500464668406226757 0 1 0 0 1 0 +-4469615932557529717 0 1 0 0 1 0 +-2676464305117434926 0 1 0 0 1 0 +-3455317119139662830 0 1 0 0 1 0 +5588544998641306169 0 1 0 0 1 0 +-9173414302980908256 0 1 0 0 1 0 +7212851845568196641 0 1 0 0 1 0 +-3704547074519116929 0 1 0 0 1 0 +-8350064335402276481 0 1 0 0 1 0 +2766908592531260293 0 1 0 0 1 0 +-8500464668406226757 0 1 0 0 1 0 +-4469615932557529717 0 1 0 0 1 0 +-2676464305117434926 0 1 0 0 1 0 +-3455317119139662830 0 1 0 0 1 0 +5588544998641306169 0 1 0 0 1 0 +-9173414302980908256 0 1 0 0 1 0 +7212851845568196641 0 1 0 0 1 0 +-3704547074519116929 0 1 0 0 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -184,127 +336,14 @@ QUERY id: 0 WHERE FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 ARGUMENTS - LIST id: 5, nodes: 3 + LIST id: 5, nodes: 2 COLUMN id: 6, column_name: B, result_type: UInt8, source_id: 3 FUNCTION id: 7, function_name: or, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 8, nodes: 3 - COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 - COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 - COLUMN id: 12, column_name: A, result_type: UInt8, source_id: 3 -7046366705027993992 0 1 1 0 0 0 -5876398964123690873 0 1 1 0 0 0 --4849840134697008981 0 1 1 0 0 0 -8219785106025342771 0 1 1 0 0 0 -7106055360641449065 0 1 1 0 0 0 -7782502397032484090 0 1 1 0 0 0 -1587022221789919998 0 1 1 0 0 0 --5488510425784433218 0 1 1 0 0 0 --3751993301879274716 0 1 1 0 0 0 -3995320070609749801 0 1 1 0 0 0 -7046366705027993992 0 1 1 0 0 0 -5876398964123690873 0 1 1 0 0 0 --4849840134697008981 0 1 1 0 0 0 -8219785106025342771 0 1 1 0 0 0 -7106055360641449065 0 1 1 0 0 0 -7782502397032484090 0 1 1 0 0 0 -1587022221789919998 0 1 1 0 0 0 --5488510425784433218 0 1 1 0 0 0 --3751993301879274716 0 1 1 0 0 0 -3995320070609749801 0 1 1 0 0 0 -QUERY id: 0 - PROJECTION COLUMNS - count() UInt64 - PROJECTION - LIST id: 1, nodes: 1 - FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.x - WHERE - FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 5, nodes: 2 - COLUMN id: 6, column_name: C, result_type: UInt8, source_id: 3 - COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 -331058316081182921 0 1 1 0 0 1 --2878539180531591201 0 1 1 0 0 1 -5634271855584736521 0 1 1 0 0 1 -602708131544673121 0 1 1 0 0 1 --7538506014351088906 0 1 1 0 0 1 --7767907992783882398 0 1 1 0 0 1 -3427035445395945973 0 1 1 0 0 1 -1130846464892565445 0 1 1 0 0 1 --7229259894257180861 0 1 1 0 0 1 --9156504923995237243 0 1 1 0 0 1 -331058316081182921 0 1 1 0 0 1 --2878539180531591201 0 1 1 0 0 1 -5634271855584736521 0 1 1 0 0 1 -602708131544673121 0 1 1 0 0 1 --7538506014351088906 0 1 1 0 0 1 --7767907992783882398 0 1 1 0 0 1 -3427035445395945973 0 1 1 0 0 1 -1130846464892565445 0 1 1 0 0 1 --7229259894257180861 0 1 1 0 0 1 --9156504923995237243 0 1 1 0 0 1 -QUERY id: 0 - PROJECTION COLUMNS - count() UInt64 - PROJECTION - LIST id: 1, nodes: 1 - FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.x - WHERE - FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 5, nodes: 3 - FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 7, nodes: 2 - COLUMN id: 8, column_name: F, result_type: UInt8, source_id: 3 - COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 - COLUMN id: 10, column_name: B, result_type: UInt8, source_id: 3 - COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 -331058316081182921 0 1 1 0 0 1 --2878539180531591201 0 1 1 0 0 1 -5634271855584736521 0 1 1 0 0 1 -602708131544673121 0 1 1 0 0 1 --7538506014351088906 0 1 1 0 0 1 --7767907992783882398 0 1 1 0 0 1 -3427035445395945973 0 1 1 0 0 1 -1130846464892565445 0 1 1 0 0 1 --7229259894257180861 0 1 1 0 0 1 --9156504923995237243 0 1 1 0 0 1 -331058316081182921 0 1 1 0 0 1 --2878539180531591201 0 1 1 0 0 1 -5634271855584736521 0 1 1 0 0 1 -602708131544673121 0 1 1 0 0 1 --7538506014351088906 0 1 1 0 0 1 --7767907992783882398 0 1 1 0 0 1 -3427035445395945973 0 1 1 0 0 1 -1130846464892565445 0 1 1 0 0 1 --7229259894257180861 0 1 1 0 0 1 --9156504923995237243 0 1 1 0 0 1 -QUERY id: 0 - PROJECTION COLUMNS - count() UInt64 - PROJECTION - LIST id: 1, nodes: 1 - FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.x - WHERE - FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 5, nodes: 3 - FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 7, nodes: 2 - COLUMN id: 8, column_name: F, result_type: UInt8, source_id: 3 - COLUMN id: 9, column_name: E, result_type: UInt8, source_id: 3 - COLUMN id: 10, column_name: B, result_type: UInt8, source_id: 3 - COLUMN id: 11, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 -8350064335402276481 0 1 0 0 1 0 2766908592531260293 0 1 0 0 1 0 -8500464668406226757 0 1 0 0 1 0 @@ -337,52 +376,13 @@ QUERY id: 0 FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 5, nodes: 2 - FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 + COLUMN id: 6, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 7, function_name: or, function_type: ordinary, result_type: UInt8 ARGUMENTS - LIST id: 7, nodes: 3 - COLUMN id: 8, column_name: E, result_type: UInt8, source_id: 3 - COLUMN id: 9, column_name: D, result_type: UInt8, source_id: 3 - COLUMN id: 10, column_name: C, result_type: UInt8, source_id: 3 - COLUMN id: 11, column_name: B, result_type: UInt8, source_id: 3 --8350064335402276481 0 1 0 0 1 0 -2766908592531260293 0 1 0 0 1 0 --8500464668406226757 0 1 0 0 1 0 --4469615932557529717 0 1 0 0 1 0 --2676464305117434926 0 1 0 0 1 0 --3455317119139662830 0 1 0 0 1 0 -5588544998641306169 0 1 0 0 1 0 --9173414302980908256 0 1 0 0 1 0 -7212851845568196641 0 1 0 0 1 0 --3704547074519116929 0 1 0 0 1 0 --8350064335402276481 0 1 0 0 1 0 -2766908592531260293 0 1 0 0 1 0 --8500464668406226757 0 1 0 0 1 0 --4469615932557529717 0 1 0 0 1 0 --2676464305117434926 0 1 0 0 1 0 --3455317119139662830 0 1 0 0 1 0 -5588544998641306169 0 1 0 0 1 0 --9173414302980908256 0 1 0 0 1 0 -7212851845568196641 0 1 0 0 1 0 --3704547074519116929 0 1 0 0 1 0 -QUERY id: 0 - PROJECTION COLUMNS - count() UInt64 - PROJECTION - LIST id: 1, nodes: 1 - FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 - JOIN TREE - TABLE id: 3, alias: __table1, table_name: default.x - WHERE - FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 5, nodes: 2 - FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 7, nodes: 3 - COLUMN id: 8, column_name: E, result_type: UInt8, source_id: 3 - COLUMN id: 9, column_name: D, result_type: UInt8, source_id: 3 - COLUMN id: 10, column_name: C, result_type: UInt8, source_id: 3 - COLUMN id: 11, column_name: B, result_type: UInt8, source_id: 3 + LIST id: 8, nodes: 3 + COLUMN id: 9, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 7296788055532769937 0 1 0 0 0 0 -8085083078325036349 0 1 0 0 0 0 8056746393406743698 0 1 0 0 0 0 @@ -415,22 +415,22 @@ QUERY id: 0 FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 5, nodes: 2 - FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 + FUNCTION id: 6, function_name: equals, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 7, nodes: 2 - COLUMN id: 8, column_name: B, result_type: UInt8, source_id: 3 - COLUMN id: 9, column_name: A, result_type: UInt8, source_id: 3 - FUNCTION id: 10, function_name: equals, function_type: ordinary, result_type: UInt8 + FUNCTION id: 8, function_name: sipHash64, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 9, nodes: 1 + COLUMN id: 10, column_name: C, result_type: UInt8, source_id: 3 + FUNCTION id: 11, function_name: sipHash64, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 12, nodes: 1 + COLUMN id: 13, column_name: D, result_type: UInt8, source_id: 3 + FUNCTION id: 14, function_name: or, function_type: ordinary, result_type: UInt8 ARGUMENTS - LIST id: 11, nodes: 2 - FUNCTION id: 12, function_name: sipHash64, function_type: ordinary, result_type: UInt64 - ARGUMENTS - LIST id: 13, nodes: 1 - COLUMN id: 14, column_name: C, result_type: UInt8, source_id: 3 - FUNCTION id: 15, function_name: sipHash64, function_type: ordinary, result_type: UInt64 - ARGUMENTS - LIST id: 16, nodes: 1 - COLUMN id: 17, column_name: D, result_type: UInt8, source_id: 3 + LIST id: 15, nodes: 2 + COLUMN id: 16, column_name: A, result_type: UInt8, source_id: 3 + COLUMN id: 17, column_name: B, result_type: UInt8, source_id: 3 2589067852420946093 0 0 0 0 1 1 566510304360553038 0 0 0 0 1 1 4712156778925374545 0 0 0 0 1 1 @@ -562,23 +562,24 @@ QUERY id: 0 WHERE FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 ARGUMENTS - LIST id: 5, nodes: 3 - FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 + LIST id: 5, nodes: 4 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 8, function_name: or, function_type: ordinary, result_type: UInt8 ARGUMENTS - LIST id: 7, nodes: 3 - FUNCTION id: 8, function_name: and, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 9, nodes: 2 - COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 - COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 + LIST id: 9, nodes: 3 + COLUMN id: 10, column_name: C, result_type: UInt8, source_id: 3 FUNCTION id: 11, function_name: and, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 12, nodes: 2 COLUMN id: 13, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 13, column_name: E, result_type: UInt8, source_id: 3 - COLUMN id: 14, column_name: C, result_type: UInt8, source_id: 3 - COLUMN id: 15, column_name: A, result_type: UInt8, source_id: 3 - COLUMN id: 16, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 14, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 15, nodes: 2 + COLUMN id: 16, column_name: F, result_type: UInt8, source_id: 3 + COLUMN id: 16, column_name: F, result_type: UInt8, source_id: 3 -9220342792229035180 1 1 0 1 0 1 -9218747728814747926 1 1 0 0 0 1 -9220342792229035180 1 1 0 1 0 1 -9217081065560798881 1 0 0 1 0 1 -9220342792229035180 1 1 0 1 0 1 -9211856624818647543 1 1 1 0 0 0 @@ -615,21 +616,21 @@ QUERY id: 0 FUNCTION id: 6, function_name: and, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 7, nodes: 2 - FUNCTION id: 8, function_name: or, function_type: ordinary, result_type: UInt8 + FUNCTION id: 8, function_name: equals, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 9, nodes: 2 - FUNCTION id: 10, function_name: equals, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 11, nodes: 2 - COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 5 - CONSTANT id: 13, constant_value: UInt64_1, constant_value_type: UInt8 + COLUMN id: 10, column_name: A, result_type: UInt8, source_id: 4 + COLUMN id: 11, column_name: A, result_type: UInt8, source_id: 5 + FUNCTION id: 12, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 2 FUNCTION id: 14, function_name: equals, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 15, nodes: 2 COLUMN id: 16, column_name: B, result_type: UInt8, source_id: 4 CONSTANT id: 17, constant_value: UInt64_1, constant_value_type: UInt8 - FUNCTION id: 18, function_name: equals, function_type: ordinary, result_type: UInt8 - ARGUMENTS - LIST id: 19, nodes: 2 - COLUMN id: 20, column_name: A, result_type: UInt8, source_id: 4 - COLUMN id: 21, column_name: A, result_type: UInt8, source_id: 5 + FUNCTION id: 18, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 19, nodes: 2 + COLUMN id: 20, column_name: C, result_type: UInt8, source_id: 5 + CONSTANT id: 21, constant_value: UInt64_1, constant_value_type: UInt8 From 5cb5e118114e0f9e0f3a554d89a99d9d808ab7bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 7 Nov 2024 11:25:41 +0000 Subject: [PATCH 117/502] Add setting to disable optimization --- .../Passes/LogicalExpressionOptimizerPass.cpp | 27 +++++--- src/Core/Settings.cpp | 5 ++ ...2_common_expression_optimization.reference | 42 ++++++++++++ .../03262_common_expression_optimization.sql | 66 ++++++++++--------- 4 files changed, 101 insertions(+), 39 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index a30b44fc4db..897d232ff47 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -18,6 +18,7 @@ namespace Setting { extern const SettingsUInt64 optimize_min_equality_disjunction_chain_length; extern const SettingsUInt64 optimize_min_inequality_conjunction_chain_length; + extern const SettingsBool optimize_extract_common_expressions; } namespace ErrorCodes @@ -445,6 +446,15 @@ void tryOptimizeCommonExpressionsInAnd(QueryTreeNodePtr & node, const ContextPtr root_node->resolveAsFunction(and_function_resolver); } +void tryOptimizeCommonExpressions(QueryTreeNodePtr & node, FunctionNode& function_node, const ContextPtr & context) +{ + chassert(node.get() == &function_node); + if (function_node.getFunctionName() == "or") + tryOptimizeCommonExpressionsInOr(node, context); + else if (function_node.getFunctionName() == "and") + tryOptimizeCommonExpressionsInAnd(node, context); +} + /// Visitor that optimizes logical expressions _only_ in JOIN ON section class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext @@ -504,10 +514,11 @@ public: if (need_rerun_resolve) rerunFunctionResolve(function_node, getContext()); - if (function_node->getFunctionName() == "or") - tryOptimizeCommonExpressionsInOr(node, getContext()); - else if (function_node->getFunctionName() == "and") - tryOptimizeCommonExpressionsInAnd(node, getContext()); + // The optimization only makes sense on the top level + if (node != join_node->getJoinExpression() || !getSettings()[Setting::optimize_extract_common_expressions]) + return; + + tryOptimizeCommonExpressions(node, *function_node, getContext()); } private: @@ -741,6 +752,9 @@ public: void leaveImpl(QueryTreeNodePtr & node) { + if (!getSettings()[Setting::optimize_extract_common_expressions]) + return; + auto * query_node = node->as(); if (!query_node) return; @@ -753,10 +767,7 @@ public: if (!function_node) return; - if (function_node->getFunctionName() == "or") - tryOptimizeCommonExpressionsInOr(where_node, getContext()); - else if (function_node->getFunctionName() == "and") - tryOptimizeCommonExpressionsInAnd(where_node, getContext()); + tryOptimizeCommonExpressions(where_node, *function_node, getContext()); } private: diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index bdfec563397..e4f7a14985d 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -5721,6 +5721,11 @@ If enabled, MongoDB tables will return an error when a MongoDB query cannot be b Allow writing simple SELECT queries without the leading SELECT keyword, which makes it simple for calculator-style usage, e.g. `1 + 2` becomes a valid query. In `clickhouse-local` it is enabled by default and can be explicitly disabled. +)", 0) \ + DECLARE(Bool, optimize_extract_common_expressions, true, R"( +Allow extracting common expressions from disjunctions in WHERE and ON expressions. A logical expression like `(A AND B) OR (A AND C)` can be rewritten to `A AND (B OR C)`, which might help to utilize: +- indices in simple filtering expresssions +- cross to inner join optimization )", 0) \ \ \ diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index a6c332f18de..a56f8b95de9 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -1,3 +1,45 @@ +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: A, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 10, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + COLUMN id: 8, column_name: A, result_type: UInt8, source_id: 3 + COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 3 + SETTINGS optimize_extract_common_expressions=0 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + FUNCTION id: 7, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: C, result_type: UInt8, source_id: 3 + SETTINGS optimize_extract_common_expressions=1 -4890149726841642150 1 1 1 0 0 0 6142641079032712283 1 1 1 0 0 0 -558050477524086008 1 1 1 0 0 0 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index 14728fa1fc4..c0d2fed9398 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -1,68 +1,72 @@ --- Tags: no-random-settings +SET enable_analyzer = 1; DROP TABLE IF EXISTS x; CREATE TABLE x (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) ENGINE = MergeTree ORDER BY x; INSERT INTO x SELECT x, A%2 AS A, B%2 AS B, C%2 AS C, D%2 AS D, E%2 AS E, F%2 AS F FROM generateRandom('x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8', 42) LIMIT 2000; +-- Verify that optimization optimization setting works as expected +EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND B) OR (A AND C) SETTINGS optimize_extract_common_expressions = 0; +EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND B) OR (A AND C) SETTINGS optimize_extract_common_expressions = 1; -SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +-- Test multiple cases +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND C AND F)); -SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)); -SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)); -SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)); -SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))); -- Without AND as a root -SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR (B AND C AND F)); -SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C AND E) OR (B AND C AND F)); -SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)); -SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)); -SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))); -- Complex expression -SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))); -- Flattening is only happening if something can be extracted -SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))); -SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))); -- Duplicates -SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 0; -SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS enable_analyzer = 1; +SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)); DROP TABLE IF EXISTS y; @@ -71,6 +75,6 @@ INSERT INTO y SELECT x, A%2 AS A, B%2 AS B, C%2 AS C, D%2 AS D, E%2 AS E, F%2 AS -- JOIN expressions -- As the optimization code is shared between ON and WHERE, it is enough to test that the optimization is done also in ON -SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, enable_analyzer = 0; -SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, enable_analyzer = 1; +SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, optimize_extract_common_expressions = 0; +SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1); From 28491937d2e082dc656fa007f4fa99a1997b9e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 7 Nov 2024 11:26:11 +0000 Subject: [PATCH 118/502] Add test to check that optimization only happens on top level --- ...2_common_expression_optimization.reference | 88 +++++++++++++++++++ .../03262_common_expression_optimization.sql | 7 ++ 2 files changed, 95 insertions(+) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index a56f8b95de9..ca7a0d8f908 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -622,6 +622,36 @@ QUERY id: 0 LIST id: 15, nodes: 2 COLUMN id: 16, column_name: F, result_type: UInt8, source_id: 3 COLUMN id: 16, column_name: F, result_type: UInt8, source_id: 3 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + FUNCTION id: 7, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 10, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + FUNCTION id: 12, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 2 + COLUMN id: 14, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 15, column_name: D, result_type: UInt8, source_id: 3 + FUNCTION id: 16, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 17, nodes: 2 + COLUMN id: 14, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 18, column_name: E, result_type: UInt8, source_id: 3 -9220342792229035180 1 1 0 1 0 1 -9218747728814747926 1 1 0 0 0 1 -9220342792229035180 1 1 0 1 0 1 -9217081065560798881 1 0 0 1 0 1 -9220342792229035180 1 1 0 1 0 1 -9211856624818647543 1 1 1 0 0 0 @@ -676,3 +706,61 @@ QUERY id: 0 LIST id: 19, nodes: 2 COLUMN id: 20, column_name: C, result_type: UInt8, source_id: 5 CONSTANT id: 21, constant_value: UInt64_1, constant_value_type: UInt8 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + JOIN id: 3, strictness: ALL, kind: INNER + LEFT TABLE EXPRESSION + TABLE id: 4, alias: __table1, table_name: default.x + RIGHT TABLE EXPRESSION + TABLE id: 5, alias: __table2, table_name: default.y + JOIN EXPRESSION + FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + FUNCTION id: 8, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 9, nodes: 2 + COLUMN id: 10, column_name: A, result_type: UInt8, source_id: 4 + COLUMN id: 11, column_name: A, result_type: UInt8, source_id: 5 + FUNCTION id: 12, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 2 + FUNCTION id: 14, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 15, nodes: 2 + COLUMN id: 16, column_name: B, result_type: UInt8, source_id: 4 + COLUMN id: 17, column_name: B, result_type: UInt8, source_id: 5 + FUNCTION id: 18, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 19, nodes: 2 + FUNCTION id: 20, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 21, nodes: 2 + FUNCTION id: 22, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 23, nodes: 2 + COLUMN id: 24, column_name: C, result_type: UInt8, source_id: 4 + COLUMN id: 25, column_name: C, result_type: UInt8, source_id: 5 + FUNCTION id: 26, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 27, nodes: 2 + COLUMN id: 28, column_name: D, result_type: UInt8, source_id: 4 + COLUMN id: 29, column_name: D, result_type: UInt8, source_id: 5 + FUNCTION id: 30, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 31, nodes: 2 + FUNCTION id: 32, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 33, nodes: 2 + COLUMN id: 34, column_name: C, result_type: UInt8, source_id: 4 + COLUMN id: 35, column_name: C, result_type: UInt8, source_id: 5 + FUNCTION id: 36, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 37, nodes: 2 + COLUMN id: 38, column_name: E, result_type: UInt8, source_id: 4 + COLUMN id: 39, column_name: E, result_type: UInt8, source_id: 5 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index c0d2fed9398..ee468eef28d 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -69,6 +69,10 @@ SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)); +-- Check that optimization only happen on top level, (C AND D) OR (C AND E) shouldn't be optimized +EXPLAIN QUERY TREE SELECT count() FROM x WHERE A OR (B AND ((C AND D) OR (C AND E))); + + DROP TABLE IF EXISTS y; CREATE TABLE y (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) ENGINE = MergeTree ORDER BY x; INSERT INTO y SELECT x, A%2 AS A, B%2 AS B, C%2 AS C, D%2 AS D, E%2 AS E, F%2 AS F FROM generateRandom('x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8', 43) LIMIT 2000; @@ -78,3 +82,6 @@ INSERT INTO y SELECT x, A%2 AS A, B%2 AS B, C%2 AS C, D%2 AS D, E%2 AS E, F%2 AS SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, optimize_extract_common_expressions = 0; SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE SELECT count() FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1); + +-- Check that optimization only happen on top level, (x.C = y.C AND x.D = y.D) OR (x.C = y.C AND x.E = y.E) shouldn't be optimized +EXPLAIN QUERY TREE SELECT count() FROM x INNER JOIN y ON (x.A = y.A) OR ((x.B = y.B) AND ((x.C = y.C AND x.D = y.D) OR (x.C = y.C AND x.E = y.E))); From 6e6a1a485f5048a61a8a4e74f68c6d6dad92de90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 7 Nov 2024 12:52:18 +0000 Subject: [PATCH 119/502] Fix typo --- src/Core/Settings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index e4f7a14985d..c5d17a08764 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -5724,7 +5724,7 @@ In `clickhouse-local` it is enabled by default and can be explicitly disabled. )", 0) \ DECLARE(Bool, optimize_extract_common_expressions, true, R"( Allow extracting common expressions from disjunctions in WHERE and ON expressions. A logical expression like `(A AND B) OR (A AND C)` can be rewritten to `A AND (B OR C)`, which might help to utilize: -- indices in simple filtering expresssions +- indices in simple filtering expressions - cross to inner join optimization )", 0) \ \ From 4f6b5c09e081ed1eee0a199dc3c9bf3b6eb7a3f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 7 Nov 2024 12:52:36 +0000 Subject: [PATCH 120/502] Add new settings to settings changes history --- src/Core/SettingsChangesHistory.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index f4a8fd198c5..db4c225877a 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -81,6 +81,7 @@ static std::initializer_list Date: Thu, 14 Nov 2024 13:34:00 +0000 Subject: [PATCH 121/502] Disable optimization by default --- src/Core/Settings.cpp | 2 +- src/Core/SettingsChangesHistory.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index c5d17a08764..d4f6fcc412e 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -5722,7 +5722,7 @@ Allow writing simple SELECT queries without the leading SELECT keyword, which ma In `clickhouse-local` it is enabled by default and can be explicitly disabled. )", 0) \ - DECLARE(Bool, optimize_extract_common_expressions, true, R"( + DECLARE(Bool, optimize_extract_common_expressions, false, R"( Allow extracting common expressions from disjunctions in WHERE and ON expressions. A logical expression like `(A AND B) OR (A AND C)` can be rewritten to `A AND (B OR C)`, which might help to utilize: - indices in simple filtering expressions - cross to inner join optimization diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index db4c225877a..e08a4ac841c 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -81,7 +81,7 @@ static std::initializer_list Date: Thu, 14 Nov 2024 14:03:32 +0000 Subject: [PATCH 122/502] Fix errors --- .../ObjectStorage/DataLakes/IcebergMetadata.h | 12 +----------- .../ObjectStorage/StorageObjectStorageSource.cpp | 1 - tests/integration/test_storage_iceberg/test.py | 10 +++++----- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 6f1058e3cf3..6d82f3506fe 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -17,16 +17,6 @@ #include #include -template <> -struct std::hash> -{ - size_t operator()(const std::pair & p) const - { - std::hash hash; - return hash(static_cast(p.first) << 32 | static_cast(p.second)); - } -}; - namespace DB { @@ -98,7 +88,7 @@ public: private: std::unordered_map iceberg_table_schemas_by_ids; std::unordered_map> clickhouse_table_schemas_by_ids; - std::unordered_map, std::shared_ptr> transform_dags_by_ids; + std::map, std::shared_ptr> transform_dags_by_ids; NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr & schema); DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type); diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index fd0a0e13710..ac6ea89aba7 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -374,7 +374,6 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade Block initial_header = read_from_format_info.format_header; - auto relative_path = object_info->relative_path; if (auto initial_schema = configuration->getInitialSchemaByPath(object_info->getPath())) { Block sample_header; diff --git a/tests/integration/test_storage_iceberg/test.py b/tests/integration/test_storage_iceberg/test.py index 1c807b8c45e..828618871bd 100644 --- a/tests/integration/test_storage_iceberg/test.py +++ b/tests/integration/test_storage_iceberg/test.py @@ -182,12 +182,12 @@ def get_creation_expression( cluster, format="Parquet", table_function=False, - allow_dynamic_metadata_for_datalakes=False, + allow_dynamic_metadata_for_data_lakes=False, **kwargs, ): allow_dynamic_metadata_for_datalakes_suffix = ( - " SETTINGS allow_dynamic_metadata_for_datalakes = 1" - if allow_dynamic_metadata_for_datalakes + " SETTINGS allow_dynamic_metadata_for_data_lakes = 1" + if allow_dynamic_metadata_for_data_lakes else "" ) @@ -654,7 +654,7 @@ def test_evolved_schema_simple( TABLE_NAME, started_cluster, table_function=is_table_function, - allow_dynamic_metadata_for_datalakes=True, + allow_dynamic_metadata_for_data_lakes=True, ) table_select_expression = ( @@ -1008,7 +1008,7 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): TABLE_NAME, started_cluster, table_function=False, - allow_dynamic_metadata_for_datalakes=False, + allow_dynamic_metadata_for_data_lakes=False, ) ) From c8f6eb4f5e8295cfa56af26ff81ffaf8a3fa5ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 14 Nov 2024 14:25:57 +0000 Subject: [PATCH 123/502] Turn optimization on in the test by default --- .../03262_common_expression_optimization.sql | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index ee468eef28d..51e0b1d9859 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -1,4 +1,5 @@ SET enable_analyzer = 1; +SET optimize_extract_common_expressions = 1; DROP TABLE IF EXISTS x; CREATE TABLE x (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) ENGINE = MergeTree ORDER BY x; @@ -10,63 +11,63 @@ EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND B) OR (A AND C) SETTINGS o -- Test multiple cases SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND C AND F)); SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)); SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)); SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)); SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))); -- Without AND as a root SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR (B AND C AND F)); SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C AND E) OR (B AND C AND F)); SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)); SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)); SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))); -- Complex expression SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))); -- Flattening is only happening if something can be extracted SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))); SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))); -- Duplicates SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)); -- Check that optimization only happen on top level, (C AND D) OR (C AND E) shouldn't be optimized From 0e14b49298932bae829aa5d1ff848b01e9e2e5af Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 14 Nov 2024 19:56:51 +0100 Subject: [PATCH 124/502] Support auth header --- src/Databases/Iceberg/DatabaseIceberg.cpp | 28 +++++++++++++++++++ src/Databases/Iceberg/DatabaseIceberg.h | 5 +++- .../Iceberg/DatabaseIcebergSettings.cpp | 1 + 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 311ae9bf2dc..39ed6118af1 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -1,6 +1,8 @@ #include #if USE_AVRO +#include + #include #include #include @@ -27,6 +29,7 @@ namespace DB namespace DatabaseIcebergSetting { extern const DatabaseIcebergSettingsString storage_endpoint; + extern const DatabaseIcebergSettingsString auth_header; extern const DatabaseIcebergSettingsDatabaseIcebergCatalogType catalog_type; extern const DatabaseIcebergSettingsDatabaseIcebergStorageType storage_type; } @@ -52,6 +55,20 @@ namespace auto namespace_name = name.substr(0, name.size() - table_name.size() - 1); return {namespace_name, table_name}; } + + void setCredentials(Poco::Net::HTTPBasicCredentials & credentials, const Poco::URI & request_uri) + { + const auto & user_info = request_uri.getUserInfo(); + if (!user_info.empty()) + { + std::size_t n = user_info.find(':'); + if (n != std::string::npos) + { + credentials.setUsername(user_info.substr(0, n)); + credentials.setPassword(user_info.substr(n + 1)); + } + } + } } DatabaseIceberg::DatabaseIceberg( @@ -65,6 +82,17 @@ DatabaseIceberg::DatabaseIceberg( , database_engine_definition(database_engine_definition_) , log(getLogger("DatabaseIceberg(" + database_name_ + ")")) { + setCredentials(credentials, Poco::URI(url)); + + const auto auth_header = settings[DatabaseIcebergSetting::auth_header].value; + if (!auth_header.empty()) + { + auto pos = auth_header.find(':'); + if (pos == std::string::npos) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected format of auth header"); + headers.emplace_back(auth_header.substr(0, pos), auth_header.substr(pos + 1)); + } + } std::unique_ptr DatabaseIceberg::getCatalog(ContextPtr context_) const diff --git a/src/Databases/Iceberg/DatabaseIceberg.h b/src/Databases/Iceberg/DatabaseIceberg.h index f7b7544f80a..230920b1190 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.h +++ b/src/Databases/Iceberg/DatabaseIceberg.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -53,8 +54,10 @@ private: const DatabaseIcebergSettings settings; /// Database engine definition taken from initial CREATE DATABASE query. const ASTPtr database_engine_definition; - const LoggerPtr log; + /// Crendetials to authenticate Iceberg Catalog. + Poco::Net::HTTPBasicCredentials credentials; + HTTPHeaderEntries headers; std::unique_ptr getCatalog(ContextPtr context_) const; std::shared_ptr getConfiguration() const; diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp index 8383c372a52..5017686b0d7 100644 --- a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp @@ -17,6 +17,7 @@ namespace ErrorCodes #define DATABASE_ICEBERG_RELATED_SETTINGS(DECLARE, ALIAS) \ DECLARE(DatabaseIcebergCatalogType, catalog_type, DatabaseIcebergCatalogType::REST, "Catalog type", 0) \ DECLARE(DatabaseIcebergStorageType, storage_type, DatabaseIcebergStorageType::S3, "Storage type: S3, Local, Azure, HDFS", 0) \ + DECLARE(String, auth_header, "", "Authorization header of format 'Authorization: '", 0) \ DECLARE(String, storage_endpoint, "", "Object storage endpoint", 0) \ #define LIST_OF_DATABASE_ICEBERG_SETTINGS(M, ALIAS) \ From 32ff7d2722c8ec70e7b692d4980d09760ddbea93 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 15 Nov 2024 11:41:54 +0100 Subject: [PATCH 125/502] Fix build and test --- src/Databases/Iceberg/DatabaseIceberg.cpp | 12 ++++++++++++ tests/integration/test_database_iceberg/test.py | 4 +--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 39ed6118af1..8f46baf0405 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -110,22 +110,34 @@ std::shared_ptr DatabaseIceberg::getConfigu { switch (settings[DatabaseIcebergSetting::storage_type].value) { +#if USE_AWS_S3 case DB::DatabaseIcebergStorageType::S3: { return std::make_shared(); } +#endif +#if USE_AZURE_BLOB_STORAGE case DB::DatabaseIcebergStorageType::Azure: { return std::make_shared(); } +#endif +#if USE_HDFS case DB::DatabaseIcebergStorageType::HDFS: { return std::make_shared(); } +#endif case DB::DatabaseIcebergStorageType::Local: { return std::make_shared(); } +#if !USE_AWS_S3 || !USE_AZURE_BLOB_STORAGE || !USE_HDFS + default: + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Server does not contain support for storage type {}", + settings[DatabaseIcebergSetting::storage_type].value); +#endif } } diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 310880c282a..51d89444c2e 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -242,13 +242,11 @@ def test_many_namespaces(started_cluster): f"{root_namespace_1}.B.C", f"{root_namespace_1}.B.C.D", f"{root_namespace_1}.B.C.D.E", - f"{root_namespace_1}.B.C.D.E.F", - f"{root_namespace_1}.B.C.D.E.FF", f"{root_namespace_2}", f"{root_namespace_2}.C", f"{root_namespace_2}.CC", ] - tables = ["A", "B", "C", "D", "E", "F"] + tables = ["A", "B", "C"] catalog = load_catalog_impl(started_cluster) for namespace in namespaces: From 6dbe20839afe23f89043bd78048f0c1a6993bb8b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 Nov 2024 16:02:07 +0000 Subject: [PATCH 126/502] Fix nulls direction check. --- .../Optimizations/optimizeReadInOrder.cpp | 2 +- .../03257_reverse_sorting_key.reference | 33 +++++++++++-------- .../0_stateless/03257_reverse_sorting_key.sql | 4 +++ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index a0921359790..50154c25842 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -426,7 +426,7 @@ SortingInputOrder buildInputOrderFromSortDescription( /// Since sorting key columns are always sorted with NULLS LAST, reading in order /// supported only for ASC NULLS LAST ("in order"), and DESC NULLS FIRST ("reverse") const auto column_is_nullable = sorting_key.data_types[next_sort_key]->isNullable(); - if (column_is_nullable && sort_column_description.nulls_direction != 1) + if (column_is_nullable && sort_column_description.nulls_direction != sort_column_description.direction) break; /// Direction for current sort key. diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key.reference b/tests/queries/0_stateless/03257_reverse_sorting_key.reference index 5371ee21f44..ef51ac39dea 100644 --- a/tests/queries/0_stateless/03257_reverse_sorting_key.reference +++ b/tests/queries/0_stateless/03257_reverse_sorting_key.reference @@ -1,22 +1,25 @@ 3 8 +Sorting (Sorting for ORDER BY) +Prefix sort description: __table1.i DESC +Result sort description: __table1.i DESC (Expression) ExpressionTransform (Limit) Limit (Sorting) - MergeSortingTransform - LimitsCheckingTransform - PartialSortingTransform - (Expression) - ExpressionTransform - (ReadFromMergeTree) - MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 99 98 97 96 95 +Sorting (Sorting for ORDER BY) +Prefix sort description: __table1.i ASC +Result sort description: __table1.i ASC (Expression) ExpressionTransform (Limit) @@ -34,22 +37,26 @@ ExpressionTransform 4 3 1003 6 +Sorting (Sorting for ORDER BY) +Prefix sort description: __table1.i ASC, __table1.j DESC +Result sort description: __table1.i ASC, __table1.j DESC (Expression) ExpressionTransform (Limit) Limit (Sorting) - FinishSortingTransform - PartialSortingTransform - (Expression) - ExpressionTransform - (ReadFromMergeTree) - MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 0 1090 0 1080 0 1070 0 1060 0 1050 +Sorting (Sorting for ORDER BY) +Prefix sort description: __table1.i ASC +Result sort description: __table1.i ASC, __table1.j ASC (Expression) ExpressionTransform (Limit) diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key.sql b/tests/queries/0_stateless/03257_reverse_sorting_key.sql index 6aaeb2bacd6..9d2f279026f 100644 --- a/tests/queries/0_stateless/03257_reverse_sorting_key.sql +++ b/tests/queries/0_stateless/03257_reverse_sorting_key.sql @@ -17,10 +17,12 @@ select * from x1 where i = 3; select count() from x1 where i between 3 and 10; +select trimLeft(explain) from (explain actions=1 select * from x1 order by i desc limit 5 settings max_threads=1, enable_analyzer=1) where explain ilike '%sort%'; explain pipeline select * from x1 order by i desc limit 5 settings max_threads=1; select * from x1 order by i desc limit 5; +select trimLeft(explain) from (explain actions=1 select * from x1 order by i limit 5 settings max_threads=1, enable_analyzer=1) where explain ilike '%sort%'; explain pipeline select * from x1 order by i limit 5 settings max_threads=1; select * from x1 order by i limit 5; @@ -35,10 +37,12 @@ select * from x2 where j = 1003; select count() from x2 where i between 3 and 10 and j between 1003 and 1008; +select trimLeft(explain) from (explain actions=1 select * from x2 order by i, j desc limit 5 settings max_threads=1, enable_analyzer=1) where explain ilike '%sort%'; explain pipeline select * from x2 order by i, j desc limit 5 settings max_threads=1; select * from x2 order by i, j desc limit 5; +select trimLeft(explain) from (explain actions=1 select * from x2 order by i, j limit 5 settings max_threads=1, enable_analyzer=1) where explain ilike '%sort%'; explain pipeline select * from x2 order by i, j limit 5 settings max_threads=1; select * from x2 order by i, j limit 5; From 17e7b287a231e7fb0e9f93d5062d46b5cff569bf Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Sat, 16 Nov 2024 17:07:01 -0300 Subject: [PATCH 127/502] initial working implementation --- src/Functions/array/arrayPrAUC.cpp | 198 ++++++++++++++++++ .../0_stateless/03272_array_pr_auc.reference | 2 + .../0_stateless/03272_array_pr_auc.sql | 2 + 3 files changed, 202 insertions(+) create mode 100644 src/Functions/array/arrayPrAUC.cpp create mode 100644 tests/queries/0_stateless/03272_array_pr_auc.reference create mode 100644 tests/queries/0_stateless/03272_array_pr_auc.sql diff --git a/src/Functions/array/arrayPrAUC.cpp b/src/Functions/array/arrayPrAUC.cpp new file mode 100644 index 00000000000..20bca480d25 --- /dev/null +++ b/src/Functions/array/arrayPrAUC.cpp @@ -0,0 +1,198 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; + extern const int BAD_ARGUMENTS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + + +/** The function takes two arrays: scores and labels. + */ + +class FunctionArrayPrAUC : public IFunction +{ +public: + static constexpr auto name = "arrayPrAUC"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + +private: + static Float64 apply( + const IColumn & scores, + const IColumn & labels, + ColumnArray::Offset current_offset, + ColumnArray::Offset next_offset, + bool scale) + { + struct ScoreLabel + { + Float64 score; + bool label; + }; + + size_t size = next_offset - current_offset; + PODArrayWithStackMemory sorted_labels(size); + + for (size_t i = 0; i < size; ++i) + { + bool label = labels.getFloat64(current_offset + i) > 0; + sorted_labels[i].score = scores.getFloat64(current_offset + i); + sorted_labels[i].label = label; + } + + /// Sorting scores in descending order to traverse the Precision Recall curve from left to right + std::sort(sorted_labels.begin(), sorted_labels.end(), [](const auto & lhs, const auto & rhs) { return lhs.score > rhs.score; }); + + Float64 area = 0.0; + Float64 prev_score = sorted_labels[0].score; + Float64 curr_precision = 1.0; + size_t prev_tp = 0; + size_t curr_fp = 0, curr_tp = 0; + for (size_t i = 0; i < size; ++i) + { + /// Only increment the area when the score changes + if (sorted_labels[i].score != prev_score) + { + curr_precision = (curr_tp + curr_fp) > 0 ? static_cast(curr_tp) / (curr_tp + curr_fp) : 1.0; + /// Sum precision * recall rectangle. Since recall is (TP / (TP + FN)) and TP + FN = fixed, we can divide by size later. + area += curr_precision * (curr_tp - prev_tp); + prev_tp = curr_tp; + prev_score = sorted_labels[i].score; + } + + if (sorted_labels[i].label) + curr_tp += 1; /// The curve moves one step up. + else + curr_fp += 1; /// The curve moves one step right. + } + + curr_precision = static_cast(curr_tp) / (curr_tp + curr_fp); + area += curr_precision * (curr_tp - prev_tp); + + if (scale) /// It doesn't make sense to not normalize + { + if (curr_tp == 0) + return std::numeric_limits::quiet_NaN(); + return area / curr_tp; + } + return area; + } + + static void vector( + const IColumn & scores, + const IColumn & labels, + const ColumnArray::Offsets & offsets, + PaddedPODArray & result, + size_t input_rows_count, + bool scale) + { + result.resize(input_rows_count); + + ColumnArray::Offset current_offset = 0; + for (size_t i = 0; i < input_rows_count; ++i) + { + auto next_offset = offsets[i]; + result[i] = apply(scores, labels, current_offset, next_offset, scale); + current_offset = next_offset; + } + } + +public: + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + size_t number_of_arguments = arguments.size(); + + if (number_of_arguments < 2 || number_of_arguments > 3) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3", + getName(), number_of_arguments); + + for (size_t i = 0; i < 2; ++i) + { + const DataTypeArray * array_type = checkAndGetDataType(arguments[i].type.get()); + if (!array_type) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The two first arguments for function {} must be of type Array.", getName()); + + const auto & nested_type = array_type->getNestedType(); + if (!isNativeNumber(nested_type) && !isEnum(nested_type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{} cannot process values of type {}", getName(), nested_type->getName()); + } + + if (number_of_arguments == 3) + { + if (!isBool(arguments[2].type) || arguments[2].column.get() == nullptr || !isColumnConst(*arguments[2].column)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Third argument (scale) for function {} must be of type const Bool.", getName()); + } + + return std::make_shared(); + } + + DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override + { + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + size_t number_of_arguments = arguments.size(); + + ColumnPtr col1 = arguments[0].column->convertToFullColumnIfConst(); + ColumnPtr col2 = arguments[1].column->convertToFullColumnIfConst(); + + const ColumnArray * col_array1 = checkAndGetColumn(col1.get()); + if (!col_array1) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of first argument of function {}", arguments[0].column->getName(), getName()); + + const ColumnArray * col_array2 = checkAndGetColumn(col2.get()); + if (!col_array2) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of second argument of function {}", arguments[1].column->getName(), getName()); + + if (!col_array1->hasEqualOffsets(*col_array2)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Array arguments for function {} must have equal sizes", getName()); + + /// Handle third argument for scale (if passed, otherwise default to true) + bool scale = true; + if (number_of_arguments == 3 && input_rows_count > 0) + scale = arguments[2].column->getBool(0); + + auto col_res = ColumnVector::create(); + + vector( + col_array1->getData(), + col_array2->getData(), + col_array1->getOffsets(), + col_res->getData(), + input_rows_count, + scale); + + return col_res; + } +}; + + +REGISTER_FUNCTION(ArrayPrAUC) +{ + factory.registerFunction(); +} + +} diff --git a/tests/queries/0_stateless/03272_array_pr_auc.reference b/tests/queries/0_stateless/03272_array_pr_auc.reference new file mode 100644 index 00000000000..175c333eee4 --- /dev/null +++ b/tests/queries/0_stateless/03272_array_pr_auc.reference @@ -0,0 +1,2 @@ +0.83333333 +0.80555555 diff --git a/tests/queries/0_stateless/03272_array_pr_auc.sql b/tests/queries/0_stateless/03272_array_pr_auc.sql new file mode 100644 index 00000000000..67c556db6e7 --- /dev/null +++ b/tests/queries/0_stateless/03272_array_pr_auc.sql @@ -0,0 +1,2 @@ +SELECT FLOOR(arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]), 8); +SELECT FLOOR(arrayPrAUC([0.1, 0.4, 0.4, 0.35, 0.8], [0, 0, 1, 1, 1]), 8); From 5d8349fd7712a9ca4633b99585ba1bf783919715 Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Sat, 16 Nov 2024 17:55:29 -0300 Subject: [PATCH 128/502] removed roc auc logic --- src/Functions/array/arrayPrAUC.cpp | 127 +++++++++++++---------------- 1 file changed, 58 insertions(+), 69 deletions(-) diff --git a/src/Functions/array/arrayPrAUC.cpp b/src/Functions/array/arrayPrAUC.cpp index 20bca480d25..f99519388ae 100644 --- a/src/Functions/array/arrayPrAUC.cpp +++ b/src/Functions/array/arrayPrAUC.cpp @@ -1,9 +1,9 @@ -#include -#include -#include #include -#include +#include +#include +#include #include +#include namespace DB @@ -11,10 +11,10 @@ namespace DB namespace ErrorCodes { - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; - extern const int BAD_ARGUMENTS; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; +extern const int BAD_ARGUMENTS; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -28,12 +28,8 @@ public: static FunctionPtr create(ContextPtr) { return std::make_shared(); } private: - static Float64 apply( - const IColumn & scores, - const IColumn & labels, - ColumnArray::Offset current_offset, - ColumnArray::Offset next_offset, - bool scale) + static Float64 + apply(const IColumn & scores, const IColumn & labels, ColumnArray::Offset current_offset, ColumnArray::Offset next_offset) { struct ScoreLabel { @@ -54,39 +50,45 @@ private: /// Sorting scores in descending order to traverse the Precision Recall curve from left to right std::sort(sorted_labels.begin(), sorted_labels.end(), [](const auto & lhs, const auto & rhs) { return lhs.score > rhs.score; }); - Float64 area = 0.0; - Float64 prev_score = sorted_labels[0].score; - Float64 curr_precision = 1.0; size_t prev_tp = 0; size_t curr_fp = 0, curr_tp = 0; + Float64 curr_precision = 1.0; + Float64 area = 0.0; + for (size_t i = 0; i < size; ++i) { - /// Only increment the area when the score changes - if (sorted_labels[i].score != prev_score) + if (curr_tp != prev_tp) { - curr_precision = (curr_tp + curr_fp) > 0 ? static_cast(curr_tp) / (curr_tp + curr_fp) : 1.0; - /// Sum precision * recall rectangle. Since recall is (TP / (TP + FN)) and TP + FN = fixed, we can divide by size later. + /* Precision = TP / (TP + FP) and Recall = TP / (TP + FN) + * + * Instead of calculating + * d_Area = Precision_n * (Recall_n - Recall_{n-1}), + * we can calculate + * d_Area = Precision_n * (TP_n - TP_{n-1}) + * and later divide it by (TP + FN), since + */ + curr_precision = static_cast(curr_tp) / (curr_tp + curr_fp); area += curr_precision * (curr_tp - prev_tp); prev_tp = curr_tp; - prev_score = sorted_labels[i].score; } if (sorted_labels[i].label) - curr_tp += 1; /// The curve moves one step up. + curr_tp += 1; else - curr_fp += 1; /// The curve moves one step right. + curr_fp += 1; } - curr_precision = static_cast(curr_tp) / (curr_tp + curr_fp); + curr_precision = (curr_tp + curr_fp) > 0 ? static_cast(curr_tp) / (curr_tp + curr_fp) : 1.0; area += curr_precision * (curr_tp - prev_tp); - if (scale) /// It doesn't make sense to not normalize - { - if (curr_tp == 0) - return std::numeric_limits::quiet_NaN(); - return area / curr_tp; - } - return area; + /// If there were no labels, return NaN + if (curr_tp == 0 && curr_fp == 0) + return std::numeric_limits::quiet_NaN(); + /// If there were no positive labels, the only point of the curve is (0, 1) and AUC is 0 + if (curr_tp == 0) + return 0.0; + /// Finally, divide it by total number of positive labels (TP + FN) + return area / curr_tp; } static void vector( @@ -94,8 +96,7 @@ private: const IColumn & labels, const ColumnArray::Offsets & offsets, PaddedPODArray & result, - size_t input_rows_count, - bool scale) + size_t input_rows_count) { result.resize(input_rows_count); @@ -103,7 +104,7 @@ private: for (size_t i = 0; i < input_rows_count; ++i) { auto next_offset = offsets[i]; - result[i] = apply(scores, labels, current_offset, next_offset, scale); + result[i] = apply(scores, labels, current_offset, next_offset); current_offset = next_offset; } } @@ -120,70 +121,58 @@ public: { size_t number_of_arguments = arguments.size(); - if (number_of_arguments < 2 || number_of_arguments > 3) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3", - getName(), number_of_arguments); + if (number_of_arguments < 2 || number_of_arguments > 2) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be 2.", + getName(), + number_of_arguments); for (size_t i = 0; i < 2; ++i) { const DataTypeArray * array_type = checkAndGetDataType(arguments[i].type.get()); if (!array_type) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The two first arguments for function {} must be of type Array.", getName()); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The two first arguments for function {} must be of type Array.", getName()); const auto & nested_type = array_type->getNestedType(); if (!isNativeNumber(nested_type) && !isEnum(nested_type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{} cannot process values of type {}", getName(), nested_type->getName()); - } - - if (number_of_arguments == 3) - { - if (!isBool(arguments[2].type) || arguments[2].column.get() == nullptr || !isColumnConst(*arguments[2].column)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Third argument (scale) for function {} must be of type const Bool.", getName()); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{} cannot process values of type {}", getName(), nested_type->getName()); } return std::make_shared(); } - DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override - { - return std::make_shared(); - } + DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override { return std::make_shared(); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - size_t number_of_arguments = arguments.size(); - ColumnPtr col1 = arguments[0].column->convertToFullColumnIfConst(); ColumnPtr col2 = arguments[1].column->convertToFullColumnIfConst(); const ColumnArray * col_array1 = checkAndGetColumn(col1.get()); if (!col_array1) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, - "Illegal column {} of first argument of function {}", arguments[0].column->getName(), getName()); + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of first argument of function {}", + arguments[0].column->getName(), + getName()); const ColumnArray * col_array2 = checkAndGetColumn(col2.get()); if (!col_array2) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, - "Illegal column {} of second argument of function {}", arguments[1].column->getName(), getName()); + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of second argument of function {}", + arguments[1].column->getName(), + getName()); if (!col_array1->hasEqualOffsets(*col_array2)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Array arguments for function {} must have equal sizes", getName()); - /// Handle third argument for scale (if passed, otherwise default to true) - bool scale = true; - if (number_of_arguments == 3 && input_rows_count > 0) - scale = arguments[2].column->getBool(0); - auto col_res = ColumnVector::create(); - vector( - col_array1->getData(), - col_array2->getData(), - col_array1->getOffsets(), - col_res->getData(), - input_rows_count, - scale); + vector(col_array1->getData(), col_array2->getData(), col_array1->getOffsets(), col_res->getData(), input_rows_count); return col_res; } From f5bf410893ef06f3ff94f6a1ca23e73d6f76a823 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 18 Nov 2024 15:18:34 +0000 Subject: [PATCH 129/502] Try to fix data race --- .../DataLakes/IcebergMetadata.cpp | 82 ++++++++++--------- .../ObjectStorage/DataLakes/IcebergMetadata.h | 17 ++-- 2 files changed, 55 insertions(+), 44 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 7e23b9df272..cce72c571c9 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -1,3 +1,4 @@ +#include #include "config.h" #if USE_AVRO @@ -51,26 +52,47 @@ extern const int UNSUPPORTED_METHOD; extern const int LOGICAL_ERROR; } +Int32 parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor) +{ + Int32 format_version = metadata_object->getValue("format-version"); + if (format_version == 2) + { + auto fields = metadata_object->get("schemas").extract(); + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + schema_processor.addIcebergTableSchema(field); + } + return metadata_object->getValue("current-schema-id"); + } + else + { + auto schema = metadata_object->getObject("schema"); + schema_processor.addIcebergTableSchema(schema); + return schema->getValue("schema-id"); + } +} + IcebergMetadata::IcebergMetadata( ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, - DB::ContextPtr context_, + const DB::ContextPtr & context_, Int32 metadata_version_, Int32 format_version_, String manifest_list_file_, - Int32 current_schema_id_, - IcebergSchemaProcessor schema_processor_) + const Poco::JSON::Object::Ptr & object) : WithContext(context_) - , object_storage(object_storage_) - , configuration(configuration_) + , object_storage(std::move(object_storage_)) + , configuration(std::move(configuration_)) , metadata_version(metadata_version_) , format_version(format_version_) , manifest_list_file(std::move(manifest_list_file_)) - , current_schema_id(current_schema_id_) - , schema_processor(schema_processor_) - , schema(*schema_processor.getClickhouseTableSchemaById(current_schema_id)) + , schema_processor(IcebergSchemaProcessor()) , log(getLogger("IcebergMetadata")) { + auto schema_id = parseTableSchema(object, schema_processor); + schema = *(schema_processor.getClickhouseTableSchemaById(schema_id)); + current_schema_id = schema_id; } namespace @@ -119,27 +141,6 @@ bool operator==(const Poco::JSON::Object & first, const Poco::JSON::Object & sec } return first_string_stream.str() == second_string_stream.str(); } - -Int32 parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor) -{ - Int32 format_version = metadata_object->getValue("format-version"); - if (format_version == 2) - { - auto fields = metadata_object->get("schemas").extract(); - for (size_t i = 0; i != fields->size(); ++i) - { - auto field = fields->getObject(static_cast(i)); - schema_processor.addIcebergTableSchema(field); - } - return metadata_object->getValue("current-schema-id"); - } - else - { - auto schema = metadata_object->getObject("schema"); - schema_processor.addIcebergTableSchema(schema); - return schema->getValue("schema-id"); - } -} } @@ -350,7 +351,7 @@ IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr { throw Exception( ErrorCodes::BAD_ARGUMENTS, - "Canßnot add a column with id {} with required values to the table during schema evolution, old schema id is {}, new " + "Cannot add a column with id {} with required values to the table during schema evolution, old schema id is {}, new " "schema id is {}", id, current_old_id, @@ -366,6 +367,7 @@ IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id) { + std::lock_guard lock(mutex); current_old_id = old_id; current_new_id = new_id; SCOPE_EXIT({ @@ -450,9 +452,8 @@ MutableColumns parseAvro(avro::DataFileReaderBase & file_reader, const Block & h * 1) v.metadata.json, where V - metadata version. * 2) -.metadata.json, where V - metadata version */ -std::pair getMetadataFileAndVersion( - ObjectStoragePtr object_storage, - const StorageObjectStorage::Configuration & configuration) +std::pair +getMetadataFileAndVersion(const ObjectStoragePtr & object_storage, const StorageObjectStorage::Configuration & configuration) { const auto metadata_files = listFiles(*object_storage, configuration, "metadata", ".metadata.json"); if (metadata_files.empty()) @@ -486,8 +487,8 @@ std::pair getMetadataFileAndVersion( } -DataLakeMetadataPtr -IcebergMetadata::create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context) +DataLakeMetadataPtr IcebergMetadata::create( + const ObjectStoragePtr & object_storage, const ConfigurationObserverPtr & configuration, const ContextPtr & local_context) { auto configuration_ptr = configuration.lock(); @@ -509,7 +510,6 @@ IcebergMetadata::create(ObjectStoragePtr object_storage, ConfigurationObserverPt IcebergSchemaProcessor schema_processor; auto format_version = object->getValue("format-version"); - auto schema_id = parseTableSchema(object, schema_processor); auto snapshots = object->get("snapshots").extract(); @@ -527,8 +527,11 @@ IcebergMetadata::create(ObjectStoragePtr object_storage, ConfigurationObserverPt } } - return std::make_unique( - object_storage, configuration_ptr, local_context, metadata_version, format_version, manifest_list_file, schema_id, schema_processor); + auto ptr = std::make_unique( + object_storage, configuration_ptr, local_context, metadata_version, format_version, manifest_list_file, object); + + + return ptr; } /** @@ -556,8 +559,11 @@ IcebergMetadata::create(ObjectStoragePtr object_storage, ConfigurationObserverPt * │ 1 │ 2252246380142525104 │ ('/iceberg_data/db/table_name/data/a=2/00000-1-c9535a00-2f4f-405c-bcfa-6d4f9f477235-00003.parquet','PARQUET',(2),1,631,67108864,[(1,46),(2,48)],[(1,1),(2,1)],[(1,0),(2,0)],[],[(1,'\0\0\0\0\0\0\0'),(2,'3')],[(1,'\0\0\0\0\0\0\0'),(2,'3')],NULL,[4],0) │ * └────────┴─────────────────────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ */ + + Strings IcebergMetadata::getDataFiles() const { + std::lock_guard lock(get_data_files_mutex); if (!data_files.empty()) return data_files; diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 6d82f3506fe..45b5b2caf80 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include "config.h" @@ -103,6 +104,8 @@ private: Int32 current_old_id = -1; Int32 current_new_id = -1; + + std::mutex mutex; }; @@ -163,12 +166,11 @@ public: IcebergMetadata( ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, - ContextPtr context_, + const DB::ContextPtr & context_, Int32 metadata_version_, Int32 format_version_, String manifest_list_file_, - Int32 current_schema_id_, - IcebergSchemaProcessor schema_processor); + const Poco::JSON::Object::Ptr& object); /// Get data files. On first request it reads manifest_list file and iterates through manifest files to find all data files. /// All subsequent calls will return saved list of files (because it cannot be changed without changing metadata file) @@ -187,7 +189,8 @@ public: return iceberg_metadata && getVersion() == iceberg_metadata->getVersion(); } - static DataLakeMetadataPtr create(ObjectStoragePtr object_storage, ConfigurationObserverPtr configuration, ContextPtr local_context); + static DataLakeMetadataPtr + create(const ObjectStoragePtr & object_storage, const ConfigurationObserverPtr & configuration, const ContextPtr & local_context); size_t getVersion() const { return metadata_version; } @@ -215,14 +218,16 @@ private: Int32 metadata_version; Int32 format_version; String manifest_list_file; - const Int32 current_schema_id; + Int32 current_schema_id; mutable Strings data_files; std::unordered_map column_name_to_physical_name; DataLakePartitionColumns partition_columns; - mutable IcebergSchemaProcessor schema_processor; NamesAndTypesList schema; + mutable IcebergSchemaProcessor schema_processor; LoggerPtr log; + mutable std::mutex get_data_files_mutex; + std::optional getSchemaVersionByFileIfOutdated(String data_path) const { auto schema_id = schema_id_by_data_file.find(data_path); From 0d26ab3288cfa11a59beca3bdfe529644d0e48d5 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:55:36 +0100 Subject: [PATCH 130/502] Avoid possible race-condition + fix tests. --- src/Interpreters/ProcessList.cpp | 40 +++++++++++++++++++------------- src/Interpreters/ProcessList.h | 6 +++-- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index f85a05979b4..0ab4d292e39 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -458,16 +458,20 @@ void QueryStatus::ExecutorHolder::remove() CancellationCode QueryStatus::cancelQuery(CancelReason reason, std::exception_ptr exception) { - if (is_killed.exchange(true)) - return CancellationCode::CancelSent; - { - std::lock_guard lock{cancellation_exception_mutex}; + // Lock the mutex to protect the critical section + std::lock_guard lock(cancel_mutex); + + if (is_killed) + return CancellationCode::CancelSent; + + is_killed = true; + if (!cancellation_exception) cancellation_exception = exception; - } - std::atomic_exchange(&cancel_reason, reason); + cancel_reason = reason; + } std::vector executors_snapshot; @@ -495,20 +499,25 @@ CancellationCode QueryStatus::cancelQuery(CancelReason reason, std::exception_pt return CancellationCode::CancelSent; } -void QueryStatus::throwProperExceptionIfNeeded(const UInt64 & max_execution_time, const UInt64 & elapsed_ns) +void QueryStatus::throwProperExceptionIfNeeded(const UInt64 & max_execution_time, const UInt64 & elapsed_ns) const { if (is_killed.load()) { - String additional_error_part; - if (!elapsed_ns) - additional_error_part = fmt::format("elapsed {} ms,", static_cast(elapsed_ns) / 1000000000ULL); - - if (cancel_reason == CancelReason::TIMEOUT) - throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout exceeded: {} maximum: {} ms", additional_error_part, max_execution_time / 1000.0); - throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); + throwProperException(max_execution_time, elapsed_ns); } } +void QueryStatus::throwProperException(const UInt64 & max_execution_time, const UInt64 & elapsed_ns) const +{ + String additional_error_part; + if (!elapsed_ns) + additional_error_part = fmt::format("elapsed {} ms,", static_cast(elapsed_ns) / 1000000000ULL); + + if (cancel_reason == CancelReason::TIMEOUT) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout exceeded: {} maximum: {} ms", additional_error_part, max_execution_time / 1000.0); + throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); +} + void QueryStatus::addPipelineExecutor(PipelineExecutor * e) { /// In case of asynchronous distributed queries it is possible to call @@ -551,8 +560,7 @@ void QueryStatus::throwQueryWasCancelled() const std::lock_guard lock{cancellation_exception_mutex}; if (cancellation_exception) std::rethrow_exception(cancellation_exception); - else - throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); + throwProperException(limits.max_execution_time.totalMilliseconds()); } bool QueryStatus::checkTimeLimitSoft() diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index bc986a274d1..ba7e1ec00af 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -118,7 +118,7 @@ protected: std::atomic is_killed { false }; std::atomic cancel_reason { CancelReason::UNDEFINED }; - std::exception_ptr cancellation_exception TSA_GUARDED_BY(cancellation_exception_mutex); + std::exception_ptr cancellation_exception; mutable std::mutex cancellation_exception_mutex; /// All data to the client already had been sent. @@ -142,6 +142,7 @@ protected: [[noreturn]] void throwQueryWasCancelled() const; mutable std::mutex executors_mutex; + mutable std::mutex cancel_mutex; struct ExecutorHolder { @@ -239,7 +240,8 @@ public: QueryStatusInfo getInfo(bool get_thread_list = false, bool get_profile_events = false, bool get_settings = false) const; - void throwProperExceptionIfNeeded(const UInt64 & max_execution_time, const UInt64 & elapsed_ns = 0); + void throwProperExceptionIfNeeded(const UInt64 & max_execution_time, const UInt64 & elapsed_ns = 0) const; + [[noreturn]] void throwProperException(const UInt64 & max_execution_time, const UInt64 & elapsed_ns = 0) const; /// Cancels the current query. /// Optional argument `exception` allows to set an exception which checkTimeLimit() will throw instead of "QUERY_WAS_CANCELLED". From 3cfd9a72ea0efaac8e0c7fa5c34135ea4bba1aa8 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:56:13 +0100 Subject: [PATCH 131/502] Remove flaky setting from test. --- .../0_stateless/01290_max_execution_speed_distributed.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql index 864467c0942..d0dc554f425 100644 --- a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql +++ b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql @@ -1,7 +1,6 @@ -- Tags: distributed SET max_execution_speed = 1000000; -SET max_execution_time = 1000; SET timeout_before_checking_execution_speed = 0; SET max_block_size = 100; From 9d168c6cb5c1797a701ca4301246b94d98b89ed6 Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Mon, 18 Nov 2024 17:22:51 -0300 Subject: [PATCH 132/502] add docs on function --- src/Functions/array/arrayPrAUC.cpp | 69 ++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/src/Functions/array/arrayPrAUC.cpp b/src/Functions/array/arrayPrAUC.cpp index f99519388ae..1dbe8db6420 100644 --- a/src/Functions/array/arrayPrAUC.cpp +++ b/src/Functions/array/arrayPrAUC.cpp @@ -19,6 +19,50 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; /** The function takes two arrays: scores and labels. + * Label can be one of two values: positive and negative. + * Score can be arbitrary number. + * + * These values are considered as the output of classifier. We have some true labels for objects. + * And classifier assigns some scores to objects that predict these labels in the following way: + * - we can define arbitrary threshold on score and predict that the label is positive if the score is greater than the threshold: + * + * f(object) = score + * predicted_label = score > threshold + * + * This way classifier may predict positive or negative value correctly - true positive (tp) or true negative (tn) or have false positive (fp) or false negative (fn) result. + * Varying the threshold we can get different probabilities of false positive or false negatives or true positives, etc... + * + * We can also calculate the Precision and the Recall: + * + * Precision is the ratio `tp / (tp + fp)` where `tp` is the number of true positives and fp `the` number of false positives. + * It represents how often the classifier is correct when giving a positive result. + * Precision = P(label = positive | prediction = positive) + * + * Recall is the ratio `tp / (tp + fn)` where `tp` is the number of true positives and `fn` the number of false negatives. + * It represents the probability of the classifier to give positive result if the object has positive label. + * Recall = P(score > threshold | label = positive) + * + * We can draw a curve of values of Precision and Recall with different threshold on [0..1] x [0..1] unit square. + * This curve is named "Precision Recall curve" (PR). For the curve we can calculate, literally, Area Under the Curve, that will be in the range of [0..1]. + * + * Let's look at the example: + * arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); + * + * 1. We have pairs: (-, 0.1), (-, 0.4), (+, 0.35), (+, 0.8) + * + * 2. Let's sort by score descending: (+, 0.8), (-, 0.4), (+, 0.35), (-, 0.1) + * + * 3. Let's draw the points: + * + * threshold = 0.8, TP = 0, FP = 0, FN = 2, Recall = 0.0, Precision = 1 + * threshold = 0.4, TP = 1, FP = 0, FN = 1, Recall = 0.5, Precision = 1 + * threshold = 0.35, TP = 1, FP = 1, FN = 1, Recall = 0.5, Precision = 0.5 + * threshold = 0.1, TP = 2, FP = 1, FN = 0, Recall = 1.0, Precision = 0.666 + * threshold = 0, TP = 2, FP = 2, FN = 0, Recall = 1.0, Precision = 0.5 + * + * The "curve" will be present by a line that moves one step either towards right or top on each threshold change. + * This implementation is not interpolated and does not use the trapezoidal rule. + * Each increment in area is calculated using `(R_n - R_{n-1}) * P_n`, which is equivalent to the right Riemann sum. */ class FunctionArrayPrAUC : public IFunction @@ -50,14 +94,17 @@ private: /// Sorting scores in descending order to traverse the Precision Recall curve from left to right std::sort(sorted_labels.begin(), sorted_labels.end(), [](const auto & lhs, const auto & rhs) { return lhs.score > rhs.score; }); - size_t prev_tp = 0; - size_t curr_fp = 0, curr_tp = 0; + size_t prev_tp = 0, curr_tp = 0; + size_t curr_p = 0; + + Float64 prev_score = sorted_labels[0].score; Float64 curr_precision = 1.0; + Float64 area = 0.0; for (size_t i = 0; i < size; ++i) { - if (curr_tp != prev_tp) + if (sorted_labels[i].score != prev_score) { /* Precision = TP / (TP + FP) and Recall = TP / (TP + FN) * @@ -65,29 +112,25 @@ private: * d_Area = Precision_n * (Recall_n - Recall_{n-1}), * we can calculate * d_Area = Precision_n * (TP_n - TP_{n-1}) - * and later divide it by (TP + FN), since + * and later divide it by (TP + FN), since (TP + FN) is constant and equal to total positive labels. */ - curr_precision = static_cast(curr_tp) / (curr_tp + curr_fp); + curr_precision = static_cast(curr_tp) / curr_p; area += curr_precision * (curr_tp - prev_tp); prev_tp = curr_tp; } if (sorted_labels[i].label) curr_tp += 1; - else - curr_fp += 1; + curr_p += 1; } - curr_precision = (curr_tp + curr_fp) > 0 ? static_cast(curr_tp) / (curr_tp + curr_fp) : 1.0; + curr_precision = curr_p > 0 ? static_cast(curr_tp) / curr_p : 1.0; area += curr_precision * (curr_tp - prev_tp); - /// If there were no labels, return NaN - if (curr_tp == 0 && curr_fp == 0) - return std::numeric_limits::quiet_NaN(); - /// If there were no positive labels, the only point of the curve is (0, 1) and AUC is 0 + /// If there were no positive labels, Recall did not change and the area is 0 if (curr_tp == 0) return 0.0; - /// Finally, divide it by total number of positive labels (TP + FN) + /// Finally, we assume that we've traversed the whole curve and total positive labels (TP + FN) is curr_tp return area / curr_tp; } From b5597381970c1225bbf6279090401f2cf0d9dfbd Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Mon, 18 Nov 2024 17:39:38 -0300 Subject: [PATCH 133/502] mention arrayPrAUC on docs --- .../functions/array-functions.md | 35 +++++++++++++++++++ tests/fuzz/all.dict | 1 + tests/fuzz/dictionaries/functions.dict | 1 + .../aspell-ignore/en/aspell-dict.txt | 1 + 4 files changed, 38 insertions(+) diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index 5957b45a881..02506e0c56e 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -2157,6 +2157,41 @@ Result: └───────────────────────────────────────────────┘ ``` +## arrayPrAUC + +Calculate AUC (Area Under the Curve) for the Precision Recall curve. + +**Syntax** + +``` sql +arrayPrAUC(arr_scores, arr_labels) +``` + +**Arguments** + +- `arr_scores` — scores prediction model gives. +- `arr_labels` — labels of samples, usually 1 for positive sample and 0 for negative sample. + +**Returned value** + +Returns PR-AUC value with type Float64. + +**Example** + +Query: + +``` sql +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); +``` + +Result: + +``` text +┌─arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1])─┐ +│ 0.8333333333333333 │ +└───────────────────────────────────────────────┘ +``` + ## arrayMap(func, arr1, ...) Returns an array obtained from the original arrays by application of `func(arr1[i], ..., arrN[i])` for each element. Arrays `arr1` ... `arrN` must have the same number of elements. diff --git a/tests/fuzz/all.dict b/tests/fuzz/all.dict index 30af3746fca..28774fd8f09 100644 --- a/tests/fuzz/all.dict +++ b/tests/fuzz/all.dict @@ -1254,6 +1254,7 @@ "arrayPartialSort" "arrayPopBack" "arrayPopFront" +"arrayPrAUC" "arrayProduct" "arrayPushBack" "arrayPushFront" diff --git a/tests/fuzz/dictionaries/functions.dict b/tests/fuzz/dictionaries/functions.dict index e562595fb67..ddc2db582e7 100644 --- a/tests/fuzz/dictionaries/functions.dict +++ b/tests/fuzz/dictionaries/functions.dict @@ -567,6 +567,7 @@ "arrayPartialSort" "arrayPopBack" "arrayPopFront" +"arrayPrAUC" "arrayProduct" "arrayPushBack" "arrayPushFront" diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index a0d4d1d349e..d20a5c75f73 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1264,6 +1264,7 @@ arrayPartialShuffle arrayPartialSort arrayPopBack arrayPopFront +arrayPrAUC arrayProduct arrayPushBack arrayPushFront From f3f292d1e96d33d0bc823f686d29f45285ffbae0 Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Mon, 18 Nov 2024 18:30:17 -0300 Subject: [PATCH 134/502] add tests, one still failing --- .../0_stateless/03272_array_pr_auc.reference | 23 +++++++++++-- .../0_stateless/03272_array_pr_auc.sql | 32 +++++++++++++++++-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/03272_array_pr_auc.reference b/tests/queries/0_stateless/03272_array_pr_auc.reference index 175c333eee4..111c371c932 100644 --- a/tests/queries/0_stateless/03272_array_pr_auc.reference +++ b/tests/queries/0_stateless/03272_array_pr_auc.reference @@ -1,2 +1,21 @@ -0.83333333 -0.80555555 +0.8333333333333333 +0.8333333333333333 +0.8333333333333333 +0.8333333333333333 +0.8333333333333333 +0.8333333333333333 +0.8333333333333333 +0.8333333333333333 +0.8333333333333333 +0.5 +0.5 +0.5 +0.5 +0.5 +0.8333333333333333 +0.8055555555555555 +0.5 +0.3666666666666667 +0.29166666666666663 +0.56875 +0.43333333333333335 diff --git a/tests/queries/0_stateless/03272_array_pr_auc.sql b/tests/queries/0_stateless/03272_array_pr_auc.sql index 67c556db6e7..b334a7606e1 100644 --- a/tests/queries/0_stateless/03272_array_pr_auc.sql +++ b/tests/queries/0_stateless/03272_array_pr_auc.sql @@ -1,2 +1,30 @@ -SELECT FLOOR(arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]), 8); -SELECT FLOOR(arrayPrAUC([0.1, 0.4, 0.4, 0.35, 0.8], [0, 0, 1, 1, 1]), 8); +-- type correctness tests +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast([0, 0, 1, 1] as Array(Int8))); +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast([-1, -1, 1, 1] as Array(Int8))); +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast(['false', 'false', 'true', 'true'] as Array(Enum8('false' = 0, 'true' = 1)))); +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast(['false', 'false', 'true', 'true'] as Array(Enum8('false' = -1, 'true' = 1)))); +select arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt8)), [0, 0, 1, 1]); +select arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt16)), [0, 0, 1, 1]); +select arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt32)), [0, 0, 1, 1]); +select arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt64)), [0, 0, 1, 1]); +select arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int8)), [0, 0, 1, 1]); +select arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int16)), [0, 0, 1, 1]); +select arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int32)), [0, 0, 1, 1]); +select arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int64)), [0, 0, 1, 1]); +select arrayPrAUC(cast([-0.1, -0.4, -0.35, -0.8] as Array(Float32)) , [0, 0, 1, 1]); + +-- output value correctness test +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); +select arrayPrAUC([0.1, 0.4, 0.4, 0.35, 0.8], [0, 0, 1, 1, 1]); +select arrayPrAUC([0.1, 0.35, 0.4, 0.8], [1, 0, 1, 0]); +select arrayPrAUC([0.1, 0.35, 0.4, 0.4, 0.8], [1, 0, 1, 0, 0]); +select arrayPrAUC([0, 3, 5, 6, 7.5, 8], [1, 0, 1, 0, 0, 0]); +select arrayPrAUC([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [1, 0, 1, 0, 0, 0, 1, 0, 0, 1]); +select arrayPrAUC([0, 1, 1, 2, 2, 2, 3, 3, 3, 3], [1, 0, 1, 0, 0, 0, 1, 0, 0, 1]); + +-- negative tests +-- select arrayPrAUC([0, 0, 1, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +-- select arrayPrAUC([0.1, 0.35], [0, 0, 1, 1]); -- { serverError BAD_ARGUMENTS } +-- select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0.0, 0.0, 1.0, 1.0]); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +-- select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1], [1, 1, 0, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } \ No newline at end of file From 6642a562ca6094ac6f9c5b44f9b8eaf4419e6404 Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Mon, 18 Nov 2024 18:49:47 -0300 Subject: [PATCH 135/502] add edge cases tests --- .../0_stateless/03272_array_pr_auc.reference | 12 ++++++++++++ tests/queries/0_stateless/03272_array_pr_auc.sql | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/tests/queries/0_stateless/03272_array_pr_auc.reference b/tests/queries/0_stateless/03272_array_pr_auc.reference index 111c371c932..db155834700 100644 --- a/tests/queries/0_stateless/03272_array_pr_auc.reference +++ b/tests/queries/0_stateless/03272_array_pr_auc.reference @@ -19,3 +19,15 @@ 0.29166666666666663 0.56875 0.43333333333333335 +1 +0 +0 +1 +1 +0 +0.5 +1 +0.5 +0.8333333333333333 +1 +0.5 diff --git a/tests/queries/0_stateless/03272_array_pr_auc.sql b/tests/queries/0_stateless/03272_array_pr_auc.sql index b334a7606e1..73f57211330 100644 --- a/tests/queries/0_stateless/03272_array_pr_auc.sql +++ b/tests/queries/0_stateless/03272_array_pr_auc.sql @@ -22,6 +22,19 @@ select arrayPrAUC([0.1, 0.35, 0.4, 0.4, 0.8], [1, 0, 1, 0, 0]); select arrayPrAUC([0, 3, 5, 6, 7.5, 8], [1, 0, 1, 0, 0, 0]); select arrayPrAUC([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [1, 0, 1, 0, 0, 0, 1, 0, 0, 1]); select arrayPrAUC([0, 1, 1, 2, 2, 2, 3, 3, 3, 3], [1, 0, 1, 0, 0, 0, 1, 0, 0, 1]); +-- edge cases +SELECT arrayPrAUC([1], [1]); +SELECT arrayPrAUC([1], [0]); +SELECT arrayPrAUC([0], [0]); +SELECT arrayPrAUC([0], [1]); +SELECT arrayPrAUC([1, 1], [1, 1]); +SELECT arrayPrAUC([1, 1], [0, 0]); +SELECT arrayPrAUC([1, 1], [0, 1]); +SELECT arrayPrAUC([0, 1], [0, 1]); +SELECT arrayPrAUC([1, 0], [0, 1]); +SELECT arrayPrAUC([0, 0, 1], [0, 1, 1]); +SELECT arrayPrAUC([0, 1, 1], [0, 1, 1]); +SELECT arrayPrAUC([0, 1, 1], [0, 0, 1]); -- negative tests -- select arrayPrAUC([0, 0, 1, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } From 7d64f4f3d5da94204f3b01696d0037cb563b926b Mon Sep 17 00:00:00 2001 From: pufit Date: Mon, 18 Nov 2024 13:59:24 -0800 Subject: [PATCH 136/502] fix conflicts --- src/Interpreters/Context.cpp | 4 ++-- src/Interpreters/Context.h | 4 ++-- src/Parsers/Access/ParserSettingsProfileElement.cpp | 4 ++-- src/Parsers/CommonParsers.h | 9 +++++++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index d2aad0a52d8..6644d116a4b 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2501,7 +2501,7 @@ void Context::applySettingsChanges(const SettingsChanges & changes) applySettingsChangesWithLock(changes, lock); } -void Context::checkSettingsConstraintsWithLock(const SettingsProfileElements & profile_elements, SettingSource source) +void Context::checkSettingsConstraintsWithLock(const AlterSettingsProfileElements & profile_elements, SettingSource source) { getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(*settings, profile_elements, source); if (getApplicationType() == ApplicationType::LOCAL || getApplicationType() == ApplicationType::SERVER) @@ -2541,7 +2541,7 @@ void Context::checkMergeTreeSettingsConstraintsWithLock(const MergeTreeSettings getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(merge_tree_settings, changes); } -void Context::checkSettingsConstraints(const SettingsProfileElements & profile_elements, SettingSource source) +void Context::checkSettingsConstraints(const AlterSettingsProfileElements & profile_elements, SettingSource source) { SharedLockGuard lock(mutex); checkSettingsConstraintsWithLock(profile_elements, source); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index e567d963ef9..d659322ae72 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -864,7 +864,7 @@ public: void applySettingsChanges(const SettingsChanges & changes); /// Checks the constraints. - void checkSettingsConstraints(const SettingsProfileElements & profile_elements, SettingSource source); + void checkSettingsConstraints(const AlterSettingsProfileElements & profile_elements, SettingSource source); void checkSettingsConstraints(const SettingChange & change, SettingSource source); void checkSettingsConstraints(const SettingsChanges & changes, SettingSource source); void checkSettingsConstraints(SettingsChanges & changes, SettingSource source); @@ -1410,7 +1410,7 @@ private: void setCurrentDatabaseWithLock(const String & name, const std::lock_guard & lock); - void checkSettingsConstraintsWithLock(const SettingsProfileElements & profile_elements, SettingSource source); + void checkSettingsConstraintsWithLock(const AlterSettingsProfileElements & profile_elements, SettingSource source); void checkSettingsConstraintsWithLock(const SettingChange & change, SettingSource source); diff --git a/src/Parsers/Access/ParserSettingsProfileElement.cpp b/src/Parsers/Access/ParserSettingsProfileElement.cpp index 965f7e5ccf6..b63192fc4ae 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.cpp +++ b/src/Parsers/Access/ParserSettingsProfileElement.cpp @@ -334,9 +334,9 @@ bool ParserAlterSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Exp submode = "ALL PROFILES"; else if (ParserKeyword{Keyword::ALL_SETTINGS}.ignore(pos, expected)) submode = "ALL SETTINGS"; - else if (ParserKeyword{Keyword::PROFILES}.ignore(pos, expected) || ParserKeyword{"PROFILE"}.ignore(pos, expected)) + else if (ParserKeyword{Keyword::PROFILES}.ignore(pos, expected) || ParserKeyword{Keyword::PROFILE}.ignore(pos, expected)) submode = "PROFILES"; - else if (ParserKeyword{Keyword::SETTINGS}.ignore(pos, expected) || ParserKeyword{"SETTING"}.ignore(pos, expected)) + else if (ParserKeyword{Keyword::SETTINGS}.ignore(pos, expected) || ParserKeyword{Keyword::SETTING}.ignore(pos, expected)) submode = "SETTINGS"; } diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index c02f8d06323..d1818e4a8cf 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -10,22 +10,25 @@ namespace DB { #define APPLY_FOR_PARSER_KEYWORDS(MR_MACROS) \ + MR_MACROS(ADD, "ADD") \ MR_MACROS(ADD_COLUMN, "ADD COLUMN") \ MR_MACROS(ADD_CONSTRAINT, "ADD CONSTRAINT") \ MR_MACROS(ADD_INDEX, "ADD INDEX") \ MR_MACROS(ADD_PROJECTION, "ADD PROJECTION") \ MR_MACROS(ADD_STATISTICS, "ADD STATISTICS") \ - MR_MACROS(ADD, "ADD") \ MR_MACROS(ADMIN_OPTION_FOR, "ADMIN OPTION FOR") \ MR_MACROS(AFTER, "AFTER") \ MR_MACROS(ALGORITHM, "ALGORITHM") \ MR_MACROS(ALIAS, "ALIAS") \ MR_MACROS(ALL, "ALL") \ + MR_MACROS(ALL_PROFILES, "ALL_PROFILES") \ + MR_MACROS(ALL_SETTINGS, "ALL SETTINGS") \ MR_MACROS(ALTER_COLUMN, "ALTER COLUMN") \ MR_MACROS(ALTER_DATABASE, "ALTER DATABASE") \ MR_MACROS(ALTER_LIVE_VIEW, "ALTER LIVE VIEW") \ MR_MACROS(ALTER_POLICY, "ALTER POLICY") \ MR_MACROS(ALTER_PROFILE, "ALTER PROFILE") \ + MR_MACROS(ALTER_PROFILES, "ALTER PROFILES") \ MR_MACROS(ALTER_QUOTA, "ALTER QUOTA") \ MR_MACROS(ALTER_ROLE, "ALTER ROLE") \ MR_MACROS(ALTER_ROW_POLICY, "ALTER ROW POLICY") \ @@ -379,6 +382,7 @@ namespace DB MR_MACROS(PRIMARY_KEY, "PRIMARY KEY") \ MR_MACROS(PRIMARY, "PRIMARY") \ MR_MACROS(PROFILE, "PROFILE") \ + MR_MACROS(PROFILES, "PROFILES") \ MR_MACROS(PROJECTION, "PROJECTION") \ MR_MACROS(PROTOBUF, "Protobuf") \ MR_MACROS(PULL, "PULL") \ @@ -443,7 +447,8 @@ namespace DB MR_MACROS(SET_ROLE_DEFAULT, "SET ROLE DEFAULT") \ MR_MACROS(SET_ROLE, "SET ROLE") \ MR_MACROS(SET_TRANSACTION_SNAPSHOT, "SET TRANSACTION SNAPSHOT") \ - MR_MACROS(SET, "SET") \ + MR_MACROS(SET, "SET") \ + MR_MACROS(SETTING, "SETTING") \ MR_MACROS(SETTINGS, "SETTINGS") \ MR_MACROS(SHOW_ACCESS, "SHOW ACCESS") \ MR_MACROS(SHOW_CREATE, "SHOW CREATE") \ From 026aa8b2ee6b0ae7a64d03e7246f0d1cacc070bc Mon Sep 17 00:00:00 2001 From: pufit Date: Mon, 18 Nov 2024 14:03:28 -0800 Subject: [PATCH 137/502] fix keyword --- src/Parsers/CommonParsers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index d1818e4a8cf..032aef9d733 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -21,7 +21,7 @@ namespace DB MR_MACROS(ALGORITHM, "ALGORITHM") \ MR_MACROS(ALIAS, "ALIAS") \ MR_MACROS(ALL, "ALL") \ - MR_MACROS(ALL_PROFILES, "ALL_PROFILES") \ + MR_MACROS(ALL_PROFILES, "ALL PROFILES") \ MR_MACROS(ALL_SETTINGS, "ALL SETTINGS") \ MR_MACROS(ALTER_COLUMN, "ALTER COLUMN") \ MR_MACROS(ALTER_DATABASE, "ALTER DATABASE") \ From 5b8f14115a9e7affce9f2d7e9f01ef8d168d589b Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Mon, 18 Nov 2024 19:07:16 -0300 Subject: [PATCH 138/502] add negative tests --- tests/queries/0_stateless/03272_array_pr_auc.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/03272_array_pr_auc.sql b/tests/queries/0_stateless/03272_array_pr_auc.sql index 73f57211330..0d542a55c69 100644 --- a/tests/queries/0_stateless/03272_array_pr_auc.sql +++ b/tests/queries/0_stateless/03272_array_pr_auc.sql @@ -37,7 +37,8 @@ SELECT arrayPrAUC([0, 1, 1], [0, 1, 1]); SELECT arrayPrAUC([0, 1, 1], [0, 0, 1]); -- negative tests --- select arrayPrAUC([0, 0, 1, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } --- select arrayPrAUC([0.1, 0.35], [0, 0, 1, 1]); -- { serverError BAD_ARGUMENTS } --- select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0.0, 0.0, 1.0, 1.0]); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } --- select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1], [1, 1, 0, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } \ No newline at end of file +select arrayPrAUC([], []); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select arrayPrAUC([0, 0, 1, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +select arrayPrAUC([0.1, 0.35], [0, 0, 1, 1]); -- { serverError BAD_ARGUMENTS } +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], []); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1], [1, 1, 0, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } \ No newline at end of file From 38bd227047b8d77185b91cc53edfba55be4f23bf Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Mon, 18 Nov 2024 20:07:45 -0300 Subject: [PATCH 139/502] fix prev_score not updating + clarify docs --- src/Functions/array/arrayPrAUC.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Functions/array/arrayPrAUC.cpp b/src/Functions/array/arrayPrAUC.cpp index 1dbe8db6420..798fe16b9c7 100644 --- a/src/Functions/array/arrayPrAUC.cpp +++ b/src/Functions/array/arrayPrAUC.cpp @@ -60,9 +60,12 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; * threshold = 0.1, TP = 2, FP = 1, FN = 0, Recall = 1.0, Precision = 0.666 * threshold = 0, TP = 2, FP = 2, FN = 0, Recall = 1.0, Precision = 0.5 * - * The "curve" will be present by a line that moves one step either towards right or top on each threshold change. - * This implementation is not interpolated and does not use the trapezoidal rule. - * Each increment in area is calculated using `(R_n - R_{n-1}) * P_n`, which is equivalent to the right Riemann sum. + * This PR-AUC uses the right Riemann sum to calculate the AUC. + * Each increment in area is calculated using `(R_n - R_{n-1}) * P_n`, + * where `R_n` is the Recall at the `n`-th point and `P_n` is the Precision at the `n`-th point. + * + * This implementation is not interpolated and is different from computing the AUC with the trapezoidal rule, + * which uses linear interpolation and can be too optimistic for the Precision Recall AUC metric. */ class FunctionArrayPrAUC : public IFunction @@ -106,17 +109,19 @@ private: { if (sorted_labels[i].score != prev_score) { - /* Precision = TP / (TP + FP) and Recall = TP / (TP + FN) + /* Precision = TP / (TP + FP) + * Recall = TP / (TP + FN) * - * Instead of calculating - * d_Area = Precision_n * (Recall_n - Recall_{n-1}), - * we can calculate - * d_Area = Precision_n * (TP_n - TP_{n-1}) - * and later divide it by (TP + FN), since (TP + FN) is constant and equal to total positive labels. + * Instead of calculating + * d_Area = Precision_n * (Recall_n - Recall_{n-1}), + * we can calculate + * d_Area = Precision_n * (TP_n - TP_{n-1}) + * and later divide it by (TP + FN), since (TP + FN) is constant and equal to total positive labels. */ curr_precision = static_cast(curr_tp) / curr_p; area += curr_precision * (curr_tp - prev_tp); prev_tp = curr_tp; + prev_score = sorted_labels[i].score; } if (sorted_labels[i].label) From 74e3e798b5f68fbac1b0f37cf05e997832f43fe0 Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Mon, 18 Nov 2024 20:12:19 -0300 Subject: [PATCH 140/502] round test queries to prevent float trunc errors --- .../0_stateless/03272_array_pr_auc.reference | 30 ++++----- .../0_stateless/03272_array_pr_auc.sql | 66 +++++++++---------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/tests/queries/0_stateless/03272_array_pr_auc.reference b/tests/queries/0_stateless/03272_array_pr_auc.reference index db155834700..ee37e657593 100644 --- a/tests/queries/0_stateless/03272_array_pr_auc.reference +++ b/tests/queries/0_stateless/03272_array_pr_auc.reference @@ -1,24 +1,24 @@ -0.8333333333333333 -0.8333333333333333 -0.8333333333333333 -0.8333333333333333 -0.8333333333333333 -0.8333333333333333 -0.8333333333333333 -0.8333333333333333 -0.8333333333333333 +0.8333333333 +0.8333333333 +0.8333333333 +0.8333333333 +0.8333333333 +0.8333333333 +0.8333333333 +0.8333333333 +0.8333333333 0.5 0.5 0.5 0.5 0.5 -0.8333333333333333 -0.8055555555555555 +0.8333333333 +0.8055555555 0.5 -0.3666666666666667 -0.29166666666666663 +0.3666666666 +0.2916666666 0.56875 -0.43333333333333335 +0.4333333333 1 0 0 @@ -28,6 +28,6 @@ 0.5 1 0.5 -0.8333333333333333 +0.8333333333 1 0.5 diff --git a/tests/queries/0_stateless/03272_array_pr_auc.sql b/tests/queries/0_stateless/03272_array_pr_auc.sql index 0d542a55c69..44b9e3bd2d6 100644 --- a/tests/queries/0_stateless/03272_array_pr_auc.sql +++ b/tests/queries/0_stateless/03272_array_pr_auc.sql @@ -1,40 +1,40 @@ -- type correctness tests -select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); -select arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast([0, 0, 1, 1] as Array(Int8))); -select arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast([-1, -1, 1, 1] as Array(Int8))); -select arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast(['false', 'false', 'true', 'true'] as Array(Enum8('false' = 0, 'true' = 1)))); -select arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast(['false', 'false', 'true', 'true'] as Array(Enum8('false' = -1, 'true' = 1)))); -select arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt8)), [0, 0, 1, 1]); -select arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt16)), [0, 0, 1, 1]); -select arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt32)), [0, 0, 1, 1]); -select arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt64)), [0, 0, 1, 1]); -select arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int8)), [0, 0, 1, 1]); -select arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int16)), [0, 0, 1, 1]); -select arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int32)), [0, 0, 1, 1]); -select arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int64)), [0, 0, 1, 1]); -select arrayPrAUC(cast([-0.1, -0.4, -0.35, -0.8] as Array(Float32)) , [0, 0, 1, 1]); +select floor(arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]), 10); +select floor(arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast([0, 0, 1, 1] as Array(Int8))), 10); +select floor(arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast([-1, -1, 1, 1] as Array(Int8))), 10); +select floor(arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast(['false', 'false', 'true', 'true'] as Array(Enum8('false' = 0, 'true' = 1)))), 10); +select floor(arrayPrAUC([0.1, 0.4, 0.35, 0.8], cast(['false', 'false', 'true', 'true'] as Array(Enum8('false' = -1, 'true' = 1)))), 10); +select floor(arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt8)), [0, 0, 1, 1]), 10); +select floor(arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt16)), [0, 0, 1, 1]), 10); +select floor(arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt32)), [0, 0, 1, 1]), 10); +select floor(arrayPrAUC(cast([10, 40, 35, 80] as Array(UInt64)), [0, 0, 1, 1]), 10); +select floor(arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int8)), [0, 0, 1, 1]), 10); +select floor(arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int16)), [0, 0, 1, 1]), 10); +select floor(arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int32)), [0, 0, 1, 1]), 10); +select floor(arrayPrAUC(cast([-10, -40, -35, -80] as Array(Int64)), [0, 0, 1, 1]), 10); +select floor(arrayPrAUC(cast([-0.1, -0.4, -0.35, -0.8] as Array(Float32)) , [0, 0, 1, 1]), 10); -- output value correctness test -select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); -select arrayPrAUC([0.1, 0.4, 0.4, 0.35, 0.8], [0, 0, 1, 1, 1]); -select arrayPrAUC([0.1, 0.35, 0.4, 0.8], [1, 0, 1, 0]); -select arrayPrAUC([0.1, 0.35, 0.4, 0.4, 0.8], [1, 0, 1, 0, 0]); -select arrayPrAUC([0, 3, 5, 6, 7.5, 8], [1, 0, 1, 0, 0, 0]); -select arrayPrAUC([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [1, 0, 1, 0, 0, 0, 1, 0, 0, 1]); -select arrayPrAUC([0, 1, 1, 2, 2, 2, 3, 3, 3, 3], [1, 0, 1, 0, 0, 0, 1, 0, 0, 1]); +select floor(arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]), 10); +select floor(arrayPrAUC([0.1, 0.4, 0.4, 0.35, 0.8], [0, 0, 1, 1, 1]), 10); +select floor(arrayPrAUC([0.1, 0.35, 0.4, 0.8], [1, 0, 1, 0]), 10); +select floor(arrayPrAUC([0.1, 0.35, 0.4, 0.4, 0.8], [1, 0, 1, 0, 0]), 10); +select floor(arrayPrAUC([0, 3, 5, 6, 7.5, 8], [1, 0, 1, 0, 0, 0]), 10); +select floor(arrayPrAUC([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [1, 0, 1, 0, 0, 0, 1, 0, 0, 1]), 10); +select floor(arrayPrAUC([0, 1, 1, 2, 2, 2, 3, 3, 3, 3], [1, 0, 1, 0, 0, 0, 1, 0, 0, 1]), 10); -- edge cases -SELECT arrayPrAUC([1], [1]); -SELECT arrayPrAUC([1], [0]); -SELECT arrayPrAUC([0], [0]); -SELECT arrayPrAUC([0], [1]); -SELECT arrayPrAUC([1, 1], [1, 1]); -SELECT arrayPrAUC([1, 1], [0, 0]); -SELECT arrayPrAUC([1, 1], [0, 1]); -SELECT arrayPrAUC([0, 1], [0, 1]); -SELECT arrayPrAUC([1, 0], [0, 1]); -SELECT arrayPrAUC([0, 0, 1], [0, 1, 1]); -SELECT arrayPrAUC([0, 1, 1], [0, 1, 1]); -SELECT arrayPrAUC([0, 1, 1], [0, 0, 1]); +SELECT floor(arrayPrAUC([1], [1]), 10); +SELECT floor(arrayPrAUC([1], [0]), 10); +SELECT floor(arrayPrAUC([0], [0]), 10); +SELECT floor(arrayPrAUC([0], [1]), 10); +SELECT floor(arrayPrAUC([1, 1], [1, 1]), 10); +SELECT floor(arrayPrAUC([1, 1], [0, 0]), 10); +SELECT floor(arrayPrAUC([1, 1], [0, 1]), 10); +SELECT floor(arrayPrAUC([0, 1], [0, 1]), 10); +SELECT floor(arrayPrAUC([1, 0], [0, 1]), 10); +SELECT floor(arrayPrAUC([0, 0, 1], [0, 1, 1]), 10); +SELECT floor(arrayPrAUC([0, 1, 1], [0, 1, 1]), 10); +SELECT floor(arrayPrAUC([0, 1, 1], [0, 0, 1]), 10); -- negative tests select arrayPrAUC([], []); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } From 9cdb870c420beb5a35c0ad7f61e0d3414dabc4cb Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Mon, 18 Nov 2024 20:29:32 -0300 Subject: [PATCH 141/502] nit --- tests/queries/0_stateless/03272_array_pr_auc.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03272_array_pr_auc.sql b/tests/queries/0_stateless/03272_array_pr_auc.sql index 44b9e3bd2d6..682a2e2315a 100644 --- a/tests/queries/0_stateless/03272_array_pr_auc.sql +++ b/tests/queries/0_stateless/03272_array_pr_auc.sql @@ -22,6 +22,7 @@ select floor(arrayPrAUC([0.1, 0.35, 0.4, 0.4, 0.8], [1, 0, 1, 0, 0]), 10); select floor(arrayPrAUC([0, 3, 5, 6, 7.5, 8], [1, 0, 1, 0, 0, 0]), 10); select floor(arrayPrAUC([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [1, 0, 1, 0, 0, 0, 1, 0, 0, 1]), 10); select floor(arrayPrAUC([0, 1, 1, 2, 2, 2, 3, 3, 3, 3], [1, 0, 1, 0, 0, 0, 1, 0, 0, 1]), 10); + -- edge cases SELECT floor(arrayPrAUC([1], [1]), 10); SELECT floor(arrayPrAUC([1], [0]), 10); @@ -41,4 +42,4 @@ select arrayPrAUC([], []); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } select arrayPrAUC([0, 0, 1, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select arrayPrAUC([0.1, 0.35], [0, 0, 1, 1]); -- { serverError BAD_ARGUMENTS } select arrayPrAUC([0.1, 0.4, 0.35, 0.8], []); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1], [1, 1, 0, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } \ No newline at end of file +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1], [1, 1, 0, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } From 31f6a5c32398eef2b7a49117db037e908015339b Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Mon, 18 Nov 2024 22:41:49 -0300 Subject: [PATCH 142/502] add performance test --- tests/performance/array_pr_auc.xml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/performance/array_pr_auc.xml diff --git a/tests/performance/array_pr_auc.xml b/tests/performance/array_pr_auc.xml new file mode 100644 index 00000000000..cd5155da992 --- /dev/null +++ b/tests/performance/array_pr_auc.xml @@ -0,0 +1,3 @@ + + SELECT avg(ifNotFinite(arrayPrAUC(arrayMap(x -> rand(x) / 0x100000000, range(2 + rand() % 100)), arrayMap(x -> rand(x) % 2, range(2 + rand() % 100))), 0)) FROM numbers(100000) + From 2a8bb871076f326ff2fa8dbadaa0015af4c0c055 Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Mon, 18 Nov 2024 23:07:03 -0300 Subject: [PATCH 143/502] improve comments --- src/Functions/array/arrayPrAUC.cpp | 34 ++++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/Functions/array/arrayPrAUC.cpp b/src/Functions/array/arrayPrAUC.cpp index 798fe16b9c7..580780a9bee 100644 --- a/src/Functions/array/arrayPrAUC.cpp +++ b/src/Functions/array/arrayPrAUC.cpp @@ -29,21 +29,24 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; * f(object) = score * predicted_label = score > threshold * - * This way classifier may predict positive or negative value correctly - true positive (tp) or true negative (tn) or have false positive (fp) or false negative (fn) result. + * This way classifier may predict positive or negative value correctly - true positive (tp) or true negative (tn) + * or have false positive (fp) or false negative (fn) result. * Varying the threshold we can get different probabilities of false positive or false negatives or true positives, etc... * * We can also calculate the Precision and the Recall: * - * Precision is the ratio `tp / (tp + fp)` where `tp` is the number of true positives and fp `the` number of false positives. + * Precision is the ratio `tp / (tp + fp)` where `tp` is the number of true positives and `fp` the number of false positives. * It represents how often the classifier is correct when giving a positive result. - * Precision = P(label = positive | prediction = positive) + * Precision = P(label = positive | score > threshold) * * Recall is the ratio `tp / (tp + fn)` where `tp` is the number of true positives and `fn` the number of false negatives. * It represents the probability of the classifier to give positive result if the object has positive label. * Recall = P(score > threshold | label = positive) * * We can draw a curve of values of Precision and Recall with different threshold on [0..1] x [0..1] unit square. - * This curve is named "Precision Recall curve" (PR). For the curve we can calculate, literally, Area Under the Curve, that will be in the range of [0..1]. + * This curve is named "Precision Recall curve" (PR). + * + * For the curve we can calculate, literally, Area Under the Curve, that will be in the range of [0..1]. * * Let's look at the example: * arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); @@ -60,12 +63,12 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; * threshold = 0.1, TP = 2, FP = 1, FN = 0, Recall = 1.0, Precision = 0.666 * threshold = 0, TP = 2, FP = 2, FN = 0, Recall = 1.0, Precision = 0.5 * - * This PR-AUC uses the right Riemann sum to calculate the AUC. - * Each increment in area is calculated using `(R_n - R_{n-1}) * P_n`, - * where `R_n` is the Recall at the `n`-th point and `P_n` is the Precision at the `n`-th point. + * This PR-AUC uses the right Riemann sum (see https://en.wikipedia.org/wiki/Riemann_sum) to calculate the AUC. + * That is, each increment in area is calculated using `(R_n - R_{n-1}) * P_n`, + * where `R_n` is the Recall at the `n`-th point and `P_n` is the Precision at the `n`-th point. * * This implementation is not interpolated and is different from computing the AUC with the trapezoidal rule, - * which uses linear interpolation and can be too optimistic for the Precision Recall AUC metric. + * which uses linear interpolation and can be too optimistic for the Precision Recall AUC metric. */ class FunctionArrayPrAUC : public IFunction @@ -112,11 +115,13 @@ private: /* Precision = TP / (TP + FP) * Recall = TP / (TP + FN) * - * Instead of calculating - * d_Area = Precision_n * (Recall_n - Recall_{n-1}), - * we can calculate - * d_Area = Precision_n * (TP_n - TP_{n-1}) - * and later divide it by (TP + FN), since (TP + FN) is constant and equal to total positive labels. + * Instead of calculating + * d_Area = Precision_n * (Recall_n - Recall_{n-1}), + * we can just calculate + * d_Area = Precision_n * (TP_n - TP_{n-1}) + * and later divide it by (TP + FN). + * + * This can be done because (TP + FN) is constant and equal to total positive labels. */ curr_precision = static_cast(curr_tp) / curr_p; area += curr_precision * (curr_tp - prev_tp); @@ -135,7 +140,8 @@ private: /// If there were no positive labels, Recall did not change and the area is 0 if (curr_tp == 0) return 0.0; - /// Finally, we assume that we've traversed the whole curve and total positive labels (TP + FN) is curr_tp + /// Finally, we divide by (TP + FN) to obtain the Recall + /// At this point we've traversed the whole curve and curr_tp = total positive labels (TP + FN) return area / curr_tp; } From cd11579a244721b42ab0ed1e073adc1d5bdc0cb6 Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Tue, 19 Nov 2024 06:47:15 -0300 Subject: [PATCH 144/502] nit --- src/Functions/array/arrayPrAUC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/array/arrayPrAUC.cpp b/src/Functions/array/arrayPrAUC.cpp index 580780a9bee..d8ea470b541 100644 --- a/src/Functions/array/arrayPrAUC.cpp +++ b/src/Functions/array/arrayPrAUC.cpp @@ -63,7 +63,7 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; * threshold = 0.1, TP = 2, FP = 1, FN = 0, Recall = 1.0, Precision = 0.666 * threshold = 0, TP = 2, FP = 2, FN = 0, Recall = 1.0, Precision = 0.5 * - * This PR-AUC uses the right Riemann sum (see https://en.wikipedia.org/wiki/Riemann_sum) to calculate the AUC. + * This implementation uses the right Riemann sum (see https://en.wikipedia.org/wiki/Riemann_sum) to calculate the AUC. * That is, each increment in area is calculated using `(R_n - R_{n-1}) * P_n`, * where `R_n` is the Recall at the `n`-th point and `P_n` is the Precision at the `n`-th point. * From 333bba4e6b6ed30e54544273586fc4ebd6a51c01 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:35:42 +0100 Subject: [PATCH 145/502] Add locks on exception throwing. --- src/Interpreters/ProcessList.cpp | 10 ++++++---- src/Interpreters/ProcessList.h | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 0ab4d292e39..08a54f85302 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -459,7 +459,6 @@ void QueryStatus::ExecutorHolder::remove() CancellationCode QueryStatus::cancelQuery(CancelReason reason, std::exception_ptr exception) { { - // Lock the mutex to protect the critical section std::lock_guard lock(cancel_mutex); if (is_killed) @@ -501,9 +500,12 @@ CancellationCode QueryStatus::cancelQuery(CancelReason reason, std::exception_pt void QueryStatus::throwProperExceptionIfNeeded(const UInt64 & max_execution_time, const UInt64 & elapsed_ns) const { - if (is_killed.load()) { - throwProperException(max_execution_time, elapsed_ns); + std::lock_guard lock(cancel_mutex); + if (is_killed) + { + throwProperException(max_execution_time, elapsed_ns); + } } } @@ -515,7 +517,7 @@ void QueryStatus::throwProperException(const UInt64 & max_execution_time, const if (cancel_reason == CancelReason::TIMEOUT) throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout exceeded: {} maximum: {} ms", additional_error_part, max_execution_time / 1000.0); - throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); + throwQueryWasCancelled(); } void QueryStatus::addPipelineExecutor(PipelineExecutor * e) diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index ba7e1ec00af..7ee6008d79c 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -116,7 +116,7 @@ protected: bool is_cancelling { false }; /// KILL was send to the query std::atomic is_killed { false }; - std::atomic cancel_reason { CancelReason::UNDEFINED }; + CancelReason cancel_reason { CancelReason::UNDEFINED }; std::exception_ptr cancellation_exception; mutable std::mutex cancellation_exception_mutex; From 763b5f5877048b1e707b0efcd01fc97d8e17ddaa Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 19 Nov 2024 13:02:42 +0100 Subject: [PATCH 146/502] Fix the code. --- src/Interpreters/ProcessList.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 08a54f85302..c2c280badc8 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -562,7 +562,8 @@ void QueryStatus::throwQueryWasCancelled() const std::lock_guard lock{cancellation_exception_mutex}; if (cancellation_exception) std::rethrow_exception(cancellation_exception); - throwProperException(limits.max_execution_time.totalMilliseconds()); + else + throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); } bool QueryStatus::checkTimeLimitSoft() From 4e2549bcf3b60063172796ef6df06f761dc63c18 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 19 Nov 2024 13:11:16 +0100 Subject: [PATCH 147/502] Fixes & improvements --- src/Databases/Iceberg/DatabaseIceberg.cpp | 75 ++++-- src/Databases/Iceberg/DatabaseIceberg.h | 4 +- .../Iceberg/DatabaseIcebergSettings.cpp | 2 + src/Databases/Iceberg/ICatalog.cpp | 65 +++++- src/Databases/Iceberg/ICatalog.h | 31 ++- src/Databases/Iceberg/RestCatalog.cpp | 216 +++++++++++++++--- src/Databases/Iceberg/RestCatalog.h | 34 ++- 7 files changed, 347 insertions(+), 80 deletions(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 8f46baf0405..c22d4747f83 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -28,10 +28,12 @@ namespace DB { namespace DatabaseIcebergSetting { - extern const DatabaseIcebergSettingsString storage_endpoint; - extern const DatabaseIcebergSettingsString auth_header; extern const DatabaseIcebergSettingsDatabaseIcebergCatalogType catalog_type; extern const DatabaseIcebergSettingsDatabaseIcebergStorageType storage_type; + extern const DatabaseIcebergSettingsString warehouse; + extern const DatabaseIcebergSettingsString catalog_credential; + extern const DatabaseIcebergSettingsString auth_header; + extern const DatabaseIcebergSettingsString storage_endpoint; } namespace ErrorCodes @@ -55,44 +57,55 @@ namespace auto namespace_name = name.substr(0, name.size() - table_name.size() - 1); return {namespace_name, table_name}; } - - void setCredentials(Poco::Net::HTTPBasicCredentials & credentials, const Poco::URI & request_uri) - { - const auto & user_info = request_uri.getUserInfo(); - if (!user_info.empty()) - { - std::size_t n = user_info.find(':'); - if (n != std::string::npos) - { - credentials.setUsername(user_info.substr(0, n)); - credentials.setPassword(user_info.substr(n + 1)); - } - } - } } DatabaseIceberg::DatabaseIceberg( const std::string & database_name_, const std::string & url_, const DatabaseIcebergSettings & settings_, - ASTPtr database_engine_definition_) + ASTPtr database_engine_definition_, + ContextPtr context_) : IDatabase(database_name_) , url(url_) , settings(settings_) , database_engine_definition(database_engine_definition_) , log(getLogger("DatabaseIceberg(" + database_name_ + ")")) { - setCredentials(credentials, Poco::URI(url)); - const auto auth_header = settings[DatabaseIcebergSetting::auth_header].value; + LOG_TEST(log, "Auth header: {}", auth_header); if (!auth_header.empty()) { auto pos = auth_header.find(':'); if (pos == std::string::npos) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected format of auth header"); headers.emplace_back(auth_header.substr(0, pos), auth_header.substr(pos + 1)); + + LOG_TEST(log, "Added header: {}={}", headers.back().name, headers.back().value); } + validateSettings(context_); +} + +void DatabaseIceberg::validateSettings(const ContextPtr & context_) +{ + if (settings[DatabaseIcebergSetting::warehouse].value.empty()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "`warehouse` setting cannot be empty. " + "Please specify 'SETTINGS warehouse=' in the CREATE DATABASE query"); + } + + if (!settings[DatabaseIcebergSetting::storage_type].changed) + { + auto catalog = getCatalog(context_); + const auto storage_type = catalog->getStorageType(); + if (!storage_type) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Storage type is not found in catalog config. " + "Please specify it manually via 'SETTINGS storage_type=' in CREATE DATABASE query"); + } + } } std::unique_ptr DatabaseIceberg::getCatalog(ContextPtr context_) const @@ -101,7 +114,12 @@ std::unique_ptr DatabaseIceberg::getCatalog(ContextPtr contex { case DB::DatabaseIcebergCatalogType::REST: { - return std::make_unique(getDatabaseName(), url, context_); + return std::make_unique( + settings[DatabaseIcebergSetting::warehouse].value, + url, + settings[DatabaseIcebergSetting::catalog_credential].value, + headers, + context_); } } } @@ -143,9 +161,17 @@ std::shared_ptr DatabaseIceberg::getConfigu std::string DatabaseIceberg::getStorageEndpointForTable(const Iceberg::TableMetadata & table_metadata) const { - return std::filesystem::path(settings[DatabaseIcebergSetting::storage_endpoint].value) - / table_metadata.getPath() - / ""; + auto endpoint_from_settings = settings[DatabaseIcebergSetting::storage_endpoint].value; + if (!endpoint_from_settings.empty()) + { + return std::filesystem::path(endpoint_from_settings) + / table_metadata.getLocation(/* path_only */true) + / ""; + } + else + { + return std::filesystem::path(table_metadata.getLocation(/* path_only */false)) / ""; + } } bool DatabaseIceberg::empty() const @@ -307,7 +333,8 @@ void registerDatabaseIceberg(DatabaseFactory & factory) args.database_name, url, database_settings, - database_engine_define->clone()); + database_engine_define->clone(), + args.context); }; factory.registerDatabase("Iceberg", create_fn, { .supports_arguments = true, .supports_settings = true }); } diff --git a/src/Databases/Iceberg/DatabaseIceberg.h b/src/Databases/Iceberg/DatabaseIceberg.h index 230920b1190..c9147bcd7ab 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.h +++ b/src/Databases/Iceberg/DatabaseIceberg.h @@ -22,7 +22,8 @@ public: const std::string & database_name_, const std::string & url_, const DatabaseIcebergSettings & settings_, - ASTPtr database_engine_definition_); + ASTPtr database_engine_definition_, + ContextPtr context_); String getEngineName() const override { return "Iceberg"; } @@ -59,6 +60,7 @@ private: Poco::Net::HTTPBasicCredentials credentials; HTTPHeaderEntries headers; + void validateSettings(const ContextPtr & context_); std::unique_ptr getCatalog(ContextPtr context_) const; std::shared_ptr getConfiguration() const; std::string getStorageEndpointForTable(const Iceberg::TableMetadata & table_metadata) const; diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp index 5017686b0d7..1dc1ba1f61c 100644 --- a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp @@ -17,6 +17,8 @@ namespace ErrorCodes #define DATABASE_ICEBERG_RELATED_SETTINGS(DECLARE, ALIAS) \ DECLARE(DatabaseIcebergCatalogType, catalog_type, DatabaseIcebergCatalogType::REST, "Catalog type", 0) \ DECLARE(DatabaseIcebergStorageType, storage_type, DatabaseIcebergStorageType::S3, "Storage type: S3, Local, Azure, HDFS", 0) \ + DECLARE(String, catalog_credential, "", "", 0) \ + DECLARE(String, warehouse, "", "Warehouse name inside the catalog", 0) \ DECLARE(String, auth_header, "", "Authorization header of format 'Authorization: '", 0) \ DECLARE(String, storage_endpoint, "", "Object storage endpoint", 0) \ diff --git a/src/Databases/Iceberg/ICatalog.cpp b/src/Databases/Iceberg/ICatalog.cpp index d07d9613e1a..6b6ecc85daf 100644 --- a/src/Databases/Iceberg/ICatalog.cpp +++ b/src/Databases/Iceberg/ICatalog.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include namespace DB::ErrorCodes { @@ -10,22 +12,75 @@ namespace DB::ErrorCodes namespace Iceberg { -std::string TableMetadata::getPath() const +void TableMetadata::setLocation(const std::string & location_) { if (!with_location) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Data location was not requested"); - if (location.starts_with("s3://")) - return location.substr(std::strlen("s3://")); + /// Location has format: + /// s3:///path/to/table/data. + /// We want to split s3:// and path/to/table/data. + + auto pos = location_.find("://"); + if (pos == std::string::npos) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unexpected location format: {}", location_); + + auto pos_to_bucket = pos + std::strlen("://"); + auto pos_to_path = location_.substr(pos_to_bucket).find('/'); + + if (pos_to_path == std::string::npos) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unexpected location format: {}", location_); + + pos_to_path = pos_to_bucket + pos_to_path; + + location_without_path = location_.substr(0, pos_to_path); + path = location_.substr(pos_to_path + 1); + + LOG_TEST(getLogger("TableMetadata"), + "Parsed location without path: {}, path: {}", + location_without_path, path); +} + +std::string TableMetadata::getLocation(bool path_only) const +{ + if (!with_location) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Data location was not requested"); + + if (path_only) + return path; else - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unexpected path format: {}", location); + return std::filesystem::path(location_without_path) / path; +} + +void TableMetadata::setSchema(const DB::NamesAndTypesList & schema_) +{ + if (!with_schema) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Data schema was not requested"); + + schema = schema_; } const DB::NamesAndTypesList & TableMetadata::getSchema() const { if (!with_schema) - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Data location was not requested"); + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Data schema was not requested"); + return schema; } +StorageType ICatalog::getStorageType(const std::string & location) +{ + auto pos = location.find("://"); + if (pos == std::string::npos) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unexpected path format: {}", location); + + auto storage_type_str = location.substr(0, pos); + auto storage_type = magic_enum::enum_cast(Poco::toUpper(storage_type_str)); + + if (!storage_type) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unsupported storage type: {}", storage_type_str); + + return *storage_type; +} + } diff --git a/src/Databases/Iceberg/ICatalog.h b/src/Databases/Iceberg/ICatalog.h index 7bd2cf3010c..4c61c87998d 100644 --- a/src/Databases/Iceberg/ICatalog.h +++ b/src/Databases/Iceberg/ICatalog.h @@ -1,27 +1,36 @@ #pragma once #include #include +#include namespace Iceberg { +using StorageType = DB::DatabaseIcebergStorageType; class TableMetadata { -friend class RestCatalog; - public: TableMetadata() = default; - std::string getPath() const; - - const DB::NamesAndTypesList & getSchema() const; - TableMetadata & withLocation() { with_location = true; return *this; } TableMetadata & withSchema() { with_schema = true; return *this; } + std::string getLocation(bool path_only) const; + std::string getLocation() const; + std::string getLocationWithoutPath() const; + + const DB::NamesAndTypesList & getSchema() const; + + bool requiresLocation() const { return with_location; } + bool requiresSchema() const { return with_schema; } + + void setLocation(const std::string & location_); + void setSchema(const DB::NamesAndTypesList & schema_); + private: /// starts with s3://, file://, etc - std::string location; + std::string location_without_path; + std::string path; /// column names and types DB::NamesAndTypesList schema; @@ -36,7 +45,7 @@ public: using Namespaces = std::vector; using Tables = std::vector; - explicit ICatalog(const std::string & catalog_name_) : catalog_name(catalog_name_) {} + explicit ICatalog(const std::string & warehouse_) : warehouse(warehouse_) {} virtual ~ICatalog() = default; @@ -58,8 +67,12 @@ public: const std::string & table_name, TableMetadata & result) const = 0; + virtual std::optional getStorageType() const = 0; + protected: - const std::string catalog_name; + const std::string warehouse; + + static StorageType getStorageType(const std::string & location); }; } diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index 729ed641494..cadb929ed5d 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -13,7 +13,6 @@ #include #include -#include #include namespace DB::ErrorCodes @@ -25,17 +24,178 @@ namespace DB::ErrorCodes namespace Iceberg { +static constexpr auto config_endpoint = "config"; static constexpr auto namespaces_endpoint = "namespaces"; -RestCatalog::RestCatalog( - const std::string & catalog_name_, - const std::string & base_url_, - DB::ContextPtr context_) - : ICatalog(catalog_name_) - , DB::WithContext(context_) - , base_url(base_url_) - , log(getLogger("RestCatalog(" + catalog_name_ + ")")) +std::string RestCatalog::Config::toString() const { + DB::WriteBufferFromOwnString wb; + + if (!prefix.empty()) + wb << "prefix: " << prefix.string() << ", "; + if (!default_base_location.empty()) + wb << "default_base_location: " << default_base_location << ", "; + + return wb.str(); +} + +RestCatalog::RestCatalog( + const std::string & warehouse_, + const std::string & base_url_, + const std::string & catalog_credential_, + const DB::HTTPHeaderEntries & headers_, + DB::ContextPtr context_) + : ICatalog(warehouse_) + , DB::WithContext(context_) + , log(getLogger("RestCatalog(" + warehouse_ + ")")) + , base_url(base_url_) + , headers(headers_) +{ + if (!catalog_credential_.empty()) + { + auto pos = catalog_credential_.find(':'); + if (pos == std::string::npos) + { + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, "Unexpected format of catalog credential: " + "expected client_id and client_secret separated by `:`"); + } + client_id = catalog_credential_.substr(0, pos); + client_secret = catalog_credential_.substr(pos + 1); + + /// TODO: remove before merge. + LOG_TEST(log, "Client id: {}, client secret: {}", client_id, client_secret); + } + LOG_TEST(log, "kssenii 1"); + config = loadConfig(); +} + +RestCatalog::Config RestCatalog::loadConfig() +{ + LOG_TEST(log, "kssenii 2"); + if (!client_id.empty()) + { + static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; + + const auto & context = getContext(); + const auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); + // Poco::URI::QueryParameters params = { + // {"grant_type", "client_credentials"}, + // {"scope", "PRINCIPAL_ROLE:ALL"}, + // {"client_id", client_id}, + // {"client_secret", client_secret}}; + + Poco::JSON::Object json; + json.set("grant_type", "client_credentials"); + json.set("scope", "PRINCIPAL_ROLE:ALL"); + json.set("client_id", client_id); + json.set("client_secret", client_secret); + + LOG_TEST(log, "kssenii 3"); + + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(json, oss); + std::string json_str = oss.str(); + + LOG_TEST(log, "kssenii 4"); + Poco::URI url(base_url / oauth_tokens_endpoint); + // url.setQueryParameters(params); + + LOG_TEST(log, "Writing {}: {}", url.toString(), json_str); + + auto callback = [&](std::ostream & out) + { + out << json_str; + }; + + auto wb = DB::BuilderRWBufferFromHTTP(url) + .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) + .withSettings(getContext()->getReadSettings()) + .withTimeouts(timeouts) + .withHostFilter(&getContext()->getRemoteHostFilter()) + .withOutCallback(callback) + .withSkipNotFound(false) + .withHeaders(headers) + .create(credentials); + + json_str.clear(); + readJSONObjectPossiblyInvalid(json_str, *wb); + + LOG_TEST(log, "Received token result: {}", json_str); + + Poco::JSON::Parser parser; + Poco::Dynamic::Var res_json = parser.parse(json_str); + const Poco::JSON::Object::Ptr & object = res_json.extract(); + + auto access_token = object->get("access_token").extract(); + headers.emplace_back("Authorization", "Bearer " + access_token); + } + + Poco::URI::QueryParameters params = {{"warehouse", warehouse}}; + auto buf = createReadBuffer(config_endpoint, params); + + std::string json_str; + readJSONObjectPossiblyInvalid(json_str, *buf); + + LOG_TEST(log, "Received config result: {}", json_str); + + Poco::JSON::Parser parser; + Poco::Dynamic::Var json = parser.parse(json_str); + const Poco::JSON::Object::Ptr & object = json.extract(); + + Config result; + + auto defaults_object = object->get("defaults").extract(); + parseConfig(defaults_object, result); + + auto overrides_object = object->get("overrides").extract(); + parseConfig(overrides_object, result); + + LOG_TEST(log, "Parsed config: {}", result.toString()); + return result; +} + +void RestCatalog::parseConfig(const Poco::JSON::Object::Ptr & object, Config & result) +{ + if (!object) + return; + + if (object->has("prefix")) + result.prefix = object->get("prefix").extract(); + + if (object->has("default-base-location")) + result.default_base_location = object->get("default-base-location").extract(); +} + +std::optional RestCatalog::getStorageType() const +{ + if (config.default_base_location.empty()) + return std::nullopt; + return ICatalog::getStorageType(config.default_base_location); +} + +DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( + const std::string & endpoint, + const Poco::URI::QueryParameters & params) const +{ + const auto & context = getContext(); + const auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); + + Poco::URI url(base_url / endpoint); + if (!params.empty()) + url.setQueryParameters(params); + + LOG_TEST(log, "Requesting: {}", url.toString()); + + return DB::BuilderRWBufferFromHTTP(url) + .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) + .withSettings(getContext()->getReadSettings()) + .withTimeouts(timeouts) + .withHostFilter(&getContext()->getRemoteHostFilter()) + .withSkipNotFound(false) + .withHeaders(headers) + .create(credentials); } bool RestCatalog::empty() const @@ -62,24 +222,6 @@ bool RestCatalog::empty() const } } -DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer(const std::string & endpoint, const Poco::URI::QueryParameters & params) const -{ - const auto & context = getContext(); - const auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); - - Poco::URI url(base_url / endpoint); - if (!params.empty()) - url.setQueryParameters(params); - - return DB::BuilderRWBufferFromHTTP(url) - .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) - .withSettings(getContext()->getReadSettings()) - .withTimeouts(timeouts) - .withHostFilter(&getContext()->getRemoteHostFilter()) - .withSkipNotFound(true) - .create(credentials); -} - RestCatalog::Tables RestCatalog::getTables() const { Namespaces namespaces; @@ -133,7 +275,7 @@ RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_name try { - auto buf = createReadBuffer(namespaces_endpoint, params); + auto buf = createReadBuffer(config.prefix / namespaces_endpoint, params); auto namespaces = parseNamespaces(*buf, base_namespace); LOG_TEST(log, "Loaded {} namespaces", namespaces.size()); return namespaces; @@ -142,7 +284,7 @@ RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_name { std::string message = fmt::format( "Received error while fetching list of namespaces from iceberg catalog `{}`. ", - catalog_name); + warehouse); if (e.code() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND) message += "Namespace provided in the `parent` query parameter is not found. "; @@ -195,7 +337,7 @@ RestCatalog::Namespaces RestCatalog::parseNamespaces(DB::ReadBuffer & buf, const RestCatalog::Tables RestCatalog::getTables(const std::string & base_namespace, size_t limit) const { const auto endpoint = std::string(namespaces_endpoint) + "/" + base_namespace + "/tables"; - auto buf = createReadBuffer(endpoint); + auto buf = createReadBuffer(config.prefix / endpoint); return parseTables(*buf, base_namespace, limit); } @@ -267,7 +409,7 @@ bool RestCatalog::getTableMetadataImpl( LOG_TEST(log, "Checking table {} in namespace {}", table_name, namespace_name); const auto endpoint = std::string(namespaces_endpoint) + "/" + namespace_name + "/tables/" + table_name; - auto buf = createReadBuffer(endpoint); + auto buf = createReadBuffer(config.prefix / endpoint); if (buf->eof()) { @@ -288,17 +430,17 @@ bool RestCatalog::getTableMetadataImpl( if (!metadata_object) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot parse result"); - if (result.with_location) + if (result.requiresLocation()) { - result.location = metadata_object->get("location").extract(); - - LOG_TEST(log, "Location for table {}: {}", table_name, result.location); + const auto location = metadata_object->get("location").extract(); + result.setLocation(location); + LOG_TEST(log, "Location for table {}: {}", table_name, location); } - if (result.with_schema) + if (result.requiresSchema()) { int format_version = metadata_object->getValue("format-version"); - result.schema = DB::IcebergMetadata::parseTableSchema(metadata_object, format_version, true).first; + result.setSchema(DB::IcebergMetadata::parseTableSchema(metadata_object, format_version, true).first); } return true; } diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index 12ff31ad8d1..5415ff73a7e 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -2,8 +2,10 @@ #include #include #include +#include #include #include +#include namespace DB { @@ -17,8 +19,10 @@ class RestCatalog final : public ICatalog, private DB::WithContext { public: explicit RestCatalog( - const std::string & catalog_name_, + const std::string & warehouse_, const std::string & base_url_, + const std::string & catalog_credential_, + const DB::HTTPHeaderEntries & headers_, DB::ContextPtr context_); ~RestCatalog() override = default; @@ -39,12 +43,31 @@ public: const std::string & table_name, TableMetadata & result) const override; + std::optional getStorageType() const override; + private: - const std::filesystem::path base_url; - Poco::Net::HTTPBasicCredentials credentials{}; LoggerPtr log; - DB::ReadWriteBufferFromHTTPPtr createReadBuffer(const std::string & endpoint, const Poco::URI::QueryParameters & params = {}) const; + struct Config + { + std::filesystem::path prefix; + std::string default_base_location; + + std::string toString() const; + }; + + const std::filesystem::path base_url; + DB::HTTPHeaderEntries headers; + std::string client_id; + std::string client_secret; + Config config; + + Poco::Net::HTTPBasicCredentials credentials{}; + + + DB::ReadWriteBufferFromHTTPPtr createReadBuffer( + const std::string & endpoint, + const Poco::URI::QueryParameters & params = {}) const; Poco::URI::QueryParameters createParentNamespaceParams(const std::string & base_namespace) const; @@ -63,6 +86,9 @@ private: const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const; + + Config loadConfig(); + static void parseConfig(const Poco::JSON::Object::Ptr & object, Config & result); }; } From 296717cf76f9563272d85fbc318acbbc1df46bdd Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 19 Nov 2024 14:18:30 +0000 Subject: [PATCH 148/502] Fixed bug --- .../DataLakes/IcebergMetadata.cpp | 2 +- .../integration/test_storage_iceberg/test.py | 156 ++++++++++-------- 2 files changed, 91 insertions(+), 67 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index cce72c571c9..aec142845ef 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -331,7 +331,7 @@ IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr } else if (allowPrimitiveTypeConversion(old_type, new_type)) { - node = &dag->addCast(*old_node, getSimpleType(new_type), name); + node = &dag->addCast(*old_node, getFieldType(field, "type", required), name); } outputs.push_back(node); } diff --git a/tests/integration/test_storage_iceberg/test.py b/tests/integration/test_storage_iceberg/test.py index 828618871bd..ce8c7373bbe 100644 --- a/tests/integration/test_storage_iceberg/test.py +++ b/tests/integration/test_storage_iceberg/test.py @@ -678,7 +678,7 @@ def test_evolved_schema_simple( execute_spark_query( f""" - INSERT INTO {TABLE_NAME} VALUES (4, 3.0, 7.12, ARRAY(5, 6, 7)); + INSERT INTO {TABLE_NAME} VALUES (4, NULL, 7.12, ARRAY(5, 6, 7)); """ ) @@ -691,12 +691,12 @@ def test_evolved_schema_simple( ["c", "Decimal(9, 2)"], ["d", "Array(Nullable(Int32))"], ], - [["4", "3", "7.12", "[5,6,7]"]], + [["4", "\\N", "7.12", "[5,6,7]"]], ) execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN b TYPE double; + ALTER TABLE {TABLE_NAME} ALTER COLUMN b TYPE double; """ ) @@ -709,13 +709,13 @@ def test_evolved_schema_simple( ["c", "Decimal(9, 2)"], ["d", "Array(Nullable(Int32))"], ], - [["4", "3", "7.12", "[5,6,7]"]], + [["4", "\\N", "7.12", "[5,6,7]"]], ) execute_spark_query( f""" - INSERT INTO {TABLE_NAME} VALUES (7, 5.0, 18.1, ARRAY(6, 7, 9)); - """ + INSERT INTO {TABLE_NAME} VALUES (7, 5.0, 18.1, ARRAY(6, 7, 9)); + """ ) check_schema_and_data( @@ -727,13 +727,13 @@ def test_evolved_schema_simple( ["c", "Decimal(9, 2)"], ["d", "Array(Nullable(Int32))"], ], - [["4", "3", "7.12", "[5,6,7]"], ["7", "5", "18.1", "[6,7,9]"]], + [["4", "\\N", "7.12", "[5,6,7]"], ["7", "5", "18.1", "[6,7,9]"]], ) execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN d FIRST; - """ + ALTER TABLE {TABLE_NAME} ALTER COLUMN d FIRST; + """ ) check_schema_and_data( @@ -745,13 +745,13 @@ def test_evolved_schema_simple( ["b", "Nullable(Float64)"], ["c", "Decimal(9, 2)"], ], - [["[5,6,7]", "4", "3", "7.12"], ["[6,7,9]", "7", "5", "18.1"]], + [["[5,6,7]", "4", "\\N", "7.12"], ["[6,7,9]", "7", "5", "18.1"]], ) execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN b AFTER d; - """ + ALTER TABLE {TABLE_NAME} ALTER COLUMN b AFTER d; + """ ) check_schema_and_data( @@ -763,16 +763,16 @@ def test_evolved_schema_simple( ["a", "Int32"], ["c", "Decimal(9, 2)"], ], - [["[5,6,7]", "3", "4", "7.12"], ["[6,7,9]", "5", "7", "18.1"]], + [["[5,6,7]", "\\N", "4", "7.12"], ["[6,7,9]", "5", "7", "18.1"]], ) execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} - ADD COLUMNS ( - e string - ); - """ + ALTER TABLE {TABLE_NAME} + ADD COLUMNS ( + e string + ); + """ ) check_schema_and_data( @@ -786,15 +786,15 @@ def test_evolved_schema_simple( ["e", "Nullable(String)"], ], [ - ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[5,6,7]", "\\N", "4", "7.12", "\\N"], ["[6,7,9]", "5", "7", "18.1", "\\N"], ], ) execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN c TYPE decimal(12, 2); - """ + ALTER TABLE {TABLE_NAME} ALTER COLUMN c TYPE decimal(12, 2); + """ ) check_schema_and_data( @@ -808,15 +808,15 @@ def test_evolved_schema_simple( ["e", "Nullable(String)"], ], [ - ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[5,6,7]", "\\N", "4", "7.12", "\\N"], ["[6,7,9]", "5", "7", "18.1", "\\N"], ], ) execute_spark_query( f""" - INSERT INTO {TABLE_NAME} VALUES (ARRAY(5, 6, 7), 3, -30, 7.12, 'AAA'); - """ + INSERT INTO {TABLE_NAME} VALUES (ARRAY(5, 6, 7), 3, -30, 7.12, 'AAA'); + """ ) check_schema_and_data( @@ -831,15 +831,15 @@ def test_evolved_schema_simple( ], [ ["[5,6,7]", "3", "-30", "7.12", "AAA"], - ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[5,6,7]", "\\N", "4", "7.12", "\\N"], ["[6,7,9]", "5", "7", "18.1", "\\N"], ], ) execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN a TYPE BIGINT; - """ + ALTER TABLE {TABLE_NAME} ALTER COLUMN a TYPE BIGINT; + """ ) check_schema_and_data( @@ -854,15 +854,15 @@ def test_evolved_schema_simple( ], [ ["[5,6,7]", "3", "-30", "7.12", "AAA"], - ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[5,6,7]", "\\N", "4", "7.12", "\\N"], ["[6,7,9]", "5", "7", "18.1", "\\N"], ], ) execute_spark_query( f""" - INSERT INTO {TABLE_NAME} VALUES (ARRAY(), 3.0, 12, -9.13, 'BBB'); - """ + INSERT INTO {TABLE_NAME} VALUES (ARRAY(), 3.0, 12, -9.13, 'BBB'); + """ ) check_schema_and_data( @@ -878,15 +878,15 @@ def test_evolved_schema_simple( [ ["[]", "3", "12", "-9.13", "BBB"], ["[5,6,7]", "3", "-30", "7.12", "AAA"], - ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[5,6,7]", "\\N", "4", "7.12", "\\N"], ["[6,7,9]", "5", "7", "18.1", "\\N"], ], ) execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN a DROP NOT NULL;; - """ + ALTER TABLE {TABLE_NAME} ALTER COLUMN a DROP NOT NULL; + """ ) check_schema_and_data( @@ -902,15 +902,15 @@ def test_evolved_schema_simple( [ ["[]", "3", "12", "-9.13", "BBB"], ["[5,6,7]", "3", "-30", "7.12", "AAA"], - ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[5,6,7]", "\\N", "4", "7.12", "\\N"], ["[6,7,9]", "5", "7", "18.1", "\\N"], ], ) execute_spark_query( f""" - INSERT INTO {TABLE_NAME} VALUES (NULL, 3.4, NULL, -9.13, NULL); - """ + INSERT INTO {TABLE_NAME} VALUES (NULL, 3.4, NULL, -9.13, NULL); + """ ) check_schema_and_data( @@ -927,15 +927,15 @@ def test_evolved_schema_simple( ["[]", "3", "12", "-9.13", "BBB"], ["[]", "3.4", "\\N", "-9.13", "\\N"], ["[5,6,7]", "3", "-30", "7.12", "AAA"], - ["[5,6,7]", "3", "4", "7.12", "\\N"], + ["[5,6,7]", "\\N", "4", "7.12", "\\N"], ["[6,7,9]", "5", "7", "18.1", "\\N"], ], ) execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} DROP COLUMN d; - """ + ALTER TABLE {TABLE_NAME} DROP COLUMN d; + """ ) check_schema_and_data( @@ -949,10 +949,34 @@ def test_evolved_schema_simple( ], [ ["3", "-30", "7.12", "AAA"], - ["3", "4", "7.12", "\\N"], ["3", "12", "-9.13", "BBB"], ["3.4", "\\N", "-9.13", "\\N"], ["5", "7", "18.1", "\\N"], + ["\\N", "4", "7.12", "\\N"], + ], + ) + + execute_spark_query( + f""" + ALTER TABLE {TABLE_NAME} RENAME COLUMN a TO f; + """ + ) + + check_schema_and_data( + instance, + table_select_expression, + [ + ["b", "Nullable(Float64)"], + ["f", "Nullable(Int64)"], + ["c", "Decimal(12, 2)"], + ["e", "Nullable(String)"], + ], + [ + ["3", "-30", "7.12", "AAA"], + ["3", "12", "-9.13", "BBB"], + ["3.4", "\\N", "-9.13", "\\N"], + ["5", "7", "18.1", "\\N"], + ["\\N", "4", "7.12", "\\N"], ], ) @@ -1044,7 +1068,7 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN b TYPE double; + ALTER TABLE {TABLE_NAME} ALTER COLUMN b TYPE double; """ ) @@ -1062,8 +1086,8 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - INSERT INTO {TABLE_NAME} VALUES (7, 5.0, 18.1, ARRAY(6, 7, 9)); - """ + INSERT INTO {TABLE_NAME} VALUES (7, 5.0, 18.1, ARRAY(6, 7, 9)); + """ ) check_schema_and_data( @@ -1080,8 +1104,8 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN d FIRST; - """ + ALTER TABLE {TABLE_NAME} ALTER COLUMN d FIRST; + """ ) check_schema_and_data( @@ -1098,8 +1122,8 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN b AFTER d; - """ + ALTER TABLE {TABLE_NAME} ALTER COLUMN b AFTER d; + """ ) check_schema_and_data( @@ -1116,11 +1140,11 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} - ADD COLUMNS ( - e string - ); - """ + ALTER TABLE {TABLE_NAME} + ADD COLUMNS ( + e string + ); + """ ) check_schema_and_data( @@ -1137,8 +1161,8 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN c TYPE decimal(12, 2); - """ + ALTER TABLE {TABLE_NAME} ALTER COLUMN c TYPE decimal(12, 2); + """ ) check_schema_and_data( @@ -1155,8 +1179,8 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - INSERT INTO {TABLE_NAME} VALUES (ARRAY(5, 6, 7), 3, -30, 7.12, 'AAA'); - """ + INSERT INTO {TABLE_NAME} VALUES (ARRAY(5, 6, 7), 3, -30, 7.12, 'AAA'); + """ ) check_schema_and_data( @@ -1177,8 +1201,8 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN a TYPE BIGINT; - """ + ALTER TABLE {TABLE_NAME} ALTER COLUMN a TYPE BIGINT; + """ ) check_schema_and_data( @@ -1199,8 +1223,8 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - INSERT INTO {TABLE_NAME} VALUES (ARRAY(), 3.0, 12, -9.13, 'BBB'); - """ + INSERT INTO {TABLE_NAME} VALUES (ARRAY(), 3.0, 12, -9.13, 'BBB'); + """ ) check_schema_and_data( @@ -1222,8 +1246,8 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} ALTER COLUMN a DROP NOT NULL; - """ + ALTER TABLE {TABLE_NAME} ALTER COLUMN a DROP NOT NULL; + """ ) check_schema_and_data( @@ -1245,8 +1269,8 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - INSERT INTO {TABLE_NAME} VALUES (NULL, 3.4, NULL, -9.13, NULL); - """ + INSERT INTO {TABLE_NAME} VALUES (NULL, 3.4, NULL, -9.13, NULL); + """ ) check_schema_and_data( @@ -1269,8 +1293,8 @@ def test_not_evolved_schema(started_cluster, format_version, storage_type): execute_spark_query( f""" - ALTER TABLE {TABLE_NAME} DROP COLUMN d; - """ + ALTER TABLE {TABLE_NAME} DROP COLUMN d; + """ ) error = instance.query_and_get_error(f"SELECT * FROM {TABLE_NAME} ORDER BY ALL") From 7b1456f17a67663c945a338466101b1d583b942f Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 19 Nov 2024 15:23:41 +0000 Subject: [PATCH 149/502] Launch CI From b34dfca0c56f63fb8caa655a28ffe6b860379936 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 19 Nov 2024 14:52:10 +0100 Subject: [PATCH 150/502] Speedup tables loading --- src/Databases/Iceberg/DatabaseIceberg.cpp | 47 +++++++++-- src/Databases/Iceberg/DatabaseIceberg.h | 4 +- src/Databases/Iceberg/RestCatalog.cpp | 83 +++++++++++++++---- src/Databases/Iceberg/RestCatalog.h | 8 +- .../ObjectStorage/StorageObjectStorage.cpp | 6 +- .../ObjectStorage/StorageObjectStorage.h | 3 +- 6 files changed, 121 insertions(+), 30 deletions(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index c22d4747f83..d701e73d429 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -23,6 +23,13 @@ #include #include +namespace CurrentMetrics +{ + extern const Metric MergeTreeDataSelectExecutorThreads; + extern const Metric MergeTreeDataSelectExecutorThreadsActive; + extern const Metric MergeTreeDataSelectExecutorThreadsScheduled; +} + namespace DB { @@ -108,20 +115,24 @@ void DatabaseIceberg::validateSettings(const ContextPtr & context_) } } -std::unique_ptr DatabaseIceberg::getCatalog(ContextPtr context_) const +std::shared_ptr DatabaseIceberg::getCatalog(ContextPtr) const { + if (catalog_impl) + return catalog_impl; + switch (settings[DatabaseIcebergSetting::catalog_type].value) { case DB::DatabaseIcebergCatalogType::REST: { - return std::make_unique( + catalog_impl = std::make_shared( settings[DatabaseIcebergSetting::warehouse].value, url, settings[DatabaseIcebergSetting::catalog_credential].value, headers, - context_); + Context::getGlobalContextInstance()); } } + return catalog_impl; } std::shared_ptr DatabaseIceberg::getConfiguration() const @@ -220,7 +231,10 @@ StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_ /* constraints */ConstraintsDescription{}, /* comment */"", getFormatSettings(context_), - LoadingStrictnessLevel::CREATE); + LoadingStrictnessLevel::CREATE, + /* distributed_processing */false, + /* partition_by */nullptr, + /* lazy_init */true); } DatabaseTablesIteratorPtr DatabaseIceberg::getTablesIterator( @@ -230,15 +244,34 @@ DatabaseTablesIteratorPtr DatabaseIceberg::getTablesIterator( { Tables tables; auto catalog = getCatalog(context_); - for (const auto & table_name : catalog->getTables()) + + auto iceberg_tables = catalog->getTables(); + size_t num_threads = std::min(10, iceberg_tables.size()); + ThreadPool pool( + CurrentMetrics::MergeTreeDataSelectExecutorThreads, + CurrentMetrics::MergeTreeDataSelectExecutorThreadsActive, + CurrentMetrics::MergeTreeDataSelectExecutorThreadsScheduled, + num_threads); + + DB::ThreadPoolCallbackRunnerLocal runner(pool, "RestCatalog"); + std::mutex mutexx; + + for (const auto & table_name : iceberg_tables) { if (filter_by_table_name && !filter_by_table_name(table_name)) continue; - auto storage = tryGetTable(table_name, context_); - tables.emplace(table_name, storage); + runner([&]{ + auto storage = tryGetTable(table_name, context_); + { + std::lock_guard lock(mutexx); + [[maybe_unused]] bool inserted = tables.emplace(table_name, storage).second; + chassert(inserted); + } + }); } + runner.waitForAllToFinishAndRethrowFirstError(); return std::make_unique(tables, getDatabaseName()); } diff --git a/src/Databases/Iceberg/DatabaseIceberg.h b/src/Databases/Iceberg/DatabaseIceberg.h index c9147bcd7ab..a355e316c1a 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.h +++ b/src/Databases/Iceberg/DatabaseIceberg.h @@ -60,8 +60,10 @@ private: Poco::Net::HTTPBasicCredentials credentials; HTTPHeaderEntries headers; + mutable std::shared_ptr catalog_impl; + void validateSettings(const ContextPtr & context_); - std::unique_ptr getCatalog(ContextPtr context_) const; + std::shared_ptr getCatalog(ContextPtr context_) const; std::shared_ptr getConfiguration() const; std::string getStorageEndpointForTable(const Iceberg::TableMetadata & table_metadata) const; }; diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index cadb929ed5d..76c5e897af9 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -9,6 +10,7 @@ #include #include +#include #include #include @@ -21,6 +23,13 @@ namespace DB::ErrorCodes extern const int LOGICAL_ERROR; } +namespace CurrentMetrics +{ + extern const Metric MergeTreeDataSelectExecutorThreads; + extern const Metric MergeTreeDataSelectExecutorThreadsActive; + extern const Metric MergeTreeDataSelectExecutorThreadsScheduled; +} + namespace Iceberg { @@ -79,11 +88,6 @@ RestCatalog::Config RestCatalog::loadConfig() const auto & context = getContext(); const auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); - // Poco::URI::QueryParameters params = { - // {"grant_type", "client_credentials"}, - // {"scope", "PRINCIPAL_ROLE:ALL"}, - // {"client_id", client_id}, - // {"client_secret", client_secret}}; Poco::JSON::Object json; json.set("grant_type", "client_credentials"); @@ -91,16 +95,33 @@ RestCatalog::Config RestCatalog::loadConfig() json.set("client_id", client_id); json.set("client_secret", client_secret); - LOG_TEST(log, "kssenii 3"); - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); Poco::JSON::Stringifier::stringify(json, oss); std::string json_str = oss.str(); - LOG_TEST(log, "kssenii 4"); + Poco::URI::QueryParameters params = { + {"Content-Type", "application/x-www-form-urlencoded"}, + {"Content-Length", DB::toString(json_str.size())} + }; + // {"grant_type", "client_credentials"}, + // {"scope", "PRINCIPAL_ROLE:ALL"}, + // {"client_id", client_id}, + // {"client_secret", client_secret}}; + + + // LOG_TEST(log, "kssenii 3"); + + // DB::HTMLForm form(context->getSettingsRef()); + // form.add("grant_type", "client_credentials"); + // form.add("scope", "PRINCIPAL_ROLE:ALL"); + // form.add("client_id", client_id); + // form.add("client_secret", client_secret); + + + LOG_TEST(log, "kssenii 4"); Poco::URI url(base_url / oauth_tokens_endpoint); - // url.setQueryParameters(params); + url.setQueryParameters(params); LOG_TEST(log, "Writing {}: {}", url.toString(), json_str); @@ -211,7 +232,7 @@ bool RestCatalog::empty() const }; Namespaces namespaces; - getNamespacesRecursive("", namespaces, stop_condition); + getNamespacesRecursive("", namespaces, stop_condition, {}); return found_table; } @@ -224,19 +245,42 @@ bool RestCatalog::empty() const RestCatalog::Tables RestCatalog::getTables() const { - Namespaces namespaces; - getNamespacesRecursive("", namespaces, {}); + size_t num_threads = 10; + ThreadPool pool( + CurrentMetrics::MergeTreeDataSelectExecutorThreads, + CurrentMetrics::MergeTreeDataSelectExecutorThreadsActive, + CurrentMetrics::MergeTreeDataSelectExecutorThreadsScheduled, + num_threads); + + DB::ThreadPoolCallbackRunnerLocal runner(pool, "RestCatalog"); Tables tables; - for (const auto & current_namespace : namespaces) + std::mutex mutex; + + auto func = [&](const std::string & current_namespace) { - auto tables_in_namespace = getTables(current_namespace); - std::move(tables_in_namespace.begin(), tables_in_namespace.end(), std::back_inserter(tables)); - } + runner( + [&]{ + auto tables_in_namespace = getTables(current_namespace); + { + std::lock_guard lock(mutex); + std::move(tables_in_namespace.begin(), tables_in_namespace.end(), std::back_inserter(tables)); + } + }); + }; + + Namespaces namespaces; + getNamespacesRecursive("", namespaces, {}, func); + + runner.waitForAllToFinishAndRethrowFirstError(); return tables; } -void RestCatalog::getNamespacesRecursive(const std::string & base_namespace, Namespaces & result, StopCondition stop_condition) const +void RestCatalog::getNamespacesRecursive( + const std::string & base_namespace, + Namespaces & result, + StopCondition stop_condition, + ExecuteFunc func) const { auto namespaces = getNamespaces(base_namespace); result.reserve(result.size() + namespaces.size()); @@ -249,7 +293,10 @@ void RestCatalog::getNamespacesRecursive(const std::string & base_namespace, Nam if (stop_condition && stop_condition(current_namespace)) break; - getNamespacesRecursive(current_namespace, result, stop_condition); + if (func) + func(current_namespace); + + getNamespacesRecursive(current_namespace, result, stop_condition, func); } } diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index 5415ff73a7e..067f67ab493 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -72,7 +72,13 @@ private: Poco::URI::QueryParameters createParentNamespaceParams(const std::string & base_namespace) const; using StopCondition = std::function; - void getNamespacesRecursive(const std::string & base_namespace, Namespaces & result, StopCondition stop_condition) const; + using ExecuteFunc = std::function; + + void getNamespacesRecursive( + const std::string & base_namespace, + Namespaces & result, + StopCondition stop_condition, + ExecuteFunc func) const; Namespaces getNamespaces(const std::string & base_namespace) const; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index fd2fe0400bb..d7faf487188 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -80,7 +80,8 @@ StorageObjectStorage::StorageObjectStorage( std::optional format_settings_, LoadingStrictnessLevel mode, bool distributed_processing_, - ASTPtr partition_by_) + ASTPtr partition_by_, + bool lazy_init) : IStorage(table_id_) , configuration(configuration_) , object_storage(object_storage_) @@ -91,7 +92,8 @@ StorageObjectStorage::StorageObjectStorage( { try { - configuration->update(object_storage, context); + if (!lazy_init) + configuration->update(object_storage, context); } catch (...) { diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index e2bb41a4935..0df0bbdefbd 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -59,7 +59,8 @@ public: std::optional format_settings_, LoadingStrictnessLevel mode, bool distributed_processing_ = false, - ASTPtr partition_by_ = nullptr); + ASTPtr partition_by_ = nullptr, + bool lazy_init = false); String getName() const override; From 7b4aabbb718bdb10629e6e1ed659bfe7f1a33059 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 19 Nov 2024 17:56:55 +0100 Subject: [PATCH 151/502] Support generating token on clickhouse server side --- src/Databases/Iceberg/RestCatalog.cpp | 40 ++++++++------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index 76c5e897af9..b4ab45c3cac 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -75,13 +76,11 @@ RestCatalog::RestCatalog( /// TODO: remove before merge. LOG_TEST(log, "Client id: {}, client secret: {}", client_id, client_secret); } - LOG_TEST(log, "kssenii 1"); config = loadConfig(); } RestCatalog::Config RestCatalog::loadConfig() { - LOG_TEST(log, "kssenii 2"); if (!client_id.empty()) { static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; @@ -100,44 +99,29 @@ RestCatalog::Config RestCatalog::loadConfig() Poco::JSON::Stringifier::stringify(json, oss); std::string json_str = oss.str(); - Poco::URI::QueryParameters params = { - {"Content-Type", "application/x-www-form-urlencoded"}, - {"Content-Length", DB::toString(json_str.size())} - }; - // {"grant_type", "client_credentials"}, - // {"scope", "PRINCIPAL_ROLE:ALL"}, - // {"client_id", client_id}, - // {"client_secret", client_secret}}; + auto current_headers = headers; + current_headers.emplace_back("Content-Type", "application/x-www-form-urlencoded"); + current_headers.emplace_back("Accepts", "application/json; charset=UTF-8"); - - // LOG_TEST(log, "kssenii 3"); - - // DB::HTMLForm form(context->getSettingsRef()); - // form.add("grant_type", "client_credentials"); - // form.add("scope", "PRINCIPAL_ROLE:ALL"); - // form.add("client_id", client_id); - // form.add("client_secret", client_secret); - - - LOG_TEST(log, "kssenii 4"); Poco::URI url(base_url / oauth_tokens_endpoint); + Poco::URI::QueryParameters params = { + {"grant_type", "client_credentials"}, + {"scope", "PRINCIPAL_ROLE:ALL"}, + {"client_id", client_id}, + {"client_secret", client_secret}, + }; url.setQueryParameters(params); LOG_TEST(log, "Writing {}: {}", url.toString(), json_str); - auto callback = [&](std::ostream & out) - { - out << json_str; - }; - auto wb = DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) + .withMethod(Poco::Net::HTTPRequest::HTTP_POST) .withSettings(getContext()->getReadSettings()) .withTimeouts(timeouts) .withHostFilter(&getContext()->getRemoteHostFilter()) - .withOutCallback(callback) .withSkipNotFound(false) - .withHeaders(headers) + .withHeaders(current_headers) .create(credentials); json_str.clear(); From 599d977d423b7c5d78f5d55789d43c1db1bd9832 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 19 Nov 2024 19:16:53 +0100 Subject: [PATCH 152/502] Update token lazily --- src/Databases/Iceberg/DatabaseIceberg.cpp | 14 +- src/Databases/Iceberg/DatabaseIceberg.h | 1 - src/Databases/Iceberg/RestCatalog.cpp | 208 +++++++++++++--------- src/Databases/Iceberg/RestCatalog.h | 7 +- 4 files changed, 133 insertions(+), 97 deletions(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index d701e73d429..6241c0d8423 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -78,18 +78,6 @@ DatabaseIceberg::DatabaseIceberg( , database_engine_definition(database_engine_definition_) , log(getLogger("DatabaseIceberg(" + database_name_ + ")")) { - const auto auth_header = settings[DatabaseIcebergSetting::auth_header].value; - LOG_TEST(log, "Auth header: {}", auth_header); - if (!auth_header.empty()) - { - auto pos = auth_header.find(':'); - if (pos == std::string::npos) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected format of auth header"); - headers.emplace_back(auth_header.substr(0, pos), auth_header.substr(pos + 1)); - - LOG_TEST(log, "Added header: {}={}", headers.back().name, headers.back().value); - } - validateSettings(context_); } @@ -128,7 +116,7 @@ std::shared_ptr DatabaseIceberg::getCatalog(ContextPtr) const settings[DatabaseIcebergSetting::warehouse].value, url, settings[DatabaseIcebergSetting::catalog_credential].value, - headers, + settings[DatabaseIcebergSetting::auth_header], Context::getGlobalContextInstance()); } } diff --git a/src/Databases/Iceberg/DatabaseIceberg.h b/src/Databases/Iceberg/DatabaseIceberg.h index a355e316c1a..145f033ad6a 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.h +++ b/src/Databases/Iceberg/DatabaseIceberg.h @@ -58,7 +58,6 @@ private: const LoggerPtr log; /// Crendetials to authenticate Iceberg Catalog. Poco::Net::HTTPBasicCredentials credentials; - HTTPHeaderEntries headers; mutable std::shared_ptr catalog_impl; diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index b4ab45c3cac..d336c4af707 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -49,94 +49,49 @@ std::string RestCatalog::Config::toString() const return wb.str(); } -RestCatalog::RestCatalog( - const std::string & warehouse_, - const std::string & base_url_, - const std::string & catalog_credential_, - const DB::HTTPHeaderEntries & headers_, - DB::ContextPtr context_) - : ICatalog(warehouse_) - , DB::WithContext(context_) - , log(getLogger("RestCatalog(" + warehouse_ + ")")) - , base_url(base_url_) - , headers(headers_) +static std::pair parseCatalogCredential(const std::string & catalog_credential) { - if (!catalog_credential_.empty()) + std::string client_id, client_secret; + if (!catalog_credential.empty()) { - auto pos = catalog_credential_.find(':'); + auto pos = catalog_credential.find(':'); if (pos == std::string::npos) { throw DB::Exception( DB::ErrorCodes::BAD_ARGUMENTS, "Unexpected format of catalog credential: " "expected client_id and client_secret separated by `:`"); } - client_id = catalog_credential_.substr(0, pos); - client_secret = catalog_credential_.substr(pos + 1); + client_id = catalog_credential.substr(0, pos); + client_secret = catalog_credential.substr(pos + 1); + } + return std::pair(client_id, client_secret); +} - /// TODO: remove before merge. - LOG_TEST(log, "Client id: {}, client secret: {}", client_id, client_secret); +RestCatalog::RestCatalog( + const std::string & warehouse_, + const std::string & base_url_, + const std::string & catalog_credential_, + const std::string & auth_header_, + DB::ContextPtr context_) + : ICatalog(warehouse_) + , DB::WithContext(context_) + , log(getLogger("RestCatalog(" + warehouse_ + ")")) + , base_url(base_url_) +{ + std::tie(client_id, client_secret) = parseCatalogCredential(catalog_credential_); + if (!auth_header_.empty()) + { + auto pos = auth_header_.find(':'); + if (pos == std::string::npos) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Unexpected format of auth header"); + + auth_header = DB::HTTPHeaderEntry(auth_header_.substr(0, pos), auth_header_.substr(pos + 1)); } config = loadConfig(); } RestCatalog::Config RestCatalog::loadConfig() { - if (!client_id.empty()) - { - static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; - - const auto & context = getContext(); - const auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); - - Poco::JSON::Object json; - json.set("grant_type", "client_credentials"); - json.set("scope", "PRINCIPAL_ROLE:ALL"); - json.set("client_id", client_id); - json.set("client_secret", client_secret); - - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - oss.exceptions(std::ios::failbit); - Poco::JSON::Stringifier::stringify(json, oss); - std::string json_str = oss.str(); - - auto current_headers = headers; - current_headers.emplace_back("Content-Type", "application/x-www-form-urlencoded"); - current_headers.emplace_back("Accepts", "application/json; charset=UTF-8"); - - Poco::URI url(base_url / oauth_tokens_endpoint); - Poco::URI::QueryParameters params = { - {"grant_type", "client_credentials"}, - {"scope", "PRINCIPAL_ROLE:ALL"}, - {"client_id", client_id}, - {"client_secret", client_secret}, - }; - url.setQueryParameters(params); - - LOG_TEST(log, "Writing {}: {}", url.toString(), json_str); - - auto wb = DB::BuilderRWBufferFromHTTP(url) - .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) - .withMethod(Poco::Net::HTTPRequest::HTTP_POST) - .withSettings(getContext()->getReadSettings()) - .withTimeouts(timeouts) - .withHostFilter(&getContext()->getRemoteHostFilter()) - .withSkipNotFound(false) - .withHeaders(current_headers) - .create(credentials); - - json_str.clear(); - readJSONObjectPossiblyInvalid(json_str, *wb); - - LOG_TEST(log, "Received token result: {}", json_str); - - Poco::JSON::Parser parser; - Poco::Dynamic::Var res_json = parser.parse(json_str); - const Poco::JSON::Object::Ptr & object = res_json.extract(); - - auto access_token = object->get("access_token").extract(); - headers.emplace_back("Authorization", "Bearer " + access_token); - } - Poco::URI::QueryParameters params = {{"warehouse", warehouse}}; auto buf = createReadBuffer(config_endpoint, params); @@ -173,6 +128,78 @@ void RestCatalog::parseConfig(const Poco::JSON::Object::Ptr & object, Config & r result.default_base_location = object->get("default-base-location").extract(); } +DB::HTTPHeaderEntries RestCatalog::getHeaders(bool update_token) const +{ + if (auth_header.has_value()) + { + return DB::HTTPHeaderEntries{auth_header.value()}; + } + + if (!access_token.has_value() || update_token) + { + access_token = retrieveAccessToken(); + } + + DB::HTTPHeaderEntries headers; + headers.emplace_back("Authorization", "Bearer " + access_token.value()); + return headers; +} + +std::string RestCatalog::retrieveAccessToken() const +{ + static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; + + const auto & context = getContext(); + const auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); + + Poco::JSON::Object json; + json.set("grant_type", "client_credentials"); + json.set("scope", "PRINCIPAL_ROLE:ALL"); + json.set("client_id", client_id); + json.set("client_secret", client_secret); + + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(json, oss); + std::string json_str = oss.str(); + + DB::HTTPHeaderEntries headers; + headers.emplace_back("Content-Type", "application/x-www-form-urlencoded"); + headers.emplace_back("Accepts", "application/json; charset=UTF-8"); + + Poco::URI url(base_url / oauth_tokens_endpoint); + Poco::URI::QueryParameters params = { + {"grant_type", "client_credentials"}, + {"scope", "PRINCIPAL_ROLE:ALL"}, + {"client_id", client_id}, + {"client_secret", client_secret}, + }; + url.setQueryParameters(params); + + LOG_TEST(log, "Writing {}: {}", url.toString(), json_str); + + auto wb = DB::BuilderRWBufferFromHTTP(url) + .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) + .withMethod(Poco::Net::HTTPRequest::HTTP_POST) + .withSettings(getContext()->getReadSettings()) + .withTimeouts(timeouts) + .withHostFilter(&getContext()->getRemoteHostFilter()) + .withSkipNotFound(false) + .withHeaders(headers) + .create(credentials); + + json_str.clear(); + readJSONObjectPossiblyInvalid(json_str, *wb); + + LOG_TEST(log, "Received token result: {}", json_str); + + Poco::JSON::Parser parser; + Poco::Dynamic::Var res_json = parser.parse(json_str); + const Poco::JSON::Object::Ptr & object = res_json.extract(); + + return object->get("access_token").extract(); +} + std::optional RestCatalog::getStorageType() const { if (config.default_base_location.empty()) @@ -191,16 +218,35 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( if (!params.empty()) url.setQueryParameters(params); + auto headers = getHeaders(false); + LOG_TEST(log, "Requesting: {}", url.toString()); - return DB::BuilderRWBufferFromHTTP(url) - .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) - .withSettings(getContext()->getReadSettings()) - .withTimeouts(timeouts) - .withHostFilter(&getContext()->getRemoteHostFilter()) - .withSkipNotFound(false) - .withHeaders(headers) - .create(credentials); + try + { + return DB::BuilderRWBufferFromHTTP(url) + .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) + .withSettings(getContext()->getReadSettings()) + .withTimeouts(timeouts) + .withHeaders(headers) + .withHostFilter(&getContext()->getRemoteHostFilter()) + .withDelayInit(false) + .withSkipNotFound(false) + .create(credentials); + } + catch (...) + { + auto new_headers = getHeaders(true); + return DB::BuilderRWBufferFromHTTP(url) + .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) + .withSettings(getContext()->getReadSettings()) + .withTimeouts(timeouts) + .withHeaders(new_headers) + .withHostFilter(&getContext()->getRemoteHostFilter()) + .withDelayInit(false) + .withSkipNotFound(false) + .create(credentials); + } } bool RestCatalog::empty() const diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index 067f67ab493..116b4253684 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -22,7 +22,7 @@ public: const std::string & warehouse_, const std::string & base_url_, const std::string & catalog_credential_, - const DB::HTTPHeaderEntries & headers_, + const std::string & auth_header_, DB::ContextPtr context_); ~RestCatalog() override = default; @@ -57,7 +57,8 @@ private: }; const std::filesystem::path base_url; - DB::HTTPHeaderEntries headers; + std::optional auth_header; + mutable std::optional access_token; std::string client_id; std::string client_secret; Config config; @@ -94,6 +95,8 @@ private: TableMetadata & result) const; Config loadConfig(); + std::string retrieveAccessToken() const; + DB::HTTPHeaderEntries getHeaders(bool update_token = false) const; static void parseConfig(const Poco::JSON::Object::Ptr & object, Config & result); }; From 7599020b8758f4e835a57d30d48b868a72182434 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 19 Nov 2024 20:14:23 +0100 Subject: [PATCH 153/502] Fix the issue in the comment above. --- src/Interpreters/CancellationChecker.cpp | 4 ++-- src/Interpreters/CancellationChecker.h | 2 +- src/Interpreters/ProcessList.cpp | 24 +++++++++++------------- src/Interpreters/ProcessList.h | 3 +-- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp index 4554bea301b..e0c1f88f89d 100644 --- a/src/Interpreters/CancellationChecker.cpp +++ b/src/Interpreters/CancellationChecker.cpp @@ -42,7 +42,7 @@ void CancellationChecker::terminateThread() cond_var.notify_all(); } -void CancellationChecker::cancelTask(QueryToTrack task, [[maybe_unused]]CancelReason reason) +void CancellationChecker::cancelTask(QueryToTrack task) { if (task.query) { @@ -123,7 +123,7 @@ void CancellationChecker::workerFunction() if ((end_time_ms <= now_ms && duration_milliseconds.count() != 0)) { LOG_TRACE(getLogger("CancellationChecker"), "Cancelling the task because of the timeout: {} ms, query: {}", duration, next_task.query->getInfo().query); - cancelTask(next_task, CancelReason::TIMEOUT); + cancelTask(next_task); querySet.erase(next_task); continue; diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index e890fc9aec3..3addc2a0cc2 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -46,7 +46,7 @@ private: std::condition_variable cond_var; // Function to execute when a task's endTime is reached - void cancelTask(QueryToTrack task, CancelReason reason); + void cancelTask(QueryToTrack task); bool removeQueryFromSet(std::shared_ptr query); public: diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index c2c280badc8..f2382177ddf 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -498,28 +498,26 @@ CancellationCode QueryStatus::cancelQuery(CancelReason reason, std::exception_pt return CancellationCode::CancelSent; } -void QueryStatus::throwProperExceptionIfNeeded(const UInt64 & max_execution_time, const UInt64 & elapsed_ns) const +void QueryStatus::throwProperExceptionIfNeeded(const UInt64 & max_execution_time, const UInt64 & elapsed_ns) { { std::lock_guard lock(cancel_mutex); if (is_killed) { - throwProperException(max_execution_time, elapsed_ns); + String additional_error_part; + if (!elapsed_ns) + additional_error_part = fmt::format("elapsed {} ms, ", static_cast(elapsed_ns) / 1000000000ULL); + + if (cancel_reason == CancelReason::TIMEOUT) + { + cancel_reason = CancelReason::UNDEFINED; // We can assign only CancelReason::TIMEOUT to cancel_reason, so we need to assign it back to UNDEFINED + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout exceeded: {}maximum: {} ms", additional_error_part, max_execution_time / 1000.0); + } + throwQueryWasCancelled(); } } } -void QueryStatus::throwProperException(const UInt64 & max_execution_time, const UInt64 & elapsed_ns) const -{ - String additional_error_part; - if (!elapsed_ns) - additional_error_part = fmt::format("elapsed {} ms,", static_cast(elapsed_ns) / 1000000000ULL); - - if (cancel_reason == CancelReason::TIMEOUT) - throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout exceeded: {} maximum: {} ms", additional_error_part, max_execution_time / 1000.0); - throwQueryWasCancelled(); -} - void QueryStatus::addPipelineExecutor(PipelineExecutor * e) { /// In case of asynchronous distributed queries it is possible to call diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index a5effc9cb51..1d4bcf8b758 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -240,8 +240,7 @@ public: QueryStatusInfo getInfo(bool get_thread_list = false, bool get_profile_events = false, bool get_settings = false) const; - void throwProperExceptionIfNeeded(const UInt64 & max_execution_time, const UInt64 & elapsed_ns = 0) const; - [[noreturn]] void throwProperException(const UInt64 & max_execution_time, const UInt64 & elapsed_ns = 0) const; + void throwProperExceptionIfNeeded(const UInt64 & max_execution_time, const UInt64 & elapsed_ns = 0); /// Cancels the current query. /// Optional argument `exception` allows to set an exception which checkTimeLimit() will throw instead of "QUERY_WAS_CANCELLED". From 90b7ae871ed9e4e418eec37f5ccfcd25ecae264e Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 19 Nov 2024 22:23:05 +0000 Subject: [PATCH 154/502] Add inferred format name to create query in File/S3/URL/HDFS/Azure engines --- src/Interpreters/InterpreterCreateQuery.cpp | 21 ++++- src/Storages/IStorage.h | 4 + .../ObjectStorage/Azure/Configuration.cpp | 54 ++++++++----- .../ObjectStorage/Azure/Configuration.h | 3 +- .../ObjectStorage/HDFS/Configuration.cpp | 16 ++-- .../ObjectStorage/HDFS/Configuration.h | 3 +- .../ObjectStorage/Local/Configuration.h | 2 +- .../ObjectStorage/S3/Configuration.cpp | 55 +++++++------ src/Storages/ObjectStorage/S3/Configuration.h | 3 +- .../ObjectStorage/StorageObjectStorage.cpp | 5 ++ .../ObjectStorage/StorageObjectStorage.h | 4 +- .../StorageObjectStorageCluster.cpp | 2 +- src/Storages/StorageFile.cpp | 5 ++ src/Storages/StorageFile.h | 2 + src/Storages/StorageURL.cpp | 7 ++ src/Storages/StorageURL.h | 2 + src/TableFunctions/ITableFunctionCluster.h | 2 +- src/TableFunctions/ITableFunctionFileLike.cpp | 12 +-- src/TableFunctions/ITableFunctionFileLike.h | 2 +- .../TableFunctionObjectStorage.h | 2 +- src/TableFunctions/TableFunctionURL.cpp | 6 +- src/TableFunctions/TableFunctionURL.h | 2 +- .../test_storage_azure_blob_storage/test.py | 79 +++++++++++++++++++ tests/integration/test_storage_hdfs/test.py | 28 ++++++- ...at_inference_create_query_s3_url.reference | 14 ++++ ...3_format_inference_create_query_s3_url.sql | 57 +++++++++++++ ...rmat_inference_create_query_file.reference | 1 + ...3274_format_inference_create_query_file.sh | 19 +++++ .../queries/0_stateless/data_minio/json_data | 1 + 29 files changed, 345 insertions(+), 68 deletions(-) create mode 100644 tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference create mode 100644 tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql create mode 100644 tests/queries/0_stateless/03274_format_inference_create_query_file.reference create mode 100755 tests/queries/0_stateless/03274_format_inference_create_query_file.sh create mode 100644 tests/queries/0_stateless/data_minio/json_data diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index f6586f8bfc2..67c4d2c0413 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1181,6 +1181,22 @@ namespace source_ast->children.push_back(source_ast->elements); dict.set(dict.source, source_ast); } + + ASTs * getEngineArgsFromCreateQuery(ASTCreateQuery & create_query) + { + ASTStorage * storage_def = create_query.storage; + if (!storage_def) + return nullptr; + + if (!storage_def->engine) + return nullptr; + + const ASTFunction & engine_def = *storage_def->engine; + if (!engine_def.arguments) + return nullptr; + + return &engine_def.arguments->children; + } } void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const @@ -1870,7 +1886,10 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, mode); /// If schema wes inferred while storage creation, add columns description to create query. - addColumnsDescriptionToCreateQueryIfNecessary(query_ptr->as(), res); + auto & create_query = query_ptr->as(); + addColumnsDescriptionToCreateQueryIfNecessary(create_query, res); + if (auto * engine_args = getEngineArgsFromCreateQuery(create_query)) + res->updateEngineArgsForCreateQuery(*engine_args, getContext()); } validateVirtualColumns(*res); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 0dc48634282..e99a66b9bda 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -284,6 +284,10 @@ public: /// Returns hints for serialization of columns accorsing to statistics accumulated by storage. virtual SerializationInfoByName getSerializationHints() const { return {}; } + /// Update engine args in create query that were inferred during storage creation to avoid the same + /// inference on server restart. For example - data format inference in File/URL/S3/etc engines. + virtual void updateEngineArgsForCreateQuery(ASTs & /*args*/, const ContextPtr & /*context*/) const {} + private: StorageID storage_id; diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index d08e0d9debc..faa1554dbe8 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -283,7 +283,7 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, } void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( - ASTs & args, const String & structure_, const String & format_, ContextPtr context) + ASTs & args, const String & structure_, const String & format_, ContextPtr context, bool with_structure) { if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { @@ -295,7 +295,7 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); args.push_back(format_equal_func); } - if (collection->getOrDefault("structure", "auto") == "auto") + if (with_structure && collection->getOrDefault("structure", "auto") == "auto") { ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); @@ -319,9 +319,12 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( if (args.size() == 3) { args.push_back(format_literal); - /// Add compression = "auto" before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + if (with_structure) + { + /// Add compression = "auto" before structure argument. + args.push_back(std::make_shared("auto")); + args.push_back(structure_literal); + } } /// (connection_string, container_name, blobpath, structure) or /// (connection_string, container_name, blobpath, format) @@ -334,12 +337,15 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( { if (fourth_arg == "auto") args[3] = format_literal; - /// Add compression=auto before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + if (with_structure) + { + /// Add compression=auto before structure argument. + args.push_back(std::make_shared("auto")); + args.push_back(structure_literal); + } } /// (..., structure) -> (..., format, compression, structure) - else + else if (with_structure) { auto structure_arg = args.back(); args[3] = format_literal; @@ -362,15 +368,19 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( { if (fourth_arg == "auto") args[3] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } /// (..., account_name, account_key) -> (..., account_name, account_key, format, compression, structure) else { args.push_back(format_literal); - /// Add compression=auto before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + if (with_structure) + { + /// Add compression=auto before structure argument. + args.push_back(std::make_shared("auto")); + args.push_back(structure_literal); + } } } /// (connection_string, container_name, blobpath, format, compression, structure) or @@ -386,7 +396,7 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( { if (fourth_arg == "auto") args[3] = format_literal; - if (checkAndGetLiteralArgument(args[5], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[5], "structure") == "auto") args[5] = structure_literal; } /// (..., account_name, account_key, format) -> (..., account_name, account_key, format, compression, structure) @@ -394,12 +404,15 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( { if (sixth_arg == "auto") args[5] = format_literal; - /// Add compression=auto before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + if (with_structure) + { + /// Add compression=auto before structure argument. + args.push_back(std::make_shared("auto")); + args.push_back(structure_literal); + } } /// (..., account_name, account_key, structure) -> (..., account_name, account_key, format, compression, structure) - else + else if (with_structure) { auto structure_arg = args.back(); args[5] = format_literal; @@ -417,14 +430,15 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( /// (..., format, compression) -> (..., format, compression, structure) if (checkAndGetLiteralArgument(args[5], "format") == "auto") args[5] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } /// (storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure) else if (args.size() == 8) { if (checkAndGetLiteralArgument(args[5], "format") == "auto") args[5] = format_literal; - if (checkAndGetLiteralArgument(args[7], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[7], "structure") == "auto") args[7] = structure_literal; } } diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index 21db81802c7..72124465c46 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -76,7 +76,8 @@ public: ASTs & args, const String & structure_, const String & format_, - ContextPtr context) override; + ContextPtr context, + bool with_structure) override; protected: void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override; diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.cpp b/src/Storages/ObjectStorage/HDFS/Configuration.cpp index 6bee4154b2f..3c4897b5062 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.cpp +++ b/src/Storages/ObjectStorage/HDFS/Configuration.cpp @@ -174,7 +174,8 @@ void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, - ContextPtr context) + ContextPtr context, + bool with_structure) { if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { @@ -186,7 +187,7 @@ void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded( auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); args.push_back(format_equal_func); } - if (collection->getOrDefault("structure", "auto") == "auto") + if (with_structure && collection->getOrDefault("structure", "auto") == "auto") { ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); @@ -209,23 +210,26 @@ void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded( if (count == 1) { /// Add format=auto before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + args.push_back(format_literal); + if (with_structure) + args.push_back(structure_literal); } /// hdfs(url, format) else if (count == 2) { if (checkAndGetLiteralArgument(args[1], "format") == "auto") args.back() = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } /// hdfs(url, format, structure) /// hdfs(url, format, structure, compression_method) + /// hdfs(url, format, compression_method) else if (count >= 3) { if (checkAndGetLiteralArgument(args[1], "format") == "auto") args[1] = format_literal; - if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") args[2] = structure_literal; } } diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.h b/src/Storages/ObjectStorage/HDFS/Configuration.h index 90997292693..db8ab7f9e4d 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.h +++ b/src/Storages/ObjectStorage/HDFS/Configuration.h @@ -62,7 +62,8 @@ public: ASTs & args, const String & structure_, const String & format_, - ContextPtr context) override; + ContextPtr context, + bool with_structure) override; private: void fromNamedCollection(const NamedCollection &, ContextPtr context) override; diff --git a/src/Storages/ObjectStorage/Local/Configuration.h b/src/Storages/ObjectStorage/Local/Configuration.h index 32a095bf7de..d679b4ad724 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.h +++ b/src/Storages/ObjectStorage/Local/Configuration.h @@ -59,7 +59,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr, bool) override { return std::make_shared("/"); } - void addStructureAndFormatToArgsIfNeeded(ASTs &, const String &, const String &, ContextPtr) override { } + void addStructureAndFormatToArgsIfNeeded(ASTs &, const String &, const String &, ContextPtr, bool) override { } private: void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override; diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 02effe261d0..629628c762f 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -395,7 +395,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ } void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( - ASTs & args, const String & structure_, const String & format_, ContextPtr context) + ASTs & args, const String & structure_, const String & format_, ContextPtr context, bool with_structure) { if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { @@ -407,7 +407,7 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); args.push_back(format_equal_func); } - if (collection->getOrDefault("structure", "auto") == "auto") + if (with_structure && collection->getOrDefault("structure", "auto") == "auto") { ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); @@ -429,8 +429,9 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( if (count == 1) { /// Add format=auto before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + args.push_back(format_literal); + if (with_structure) + args.push_back(structure_literal); } /// s3(s3_url, format) or /// s3(s3_url, NOSIGN) @@ -444,11 +445,13 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( else if (checkAndGetLiteralArgument(args[1], "format") == "auto") args[1] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } /// s3(source, format, structure) or /// s3(source, access_key_id, secret_access_key) or - /// s3(source, NOSIGN, format) + /// s3(source, NOSIGN, format) or + /// s3(source, format, compression_method) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN, format name or neither. else if (count == 3) { @@ -457,26 +460,29 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[2], "format") == "auto") args[2] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg)) { if (second_arg == "auto") args[1] = format_literal; - if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") args[2] = structure_literal; } else { /// Add format and structure arguments. args.push_back(format_literal); - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } } /// s3(source, format, structure, compression_method) or /// s3(source, access_key_id, secret_access_key, format) or /// s3(source, access_key_id, secret_access_key, session_token) or - /// s3(source, NOSIGN, format, structure) + /// s3(source, NOSIGN, format, structure) or + /// s3(source, NOSIGN, format, compression_method) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN, format name or neither. else if (count == 4) { @@ -485,14 +491,14 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[2], "format") == "auto") args[2] = format_literal; - if (checkAndGetLiteralArgument(args[3], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[3], "structure") == "auto") args[3] = structure_literal; } else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg)) { if (second_arg == "auto") args[1] = format_literal; - if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") args[2] = structure_literal; } else @@ -502,18 +508,21 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[3], "format") == "auto") args[3] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } else { args.push_back(format_literal); - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } } } /// s3(source, access_key_id, secret_access_key, format, structure) or /// s3(source, access_key_id, secret_access_key, session_token, format) or - /// s3(source, NOSIGN, format, structure, compression_method) + /// s3(source, NOSIGN, format, structure, compression_method) or + /// s3(source, access_key_id, secret_access_key, format, compression) /// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN keyword name or not. else if (count == 5) { @@ -522,7 +531,7 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[2], "format") == "auto") args[2] = format_literal; - if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") args[3] = structure_literal; } else @@ -532,19 +541,21 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[3], "format") == "auto") args[3] = format_literal; - if (checkAndGetLiteralArgument(args[4], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[4], "structure") == "auto") args[4] = structure_literal; } else { if (checkAndGetLiteralArgument(args[4], "format") == "auto") args[4] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } } } /// s3(source, access_key_id, secret_access_key, format, structure, compression) or - /// s3(source, access_key_id, secret_access_key, session_token, format, structure) + /// s3(source, access_key_id, secret_access_key, session_token, format, structure) or + /// s3(source, access_key_id, secret_access_key, session_token, format, compression_method) else if (count == 6) { auto fourth_arg = checkAndGetLiteralArgument(args[3], "format/session_token"); @@ -552,14 +563,14 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[3], "format") == "auto") args[3] = format_literal; - if (checkAndGetLiteralArgument(args[4], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[4], "structure") == "auto") args[4] = structure_literal; } else { if (checkAndGetLiteralArgument(args[4], "format") == "auto") args[4] = format_literal; - if (checkAndGetLiteralArgument(args[5], "format") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[5], "format") == "auto") args[5] = structure_literal; } } @@ -568,7 +579,7 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[4], "format") == "auto") args[4] = format_literal; - if (checkAndGetLiteralArgument(args[5], "format") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[5], "format") == "auto") args[5] = structure_literal; } } diff --git a/src/Storages/ObjectStorage/S3/Configuration.h b/src/Storages/ObjectStorage/S3/Configuration.h index 87f2be1bf3e..c4614a28189 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.h +++ b/src/Storages/ObjectStorage/S3/Configuration.h @@ -91,7 +91,8 @@ public: ASTs & args, const String & structure, const String & format, - ContextPtr context) override; + ContextPtr context, + bool with_structure) override; private: void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index fd2fe0400bb..419069dbb8d 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -479,6 +479,11 @@ std::pair StorageObjectStorage::resolveSchemaAn return std::pair(columns, format); } +void StorageObjectStorage::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +{ + configuration->addStructureAndFormatToArgsIfNeeded(args, "", configuration->format, context, /*with_structure=*/false); +} + SchemaCache & StorageObjectStorage::getSchemaCache(const ContextPtr & context, const std::string & storage_type_name) { if (storage_type_name == "s3") diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index e2bb41a4935..81f9828ccc8 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -122,6 +122,8 @@ public: std::string & sample_path, const ContextPtr & context); + void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + protected: String getPathSample(StorageInMemoryMetadata metadata, ContextPtr context); @@ -179,7 +181,7 @@ public: /// Add/replace structure and format arguments in the AST arguments if they have 'auto' values. virtual void addStructureAndFormatToArgsIfNeeded( - ASTs & args, const String & structure_, const String & format_, ContextPtr context) = 0; + ASTs & args, const String & structure_, const String & format_, ContextPtr context, bool with_structure) = 0; bool withPartitionWildcard() const; bool withGlobs() const { return isPathWithGlobs() || isNamespaceWithGlobs(); } diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index 176f20cf5ca..07eecc65599 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -107,7 +107,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( ASTPtr cluster_name_arg = args.front(); args.erase(args.begin()); - configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context); + configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); args.insert(args.begin(), cluster_name_arg); } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index eefd60128a6..91263180b5c 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -2114,6 +2114,11 @@ void StorageFile::truncate( } } +void StorageFile::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +{ + if (checkAndGetLiteralArgument(evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context), "format") == "auto") + args[0] = std::make_shared(format_name); +} void registerStorageFile(StorageFactory & factory) { diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 6b21353f161..5b95d1a9f8b 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -139,6 +139,8 @@ public: bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } + void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + protected: friend class StorageFileSource; friend class StorageFileSink; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 3ba8d1fa304..af91768f79e 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -38,6 +38,8 @@ #include #include +#include + #include #include #include @@ -1570,6 +1572,11 @@ void StorageURL::processNamedCollectionResult(Configuration & configuration, con configuration.structure = collection.getOrDefault("structure", "auto"); } +void StorageURL::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +{ + TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(args, "", format_name, context, /*with_structure=*/false); +} + StorageURL::Configuration StorageURL::getConfiguration(ASTs & args, const ContextPtr & local_context) { StorageURL::Configuration configuration; diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 6f1d544629a..4f3e920afac 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -305,6 +305,8 @@ public: bool supportsDynamicSubcolumns() const override { return true; } + void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + static FormatSettings getFormatSettingsFromArgs(const StorageFactory::Arguments & args); struct Configuration : public StatelessTableEngineConfiguration diff --git a/src/TableFunctions/ITableFunctionCluster.h b/src/TableFunctions/ITableFunctionCluster.h index 744d7139d16..3378cbe87d3 100644 --- a/src/TableFunctions/ITableFunctionCluster.h +++ b/src/TableFunctions/ITableFunctionCluster.h @@ -31,7 +31,7 @@ public: ASTPtr cluster_name_arg = args.front(); args.erase(args.begin()); - Base::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context); + Base::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context, /*with_structure=*/true); args.insert(args.begin(), cluster_name_arg); } diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index 23e59494f61..b591ea97e3d 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -88,7 +88,7 @@ void ITableFunctionFileLike::parseArgumentsImpl(ASTs & args, const ContextPtr & compression_method = checkAndGetLiteralArgument(args[3], "compression_method"); } -void ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr & context) +void ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr & context, bool with_structure) { if (args.empty() || args.size() > getMaxNumberOfArguments()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function, got {}", getMaxNumberOfArguments(), args.size()); @@ -103,21 +103,23 @@ void ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(ASTs & ar if (args.size() == 1) { args.push_back(format_literal); - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } /// f(filename, format) else if (args.size() == 2) { if (checkAndGetLiteralArgument(args[1], "format") == "auto") args.back() = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } - /// f(filename, format, structure) or f(filename, format, structure, compression) + /// f(filename, format, structure) or f(filename, format, structure, compression) or f(filename, format, compression) else if (args.size() >= 3) { if (checkAndGetLiteralArgument(args[1], "format") == "auto") args[1] = format_literal; - if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") args[2] = structure_literal; } } diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index 4c97507b8d1..f9b5e30e85e 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -35,7 +35,7 @@ public: static size_t getMaxNumberOfArguments() { return max_number_of_arguments; } - static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &); + static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &, bool with_structure); protected: diff --git a/src/TableFunctions/TableFunctionObjectStorage.h b/src/TableFunctions/TableFunctionObjectStorage.h index 19cd637bd80..855acca85ee 100644 --- a/src/TableFunctions/TableFunctionObjectStorage.h +++ b/src/TableFunctions/TableFunctionObjectStorage.h @@ -139,7 +139,7 @@ public: const String & format, const ContextPtr & context) { - Configuration().addStructureAndFormatToArgsIfNeeded(args, structure, format, context); + Configuration().addStructureAndFormatToArgsIfNeeded(args, structure, format, context, /*with_structure=*/true); } protected: diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 8f4841a992b..5de3a302772 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -77,7 +77,7 @@ void TableFunctionURL::parseArgumentsImpl(ASTs & args, const ContextPtr & contex } } -void TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context) +void TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context, bool with_structure) { if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { @@ -89,7 +89,7 @@ void TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, co auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); args.push_back(format_equal_func); } - if (collection->getOrDefault("structure", "auto") == "auto") + if (with_structure && collection->getOrDefault("structure", "auto") == "auto") { ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); @@ -109,7 +109,7 @@ void TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, co args.pop_back(); } - ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context); + ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context, with_structure); if (headers_ast) args.push_back(headers_ast); diff --git a/src/TableFunctions/TableFunctionURL.h b/src/TableFunctions/TableFunctionURL.h index a1efddb84c6..d2c9d4d9ddf 100644 --- a/src/TableFunctions/TableFunctionURL.h +++ b/src/TableFunctions/TableFunctionURL.h @@ -34,7 +34,7 @@ public: ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; - static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context); + static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context, bool with_structure); protected: void parseArguments(const ASTPtr & ast, ContextPtr context) override; diff --git a/tests/integration/test_storage_azure_blob_storage/test.py b/tests/integration/test_storage_azure_blob_storage/test.py index 220ee13cb25..e22ce925f6c 100644 --- a/tests/integration/test_storage_azure_blob_storage/test.py +++ b/tests/integration/test_storage_azure_blob_storage/test.py @@ -1314,6 +1314,7 @@ def test_size_virtual_column(cluster): def test_format_detection(cluster): node = cluster.instances["node"] + connection_string = cluster.env_variables["AZURITE_CONNECTION_STRING"] storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] account_name = "devstoreaccount1" account_key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" @@ -1381,6 +1382,84 @@ def test_format_detection(cluster): assert result == expected_result + azure_query( + node, + f"create table test_format_detection engine=AzureBlobStorage('{connection_string}', 'cont', 'test_format_detection1')", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{connection_string}\\', \\'cont\\', \\'test_format_detection1\\', \\'JSON\\')\n" + ) + + azure_query( + node, + f"create or replace table test_format_detection engine=AzureBlobStorage('{connection_string}', 'cont', 'test_format_detection1', auto)", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{connection_string}\\', \\'cont\\', \\'test_format_detection1\\', \\'JSON\\')\n" + ) + + azure_query( + node, + f"create or replace table test_format_detection engine=AzureBlobStorage('{connection_string}', 'cont', 'test_format_detection1', auto, 'none')", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{connection_string}\\', \\'cont\\', \\'test_format_detection1\\', \\'JSON\\', \\'none\\')\n" + ) + + azure_query( + node, + f"create or replace table test_format_detection engine=AzureBlobStorage('{storage_account_url}', 'cont', 'test_format_detection1', '{account_name}', '{account_key}')", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{storage_account_url}\\', \\'cont\\', \\'test_format_detection1\\', \\'{account_name}\\', \\'{account_key}\\', \\'JSON\\')\n" + ) + + azure_query( + node, + f"create or replace table test_format_detection engine=AzureBlobStorage('{storage_account_url}', 'cont', 'test_format_detection1', '{account_name}', '{account_key}', auto)", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{storage_account_url}\\', \\'cont\\', \\'test_format_detection1\\', \\'{account_name}\\', \\'{account_key}\\', \\'JSON\\')\n" + ) + + azure_query( + node, + f"create or replace table test_format_detection engine=AzureBlobStorage('{storage_account_url}', 'cont', 'test_format_detection1', '{account_name}', '{account_key}', auto, 'none')", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{storage_account_url}\\', \\'cont\\', \\'test_format_detection1\\', \\'{account_name}\\', \\'{account_key}\\', \\'JSON\\', \\'none\\')\n" + ) + def test_write_to_globbed_partitioned_path(cluster): node = cluster.instances["node"] diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 366bc28d2c9..9f0afc57ad9 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -636,7 +636,7 @@ def test_multiple_inserts(started_cluster): node1.query(f"drop table test_multiple_inserts") -def test_format_detection(started_cluster): +def test_format_detection_from_file_name(started_cluster): node1.query( f"create table arrow_table (x UInt64) engine=HDFS('hdfs://hdfs1:9000/data.arrow')" ) @@ -1222,6 +1222,32 @@ def test_format_detection(started_cluster): assert expected_result == result + node.query( + f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1')" + ) + result = node.query( + f"show create table test_format_detection" + ) + assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" + + node.query("drop table test_format_detection") + node.query( + f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1', auto)" + ) + result = node.query( + f"show create table test_format_detection" + ) + assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" + + node.query("drop table test_format_detection") + node.query( + f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1', auto, 'none')" + ) + result = node.query( + f"show create table test_format_detection" + ) + assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\', \\'none\\')\n" + def test_write_to_globbed_partitioned_path(started_cluster): node = started_cluster.instances["node1"] diff --git a/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference new file mode 100644 index 00000000000..72696cef342 --- /dev/null +++ b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference @@ -0,0 +1,14 @@ +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'NOSIGN\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'JSON\', \'none\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'NOSIGN\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'NOSIGN\', \'JSON\', \'none\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'JSON\', \'none\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'\', \'JSON\', \'none\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = URL(\'http://localhost:11111/test/json_data\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = URL(\'http://localhost:11111/test/json_data\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = URL(\'http://localhost:11111/test/json_data\', \'JSON\', \'none\') diff --git a/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql new file mode 100644 index 00000000000..353b78b1e15 --- /dev/null +++ b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql @@ -0,0 +1,57 @@ +drop table if exists test; + +create table test engine=S3('http://localhost:11111/test/json_data'); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', NOSIGN); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', auto); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', auto, 'none'); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', NOSIGN, auto); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', 'test', 'testtest'); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', NOSIGN, auto, 'none'); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', 'test', 'testtest', ''); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', 'test', 'testtest', '', auto); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', 'test', 'testtest', auto, 'none'); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', 'test', 'testtest', '', auto, 'none'); +show create table test; +drop table test; + +create table test engine=URL('http://localhost:11111/test/json_data'); +show create table test; +drop table test; + +create table test engine=URL('http://localhost:11111/test/json_data', auto); +show create table test; +drop table test; + +create table test engine=URL('http://localhost:11111/test/json_data', auto, 'none'); +show create table test; +drop table test; diff --git a/tests/queries/0_stateless/03274_format_inference_create_query_file.reference b/tests/queries/0_stateless/03274_format_inference_create_query_file.reference new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/tests/queries/0_stateless/03274_format_inference_create_query_file.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/03274_format_inference_create_query_file.sh b/tests/queries/0_stateless/03274_format_inference_create_query_file.sh new file mode 100755 index 00000000000..10c0650144c --- /dev/null +++ b/tests/queries/0_stateless/03274_format_inference_create_query_file.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL -q "select 42 as a format JSONEachRow" > data_$CLICKHOUSE_TEST_UNIQUE_NAME +$CLICKHOUSE_LOCAL -nm -q " +create table test engine=File(auto, './data_$CLICKHOUSE_TEST_UNIQUE_NAME'); +show create table test; +drop table test; + +create table test engine=File(auto, './data_$CLICKHOUSE_TEST_UNIQUE_NAME', 'none'); +show create table test; +drop table test; +" | grep "JSON" -c + +rm data_$CLICKHOUSE_TEST_UNIQUE_NAME + diff --git a/tests/queries/0_stateless/data_minio/json_data b/tests/queries/0_stateless/data_minio/json_data new file mode 100644 index 00000000000..bfe33aeb603 --- /dev/null +++ b/tests/queries/0_stateless/data_minio/json_data @@ -0,0 +1 @@ +{"a" : 42} From 1fc4757c0829a9b013972858c2eddb293bd3cce8 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 19 Nov 2024 22:27:29 +0000 Subject: [PATCH 155/502] Better naming --- src/Interpreters/InterpreterCreateQuery.cpp | 3 ++- src/Storages/IStorage.h | 4 ++-- src/Storages/ObjectStorage/StorageObjectStorage.cpp | 2 +- src/Storages/ObjectStorage/StorageObjectStorage.h | 2 +- src/Storages/StorageFile.cpp | 2 +- src/Storages/StorageFile.h | 2 +- src/Storages/StorageURL.cpp | 2 +- src/Storages/StorageURL.h | 2 +- 8 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 67c4d2c0413..a8cbf83ff23 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1888,8 +1888,9 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, /// If schema wes inferred while storage creation, add columns description to create query. auto & create_query = query_ptr->as(); addColumnsDescriptionToCreateQueryIfNecessary(create_query, res); + /// Add any inferred engine args if needed. For example, data format for engines File/S3/URL/etc if (auto * engine_args = getEngineArgsFromCreateQuery(create_query)) - res->updateEngineArgsForCreateQuery(*engine_args, getContext()); + res->addInferredEngineArgsToCreateQuery(*engine_args, getContext()); } validateVirtualColumns(*res); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index e99a66b9bda..37359aad290 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -284,9 +284,9 @@ public: /// Returns hints for serialization of columns accorsing to statistics accumulated by storage. virtual SerializationInfoByName getSerializationHints() const { return {}; } - /// Update engine args in create query that were inferred during storage creation to avoid the same + /// Add engine args that were inferred during storage creation to create query to avoid the same /// inference on server restart. For example - data format inference in File/URL/S3/etc engines. - virtual void updateEngineArgsForCreateQuery(ASTs & /*args*/, const ContextPtr & /*context*/) const {} + virtual void addInferredEngineArgsToCreateQuery(ASTs & /*args*/, const ContextPtr & /*context*/) const {} private: StorageID storage_id; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 419069dbb8d..098a5549fae 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -479,7 +479,7 @@ std::pair StorageObjectStorage::resolveSchemaAn return std::pair(columns, format); } -void StorageObjectStorage::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +void StorageObjectStorage::addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const { configuration->addStructureAndFormatToArgsIfNeeded(args, "", configuration->format, context, /*with_structure=*/false); } diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 81f9828ccc8..e32dcc438cd 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -122,7 +122,7 @@ public: std::string & sample_path, const ContextPtr & context); - void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + void addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const override; protected: String getPathSample(StorageInMemoryMetadata metadata, ContextPtr context); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 91263180b5c..550ffa245e3 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -2114,7 +2114,7 @@ void StorageFile::truncate( } } -void StorageFile::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +void StorageFile::addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const { if (checkAndGetLiteralArgument(evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context), "format") == "auto") args[0] = std::make_shared(format_name); diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 5b95d1a9f8b..477d75a77a8 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -139,7 +139,7 @@ public: bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } - void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + void addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const override; protected: friend class StorageFileSource; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index af91768f79e..5607054a149 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -1572,7 +1572,7 @@ void StorageURL::processNamedCollectionResult(Configuration & configuration, con configuration.structure = collection.getOrDefault("structure", "auto"); } -void StorageURL::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +void StorageURL::addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const { TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(args, "", format_name, context, /*with_structure=*/false); } diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 4f3e920afac..7df5b90653d 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -305,7 +305,7 @@ public: bool supportsDynamicSubcolumns() const override { return true; } - void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + void addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const override; static FormatSettings getFormatSettingsFromArgs(const StorageFactory::Arguments & args); From c185b5bc3813ad22d45ac29e6a1adc8b71b07265 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 19 Nov 2024 22:57:53 +0000 Subject: [PATCH 156/502] Fix style --- tests/integration/test_storage_hdfs/test.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 9f0afc57ad9..d37b0d7218f 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -1225,28 +1225,31 @@ def test_format_detection(started_cluster): node.query( f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1')" ) - result = node.query( - f"show create table test_format_detection" + result = node.query(f"show create table test_format_detection") + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" ) - assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" node.query("drop table test_format_detection") node.query( f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1', auto)" ) - result = node.query( - f"show create table test_format_detection" + result = node.query(f"show create table test_format_detection") + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" ) - assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" node.query("drop table test_format_detection") node.query( f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1', auto, 'none')" ) - result = node.query( - f"show create table test_format_detection" + result = node.query(f"show create table test_format_detection") + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\', \\'none\\')\n" ) - assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\', \\'none\\')\n" def test_write_to_globbed_partitioned_path(started_cluster): From 8d43e3bc903b951d0ea6b1c4a3ebf8bfc82f610b Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 20 Nov 2024 01:27:41 +0100 Subject: [PATCH 157/502] Change cancellation reason for Backups timeout. --- src/Backups/BackupCoordinationStageSync.cpp | 2 +- src/Interpreters/ProcessList.cpp | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Backups/BackupCoordinationStageSync.cpp b/src/Backups/BackupCoordinationStageSync.cpp index b16306cd0c6..db0672c7c2e 100644 --- a/src/Backups/BackupCoordinationStageSync.cpp +++ b/src/Backups/BackupCoordinationStageSync.cpp @@ -747,7 +747,7 @@ void BackupCoordinationStageSync::cancelQueryIfDisconnectedTooLong() /// we don't want the watching thread to try waiting here for retries or a reconnection). /// Also we don't set the `state.host_with_error` field here because `state.host_with_error` can only be set /// AFTER creating the 'error' node (see the comment for `State`). - process_list_element->cancelQuery(CancelReason::TIMEOUT, exception); + process_list_element->cancelQuery(CancelReason::CANCELLED_BY_USER, exception); state_changed.notify_all(); } diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index f2382177ddf..970a4929d28 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -509,10 +509,7 @@ void QueryStatus::throwProperExceptionIfNeeded(const UInt64 & max_execution_time additional_error_part = fmt::format("elapsed {} ms, ", static_cast(elapsed_ns) / 1000000000ULL); if (cancel_reason == CancelReason::TIMEOUT) - { - cancel_reason = CancelReason::UNDEFINED; // We can assign only CancelReason::TIMEOUT to cancel_reason, so we need to assign it back to UNDEFINED throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout exceeded: {}maximum: {} ms", additional_error_part, max_execution_time / 1000.0); - } throwQueryWasCancelled(); } } From ebb19cd9b54e51083209259bb53c6e676962c064 Mon Sep 17 00:00:00 2001 From: pufit Date: Wed, 20 Nov 2024 01:25:11 -0500 Subject: [PATCH 158/502] fix tests --- .../Access/ASTSettingsProfileElement.cpp | 12 +--- .../Access/ParserSettingsProfileElement.cpp | 52 +++++++---------- .../01294_create_settings_profile.reference | 2 +- ...02943_alter_user_modify_profiles.reference | 34 +++++------ ...ser_modify_profiles_and_settings.reference | 50 ++++++++--------- ...02943_alter_user_modify_settings.reference | 56 +++++++++---------- 6 files changed, 94 insertions(+), 112 deletions(-) diff --git a/src/Parsers/Access/ASTSettingsProfileElement.cpp b/src/Parsers/Access/ASTSettingsProfileElement.cpp index e0c32ddfb05..3e734b9218e 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.cpp +++ b/src/Parsers/Access/ASTSettingsProfileElement.cpp @@ -131,20 +131,12 @@ bool ASTSettingsProfileElements::empty() const size_t ASTSettingsProfileElements::getNumberOfSettings() const { - size_t count = 0; - for (const auto & element : elements) - if (!element->setting_name.empty()) - ++count; - return count; + return std::count_if(elements.begin(), elements.end(), [](const auto & element){ return !element->setting_name.empty(); }); } size_t ASTSettingsProfileElements::getNumberOfProfiles() const { - size_t count = 0; - for (const auto & element : elements) - if (!element->parent_profile.empty()) - ++count; - return count; + return std::count_if(elements.begin(), elements.end(), [](const auto & element){ return !element->parent_profile.empty(); }); } diff --git a/src/Parsers/Access/ParserSettingsProfileElement.cpp b/src/Parsers/Access/ParserSettingsProfileElement.cpp index b63192fc4ae..99c26d5d766 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.cpp +++ b/src/Parsers/Access/ParserSettingsProfileElement.cpp @@ -307,53 +307,43 @@ bool ParserAlterSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Exp else { /// new style: "MODIFY SETTINGS ..., ADD PROFILES ..., DROP PROFILES ..., DROP SETTINGS ..." - std::string_view mode; - std::string_view submode; + std::string_view action; + std::string_view target; auto parse_element = [&] { if (ParserKeyword{Keyword::ADD}.ignore(pos, expected)) - { - mode = "ADD"; - submode = ""; - } + action = "ADD"; else if (ParserKeyword{Keyword::DROP}.ignore(pos, expected)) - { - mode = "DROP"; - submode = ""; - } + action = "DROP"; else if (ParserKeyword{Keyword::MODIFY}.ignore(pos, expected)) - { - mode = "MODIFY"; - submode = ""; - } + action = "MODIFY"; - if (!mode.empty()) + if (!action.empty()) { if (ParserKeyword{Keyword::ALL_PROFILES}.ignore(pos, expected)) - submode = "ALL PROFILES"; + target = "ALL PROFILES"; else if (ParserKeyword{Keyword::ALL_SETTINGS}.ignore(pos, expected)) - submode = "ALL SETTINGS"; + target = "ALL SETTINGS"; else if (ParserKeyword{Keyword::PROFILES}.ignore(pos, expected) || ParserKeyword{Keyword::PROFILE}.ignore(pos, expected)) - submode = "PROFILES"; + target = "PROFILES"; else if (ParserKeyword{Keyword::SETTINGS}.ignore(pos, expected) || ParserKeyword{Keyword::SETTING}.ignore(pos, expected)) - submode = "SETTINGS"; + target = "SETTINGS"; + else + return false; } - if (submode.empty()) - return false; - - if (submode == "PROFILES") + if (target == "PROFILES") { auto element = std::make_shared(); if (!parseProfileNameOrID(pos, expected, /* id_mode= */ false, element->parent_profile)) return false; - if (mode == "ADD") + if (action == "ADD") { add_settings.push_back(element); return true; } - if (mode == "DROP") + if (action == "DROP") { drop_settings.push_back(element); return true; @@ -361,20 +351,20 @@ bool ParserAlterSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Exp return false; } - if (submode == "SETTINGS") + if (target == "SETTINGS") { auto element = std::make_shared(); - if (mode == "ADD" || mode == "MODIFY") + if (action == "ADD" || action == "MODIFY") { if (!parseSettingNameWithValueOrConstraints(pos, expected, element->setting_name, element->value, element->min_value, element->max_value, element->writability)) return false; - if (mode == "ADD") + if (action == "ADD") add_settings.push_back(element); else modify_settings.push_back(element); return true; } - if (mode == "DROP") + if (action == "DROP") { ASTPtr name_ast; if (!ParserCompoundIdentifier{}.parse(pos, name_ast, expected)) @@ -386,13 +376,13 @@ bool ParserAlterSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Exp return false; } - if (mode == "DROP" && submode == "ALL PROFILES") + if (action == "DROP" && target == "ALL PROFILES") { drop_all_profiles = true; return true; } - if (mode == "DROP" && submode == "ALL SETTINGS") + if (action == "DROP" && target == "ALL SETTINGS") { drop_all_settings = true; return true; diff --git a/tests/queries/0_stateless/01294_create_settings_profile.reference b/tests/queries/0_stateless/01294_create_settings_profile.reference index 7e1838c8dae..38d5a8d29af 100644 --- a/tests/queries/0_stateless/01294_create_settings_profile.reference +++ b/tests/queries/0_stateless/01294_create_settings_profile.reference @@ -49,7 +49,7 @@ CREATE SETTINGS PROFILE `s4_01294` TO r1_01294 CREATE SETTINGS PROFILE `s1_01294` SETTINGS readonly = 1 CREATE SETTINGS PROFILE `s2_01294` SETTINGS readonly CONST CREATE SETTINGS PROFILE `s3_01294` SETTINGS INHERIT `readonly` -CREATE SETTINGS PROFILE `s4_01294` SETTINGS INHERIT `readonly`, INHERIT `readonly` +CREATE SETTINGS PROFILE `s4_01294` SETTINGS INHERIT `readonly` CREATE SETTINGS PROFILE `s5_01294` SETTINGS INHERIT `readonly`, readonly = 1 CREATE SETTINGS PROFILE `s6_01294` SETTINGS INHERIT `readonly`, readonly CONST -- system.settings_profiles diff --git a/tests/queries/0_stateless/02943_alter_user_modify_profiles.reference b/tests/queries/0_stateless/02943_alter_user_modify_profiles.reference index 8c435e31354..a0088584807 100644 --- a/tests/queries/0_stateless/02943_alter_user_modify_profiles.reference +++ b/tests/queries/0_stateless/02943_alter_user_modify_profiles.reference @@ -1,21 +1,21 @@ -CREATE USER test_user -CREATE USER test_user SETTINGS PROFILE profile_a -CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_b -CREATE USER test_user SETTINGS PROFILE profile_b, PROFILE profile_a, PROFILE profile_c -CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_d -CREATE USER test_user SETTINGS PROFILE profile_e -CREATE USER test_user +CREATE USER test_user IDENTIFIED WITH no_password +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS PROFILE `profile_a` +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS PROFILE `profile_a`, PROFILE `profile_b` +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS PROFILE `profile_b`, PROFILE `profile_a`, PROFILE `profile_c` +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS PROFILE `profile_a`, PROFILE `profile_d` +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS PROFILE `profile_e` +CREATE USER test_user IDENTIFIED WITH no_password CREATE ROLE test_role -CREATE ROLE test_role SETTINGS PROFILE profile_a -CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_b -CREATE ROLE test_role SETTINGS PROFILE profile_b, PROFILE profile_a, PROFILE profile_c -CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_d -CREATE ROLE test_role SETTINGS PROFILE profile_e +CREATE ROLE test_role SETTINGS PROFILE `profile_a` +CREATE ROLE test_role SETTINGS PROFILE `profile_a`, PROFILE `profile_b` +CREATE ROLE test_role SETTINGS PROFILE `profile_b`, PROFILE `profile_a`, PROFILE `profile_c` +CREATE ROLE test_role SETTINGS PROFILE `profile_a`, PROFILE `profile_d` +CREATE ROLE test_role SETTINGS PROFILE `profile_e` CREATE ROLE test_role CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_b -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_b, INHERIT profile_a, INHERIT profile_c -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_d -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_e +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a` +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_b` +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_b`, INHERIT `profile_a`, INHERIT `profile_c` +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_d` +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_e` CREATE SETTINGS PROFILE test_profile diff --git a/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference b/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference index 7ae78818372..0b7ccdce95d 100644 --- a/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference +++ b/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference @@ -1,39 +1,39 @@ -CREATE USER test_user -CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' -CREATE USER test_user -CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' -CREATE USER test_user -CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' -CREATE USER test_user -CREATE USER test_user SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' -CREATE USER test_user -CREATE USER test_user SETTINGS custom_x = 123, custom_t = 7, custom_s = \'str\' -CREATE USER test_user SETTINGS PROFILE profile_a, custom_x = 123, custom_t = 7, custom_s = \'str\' -CREATE USER test_user SETTINGS PROFILE profile_b, PROFILE profile_a, custom_x = 321, custom_s = \'str\' CONST -CREATE USER test_user +CREATE USER test_user IDENTIFIED WITH no_password +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS PROFILE `profile_a`, PROFILE `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE USER test_user IDENTIFIED WITH no_password +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS PROFILE `profile_a`, PROFILE `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE USER test_user IDENTIFIED WITH no_password +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS PROFILE `profile_a`, PROFILE `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE USER test_user IDENTIFIED WITH no_password +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS PROFILE `profile_a`, PROFILE `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE USER test_user IDENTIFIED WITH no_password +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS PROFILE `profile_a`, custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS PROFILE `profile_b`, PROFILE `profile_a`, custom_x = 321, custom_s = \'str\' CONST +CREATE USER test_user IDENTIFIED WITH no_password CREATE ROLE test_role -CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE ROLE test_role SETTINGS PROFILE `profile_a`, PROFILE `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' CREATE ROLE test_role -CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE ROLE test_role SETTINGS PROFILE `profile_a`, PROFILE `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' CREATE ROLE test_role -CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE ROLE test_role SETTINGS PROFILE `profile_a`, PROFILE `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' CREATE ROLE test_role -CREATE ROLE test_role SETTINGS PROFILE profile_a, PROFILE profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE ROLE test_role SETTINGS PROFILE `profile_a`, PROFILE `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' CREATE ROLE test_role CREATE ROLE test_role SETTINGS custom_x = 123, custom_t = 7, custom_s = \'str\' -CREATE ROLE test_role SETTINGS PROFILE profile_a, custom_x = 123, custom_t = 7, custom_s = \'str\' -CREATE ROLE test_role SETTINGS PROFILE profile_b, PROFILE profile_a, custom_x = 321, custom_s = \'str\' CONST +CREATE ROLE test_role SETTINGS PROFILE `profile_a`, custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE ROLE test_role SETTINGS PROFILE `profile_b`, PROFILE `profile_a`, custom_x = 321, custom_s = \'str\' CONST CREATE ROLE test_role CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, INHERIT profile_b, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' CREATE SETTINGS PROFILE test_profile CREATE SETTINGS PROFILE test_profile SETTINGS custom_x = 123, custom_t = 7, custom_s = \'str\' -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_a, custom_x = 123, custom_t = 7, custom_s = \'str\' -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT profile_b, INHERIT profile_a, custom_x = 321, custom_s = \'str\' CONST +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_b`, INHERIT `profile_a`, custom_x = 321, custom_s = \'str\' CONST CREATE SETTINGS PROFILE test_profile diff --git a/tests/queries/0_stateless/02943_alter_user_modify_settings.reference b/tests/queries/0_stateless/02943_alter_user_modify_settings.reference index 8ed61ef77c7..c148bfa591d 100644 --- a/tests/queries/0_stateless/02943_alter_user_modify_settings.reference +++ b/tests/queries/0_stateless/02943_alter_user_modify_settings.reference @@ -1,17 +1,17 @@ -CREATE USER test_user -CREATE USER test_user SETTINGS custom_a = 100 -CREATE USER test_user SETTINGS custom_a = 100, custom_b = 200 -CREATE USER test_user SETTINGS custom_a = 100, custom_b = 300, custom_c = 400 -CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500 -CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 1000 -CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 2000 -CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 500 MAX 2000 -CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 700 MIN 500 MAX 2000 -CREATE USER test_user SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 800 MIN 150 -CREATE USER test_user SETTINGS custom_c = 400, custom_d = 500 -CREATE USER test_user SETTINGS custom_c = 400, custom_d = 500, custom_x = 1, custom_y = 2 -CREATE USER test_user SETTINGS custom_z = 3 -CREATE USER test_user +CREATE USER test_user IDENTIFIED WITH no_password +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_a = 100 +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_a = 100, custom_b = 200 +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_a = 100, custom_b = 300, custom_c = 400 +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_b = 300, custom_c = 400, custom_d = 500 +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 1000 +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 2000 +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 500 MAX 2000 +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 700 MIN 500 MAX 2000 +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 800 MIN 150 +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_c = 400, custom_d = 500 +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_c = 400, custom_d = 500, custom_x = 1, custom_y = 2 +CREATE USER test_user IDENTIFIED WITH no_password SETTINGS custom_z = 3 +CREATE USER test_user IDENTIFIED WITH no_password CREATE ROLE test_role CREATE ROLE test_role SETTINGS custom_a = 100 CREATE ROLE test_role SETTINGS custom_a = 100, custom_b = 200 @@ -26,17 +26,17 @@ CREATE ROLE test_role SETTINGS custom_c = 400, custom_d = 500 CREATE ROLE test_role SETTINGS custom_c = 400, custom_d = 500, custom_x = 1, custom_y = 2 CREATE ROLE test_role SETTINGS custom_z = 3 CREATE ROLE test_role -CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS custom_a = 100 -CREATE SETTINGS PROFILE test_profile SETTINGS custom_a = 100, custom_b = 200 -CREATE SETTINGS PROFILE test_profile SETTINGS custom_a = 100, custom_b = 300, custom_c = 400 -CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500 -CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 1000 -CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 2000 -CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 500 MAX 2000 -CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 700 MIN 500 MAX 2000 -CREATE SETTINGS PROFILE test_profile SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 800 MIN 150 -CREATE SETTINGS PROFILE test_profile SETTINGS custom_c = 400, custom_d = 500 -CREATE SETTINGS PROFILE test_profile SETTINGS custom_c = 400, custom_d = 500, custom_x = 1, custom_y = 2 -CREATE SETTINGS PROFILE test_profile SETTINGS custom_z = 3 -CREATE SETTINGS PROFILE test_profile +CREATE SETTINGS PROFILE `test_profile` +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_a = 100 +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_a = 100, custom_b = 200 +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_a = 100, custom_b = 300, custom_c = 400 +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_b = 300, custom_c = 400, custom_d = 500 +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 1000 +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 0 MAX 2000 +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 600 MIN 500 MAX 2000 +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 700 MIN 500 MAX 2000 +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_b = 300, custom_c = 400, custom_d = 500, custom_e = 800 MIN 150 +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_c = 400, custom_d = 500 +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_c = 400, custom_d = 500, custom_x = 1, custom_y = 2 +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_z = 3 +CREATE SETTINGS PROFILE `test_profile` From 0f1cf3a3b8157f44f27be4144b866972df10e988 Mon Sep 17 00:00:00 2001 From: pufit Date: Wed, 20 Nov 2024 01:37:54 -0500 Subject: [PATCH 159/502] fix parser --- .../Access/ParserSettingsProfileElement.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Parsers/Access/ParserSettingsProfileElement.cpp b/src/Parsers/Access/ParserSettingsProfileElement.cpp index 99c26d5d766..f76b171aad7 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.cpp +++ b/src/Parsers/Access/ParserSettingsProfileElement.cpp @@ -313,11 +313,20 @@ bool ParserAlterSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Exp auto parse_element = [&] { if (ParserKeyword{Keyword::ADD}.ignore(pos, expected)) + { action = "ADD"; + target = ""; + } else if (ParserKeyword{Keyword::DROP}.ignore(pos, expected)) + { action = "DROP"; + target = ""; + } else if (ParserKeyword{Keyword::MODIFY}.ignore(pos, expected)) + { action = "MODIFY"; + target = ""; + } if (!action.empty()) { @@ -329,10 +338,11 @@ bool ParserAlterSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Exp target = "PROFILES"; else if (ParserKeyword{Keyword::SETTINGS}.ignore(pos, expected) || ParserKeyword{Keyword::SETTING}.ignore(pos, expected)) target = "SETTINGS"; - else - return false; } + if (target.empty()) + return false; + if (target == "PROFILES") { auto element = std::make_shared(); From 396544a70db883d79d58dc6fb91cc1e16ea9bd3e Mon Sep 17 00:00:00 2001 From: pufit Date: Wed, 20 Nov 2024 02:41:35 -0500 Subject: [PATCH 160/502] fix tests --- ...02943_alter_user_modify_profiles.reference | 14 +++++----- ...ser_modify_profiles_and_settings.reference | 26 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/queries/0_stateless/02943_alter_user_modify_profiles.reference b/tests/queries/0_stateless/02943_alter_user_modify_profiles.reference index a0088584807..98de0d53b2f 100644 --- a/tests/queries/0_stateless/02943_alter_user_modify_profiles.reference +++ b/tests/queries/0_stateless/02943_alter_user_modify_profiles.reference @@ -12,10 +12,10 @@ CREATE ROLE test_role SETTINGS PROFILE `profile_b`, PROFILE `profile_a`, PROFILE CREATE ROLE test_role SETTINGS PROFILE `profile_a`, PROFILE `profile_d` CREATE ROLE test_role SETTINGS PROFILE `profile_e` CREATE ROLE test_role -CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a` -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_b` -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_b`, INHERIT `profile_a`, INHERIT `profile_c` -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_d` -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_e` -CREATE SETTINGS PROFILE test_profile +CREATE SETTINGS PROFILE `test_profile` +CREATE SETTINGS PROFILE `test_profile` SETTINGS INHERIT `profile_a` +CREATE SETTINGS PROFILE `test_profile` SETTINGS INHERIT `profile_a`, INHERIT `profile_b` +CREATE SETTINGS PROFILE `test_profile` SETTINGS INHERIT `profile_b`, INHERIT `profile_a`, INHERIT `profile_c` +CREATE SETTINGS PROFILE `test_profile` SETTINGS INHERIT `profile_a`, INHERIT `profile_d` +CREATE SETTINGS PROFILE `test_profile` SETTINGS INHERIT `profile_e` +CREATE SETTINGS PROFILE `test_profile` diff --git a/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference b/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference index 0b7ccdce95d..6df5a48c865 100644 --- a/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference +++ b/tests/queries/0_stateless/02943_alter_user_modify_profiles_and_settings.reference @@ -24,16 +24,16 @@ CREATE ROLE test_role SETTINGS custom_x = 123, custom_t = 7, custom_s = \'str\' CREATE ROLE test_role SETTINGS PROFILE `profile_a`, custom_x = 123, custom_t = 7, custom_s = \'str\' CREATE ROLE test_role SETTINGS PROFILE `profile_b`, PROFILE `profile_a`, custom_x = 321, custom_s = \'str\' CONST CREATE ROLE test_role -CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' -CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' -CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' -CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' -CREATE SETTINGS PROFILE test_profile -CREATE SETTINGS PROFILE test_profile SETTINGS custom_x = 123, custom_t = 7, custom_s = \'str\' -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_a`, custom_x = 123, custom_t = 7, custom_s = \'str\' -CREATE SETTINGS PROFILE test_profile SETTINGS INHERIT `profile_b`, INHERIT `profile_a`, custom_x = 321, custom_s = \'str\' CONST -CREATE SETTINGS PROFILE test_profile +CREATE SETTINGS PROFILE `test_profile` +CREATE SETTINGS PROFILE `test_profile` SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE `test_profile` +CREATE SETTINGS PROFILE `test_profile` SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE `test_profile` +CREATE SETTINGS PROFILE `test_profile` SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE `test_profile` +CREATE SETTINGS PROFILE `test_profile` SETTINGS INHERIT `profile_a`, INHERIT `profile_b`, custom_x = 123, custom_y = 56.5, custom_w = \'www\' +CREATE SETTINGS PROFILE `test_profile` +CREATE SETTINGS PROFILE `test_profile` SETTINGS custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE SETTINGS PROFILE `test_profile` SETTINGS INHERIT `profile_a`, custom_x = 123, custom_t = 7, custom_s = \'str\' +CREATE SETTINGS PROFILE `test_profile` SETTINGS INHERIT `profile_b`, INHERIT `profile_a`, custom_x = 321, custom_s = \'str\' CONST +CREATE SETTINGS PROFILE `test_profile` From 130a1c52a252037120fdffa4d6f5b30657dba890 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov <48961922+Avogar@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:11:51 +0100 Subject: [PATCH 161/502] Update 03273_format_inference_create_query_s3_url.sql --- .../0_stateless/03273_format_inference_create_query_s3_url.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql index 353b78b1e15..5d653843ec8 100644 --- a/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql +++ b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql @@ -1,3 +1,5 @@ +-- Tags: no-fasttest + drop table if exists test; create table test engine=S3('http://localhost:11111/test/json_data'); From 34c2dd39a504f5a47129954f747fbd9ac23f3429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 20 Nov 2024 12:56:04 +0000 Subject: [PATCH 162/502] Handle duplicated expressions properly --- .../Passes/LogicalExpressionOptimizerPass.cpp | 12 +++++++ ...2_common_expression_optimization.reference | 32 +++++++++++++++++++ .../03262_common_expression_optimization.sql | 5 +++ 3 files changed, 49 insertions(+) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 897d232ff47..aa8c0f0cd75 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -195,6 +195,10 @@ std::shared_ptr getFlattenedAndOrOr(const FunctionNode & node, con if (!maybe_function || maybe_function->getFunctionName() != function_name) continue; + auto it = arguments_to_replace.find(argument); + if (it != arguments_to_replace.end()) + continue; + auto maybe_flattened = getFlattenedAndOrOr(*maybe_function, context); if (maybe_flattened) arguments_to_replace.emplace(argument, std::move(maybe_flattened->getArguments().getNodes())); @@ -212,10 +216,18 @@ std::shared_ptr getFlattenedAndOrOr(const FunctionNode & node, con for (const auto & argument : arguments) { if (auto it = arguments_to_replace.find(argument); it != arguments_to_replace.end()) + { + if (it->second.empty()) + continue; + new_arguments.insert( new_arguments.end(), std::make_move_iterator(it->second.begin()), std::make_move_iterator(it->second.end())); + it->second.clear(); + } else + { new_arguments.push_back(argument); + } } auto function_resolver = FunctionFactory::instance().get(function_name, context); diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index ca7a0d8f908..fa42fd8f56c 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -764,3 +764,35 @@ QUERY id: 0 LIST id: 37, nodes: 2 COLUMN id: 38, column_name: E, result_type: UInt8, source_id: 4 COLUMN id: 39, column_name: E, result_type: UInt8, source_id: 5 +-9220342792229035180 1 1 0 1 0 1 +-9215479184005563028 0 1 1 1 0 1 +-9211524111550600160 0 0 1 1 1 0 +-9220342792229035180 1 1 0 1 0 1 +-9215479184005563028 0 1 1 1 0 1 +-9211524111550600160 0 0 1 1 1 0 +QUERY id: 0 + PROJECTION COLUMNS + x Int64 + A UInt8 + B UInt8 + C UInt8 + D UInt8 + E UInt8 + F UInt8 + PROJECTION + LIST id: 1, nodes: 7 + COLUMN id: 2, column_name: x, result_type: Int64, source_id: 3 + COLUMN id: 4, column_name: A, result_type: UInt8, source_id: 3 + COLUMN id: 5, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 6, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 7, column_name: D, result_type: UInt8, source_id: 3 + COLUMN id: 8, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: F, result_type: UInt8, source_id: 3 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 10, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 13, column_name: E, result_type: UInt8, source_id: 3 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index 51e0b1d9859..ada294d74b9 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -86,3 +86,8 @@ EXPLAIN QUERY TREE SELECT count() FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = -- Check that optimization only happen on top level, (x.C = y.C AND x.D = y.D) OR (x.C = y.C AND x.E = y.E) shouldn't be optimized EXPLAIN QUERY TREE SELECT count() FROM x INNER JOIN y ON (x.A = y.A) OR ((x.B = y.B) AND ((x.C = y.C AND x.D = y.D) OR (x.C = y.C AND x.E = y.E))); + +-- Duplicated subexpressions, found by fuzzer +SELECT * FROM x WHERE (D AND 5) OR ((C AND E) AND (C AND E)) ORDER BY ALL LIMIT 3 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE (D AND 5) OR ((C AND E) AND (C AND E)) ORDER BY ALL LIMIT 3 SETTINGS optimize_extract_common_expressions = 1; +EXPLAIN QUERY TREE SELECT * FROM x WHERE (C AND E) OR ((C AND E) AND (C AND E)); From fd5023e2a6f111e1027d7abb6831f23c056b4e80 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 20 Nov 2024 17:32:46 +0100 Subject: [PATCH 163/502] Add comments --- src/Common/CurrentMetrics.cpp | 3 + src/Databases/Iceberg/DatabaseIceberg.cpp | 12 +-- src/Databases/Iceberg/RestCatalog.cpp | 124 ++++++++++++---------- src/Databases/Iceberg/RestCatalog.h | 24 +++-- 4 files changed, 94 insertions(+), 69 deletions(-) diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index d073e477593..33f0e10156f 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -189,6 +189,9 @@ M(BuildVectorSimilarityIndexThreads, "Number of threads in the build vector similarity index thread pool.") \ M(BuildVectorSimilarityIndexThreadsActive, "Number of threads in the build vector similarity index thread pool running a task.") \ M(BuildVectorSimilarityIndexThreadsScheduled, "Number of queued or active jobs in the build vector similarity index thread pool.") \ + M(IcebergCatalogThreads, "Number of threads in the IcebergCatalog thread pool.") \ + M(IcebergCatalogThreadsActive, "Number of threads in the IcebergCatalog thread pool running a task.") \ + M(IcebergCatalogThreadsScheduled, "Number of queued or active jobs in the IcebergCatalog thread pool.") \ \ M(DiskPlainRewritableAzureDirectoryMapSize, "Number of local-to-remote path entries in the 'plain_rewritable' in-memory map for AzureObjectStorage.") \ M(DiskPlainRewritableAzureFileCount, "Number of file entries in the 'plain_rewritable' in-memory map for AzureObjectStorage.") \ diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 6241c0d8423..c672d805e5e 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -25,9 +25,9 @@ namespace CurrentMetrics { - extern const Metric MergeTreeDataSelectExecutorThreads; - extern const Metric MergeTreeDataSelectExecutorThreadsActive; - extern const Metric MergeTreeDataSelectExecutorThreadsScheduled; + extern const Metric IcebergCatalogThreads; + extern const Metric IcebergCatalogThreadsActive; + extern const Metric IcebergCatalogThreadsScheduled; } @@ -236,9 +236,9 @@ DatabaseTablesIteratorPtr DatabaseIceberg::getTablesIterator( auto iceberg_tables = catalog->getTables(); size_t num_threads = std::min(10, iceberg_tables.size()); ThreadPool pool( - CurrentMetrics::MergeTreeDataSelectExecutorThreads, - CurrentMetrics::MergeTreeDataSelectExecutorThreadsActive, - CurrentMetrics::MergeTreeDataSelectExecutorThreadsScheduled, + CurrentMetrics::IcebergCatalogThreads, + CurrentMetrics::IcebergCatalogThreadsActive, + CurrentMetrics::IcebergCatalogThreadsScheduled, num_threads); DB::ThreadPoolCallbackRunnerLocal runner(pool, "RestCatalog"); diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index d336c4af707..0fe9e90e46f 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -26,9 +26,9 @@ namespace DB::ErrorCodes namespace CurrentMetrics { - extern const Metric MergeTreeDataSelectExecutorThreads; - extern const Metric MergeTreeDataSelectExecutorThreadsActive; - extern const Metric MergeTreeDataSelectExecutorThreadsScheduled; + extern const Metric IcebergCatalogThreads; + extern const Metric IcebergCatalogThreadsActive; + extern const Metric IcebergCatalogThreadsScheduled; } namespace Iceberg @@ -37,20 +37,14 @@ namespace Iceberg static constexpr auto config_endpoint = "config"; static constexpr auto namespaces_endpoint = "namespaces"; -std::string RestCatalog::Config::toString() const +namespace { - DB::WriteBufferFromOwnString wb; - if (!prefix.empty()) - wb << "prefix: " << prefix.string() << ", "; - if (!default_base_location.empty()) - wb << "default_base_location: " << default_base_location << ", "; - - return wb.str(); -} - -static std::pair parseCatalogCredential(const std::string & catalog_credential) +std::pair parseCatalogCredential(const std::string & catalog_credential) { + /// Parse a string of format ":" + /// into separare strings client_id and client_secret. + std::string client_id, client_secret; if (!catalog_credential.empty()) { @@ -67,6 +61,33 @@ static std::pair parseCatalogCredential(const std::str return std::pair(client_id, client_secret); } +DB::HTTPHeaderEntry parseAuthHeader(const std::string & auth_header) +{ + /// Parse a string of format "Authorization: " + /// into a key-value header "Authorization", " " + + auto pos = auth_header.find(':'); + if (pos == std::string::npos) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Unexpected format of auth header"); + + return DB::HTTPHeaderEntry(auth_header.substr(0, pos), auth_header.substr(pos + 1)); +} + +} + +std::string RestCatalog::Config::toString() const +{ + DB::WriteBufferFromOwnString wb; + + if (!prefix.empty()) + wb << "prefix: " << prefix.string() << ", "; + + if (!default_base_location.empty()) + wb << "default_base_location: " << default_base_location << ", "; + + return wb.str(); +} + RestCatalog::RestCatalog( const std::string & warehouse_, const std::string & base_url_, @@ -75,18 +96,14 @@ RestCatalog::RestCatalog( DB::ContextPtr context_) : ICatalog(warehouse_) , DB::WithContext(context_) - , log(getLogger("RestCatalog(" + warehouse_ + ")")) , base_url(base_url_) + , log(getLogger("RestCatalog(" + warehouse_ + ")")) { - std::tie(client_id, client_secret) = parseCatalogCredential(catalog_credential_); - if (!auth_header_.empty()) - { - auto pos = auth_header_.find(':'); - if (pos == std::string::npos) - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Unexpected format of auth header"); + if (!catalog_credential_.empty()) + std::tie(client_id, client_secret) = parseCatalogCredential(catalog_credential_); + else if (!auth_header_.empty()) + auth_header = parseAuthHeader(auth_header_); - auth_header = DB::HTTPHeaderEntry(auth_header_.substr(0, pos), auth_header_.substr(pos + 1)); - } config = loadConfig(); } @@ -98,7 +115,7 @@ RestCatalog::Config RestCatalog::loadConfig() std::string json_str; readJSONObjectPossiblyInvalid(json_str, *buf); - LOG_TEST(log, "Received config result: {}", json_str); + LOG_TEST(log, "Received catalog configuration settings: {}", json_str); Poco::JSON::Parser parser; Poco::Dynamic::Var json = parser.parse(json_str); @@ -107,16 +124,16 @@ RestCatalog::Config RestCatalog::loadConfig() Config result; auto defaults_object = object->get("defaults").extract(); - parseConfig(defaults_object, result); + parseCatalogConfigurationSettings(defaults_object, result); auto overrides_object = object->get("overrides").extract(); - parseConfig(overrides_object, result); + parseCatalogConfigurationSettings(overrides_object, result); - LOG_TEST(log, "Parsed config: {}", result.toString()); + LOG_TEST(log, "Parsed catalog configuration settings: {}", result.toString()); return result; } -void RestCatalog::parseConfig(const Poco::JSON::Object::Ptr & object, Config & result) +void RestCatalog::parseCatalogConfigurationSettings(const Poco::JSON::Object::Ptr & object, Config & result) { if (!object) return; @@ -135,34 +152,35 @@ DB::HTTPHeaderEntries RestCatalog::getHeaders(bool update_token) const return DB::HTTPHeaderEntries{auth_header.value()}; } - if (!access_token.has_value() || update_token) + if (!client_id.empty()) { - access_token = retrieveAccessToken(); + if (!access_token.has_value() || update_token) + { + access_token = retrieveAccessToken(); + } + + DB::HTTPHeaderEntries headers; + headers.emplace_back("Authorization", "Bearer " + access_token.value()); + return headers; } - DB::HTTPHeaderEntries headers; - headers.emplace_back("Authorization", "Bearer " + access_token.value()); - return headers; + return {}; } std::string RestCatalog::retrieveAccessToken() const { static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; - const auto & context = getContext(); - const auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); + /// TODO: + /// 1. support oauth2-server-uri + /// https://github.com/apache/iceberg/blob/918f81f3c3f498f46afcea17c1ac9cdc6913cb5c/open-api/rest-catalog-open-api.yaml#L183C82-L183C99 Poco::JSON::Object json; json.set("grant_type", "client_credentials"); - json.set("scope", "PRINCIPAL_ROLE:ALL"); + json.set("scope", "PRINCIPAL_ROLE:ALL"); /// TODO: add it into setting. json.set("client_id", client_id); json.set("client_secret", client_secret); - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - oss.exceptions(std::ios::failbit); - Poco::JSON::Stringifier::stringify(json, oss); - std::string json_str = oss.str(); - DB::HTTPHeaderEntries headers; headers.emplace_back("Content-Type", "application/x-www-form-urlencoded"); headers.emplace_back("Accepts", "application/json; charset=UTF-8"); @@ -176,23 +194,20 @@ std::string RestCatalog::retrieveAccessToken() const }; url.setQueryParameters(params); - LOG_TEST(log, "Writing {}: {}", url.toString(), json_str); - + const auto & context = getContext(); auto wb = DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) .withMethod(Poco::Net::HTTPRequest::HTTP_POST) - .withSettings(getContext()->getReadSettings()) - .withTimeouts(timeouts) - .withHostFilter(&getContext()->getRemoteHostFilter()) + .withSettings(context->getReadSettings()) + .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) + .withHostFilter(&context->getRemoteHostFilter()) .withSkipNotFound(false) .withHeaders(headers) .create(credentials); - json_str.clear(); + std::string json_str; readJSONObjectPossiblyInvalid(json_str, *wb); - LOG_TEST(log, "Received token result: {}", json_str); - Poco::JSON::Parser parser; Poco::Dynamic::Var res_json = parser.parse(json_str); const Poco::JSON::Object::Ptr & object = res_json.extract(); @@ -212,7 +227,6 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( const Poco::URI::QueryParameters & params) const { const auto & context = getContext(); - const auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); Poco::URI url(base_url / endpoint); if (!params.empty()) @@ -227,7 +241,7 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( return DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) .withSettings(getContext()->getReadSettings()) - .withTimeouts(timeouts) + .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) .withHeaders(headers) .withHostFilter(&getContext()->getRemoteHostFilter()) .withDelayInit(false) @@ -240,7 +254,7 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( return DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) .withSettings(getContext()->getReadSettings()) - .withTimeouts(timeouts) + .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) .withHeaders(new_headers) .withHostFilter(&getContext()->getRemoteHostFilter()) .withDelayInit(false) @@ -277,9 +291,9 @@ RestCatalog::Tables RestCatalog::getTables() const { size_t num_threads = 10; ThreadPool pool( - CurrentMetrics::MergeTreeDataSelectExecutorThreads, - CurrentMetrics::MergeTreeDataSelectExecutorThreadsActive, - CurrentMetrics::MergeTreeDataSelectExecutorThreadsScheduled, + CurrentMetrics::IcebergCatalogThreads, + CurrentMetrics::IcebergCatalogThreadsActive, + CurrentMetrics::IcebergCatalogThreadsScheduled, num_threads); DB::ThreadPoolCallbackRunnerLocal runner(pool, "RestCatalog"); diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index 116b4253684..6c842e1dff7 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -46,25 +46,33 @@ public: std::optional getStorageType() const override; private: - LoggerPtr log; - struct Config { + /// Prefix is a path of the catalog enpoint, + /// e.g. /v1/{prefix}/namespaces/{namespace}/tables/{table} std::filesystem::path prefix; + /// Base location is location of data in storage + /// (in filesystem or object storage). std::string default_base_location; std::string toString() const; }; const std::filesystem::path base_url; - std::optional auth_header; - mutable std::optional access_token; - std::string client_id; - std::string client_secret; + const LoggerPtr log; + + /// Catalog configuration settings from /v1/config endpoint. Config config; - Poco::Net::HTTPBasicCredentials credentials{}; + /// Auth headers of format: "Authorization": " " + std::optional auth_header; + /// Parameters for OAuth. + std::string client_id; + std::string client_secret; + mutable std::optional access_token; + + Poco::Net::HTTPBasicCredentials credentials{}; DB::ReadWriteBufferFromHTTPPtr createReadBuffer( const std::string & endpoint, @@ -97,7 +105,7 @@ private: Config loadConfig(); std::string retrieveAccessToken() const; DB::HTTPHeaderEntries getHeaders(bool update_token = false) const; - static void parseConfig(const Poco::JSON::Object::Ptr & object, Config & result); + static void parseCatalogConfigurationSettings(const Poco::JSON::Object::Ptr & object, Config & result); }; } From 4abbf29a865f0805dd125a188f7e312ba67400a1 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 20 Nov 2024 18:59:47 +0100 Subject: [PATCH 164/502] Fix style check --- src/Databases/Iceberg/RestCatalog.cpp | 1 + src/Databases/Iceberg/RestCatalog.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index 0fe9e90e46f..a34aa45b812 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -22,6 +22,7 @@ namespace DB::ErrorCodes { extern const int ICEBERG_CATALOG_ERROR; extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; } namespace CurrentMetrics diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index 6c842e1dff7..402e761b07a 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -48,7 +48,7 @@ public: private: struct Config { - /// Prefix is a path of the catalog enpoint, + /// Prefix is a path of the catalog endpoint, /// e.g. /v1/{prefix}/namespaces/{namespace}/tables/{table} std::filesystem::path prefix; /// Base location is location of data in storage From 8953babcd0622120a4489de6bd7d9887a2351b38 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 20 Nov 2024 19:49:13 +0100 Subject: [PATCH 165/502] Update 03257_reverse_sorting_key.sql --- tests/queries/0_stateless/03257_reverse_sorting_key.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key.sql b/tests/queries/0_stateless/03257_reverse_sorting_key.sql index 9d2f279026f..0777d101ca1 100644 --- a/tests/queries/0_stateless/03257_reverse_sorting_key.sql +++ b/tests/queries/0_stateless/03257_reverse_sorting_key.sql @@ -17,12 +17,12 @@ select * from x1 where i = 3; select count() from x1 where i between 3 and 10; -select trimLeft(explain) from (explain actions=1 select * from x1 order by i desc limit 5 settings max_threads=1, enable_analyzer=1) where explain ilike '%sort%'; +select trimLeft(explain) from (explain actions=1 select * from x1 order by i desc limit 5) where explain ilike '%sort%' settings max_threads=1, enable_analyzer=1; explain pipeline select * from x1 order by i desc limit 5 settings max_threads=1; select * from x1 order by i desc limit 5; -select trimLeft(explain) from (explain actions=1 select * from x1 order by i limit 5 settings max_threads=1, enable_analyzer=1) where explain ilike '%sort%'; +select trimLeft(explain) from (explain actions=1 select * from x1 order by i limit 5) where explain ilike '%sort%' settings max_threads=1, enable_analyzer=1; explain pipeline select * from x1 order by i limit 5 settings max_threads=1; select * from x1 order by i limit 5; @@ -37,12 +37,12 @@ select * from x2 where j = 1003; select count() from x2 where i between 3 and 10 and j between 1003 and 1008; -select trimLeft(explain) from (explain actions=1 select * from x2 order by i, j desc limit 5 settings max_threads=1, enable_analyzer=1) where explain ilike '%sort%'; +select trimLeft(explain) from (explain actions=1 select * from x2 order by i, j desc limit 5) where explain ilike '%sort%' settings max_threads=1, enable_analyzer=1; explain pipeline select * from x2 order by i, j desc limit 5 settings max_threads=1; select * from x2 order by i, j desc limit 5; -select trimLeft(explain) from (explain actions=1 select * from x2 order by i, j limit 5 settings max_threads=1, enable_analyzer=1) where explain ilike '%sort%'; +select trimLeft(explain) from (explain actions=1 select * from x2 order by i, j limit 5) where explain ilike '%sort%' settings max_threads=1, enable_analyzer=1; explain pipeline select * from x2 order by i, j limit 5 settings max_threads=1; select * from x2 order by i, j limit 5; From 3af404f2c950d25a3148c068da2a8b03f8f39ec7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 21 Nov 2024 15:53:12 +0000 Subject: [PATCH 166/502] Automatic style fix --- tests/integration/test_storage_iceberg/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_iceberg/test.py b/tests/integration/test_storage_iceberg/test.py index bb316aca8ee..cd79aacd534 100644 --- a/tests/integration/test_storage_iceberg/test.py +++ b/tests/integration/test_storage_iceberg/test.py @@ -601,7 +601,7 @@ def test_cluster_table_function(started_cluster, format_version, storage_type): mode=mode, format_version=format_version, ) - + files = default_upload_directory( started_cluster, storage_type, From 07f2745d7b8d704c4ae99ce78d283488ac957980 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:53:58 +0100 Subject: [PATCH 167/502] Update thread starting. --- programs/server/Server.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 7976d67bd54..1e1e7335d58 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1375,15 +1375,21 @@ try setOOMScore(oom_score, log); #endif + std::unique_ptr cancellation_task; + + SCOPE_EXIT({ + if (cancellation_task) + CancellationChecker::getInstance().terminateThread(); + }); + if (server_settings[ServerSetting::background_schedule_pool_size] > 1) { - auto cancellation_task_holder = global_context->getSchedulePool().createTask("CancellationChecker", []{ CancellationChecker::getInstance().workerFunction(); }); - auto cancellation_task = std::make_unique(std::move(cancellation_task_holder)); + auto cancellation_task_holder = global_context->getSchedulePool().createTask( + "CancellationChecker", + [] { CancellationChecker::getInstance().workerFunction(); } + ); + cancellation_task = std::make_unique(std::move(cancellation_task_holder)); (*cancellation_task)->activateAndSchedule(); - - SCOPE_EXIT({ - CancellationChecker::getInstance().terminateThread(); - }); } global_context->setRemoteHostFilter(config()); From b5f6219a6bbdd5892d95254de7a0721260cb684d Mon Sep 17 00:00:00 2001 From: Pavel Kruglov <48961922+Avogar@users.noreply.github.com> Date: Thu, 21 Nov 2024 21:44:39 +0100 Subject: [PATCH 168/502] Update 03270_object_to_json_alter.sql --- tests/queries/0_stateless/03270_object_to_json_alter.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/03270_object_to_json_alter.sql b/tests/queries/0_stateless/03270_object_to_json_alter.sql index 61db4d0d33a..1191857d484 100644 --- a/tests/queries/0_stateless/03270_object_to_json_alter.sql +++ b/tests/queries/0_stateless/03270_object_to_json_alter.sql @@ -1,3 +1,5 @@ +-- Tags: long + set allow_experimental_object_type = 1; set allow_experimental_json_type = 1; set max_block_size = 100; From 081aae87db462c6701c7c40018e5604dfa702fb4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 22 Nov 2024 11:14:16 +0000 Subject: [PATCH 169/502] Update version_date.tsv and changelogs after v24.10.3.21-stable --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v24.10.3.21-stable.md | 26 ++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 1 + 5 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v24.10.3.21-stable.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 0d75c0bd225..f3f25c1a247 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -38,7 +38,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.10.2.80" +ARG VERSION="24.10.3.21" ARG PACKAGES="clickhouse-keeper" ARG DIRECT_DOWNLOAD_URLS="" diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 5b3d86ca3e6..3bf23767150 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -35,7 +35,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.10.2.80" +ARG VERSION="24.10.3.21" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" ARG DIRECT_DOWNLOAD_URLS="" diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 2e56f676cbc..31fdcb8a490 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="24.10.2.80" +ARG VERSION="24.10.3.21" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" #docker-official-library:off diff --git a/docs/changelogs/v24.10.3.21-stable.md b/docs/changelogs/v24.10.3.21-stable.md new file mode 100644 index 00000000000..07f1920e15b --- /dev/null +++ b/docs/changelogs/v24.10.3.21-stable.md @@ -0,0 +1,26 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.10.3.21-stable (e668b927efb) FIXME as compared to v24.10.2.80-stable (96b80057159) + +#### Improvement +* Backported in [#72100](https://github.com/ClickHouse/ClickHouse/issues/72100): Fix the metadata_version record in ZooKeeper in restarting thread rather than in attach thread. [#70297](https://github.com/ClickHouse/ClickHouse/pull/70297) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)). +* Backported in [#72169](https://github.com/ClickHouse/ClickHouse/issues/72169): Forbid Dynamic/Variant types in min/max functions to avoid confusion. [#71761](https://github.com/ClickHouse/ClickHouse/pull/71761) ([Pavel Kruglov](https://github.com/Avogar)). +* Backported in [#72064](https://github.com/ClickHouse/ClickHouse/issues/72064): When retrieving data directly from a dictionary using Dictionary storage, dictionary table function, or direct SELECT from the dictionary itself, it is now enough to have `SELECT` permission or `dictGet` permission for the dictionary. This aligns with previous attempts to prevent ACL bypasses: https://github.com/ClickHouse/ClickHouse/pull/57362 and https://github.com/ClickHouse/ClickHouse/pull/65359. It also makes the latter one backward compatible. [#72051](https://github.com/ClickHouse/ClickHouse/pull/72051) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### Bug Fix (user-visible misbehavior in an official stable release) +* Backported in [#72144](https://github.com/ClickHouse/ClickHouse/issues/72144): Acquiring zero-copy shared lock before moving a part to zero-copy disk to prevent possible data loss if Keeper is unavailable. [#71845](https://github.com/ClickHouse/ClickHouse/pull/71845) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Backported in [#72088](https://github.com/ClickHouse/ClickHouse/issues/72088): Fix rows_processed column in system.s3/azure_queue_log broken in 24.6. Closes [#69975](https://github.com/ClickHouse/ClickHouse/issues/69975). [#71946](https://github.com/ClickHouse/ClickHouse/pull/71946) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#72036](https://github.com/ClickHouse/ClickHouse/issues/72036): Fix `Illegal type` error for `MergeTree` tables with binary monotonic function in `ORDER BY` when the first argument is constant. Fixes [#71941](https://github.com/ClickHouse/ClickHouse/issues/71941). [#71966](https://github.com/ClickHouse/ClickHouse/pull/71966) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backported in [#72207](https://github.com/ClickHouse/ClickHouse/issues/72207): Fixed incorrect settings order `max_parser_depth` and `max_parser_backtracks`. [#71498](https://github.com/ClickHouse/ClickHouse/pull/71498) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#72071](https://github.com/ClickHouse/ClickHouse/issues/72071): Fix client syntax highlighting that was broken in https://github.com/ClickHouse/ClickHouse/pull/71949. [#72049](https://github.com/ClickHouse/ClickHouse/pull/72049) ([Nikolay Degterinsky](https://github.com/evillique)). +* Backported in [#72095](https://github.com/ClickHouse/ClickHouse/issues/72095): Minor improvement for system.query_metric_log stateless test. [#72076](https://github.com/ClickHouse/ClickHouse/pull/72076) ([Pablo Marcos](https://github.com/pamarcos)). +* Backported in [#72184](https://github.com/ClickHouse/ClickHouse/issues/72184): Add google-cloud-cpp submodule. [#72092](https://github.com/ClickHouse/ClickHouse/pull/72092) ([Pablo Marcos](https://github.com/pamarcos)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index f69c72163c2..a7b6bf897cf 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v24.10.3.21-stable 2024-11-22 v24.10.2.80-stable 2024-11-18 v24.10.1.2812-stable 2024-11-01 v24.9.3.128-stable 2024-11-18 From e05b64064e8a15222c77a233b513852a4127b35b Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 22 Nov 2024 13:05:12 +0000 Subject: [PATCH 170/502] Resolve issues --- .../DataLakes/IcebergMetadata.cpp | 81 ++++++++----------- .../ObjectStorage/DataLakes/IcebergMetadata.h | 6 -- .../StorageObjectStorageSettings.cpp | 34 -------- 3 files changed, 35 insertions(+), 86 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 31b2ef9a87c..c63a61860df 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -270,9 +270,31 @@ bool IcebergSchemaProcessor::allowPrimitiveTypeConversion(const String & old_typ return allowed_type_conversion; } -std::shared_ptr -IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema) +std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id) { + std::lock_guard lock(mutex); + Poco::JSON::Object::Ptr old_schema, new_schema; + auto required_transform_dag_it = transform_dags_by_ids.find({old_id, new_id}); + if (required_transform_dag_it != transform_dags_by_ids.end()) + { + return required_transform_dag_it->second; + } + + if (old_id == new_id) + { + return nullptr; + } + + auto old_schema_it = iceberg_table_schemas_by_ids.find(old_id); + if (old_schema_it == iceberg_table_schemas_by_ids.end()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", old_id); + } + auto new_schema_it = iceberg_table_schemas_by_ids.find(new_id); + if (new_schema_it == iceberg_table_schemas_by_ids.end()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", new_id); + } std::unordered_map> old_schema_entries; auto old_schema_fields = old_schema->get("fields").extract(); std::shared_ptr dag = std::make_shared(); @@ -305,8 +327,8 @@ IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr ErrorCodes::UNSUPPORTED_METHOD, "Schema evolution is not supported for complex types yet, field id is {}, old schema id is {}, new schema id is {}", id, - current_old_id, - current_new_id); + old_id, + new_id); } else { @@ -321,8 +343,8 @@ IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr ErrorCodes::LOGICAL_ERROR, "Can't cast primitive type to the complex type, field id is {}, old schema id is {}, new schema id is {}", id, - current_old_id, - current_new_id); + old_id, + new_id); } String old_type = old_json->getValue("type"); String new_type = field->getValue("type"); @@ -350,59 +372,26 @@ IcebergSchemaProcessor::getSchemaTransformationDag(const Poco::JSON::Object::Ptr ErrorCodes::UNSUPPORTED_METHOD, "Adding a default column with id {} and complex type is not supported yet. Old schema id is {}, new schema id is {}", id, - current_old_id, - current_new_id); + old_id, + new_id); } if (!type->isNullable()) { throw Exception( ErrorCodes::LOGICAL_ERROR, - "Cannot add a column with id {} with required values to the table during schema evolution. This is forbidden by Iceberg format specification. Old schema id is {}, new " + "Cannot add a column with id {} with required values to the table during schema evolution. This is forbidden by " + "Iceberg format specification. Old schema id is {}, new " "schema id is {}", id, - current_old_id, - current_new_id); + old_id, + new_id); } ColumnPtr default_type_column = type->createColumnConstWithDefaultValue(0); const auto & constant = dag->addColumn({default_type_column, type, name}); outputs.push_back(&constant); } } - return dag; -} - -std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id) -{ - std::lock_guard lock(mutex); - current_old_id = old_id; - current_new_id = new_id; - SCOPE_EXIT({ - current_old_id = -1; - current_new_id = -1; - }); - Poco::JSON::Object::Ptr old_schema, new_schema; - auto required_transform_dag_it = transform_dags_by_ids.find({old_id, new_id}); - if (required_transform_dag_it != transform_dags_by_ids.end()) - { - return required_transform_dag_it->second; - } - - if (old_id == new_id) - { - return nullptr; - } - - auto old_schema_it = iceberg_table_schemas_by_ids.find(old_id); - if (old_schema_it == iceberg_table_schemas_by_ids.end()) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", old_id); - } - auto new_schema_it = iceberg_table_schemas_by_ids.find(new_id); - if (new_schema_it == iceberg_table_schemas_by_ids.end()) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", new_id); - } - return transform_dags_by_ids[{old_id, new_id}] = getSchemaTransformationDag(old_schema_it->second, new_schema_it->second); + return transform_dags_by_ids[{old_id, new_id}] = dag; } void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schema_ptr) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 45b5b2caf80..4bb55adf9da 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -95,16 +95,10 @@ private: DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type); DataTypePtr getFieldType(const Poco::JSON::Object::Ptr & field, const String & type_key, bool required); DataTypePtr getSimpleType(const String & type_name); - std::shared_ptr - getSchemaTransformationDag(const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema); bool allowPrimitiveTypeConversion(const String & old_type, const String & new_type); - const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); - Int32 current_old_id = -1; - Int32 current_new_id = -1; - std::mutex mutex; }; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp index 3bb08bc1419..b280086c28b 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp @@ -49,40 +49,6 @@ StorageObjectStorageSettings::StorageObjectStorageSettings(StorageObjectStorageS } -void StorageObjectStorageSettings::dumpToSystemEngineSettingsColumns( - MutableColumnsAndConstraints & params, - const std::string & table_name, - const std::string & database_name, - const StorageObjectStorage & storage) const -{ - MutableColumns & res_columns = params.res_columns; - - /// We cannot use setting.isValueChanged(), because we do not store initial settings in storage. - /// Therefore check if the setting was changed via table metadata. - const auto & settings_changes = storage.getInMemoryMetadataPtr()->settings_changes->as()->changes; - auto is_changed = [&](const std::string & setting_name) -> bool - { - return settings_changes.end() - != std::find_if( - settings_changes.begin(), - settings_changes.end(), - [&](const SettingChange & change) { return change.name == setting_name; }); - }; - - for (const auto & change : impl->all()) - { - size_t i = 0; - res_columns[i++]->insert(database_name); - res_columns[i++]->insert(table_name); - res_columns[i++]->insert(change.getName()); - res_columns[i++]->insert(convertFieldToString(change.getValue())); - res_columns[i++]->insert(change.getTypeName()); - res_columns[i++]->insert(is_changed(change.getName())); - res_columns[i++]->insert(change.getDescription()); - res_columns[i++]->insert(false); - } -} - StorageObjectStorageSettings::~StorageObjectStorageSettings() = default; STORAGE_OBJECT_STORAGE_SETTINGS_SUPPORTED_TYPES(StorageObjectStorageSettings, IMPLEMENT_SETTING_SUBSCRIPT_OPERATOR) From 28534272c9577085d2d415c664e831af3885df98 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 22 Nov 2024 15:30:34 +0000 Subject: [PATCH 171/502] Bring back optimization for reading subcolumns of single column in Compact parts --- .../Serializations/SerializationObject.cpp | 2 + .../MergeTree/MergeTreeReaderCompact.cpp | 37 +++++++++++++------ .../MergeTree/MergeTreeReaderCompact.h | 3 +- .../MergeTreeReaderCompactSingleBuffer.cpp | 8 +++- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 1b95fddee9f..805a11521b3 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -9,6 +9,8 @@ #include #include +#include + namespace DB { diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 7451374070c..06635c39838 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -148,7 +148,8 @@ void MergeTreeReaderCompact::readData( ColumnPtr & column, size_t rows_to_read, const InputStreamGetter & getter, - ISerialization::SubstreamsCache & cache) + ISerialization::SubstreamsCache & cache, + std::unordered_map & columns_cache_for_subcolumns) { try { @@ -171,17 +172,31 @@ void MergeTreeReaderCompact::readData( const auto & type_in_storage = name_and_type.getTypeInStorage(); const auto & name_in_storage = name_and_type.getNameInStorage(); - auto serialization = getSerializationInPart({name_in_storage, type_in_storage}); - ColumnPtr temp_column = type_in_storage->createColumn(*serialization); - - serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); - auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); - - /// TODO: Avoid extra copying. - if (column->empty()) - column = subcolumn; + if (auto cache_for_subcolumns_it = columns_cache_for_subcolumns.find(name_in_storage); cache_for_subcolumns_it != columns_cache_for_subcolumns.end()) + { + auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), cache_for_subcolumns_it->second); + /// TODO: Avoid extra copying. + if (column->empty()) + column = IColumn::mutate(subcolumn); + else + column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); + } else - column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); + { + auto serialization = getSerializationInPart({name_in_storage, type_in_storage}); + ColumnPtr temp_column = type_in_storage->createColumn(*serialization); + + serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); + auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); + + /// TODO: Avoid extra copying. + if (column->empty()) + column = subcolumn; + else + column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); + + columns_cache_for_subcolumns[name_in_storage] = temp_column; + } } else { diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index 1c6bd1474e3..f18ee8808ec 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -45,7 +45,8 @@ protected: ColumnPtr & column, size_t rows_to_read, const InputStreamGetter & getter, - ISerialization::SubstreamsCache & cache); + ISerialization::SubstreamsCache & cache, + std::unordered_map & columns_cache_for_subcolumns); void readPrefix( const NameAndTypePair & name_and_type, diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index 649bcce1188..50224eba82d 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -29,6 +29,12 @@ try /// Use cache to avoid reading the column with the same name twice. /// It may happen if there are empty array Nested in the part. ISerialization::SubstreamsCache cache; + /// If we need to read multiple subcolumns from a single column in storage, + /// we will read it this column only once and then reuse to extract all subcolumns. + /// We cannot use SubstreamsCache for it, because we may also read the full column itself + /// and it might me not empty inside res_columns (and SubstreamsCache contains the whole columns). + /// TODO: refactor the code in a way when we first read all full columns and then extract all subcolumns from them. + std::unordered_map columns_cache_for_subcolumns; for (size_t pos = 0; pos < num_columns; ++pos) { @@ -56,7 +62,7 @@ try }; readPrefix(columns_to_read[pos], buffer_getter, buffer_getter_for_prefix, columns_for_offsets[pos]); - readData(columns_to_read[pos], column, rows_to_read, buffer_getter, cache); + readData(columns_to_read[pos], column, rows_to_read, buffer_getter, cache, columns_cache_for_subcolumns); } ++from_mark; From e1d0932b98d32ec70a80710c3aff980c01ff4460 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 22 Nov 2024 15:32:09 +0000 Subject: [PATCH 172/502] Remove extra include --- src/DataTypes/Serializations/SerializationObject.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 805a11521b3..1b95fddee9f 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -9,8 +9,6 @@ #include #include -#include - namespace DB { From cb833a009aa1ff5c9aa3296d76dd22a06d39e7e5 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 22 Nov 2024 20:29:02 +0000 Subject: [PATCH 173/502] Fix tests --- src/Storages/MergeTree/MergeTreeReaderCompact.cpp | 9 ++++++--- src/Storages/MergeTree/MergeTreeReaderCompact.h | 3 ++- .../MergeTree/MergeTreeReaderCompactSingleBuffer.cpp | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 06635c39838..a4fa3a30d23 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -149,7 +149,8 @@ void MergeTreeReaderCompact::readData( size_t rows_to_read, const InputStreamGetter & getter, ISerialization::SubstreamsCache & cache, - std::unordered_map & columns_cache_for_subcolumns) + std::unordered_map & columns_cache_for_subcolumns, + const ColumnNameLevel & name_level_for_offsets) { try { @@ -172,7 +173,8 @@ void MergeTreeReaderCompact::readData( const auto & type_in_storage = name_and_type.getTypeInStorage(); const auto & name_in_storage = name_and_type.getNameInStorage(); - if (auto cache_for_subcolumns_it = columns_cache_for_subcolumns.find(name_in_storage); cache_for_subcolumns_it != columns_cache_for_subcolumns.end()) + auto cache_for_subcolumns_it = columns_cache_for_subcolumns.find(name_in_storage); + if (!name_level_for_offsets.has_value() && cache_for_subcolumns_it != columns_cache_for_subcolumns.end()) { auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), cache_for_subcolumns_it->second); /// TODO: Avoid extra copying. @@ -195,7 +197,8 @@ void MergeTreeReaderCompact::readData( else column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); - columns_cache_for_subcolumns[name_in_storage] = temp_column; + if (!name_level_for_offsets.has_value()) + columns_cache_for_subcolumns[name_in_storage] = temp_column; } } else diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index f18ee8808ec..7dbebb75a91 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -46,7 +46,8 @@ protected: size_t rows_to_read, const InputStreamGetter & getter, ISerialization::SubstreamsCache & cache, - std::unordered_map & columns_cache_for_subcolumns); + std::unordered_map & columns_cache_for_subcolumns, + const ColumnNameLevel & name_level_for_offsets); void readPrefix( const NameAndTypePair & name_and_type, diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index 50224eba82d..0c3e67d5078 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -62,7 +62,7 @@ try }; readPrefix(columns_to_read[pos], buffer_getter, buffer_getter_for_prefix, columns_for_offsets[pos]); - readData(columns_to_read[pos], column, rows_to_read, buffer_getter, cache, columns_cache_for_subcolumns); + readData(columns_to_read[pos], column, rows_to_read, buffer_getter, cache, columns_cache_for_subcolumns, columns_for_offsets[pos]); } ++from_mark; From f753b16f9d9533dea5d578d1ed3bfccd6d790850 Mon Sep 17 00:00:00 2001 From: jotosoares <165800096+jotosoares@users.noreply.github.com> Date: Sun, 24 Nov 2024 20:52:27 -0300 Subject: [PATCH 174/502] docs(view.md): remove experimental from title --- docs/en/sql-reference/statements/create/view.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index c770348bce0..141538d502c 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -154,7 +154,7 @@ This feature is deprecated and will be removed in the future. For your convenience, the old documentation is located [here](https://pastila.nl/?00f32652/fdf07272a7b54bda7e13b919264e449f.md) -## Refreshable Materialized View [Experimental] {#refreshable-materialized-view} +## Refreshable Materialized View {#refreshable-materialized-view} ```sql CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name From 1016add3d8cddc06285f02da0fc7dfcf44543b9d Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 25 Nov 2024 14:22:54 +0100 Subject: [PATCH 175/502] Add comments, fix test --- src/Databases/Iceberg/DatabaseIceberg.cpp | 2 +- src/Databases/Iceberg/ICatalog.cpp | 15 ------- src/Databases/Iceberg/ICatalog.h | 33 ++++++++++----- src/Databases/Iceberg/RestCatalog.cpp | 40 +++++++++++++------ src/Databases/Iceberg/RestCatalog.h | 6 +-- .../integration/test_database_iceberg/test.py | 5 ++- 6 files changed, 58 insertions(+), 43 deletions(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index c672d805e5e..6f06d14d30d 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -249,7 +249,7 @@ DatabaseTablesIteratorPtr DatabaseIceberg::getTablesIterator( if (filter_by_table_name && !filter_by_table_name(table_name)) continue; - runner([&]{ + runner([=, &tables, &mutexx, this]{ auto storage = tryGetTable(table_name, context_); { std::lock_guard lock(mutexx); diff --git a/src/Databases/Iceberg/ICatalog.cpp b/src/Databases/Iceberg/ICatalog.cpp index 6b6ecc85daf..db8ad443c0e 100644 --- a/src/Databases/Iceberg/ICatalog.cpp +++ b/src/Databases/Iceberg/ICatalog.cpp @@ -68,19 +68,4 @@ const DB::NamesAndTypesList & TableMetadata::getSchema() const return schema; } -StorageType ICatalog::getStorageType(const std::string & location) -{ - auto pos = location.find("://"); - if (pos == std::string::npos) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unexpected path format: {}", location); - - auto storage_type_str = location.substr(0, pos); - auto storage_type = magic_enum::enum_cast(Poco::toUpper(storage_type_str)); - - if (!storage_type) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unsupported storage type: {}", storage_type_str); - - return *storage_type; -} - } diff --git a/src/Databases/Iceberg/ICatalog.h b/src/Databases/Iceberg/ICatalog.h index 4c61c87998d..aba1884ab3e 100644 --- a/src/Databases/Iceberg/ICatalog.h +++ b/src/Databases/Iceberg/ICatalog.h @@ -7,6 +7,8 @@ namespace Iceberg { using StorageType = DB::DatabaseIcebergStorageType; +/// A class representing table metadata, +/// which was received from Catalog. class TableMetadata { public: @@ -15,23 +17,21 @@ public: TableMetadata & withLocation() { with_location = true; return *this; } TableMetadata & withSchema() { with_schema = true; return *this; } + void setLocation(const std::string & location_); std::string getLocation(bool path_only) const; - std::string getLocation() const; - std::string getLocationWithoutPath() const; + void setSchema(const DB::NamesAndTypesList & schema_); const DB::NamesAndTypesList & getSchema() const; bool requiresLocation() const { return with_location; } bool requiresSchema() const { return with_schema; } - void setLocation(const std::string & location_); - void setSchema(const DB::NamesAndTypesList & schema_); - private: - /// starts with s3://, file://, etc + /// Starts with s3://, file://, etc. + /// For example, `s3://bucket/` std::string location_without_path; + /// Path to table's data: `/path/to/table/data/` std::string path; - /// column names and types DB::NamesAndTypesList schema; bool with_location = false; @@ -39,40 +39,51 @@ private: }; +/// Base class for catalog implementation. +/// Used for communication with the catalog. class ICatalog { public: using Namespaces = std::vector; - using Tables = std::vector; explicit ICatalog(const std::string & warehouse_) : warehouse(warehouse_) {} virtual ~ICatalog() = default; + /// Does catalog have any tables? virtual bool empty() const = 0; - virtual Tables getTables() const = 0; + /// Fetch tables' names list. + /// Contains full namespaces in names. + virtual DB::Names getTables() const = 0; + /// Check that a table exists in a given namespace. virtual bool existsTable( const std::string & namespace_naem, const std::string & table_name) const = 0; + /// Get table metadata in the given namespace. + /// Throw exception if table does not exist. virtual void getTableMetadata( const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const = 0; + /// Get table metadata in the given namespace. + /// Return `false` if table does not exist, `true` otherwise. virtual bool tryGetTableMetadata( const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const = 0; + /// Get storage type, where Iceberg tables' data is stored. + /// E.g. one of S3, Azure, Local, HDFS. virtual std::optional getStorageType() const = 0; protected: + /// Name of the warehouse, + /// which is sometimes also called "catalog name". const std::string warehouse; - - static StorageType getStorageType(const std::string & location); }; } diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index a34aa45b812..345222fcea8 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -74,6 +74,21 @@ DB::HTTPHeaderEntry parseAuthHeader(const std::string & auth_header) return DB::HTTPHeaderEntry(auth_header.substr(0, pos), auth_header.substr(pos + 1)); } +StorageType parseStorageTypeFromLocation(const std::string & location) +{ + auto pos = location.find("://"); + if (pos == std::string::npos) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unexpected path format: {}", location); + + auto storage_type_str = location.substr(0, pos); + auto storage_type = magic_enum::enum_cast(Poco::toUpper(storage_type_str)); + + if (!storage_type) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unsupported storage type: {}", storage_type_str); + + return *storage_type; +} + } std::string RestCatalog::Config::toString() const @@ -220,7 +235,7 @@ std::optional RestCatalog::getStorageType() const { if (config.default_base_location.empty()) return std::nullopt; - return ICatalog::getStorageType(config.default_base_location); + return parseStorageTypeFromLocation(config.default_base_location); } DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( @@ -288,7 +303,7 @@ bool RestCatalog::empty() const } } -RestCatalog::Tables RestCatalog::getTables() const +DB::Names RestCatalog::getTables() const { size_t num_threads = 10; ThreadPool pool( @@ -299,18 +314,16 @@ RestCatalog::Tables RestCatalog::getTables() const DB::ThreadPoolCallbackRunnerLocal runner(pool, "RestCatalog"); - Tables tables; + DB::Names tables; std::mutex mutex; auto func = [&](const std::string & current_namespace) { runner( - [&]{ + [=, &tables, &mutex, this]{ auto tables_in_namespace = getTables(current_namespace); - { - std::lock_guard lock(mutex); - std::move(tables_in_namespace.begin(), tables_in_namespace.end(), std::back_inserter(tables)); - } + std::lock_guard lock(mutex); + std::move(tables_in_namespace.begin(), tables_in_namespace.end(), std::back_inserter(tables)); }); }; @@ -369,7 +382,7 @@ RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_name { auto buf = createReadBuffer(config.prefix / namespaces_endpoint, params); auto namespaces = parseNamespaces(*buf, base_namespace); - LOG_TEST(log, "Loaded {} namespaces", namespaces.size()); + LOG_TEST(log, "Loaded {} namespaces in base namespace {}", namespaces.size(), base_namespace); return namespaces; } catch (const DB::HTTPException & e) @@ -426,14 +439,14 @@ RestCatalog::Namespaces RestCatalog::parseNamespaces(DB::ReadBuffer & buf, const return namespaces; } -RestCatalog::Tables RestCatalog::getTables(const std::string & base_namespace, size_t limit) const +DB::Names RestCatalog::getTables(const std::string & base_namespace, size_t limit) const { const auto endpoint = std::string(namespaces_endpoint) + "/" + base_namespace + "/tables"; auto buf = createReadBuffer(config.prefix / endpoint); return parseTables(*buf, base_namespace, limit); } -RestCatalog::Tables RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit) const +DB::Names RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit) const { if (buf.eof()) return {}; @@ -449,12 +462,15 @@ RestCatalog::Tables RestCatalog::parseTables(DB::ReadBuffer & buf, const std::st if (!identifiers_object) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot parse result"); - Tables tables; + DB::Names tables; for (size_t i = 0; i < identifiers_object->size(); ++i) { const auto current_table_json = identifiers_object->get(static_cast(i)).extract(); const auto table_name = current_table_json->get("name").extract(); + LOG_TEST(log, "Base namespace: {}", base_namespace); + LOG_TEST(log, "Table name: {}", table_name); + tables.push_back(base_namespace + "." + table_name); if (limit && tables.size() >= limit) break; diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index 402e761b07a..5e92ec846a0 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -29,7 +29,7 @@ public: bool empty() const override; - Tables getTables() const override; + DB::Names getTables() const override; bool existsTable(const std::string & namespace_name, const std::string & table_name) const override; @@ -93,9 +93,9 @@ private: Namespaces parseNamespaces(DB::ReadBuffer & buf, const std::string & base_namespace) const; - Tables getTables(const std::string & base_namespace, size_t limit = 0) const; + DB::Names getTables(const std::string & base_namespace, size_t limit = 0) const; - Tables parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit) const; + DB::Names parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit) const; bool getTableMetadataImpl( const std::string & namespace_name, diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 51d89444c2e..c0f0087522d 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -122,7 +122,10 @@ def create_clickhouse_iceberg_database(started_cluster, node, name): f""" DROP DATABASE IF EXISTS {name}; CREATE DATABASE {name} ENGINE = Iceberg('{BASE_URL}', 'minio', 'minio123') -SETTINGS catalog_type = 'rest', storage_endpoint = 'http://minio:9000/' +SETTINGS catalog_type = 'rest', + storage_endpoint = 'http://minio:9000/warehouse', + warehouse='demo', + storage_type='s3' """ ) From deb3751a3a607459a9296c84c472ece1c33af986 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 25 Nov 2024 14:36:09 +0000 Subject: [PATCH 176/502] Fix broken tests --- .../DataLakes/IcebergMetadata.cpp | 63 +++++++++++-------- .../ObjectStorage/DataLakes/IcebergMetadata.h | 3 + 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index c63a61860df..36f901575f9 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -270,31 +270,11 @@ bool IcebergSchemaProcessor::allowPrimitiveTypeConversion(const String & old_typ return allowed_type_conversion; } -std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id) + +// Ids are passed only for error logging purposes +std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag( + const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema, Int32 old_id, Int32 new_id) { - std::lock_guard lock(mutex); - Poco::JSON::Object::Ptr old_schema, new_schema; - auto required_transform_dag_it = transform_dags_by_ids.find({old_id, new_id}); - if (required_transform_dag_it != transform_dags_by_ids.end()) - { - return required_transform_dag_it->second; - } - - if (old_id == new_id) - { - return nullptr; - } - - auto old_schema_it = iceberg_table_schemas_by_ids.find(old_id); - if (old_schema_it == iceberg_table_schemas_by_ids.end()) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", old_id); - } - auto new_schema_it = iceberg_table_schemas_by_ids.find(new_id); - if (new_schema_it == iceberg_table_schemas_by_ids.end()) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", new_id); - } std::unordered_map> old_schema_entries; auto old_schema_fields = old_schema->get("fields").extract(); std::shared_ptr dag = std::make_shared(); @@ -325,7 +305,8 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformatio { throw Exception( ErrorCodes::UNSUPPORTED_METHOD, - "Schema evolution is not supported for complex types yet, field id is {}, old schema id is {}, new schema id is {}", + "Schema evolution is not supported for complex types yet, field id is {}, old schema id is {}, new schema id " + "is {}", id, old_id, new_id); @@ -370,7 +351,8 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformatio { throw Exception( ErrorCodes::UNSUPPORTED_METHOD, - "Adding a default column with id {} and complex type is not supported yet. Old schema id is {}, new schema id is {}", + "Adding a default column with id {} and complex type is not supported yet. Old schema id is {}, new schema id is " + "{}", id, old_id, new_id); @@ -391,7 +373,34 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformatio outputs.push_back(&constant); } } - return transform_dags_by_ids[{old_id, new_id}] = dag; + return dag; +} + +std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id) +{ + if (old_id == new_id) + { + return nullptr; + } + std::lock_guard lock(mutex); + auto required_transform_dag_it = transform_dags_by_ids.find({old_id, new_id}); + if (required_transform_dag_it != transform_dags_by_ids.end()) + { + return required_transform_dag_it->second; + } + + auto old_schema_it = iceberg_table_schemas_by_ids.find(old_id); + if (old_schema_it == iceberg_table_schemas_by_ids.end()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", old_id); + } + auto new_schema_it = iceberg_table_schemas_by_ids.find(new_id); + if (new_schema_it == iceberg_table_schemas_by_ids.end()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Schema with schema-id {} is unknown", new_id); + } + return transform_dags_by_ids[{old_id, new_id}] + = getSchemaTransformationDag(old_schema_it->second, new_schema_it->second, old_id, new_id); } void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schema_ptr) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index 4bb55adf9da..fb5a7800228 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -99,6 +99,9 @@ private: bool allowPrimitiveTypeConversion(const String & old_type, const String & new_type); const Node * getDefaultNodeForField(const Poco::JSON::Object::Ptr & field); + std::shared_ptr getSchemaTransformationDag( + const Poco::JSON::Object::Ptr & old_schema, const Poco::JSON::Object::Ptr & new_schema, Int32 old_id, Int32 new_id); + std::mutex mutex; }; From 8393e4382ca490c98624fc9c46ceec45bddb48f5 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 25 Nov 2024 17:36:50 +0100 Subject: [PATCH 177/502] Better --- src/Core/ServerSettings.cpp | 2 + src/Databases/Iceberg/DatabaseIceberg.cpp | 14 +- src/Databases/Iceberg/DatabaseIceberg.h | 4 - .../Iceberg/DatabaseIcebergSettings.cpp | 1 + src/Databases/Iceberg/ICatalog.h | 2 + src/Databases/Iceberg/RestCatalog.cpp | 126 +++++++++++------- src/Databases/Iceberg/RestCatalog.h | 3 + src/Interpreters/Context.cpp | 24 ++++ src/Interpreters/Context.h | 2 + 9 files changed, 115 insertions(+), 63 deletions(-) diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 4bea23d4e90..47424986c9e 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -205,6 +205,8 @@ namespace DB DECLARE(UInt64, load_marks_threadpool_queue_size, 1000000, "Number of tasks which is possible to push into prefetches pool", 0) \ DECLARE(UInt64, threadpool_writer_pool_size, 100, "Size of background pool for write requests to object storages", 0) \ DECLARE(UInt64, threadpool_writer_queue_size, 1000000, "Number of tasks which is possible to push into background pool for write requests to object storages", 0) \ + DECLARE(UInt64, iceberg_catalog_threadpool_pool_size, 50, "Size of background pool for iceberg catalog", 0) \ + DECLARE(UInt64, iceberg_catalog_threadpool_queue_size, 1000000, "Number of tasks which is possible to push into iceberg catalog pool", 0) \ DECLARE(UInt32, allowed_feature_tier, 0, "0 - All feature tiers allowed (experimental, beta, production). 1 - Only beta and production feature tiers allowed. 2 - Only production feature tier allowed", 0) \ diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 6f06d14d30d..2d096ed4d64 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -40,6 +40,7 @@ namespace DatabaseIcebergSetting extern const DatabaseIcebergSettingsString warehouse; extern const DatabaseIcebergSettingsString catalog_credential; extern const DatabaseIcebergSettingsString auth_header; + extern const DatabaseIcebergSettingsString auth_scope; extern const DatabaseIcebergSettingsString storage_endpoint; } @@ -116,6 +117,7 @@ std::shared_ptr DatabaseIceberg::getCatalog(ContextPtr) const settings[DatabaseIcebergSetting::warehouse].value, url, settings[DatabaseIcebergSetting::catalog_credential].value, + settings[DatabaseIcebergSetting::auth_scope].value, settings[DatabaseIcebergSetting::auth_header], Context::getGlobalContextInstance()); } @@ -125,6 +127,8 @@ std::shared_ptr DatabaseIceberg::getCatalog(ContextPtr) const std::shared_ptr DatabaseIceberg::getConfiguration() const { + /// TODO: add tests for azure, local storage types. + switch (settings[DatabaseIcebergSetting::storage_type].value) { #if USE_AWS_S3 @@ -232,15 +236,9 @@ DatabaseTablesIteratorPtr DatabaseIceberg::getTablesIterator( { Tables tables; auto catalog = getCatalog(context_); + const auto iceberg_tables = catalog->getTables(); - auto iceberg_tables = catalog->getTables(); - size_t num_threads = std::min(10, iceberg_tables.size()); - ThreadPool pool( - CurrentMetrics::IcebergCatalogThreads, - CurrentMetrics::IcebergCatalogThreadsActive, - CurrentMetrics::IcebergCatalogThreadsScheduled, - num_threads); - + auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::ThreadPoolCallbackRunnerLocal runner(pool, "RestCatalog"); std::mutex mutexx; diff --git a/src/Databases/Iceberg/DatabaseIceberg.h b/src/Databases/Iceberg/DatabaseIceberg.h index 145f033ad6a..4b9929a3a76 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.h +++ b/src/Databases/Iceberg/DatabaseIceberg.h @@ -11,10 +11,6 @@ namespace DB { -/// TODO: -/// - auth: oauth, bearer token? -/// - tests with azure, hdfs, local - class DatabaseIceberg final : public IDatabase, WithContext { public: diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp index 1dc1ba1f61c..d5c13d69f9d 100644 --- a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp @@ -18,6 +18,7 @@ namespace ErrorCodes DECLARE(DatabaseIcebergCatalogType, catalog_type, DatabaseIcebergCatalogType::REST, "Catalog type", 0) \ DECLARE(DatabaseIcebergStorageType, storage_type, DatabaseIcebergStorageType::S3, "Storage type: S3, Local, Azure, HDFS", 0) \ DECLARE(String, catalog_credential, "", "", 0) \ + DECLARE(String, auth_scope, "PRINCIPAL_ROLE:ALL", "Authorization scope for client credentials or token exchange", 0) \ DECLARE(String, warehouse, "", "Warehouse name inside the catalog", 0) \ DECLARE(String, auth_header, "", "Authorization header of format 'Authorization: '", 0) \ DECLARE(String, storage_endpoint, "", "Object storage endpoint", 0) \ diff --git a/src/Databases/Iceberg/ICatalog.h b/src/Databases/Iceberg/ICatalog.h index aba1884ab3e..aa574509c8a 100644 --- a/src/Databases/Iceberg/ICatalog.h +++ b/src/Databases/Iceberg/ICatalog.h @@ -34,6 +34,8 @@ private: std::string path; DB::NamesAndTypesList schema; + std::string credentials; + bool with_location = false; bool with_schema = false; }; diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index 345222fcea8..149a26f107a 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -25,13 +25,6 @@ namespace DB::ErrorCodes extern const int BAD_ARGUMENTS; } -namespace CurrentMetrics -{ - extern const Metric IcebergCatalogThreads; - extern const Metric IcebergCatalogThreadsActive; - extern const Metric IcebergCatalogThreadsScheduled; -} - namespace Iceberg { @@ -76,15 +69,26 @@ DB::HTTPHeaderEntry parseAuthHeader(const std::string & auth_header) StorageType parseStorageTypeFromLocation(const std::string & location) { + /// Table location in catalog metadata always starts with one of s3://, file://, etc. + /// So just extract this part of the path and deduce storage type from it. + auto pos = location.find("://"); if (pos == std::string::npos) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unexpected path format: {}", location); + { + throw DB::Exception( + DB::ErrorCodes::NOT_IMPLEMENTED, + "Unexpected path format: {}", location); + } auto storage_type_str = location.substr(0, pos); auto storage_type = magic_enum::enum_cast(Poco::toUpper(storage_type_str)); if (!storage_type) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unsupported storage type: {}", storage_type_str); + { + throw DB::Exception( + DB::ErrorCodes::NOT_IMPLEMENTED, + "Unsupported storage type: {}", storage_type_str); + } return *storage_type; } @@ -108,15 +112,20 @@ RestCatalog::RestCatalog( const std::string & warehouse_, const std::string & base_url_, const std::string & catalog_credential_, + const std::string & auth_scope_, const std::string & auth_header_, DB::ContextPtr context_) : ICatalog(warehouse_) , DB::WithContext(context_) , base_url(base_url_) , log(getLogger("RestCatalog(" + warehouse_ + ")")) + , auth_scope(auth_scope_) { if (!catalog_credential_.empty()) + { std::tie(client_id, client_secret) = parseCatalogCredential(catalog_credential_); + update_token_if_expired = true; + } else if (!auth_header_.empty()) auth_header = parseAuthHeader(auth_header_); @@ -163,11 +172,16 @@ void RestCatalog::parseCatalogConfigurationSettings(const Poco::JSON::Object::Pt DB::HTTPHeaderEntries RestCatalog::getHeaders(bool update_token) const { + /// Option 1: user specified auth header manually. + /// Header has format: 'Authorization: '. if (auth_header.has_value()) { return DB::HTTPHeaderEntries{auth_header.value()}; } + /// Option 2: user provided grant_type, client_id and client_secret. + /// We would make OAuthClientCredentialsRequest + /// https://github.com/apache/iceberg/blob/3badfe0c1fcf0c0adfc7aa4a10f0b50365c48cf9/open-api/rest-catalog-open-api.yaml#L3498C5-L3498C34 if (!client_id.empty()) { if (!access_token.has_value() || update_token) @@ -191,12 +205,6 @@ std::string RestCatalog::retrieveAccessToken() const /// 1. support oauth2-server-uri /// https://github.com/apache/iceberg/blob/918f81f3c3f498f46afcea17c1ac9cdc6913cb5c/open-api/rest-catalog-open-api.yaml#L183C82-L183C99 - Poco::JSON::Object json; - json.set("grant_type", "client_credentials"); - json.set("scope", "PRINCIPAL_ROLE:ALL"); /// TODO: add it into setting. - json.set("client_id", client_id); - json.set("client_secret", client_secret); - DB::HTTPHeaderEntries headers; headers.emplace_back("Content-Type", "application/x-www-form-urlencoded"); headers.emplace_back("Accepts", "application/json; charset=UTF-8"); @@ -204,7 +212,7 @@ std::string RestCatalog::retrieveAccessToken() const Poco::URI url(base_url / oauth_tokens_endpoint); Poco::URI::QueryParameters params = { {"grant_type", "client_credentials"}, - {"scope", "PRINCIPAL_ROLE:ALL"}, + {"scope", auth_scope}, {"client_id", client_id}, {"client_secret", client_secret}, }; @@ -248,34 +256,35 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( if (!params.empty()) url.setQueryParameters(params); - auto headers = getHeaders(false); + auto create_buffer = [&](bool update_token) + { + auto headers = getHeaders(update_token); + return DB::BuilderRWBufferFromHTTP(url) + .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) + .withSettings(getContext()->getReadSettings()) + .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) + .withHostFilter(&getContext()->getRemoteHostFilter()) + .withHeaders(headers) + .withDelayInit(false) + .withSkipNotFound(false) + .create(credentials); + }; LOG_TEST(log, "Requesting: {}", url.toString()); try { - return DB::BuilderRWBufferFromHTTP(url) - .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) - .withSettings(getContext()->getReadSettings()) - .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) - .withHeaders(headers) - .withHostFilter(&getContext()->getRemoteHostFilter()) - .withDelayInit(false) - .withSkipNotFound(false) - .create(credentials); + return create_buffer(false); } - catch (...) + catch (const DB::HTTPException & e) { - auto new_headers = getHeaders(true); - return DB::BuilderRWBufferFromHTTP(url) - .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) - .withSettings(getContext()->getReadSettings()) - .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) - .withHeaders(new_headers) - .withHostFilter(&getContext()->getRemoteHostFilter()) - .withDelayInit(false) - .withSkipNotFound(false) - .create(credentials); + if (update_token_if_expired && + (e.code() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED + || e.code() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN)) + { + return create_buffer(true); + } + throw; } } @@ -292,7 +301,7 @@ bool RestCatalog::empty() const }; Namespaces namespaces; - getNamespacesRecursive("", namespaces, stop_condition, {}); + getNamespacesRecursive("", namespaces, stop_condition, /* execute_func */{}); return found_table; } @@ -305,22 +314,17 @@ bool RestCatalog::empty() const DB::Names RestCatalog::getTables() const { - size_t num_threads = 10; - ThreadPool pool( - CurrentMetrics::IcebergCatalogThreads, - CurrentMetrics::IcebergCatalogThreadsActive, - CurrentMetrics::IcebergCatalogThreadsScheduled, - num_threads); - + auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::ThreadPoolCallbackRunnerLocal runner(pool, "RestCatalog"); DB::Names tables; std::mutex mutex; - auto func = [&](const std::string & current_namespace) + auto execute_for_each_namespace = [&](const std::string & current_namespace) { runner( - [=, &tables, &mutex, this]{ + [=, &tables, &mutex, this] + { auto tables_in_namespace = getTables(current_namespace); std::lock_guard lock(mutex); std::move(tables_in_namespace.begin(), tables_in_namespace.end(), std::back_inserter(tables)); @@ -328,7 +332,11 @@ DB::Names RestCatalog::getTables() const }; Namespaces namespaces; - getNamespacesRecursive("", namespaces, {}, func); + getNamespacesRecursive( + /* base_namespace */"", /// Empty base namespace means starting from root. + namespaces, + /* stop_condition */{}, + /* execute_func */execute_for_each_namespace); runner.waitForAllToFinishAndRethrowFirstError(); return tables; @@ -468,9 +476,6 @@ DB::Names RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & bas const auto current_table_json = identifiers_object->get(static_cast(i)).extract(); const auto table_name = current_table_json->get("name").extract(); - LOG_TEST(log, "Base namespace: {}", base_namespace); - LOG_TEST(log, "Table name: {}", table_name); - tables.push_back(base_namespace + "." + table_name); if (limit && tables.size() >= limit) break; @@ -550,6 +555,25 @@ bool RestCatalog::getTableMetadataImpl( int format_version = metadata_object->getValue("format-version"); result.setSchema(DB::IcebergMetadata::parseTableSchema(metadata_object, format_version, true).first); } + + // if (result.requiresCredentials()) + // { + // try + // { + // const auto credentials_endpoint = std::filesystem::path(endpoint) / "credentials"; + // auto credentials_buf = createReadBuffer(config.prefix / credentials_endpoint); + + // String credentials_json_str; + // readJSONObjectPossiblyInvalid(credentials_json_str, *credentials_buf); + + // LOG_TEST(log, "Credentials : {}", credentials_json_str); + // } + // catch (...) + // { + // DB::tryLogCurrentException(log); + // } + // } + return true; } diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index 5e92ec846a0..570707e3aa0 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -22,6 +22,7 @@ public: const std::string & warehouse_, const std::string & base_url_, const std::string & catalog_credential_, + const std::string & auth_scope_, const std::string & auth_header_, DB::ContextPtr context_); @@ -68,8 +69,10 @@ private: std::optional auth_header; /// Parameters for OAuth. + bool update_token_if_expired = false; std::string client_id; std::string client_secret; + std::string auth_scope; mutable std::optional access_token; Poco::Net::HTTPBasicCredentials credentials{}; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 7f0ad013c1d..45aa69eb93d 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -174,6 +174,9 @@ namespace CurrentMetrics extern const Metric AttachedDictionary; extern const Metric AttachedDatabase; extern const Metric PartsActive; + extern const Metric IcebergCatalogThreads; + extern const Metric IcebergCatalogThreadsActive; + extern const Metric IcebergCatalogThreadsScheduled; } @@ -282,6 +285,8 @@ namespace ServerSetting extern const ServerSettingsUInt64 load_marks_threadpool_queue_size; extern const ServerSettingsUInt64 threadpool_writer_pool_size; extern const ServerSettingsUInt64 threadpool_writer_queue_size; + extern const ServerSettingsUInt64 iceberg_catalog_threadpool_pool_size; + extern const ServerSettingsUInt64 iceberg_catalog_threadpool_queue_size; } @@ -412,6 +417,8 @@ struct ContextSharedPart : boost::noncopyable mutable std::unique_ptr load_marks_threadpool; /// Threadpool for loading marks cache. mutable OnceFlag prefetch_threadpool_initialized; mutable std::unique_ptr prefetch_threadpool; /// Threadpool for loading marks cache. + mutable std::unique_ptr iceberg_catalog_threadpool; + mutable OnceFlag iceberg_catalog_threadpool_initialized; mutable OnceFlag build_vector_similarity_index_threadpool_initialized; mutable std::unique_ptr build_vector_similarity_index_threadpool; /// Threadpool for vector-similarity index creation. mutable UncompressedCachePtr index_uncompressed_cache TSA_GUARDED_BY(mutex); /// The cache of decompressed blocks for MergeTree indices. @@ -3254,6 +3261,23 @@ ThreadPool & Context::getLoadMarksThreadpool() const return *shared->load_marks_threadpool; } +ThreadPool & Context::getIcebergCatalogThreadpool() const +{ + callOnce(shared->iceberg_catalog_threadpool_initialized, [&] + { + auto pool_size = shared->server_settings[ServerSetting::iceberg_catalog_threadpool_pool_size]; + auto queue_size = shared->server_settings[ServerSetting::iceberg_catalog_threadpool_queue_size]; + + shared->iceberg_catalog_threadpool = std::make_unique( + CurrentMetrics::IcebergCatalogThreads, + CurrentMetrics::IcebergCatalogThreadsActive, + CurrentMetrics::IcebergCatalogThreadsScheduled, + pool_size, pool_size, queue_size); + }); + + return *shared->iceberg_catalog_threadpool; +} + void Context::setIndexUncompressedCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) { std::lock_guard lock(shared->mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 327ac0af5fd..f97bd055890 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1076,6 +1076,8 @@ public: void clearMarkCache() const; ThreadPool & getLoadMarksThreadpool() const; + ThreadPool & getIcebergCatalogThreadpool() const; + void setIndexUncompressedCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); void updateIndexUncompressedCacheConfiguration(const Poco::Util::AbstractConfiguration & config); std::shared_ptr getIndexUncompressedCache() const; From c22e45dff7e2f4ebaeaab0b25cd7b90b1105cba0 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 25 Nov 2024 18:59:41 +0100 Subject: [PATCH 178/502] Support vended credentials --- src/Databases/Iceberg/DatabaseIceberg.cpp | 29 +++++++- .../Iceberg/DatabaseIcebergSettings.cpp | 1 + .../Iceberg/DatabaseIcebergSettings.h | 1 + src/Databases/Iceberg/ICatalog.cpp | 15 +++++ src/Databases/Iceberg/ICatalog.h | 11 +++- src/Databases/Iceberg/RestCatalog.cpp | 66 +++++++++++++------ src/Databases/Iceberg/RestCatalog.h | 5 +- src/Databases/Iceberg/StorageCredentials.h | 50 ++++++++++++++ 8 files changed, 152 insertions(+), 26 deletions(-) create mode 100644 src/Databases/Iceberg/StorageCredentials.h diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 2d096ed4d64..5223bf14452 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -42,6 +42,7 @@ namespace DatabaseIcebergSetting extern const DatabaseIcebergSettingsString auth_header; extern const DatabaseIcebergSettingsString auth_scope; extern const DatabaseIcebergSettingsString storage_endpoint; + extern const DatabaseIcebergSettingsBool vended_credentials; } namespace ErrorCodes @@ -192,6 +193,11 @@ StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_ { auto catalog = getCatalog(context_); auto table_metadata = Iceberg::TableMetadata().withLocation().withSchema(); + + const bool with_vended_credentials = settings[DatabaseIcebergSetting::vended_credentials].value; + if (with_vended_credentials) + table_metadata = table_metadata.withStorageCredentials(); + auto [namespace_name, table_name] = parseTableName(name); if (!catalog->tryGetTableMetadata(namespace_name, table_name, table_metadata)) @@ -205,6 +211,25 @@ StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_ auto table_endpoint = getStorageEndpointForTable(table_metadata); args[0] = std::make_shared(table_endpoint); + /// We either fetch storage credentials from catalog " + /// "or get storage credentials from database engine arguments + /// in CREATE query (e.g. in `args`). + /// Vended credentials can be disabled in catalog itself, + /// so we have a separate setting to know whether we should even try to fetch them. + if (with_vended_credentials && args.size() == 1) + { + auto storage_credentials = table_metadata.getStorageCredentials(); + if (storage_credentials) + storage_credentials->addCredentialsToEngineArgs(args); + } + else if (args.size() == 1) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Either vended credentials need to be enabled " + "or storage credentials need to be specified in database engine arguements in CREATE query"); + } + LOG_TEST(log, "Using table endpoint: {}", table_endpoint); const auto columns = ColumnsDescription(table_metadata.getSchema()); @@ -238,7 +263,7 @@ DatabaseTablesIteratorPtr DatabaseIceberg::getTablesIterator( auto catalog = getCatalog(context_); const auto iceberg_tables = catalog->getTables(); - auto & pool = getContext()->getIcebergCatalogThreadpool(); + auto & pool = context_->getIcebergCatalogThreadpool(); DB::ThreadPoolCallbackRunnerLocal runner(pool, "RestCatalog"); std::mutex mutexx; @@ -335,7 +360,7 @@ void registerDatabaseIceberg(DatabaseFactory & factory) if (engine_args.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine `{}` must have arguments", database_engine_name); - const size_t max_args_num = 3; + const size_t max_args_num = 1; if (engine_args.size() != max_args_num) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine must have {} arguments", max_args_num); diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp index d5c13d69f9d..de04fc0bd11 100644 --- a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp @@ -18,6 +18,7 @@ namespace ErrorCodes DECLARE(DatabaseIcebergCatalogType, catalog_type, DatabaseIcebergCatalogType::REST, "Catalog type", 0) \ DECLARE(DatabaseIcebergStorageType, storage_type, DatabaseIcebergStorageType::S3, "Storage type: S3, Local, Azure, HDFS", 0) \ DECLARE(String, catalog_credential, "", "", 0) \ + DECLARE(Bool, vended_credentials, true, "Use vended credentials (storage credentials) from catalog", 0) \ DECLARE(String, auth_scope, "PRINCIPAL_ROLE:ALL", "Authorization scope for client credentials or token exchange", 0) \ DECLARE(String, warehouse, "", "Warehouse name inside the catalog", 0) \ DECLARE(String, auth_header, "", "Authorization header of format 'Authorization: '", 0) \ diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.h b/src/Databases/Iceberg/DatabaseIcebergSettings.h index 5d9d120efed..4e5bc0defba 100644 --- a/src/Databases/Iceberg/DatabaseIcebergSettings.h +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.h @@ -16,6 +16,7 @@ class SettingsChanges; #define DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(CLASS_NAME, M) \ M(CLASS_NAME, String) \ M(CLASS_NAME, UInt64) \ + M(CLASS_NAME, Bool) \ M(CLASS_NAME, DatabaseIcebergCatalogType) \ M(CLASS_NAME, DatabaseIcebergStorageType) diff --git a/src/Databases/Iceberg/ICatalog.cpp b/src/Databases/Iceberg/ICatalog.cpp index db8ad443c0e..97b2bfeeef9 100644 --- a/src/Databases/Iceberg/ICatalog.cpp +++ b/src/Databases/Iceberg/ICatalog.cpp @@ -68,4 +68,19 @@ const DB::NamesAndTypesList & TableMetadata::getSchema() const return schema; } +void TableMetadata::setStorageCredentials(std::shared_ptr credentials_) +{ + if (!with_storage_credentials) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Storage credentials were not requested"); + + storage_credentials = std::move(credentials_); +} + +std::shared_ptr TableMetadata::getStorageCredentials() const +{ + if (!with_storage_credentials) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Data schema was not requested"); + + return storage_credentials; +} } diff --git a/src/Databases/Iceberg/ICatalog.h b/src/Databases/Iceberg/ICatalog.h index aa574509c8a..9657ef6ba41 100644 --- a/src/Databases/Iceberg/ICatalog.h +++ b/src/Databases/Iceberg/ICatalog.h @@ -2,6 +2,7 @@ #include #include #include +#include namespace Iceberg { @@ -16,6 +17,7 @@ public: TableMetadata & withLocation() { with_location = true; return *this; } TableMetadata & withSchema() { with_schema = true; return *this; } + TableMetadata & withStorageCredentials() { with_storage_credentials = true; return *this; } void setLocation(const std::string & location_); std::string getLocation(bool path_only) const; @@ -23,8 +25,12 @@ public: void setSchema(const DB::NamesAndTypesList & schema_); const DB::NamesAndTypesList & getSchema() const; + void setStorageCredentials(std::shared_ptr credentials_); + std::shared_ptr getStorageCredentials() const; + bool requiresLocation() const { return with_location; } bool requiresSchema() const { return with_schema; } + bool requiresCredentials() const { return with_storage_credentials; } private: /// Starts with s3://, file://, etc. @@ -34,10 +40,12 @@ private: std::string path; DB::NamesAndTypesList schema; - std::string credentials; + /// Storage credentials, which are called "vended credentials". + std::shared_ptr storage_credentials; bool with_location = false; bool with_schema = false; + bool with_storage_credentials = false; }; @@ -88,4 +96,5 @@ protected: const std::string warehouse; }; + } diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index 149a26f107a..e5676265855 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -170,7 +171,7 @@ void RestCatalog::parseCatalogConfigurationSettings(const Poco::JSON::Object::Pt result.default_base_location = object->get("default-base-location").extract(); } -DB::HTTPHeaderEntries RestCatalog::getHeaders(bool update_token) const +DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(bool update_token) const { /// Option 1: user specified auth header manually. /// Header has format: 'Authorization: '. @@ -248,7 +249,8 @@ std::optional RestCatalog::getStorageType() const DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( const std::string & endpoint, - const Poco::URI::QueryParameters & params) const + const Poco::URI::QueryParameters & params, + const DB::HTTPHeaderEntries & headers) const { const auto & context = getContext(); @@ -258,13 +260,15 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token) { - auto headers = getHeaders(update_token); + auto result_headers = getAuthHeaders(update_token); + std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); + return DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) .withSettings(getContext()->getReadSettings()) .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) .withHostFilter(&getContext()->getRemoteHostFilter()) - .withHeaders(headers) + .withHeaders(result_headers) .withDelayInit(false) .withSkipNotFound(false) .create(credentials); @@ -521,8 +525,12 @@ bool RestCatalog::getTableMetadataImpl( { LOG_TEST(log, "Checking table {} in namespace {}", table_name, namespace_name); + DB::HTTPHeaderEntries headers; + if (result.requiresCredentials()) + headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); + const auto endpoint = std::string(namespaces_endpoint) + "/" + namespace_name + "/tables/" + table_name; - auto buf = createReadBuffer(config.prefix / endpoint); + auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); if (buf->eof()) { @@ -533,6 +541,7 @@ bool RestCatalog::getTableMetadataImpl( String json_str; readJSONObjectPossiblyInvalid(json_str, *buf); + /// TODO: remove before merge because it might contain credentials. LOG_TEST(log, "Received metadata for table {}: {}", table_name, json_str); Poco::JSON::Parser parser; @@ -543,9 +552,10 @@ bool RestCatalog::getTableMetadataImpl( if (!metadata_object) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot parse result"); + std::string location; if (result.requiresLocation()) { - const auto location = metadata_object->get("location").extract(); + location = metadata_object->get("location").extract(); result.setLocation(location); LOG_TEST(log, "Location for table {}: {}", table_name, location); } @@ -556,23 +566,37 @@ bool RestCatalog::getTableMetadataImpl( result.setSchema(DB::IcebergMetadata::parseTableSchema(metadata_object, format_version, true).first); } - // if (result.requiresCredentials()) - // { - // try - // { - // const auto credentials_endpoint = std::filesystem::path(endpoint) / "credentials"; - // auto credentials_buf = createReadBuffer(config.prefix / credentials_endpoint); + if (result.requiresCredentials() && object->has("config")) + { + auto config_object = object->get("config").extract(); + if (!config_object) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot parse config result"); - // String credentials_json_str; - // readJSONObjectPossiblyInvalid(credentials_json_str, *credentials_buf); + auto storage_type = parseStorageTypeFromLocation(location); + switch (storage_type) + { + case StorageType::S3: + { + static constexpr auto access_key_id_str = "s3.access-key-id"; + static constexpr auto secret_access_key_str = "s3.secret-access-key"; + static constexpr auto session_token_str = "s3.session-token"; - // LOG_TEST(log, "Credentials : {}", credentials_json_str); - // } - // catch (...) - // { - // DB::tryLogCurrentException(log); - // } - // } + std::string access_key_id, secret_access_key, session_token; + if (config_object->has(access_key_id_str)) + access_key_id = config_object->get(access_key_id_str).extract(); + if (config_object->has(secret_access_key_str)) + secret_access_key = config_object->get(secret_access_key_str).extract(); + if (config_object->has(session_token_str)) + session_token = config_object->get(session_token_str).extract(); + + result.setStorageCredentials( + std::make_shared(access_key_id, secret_access_key, session_token)); + break; + } + default: + break; + } + } return true; } diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index 570707e3aa0..4505e020580 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -79,7 +79,8 @@ private: DB::ReadWriteBufferFromHTTPPtr createReadBuffer( const std::string & endpoint, - const Poco::URI::QueryParameters & params = {}) const; + const Poco::URI::QueryParameters & params = {}, + const DB::HTTPHeaderEntries & headers = {}) const; Poco::URI::QueryParameters createParentNamespaceParams(const std::string & base_namespace) const; @@ -107,7 +108,7 @@ private: Config loadConfig(); std::string retrieveAccessToken() const; - DB::HTTPHeaderEntries getHeaders(bool update_token = false) const; + DB::HTTPHeaderEntries getAuthHeaders(bool update_token = false) const; static void parseCatalogConfigurationSettings(const Poco::JSON::Object::Ptr & object, Config & result); }; diff --git a/src/Databases/Iceberg/StorageCredentials.h b/src/Databases/Iceberg/StorageCredentials.h new file mode 100644 index 00000000000..0cf0ffea4a5 --- /dev/null +++ b/src/Databases/Iceberg/StorageCredentials.h @@ -0,0 +1,50 @@ +#pragma once +#include +#include +#include + +namespace DB::ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +namespace Iceberg +{ + +class IStorageCredentials +{ +public: + virtual ~IStorageCredentials() = default; + + virtual void addCredentialsToEngineArgs(DB::ASTs & engine_args) const = 0; +}; + +class S3Credentials final : public IStorageCredentials +{ +public: + S3Credentials( + const std::string & access_key_id_, + const std::string & secret_access_key_, + const std::string session_token_) + : access_key_id(access_key_id_) + , secret_access_key(secret_access_key_) + , session_token(session_token_) + {} + + void addCredentialsToEngineArgs(DB::ASTs & engine_args) const override + { + if (engine_args.size() != 1) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Storage credentials specified in AST already"); + + engine_args.push_back(std::make_shared(access_key_id)); + engine_args.push_back(std::make_shared(secret_access_key)); + engine_args.push_back(std::make_shared(session_token)); + } + +private: + std::string access_key_id; + std::string secret_access_key; + std::string session_token; +}; + +} From bec1e4af97b15bdfa55f65f1ba175b8d1ec30264 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 25 Nov 2024 19:03:22 +0100 Subject: [PATCH 179/502] Add a comment --- src/Databases/Iceberg/RestCatalog.cpp | 8 ++++++++ src/Databases/Iceberg/StorageCredentials.h | 1 + 2 files changed, 9 insertions(+) diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index e5676265855..9509f40b3d6 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -527,7 +527,15 @@ bool RestCatalog::getTableMetadataImpl( DB::HTTPHeaderEntries headers; if (result.requiresCredentials()) + { + /// Header `X-Iceberg-Access-Delegation` tells catalog to include storage credentials in LoadTableResponse. + /// Value can be one of the two: + /// 1. `vended-credentials` + /// 2. `remote-signing` + /// Currently we support only the first. + /// https://github.com/apache/iceberg/blob/3badfe0c1fcf0c0adfc7aa4a10f0b50365c48cf9/open-api/rest-catalog-open-api.yaml#L1832 headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); + } const auto endpoint = std::string(namespaces_endpoint) + "/" + namespace_name + "/tables/" + table_name; auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); diff --git a/src/Databases/Iceberg/StorageCredentials.h b/src/Databases/Iceberg/StorageCredentials.h index 0cf0ffea4a5..90c20262cf4 100644 --- a/src/Databases/Iceberg/StorageCredentials.h +++ b/src/Databases/Iceberg/StorageCredentials.h @@ -22,6 +22,7 @@ public: class S3Credentials final : public IStorageCredentials { public: + /// TODO: support region as well. S3Credentials( const std::string & access_key_id_, const std::string & secret_access_key_, From 8239663c0504c38ed7d9ffa5f64187e239a212d1 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 25 Nov 2024 19:23:19 +0100 Subject: [PATCH 180/502] Add experimental flag --- src/Core/Settings.cpp | 3 +++ src/Core/SettingsChangesHistory.cpp | 1 + src/Databases/Iceberg/DatabaseIceberg.cpp | 20 ++++++++++++++----- .../integration/test_database_iceberg/test.py | 1 + 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 2526334d290..772585d3f46 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -5533,6 +5533,9 @@ Allow to ignore schema evolution in Iceberg table engine and read all data using :::note Enabling this setting can lead to incorrect result as in case of evolved schema all data files will be read using the same schema. ::: +)", 0) \ + DECLARE(Bool, allow_experimental_database_iceberg, false, R"( +Allow experimental database engine Iceberg )", 0) \ DECLARE(Bool, allow_deprecated_error_prone_window_functions, false, R"( Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 768779c37db..490d21c44a2 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -60,6 +60,7 @@ static std::initializer_list +#include #include #include @@ -44,10 +45,15 @@ namespace DatabaseIcebergSetting extern const DatabaseIcebergSettingsString storage_endpoint; extern const DatabaseIcebergSettingsBool vended_credentials; } +namespace Setting +{ + extern const SettingsBool allow_experimental_database_iceberg; +} namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int SUPPORT_IS_DISABLED; } namespace @@ -349,6 +355,14 @@ void registerDatabaseIceberg(DatabaseFactory & factory) { auto create_fn = [](const DatabaseFactory::Arguments & args) { + if (!args.create_query.attach + && !args.context->getSettingsRef()[Setting::allow_experimental_database_iceberg]) + { + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "DatabaseIceberg engine is experimental. " + "To allow its usage, enable setting allow_experimental_database_iceberg"); + } + const auto * database_engine_define = args.create_query.storage; const auto & database_engine_name = args.engine_name; @@ -357,13 +371,9 @@ void registerDatabaseIceberg(DatabaseFactory & factory) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine `{}` must have arguments", database_engine_name); ASTs & engine_args = function_define->arguments->children; - if (engine_args.empty()) + if (engine_args.size() < 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine `{}` must have arguments", database_engine_name); - const size_t max_args_num = 1; - if (engine_args.size() != max_args_num) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine must have {} arguments", max_args_num); - for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.context); diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index c0f0087522d..2896b87f63e 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -121,6 +121,7 @@ def create_clickhouse_iceberg_database(started_cluster, node, name): node.query( f""" DROP DATABASE IF EXISTS {name}; +SET allow_experimental_database_iceberg=true; CREATE DATABASE {name} ENGINE = Iceberg('{BASE_URL}', 'minio', 'minio123') SETTINGS catalog_type = 'rest', storage_endpoint = 'http://minio:9000/warehouse', From 8e345cfccebf29a1b477aebd08caf4654b6c6552 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 25 Nov 2024 19:27:46 +0100 Subject: [PATCH 181/502] Fix style check --- src/Databases/Iceberg/DatabaseIceberg.cpp | 9 +-------- src/Databases/Iceberg/RestCatalog.cpp | 1 + 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 3083b40d118..dd559ba8b92 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -24,13 +24,6 @@ #include #include -namespace CurrentMetrics -{ - extern const Metric IcebergCatalogThreads; - extern const Metric IcebergCatalogThreadsActive; - extern const Metric IcebergCatalogThreadsScheduled; -} - namespace DB { @@ -233,7 +226,7 @@ StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_ throw Exception( ErrorCodes::BAD_ARGUMENTS, "Either vended credentials need to be enabled " - "or storage credentials need to be specified in database engine arguements in CREATE query"); + "or storage credentials need to be specified in database engine arguments in CREATE query"); } LOG_TEST(log, "Using table endpoint: {}", table_endpoint); diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index 9509f40b3d6..a3d8a2a2d5a 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -24,6 +24,7 @@ namespace DB::ErrorCodes extern const int ICEBERG_CATALOG_ERROR; extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; + extern const int NOT_IMPLEMENTED; } namespace Iceberg From 09d4d501dc49ffdaa51f17d319381fdc5ce3d5b0 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 25 Nov 2024 19:08:08 +0000 Subject: [PATCH 182/502] Add test --- tests/performance/compact_part_subcolumns.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/performance/compact_part_subcolumns.xml diff --git a/tests/performance/compact_part_subcolumns.xml b/tests/performance/compact_part_subcolumns.xml new file mode 100644 index 00000000000..234b33ab8a7 --- /dev/null +++ b/tests/performance/compact_part_subcolumns.xml @@ -0,0 +1,13 @@ + + + 1 + + + + + CREATE TABLE t_json (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=100000000000 + INSERT INTO t_json SELECT toJSONString(map(number % 10, repeat('a', number % 100))) FROM numbers(10000000) + SELECT data.k0, data.k1, data.k2, data.k3, data.k4, data.k5, data.k6, data.k7, data.k8, data.k9 FROM t_json FORMAT Null + DROP TABLE IF EXISTS t_json + From ea1827ca5846d69e71af4ebfa080fbc4f5b2df96 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 26 Nov 2024 11:11:41 +0100 Subject: [PATCH 183/502] Fix style check --- src/Databases/enableAllExperimentalSettings.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Databases/enableAllExperimentalSettings.cpp b/src/Databases/enableAllExperimentalSettings.cpp index 1be54664bc9..2b102095632 100644 --- a/src/Databases/enableAllExperimentalSettings.cpp +++ b/src/Databases/enableAllExperimentalSettings.cpp @@ -46,6 +46,7 @@ void enableAllExperimentalSettings(ContextMutablePtr context) context->setSetting("enable_zstd_qat_codec", 1); context->setSetting("allow_create_index_without_type", 1); context->setSetting("allow_experimental_s3queue", 1); + context->setSetting("allow_experimental_database_iceberg", 1); /// clickhouse-private settings context->setSetting("allow_experimental_shared_set_join", 1); From 6d935a6abe1e6ec793805cbd0a2b33f7c54aeb7e Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 26 Nov 2024 11:20:15 +0000 Subject: [PATCH 184/502] Don't allow creating empty Variant --- src/DataTypes/DataTypeVariant.cpp | 3 +++ tests/queries/0_stateless/03276_empty_variant_type.sql | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 tests/queries/0_stateless/03276_empty_variant_type.sql diff --git a/src/DataTypes/DataTypeVariant.cpp b/src/DataTypes/DataTypeVariant.cpp index cc8d04e94da..3ec1c994f5e 100644 --- a/src/DataTypes/DataTypeVariant.cpp +++ b/src/DataTypes/DataTypeVariant.cpp @@ -43,6 +43,9 @@ DataTypeVariant::DataTypeVariant(const DataTypes & variants_) for (const auto & [_, type] : name_to_type) variants.push_back(type); + if (variants.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Variant type should have at least one nested type"); + if (variants.size() > ColumnVariant::MAX_NESTED_COLUMNS) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Variant type with more than {} nested types is not allowed", ColumnVariant::MAX_NESTED_COLUMNS); } diff --git a/tests/queries/0_stateless/03276_empty_variant_type.sql b/tests/queries/0_stateless/03276_empty_variant_type.sql new file mode 100644 index 00000000000..b87ebbee764 --- /dev/null +++ b/tests/queries/0_stateless/03276_empty_variant_type.sql @@ -0,0 +1,3 @@ +set allow_experimental_variant_type=1; +create table test (v Variant()) engine=Variant(); -- {serverError BAD_ARGUMENTS} + From fb73952708caae4d1f409d6ad7cb699b6aaefa25 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:53:32 +0100 Subject: [PATCH 185/502] Cosmetic fixes --- src/Interpreters/CancellationChecker.h | 2 +- src/Interpreters/ProcessList.cpp | 1 - src/Interpreters/ProcessList.h | 9 ++++----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/CancellationChecker.h b/src/Interpreters/CancellationChecker.h index 3addc2a0cc2..cb0526b3580 100644 --- a/src/Interpreters/CancellationChecker.h +++ b/src/Interpreters/CancellationChecker.h @@ -41,7 +41,7 @@ private: // Priority queue to manage tasks based on endTime std::multiset querySet; - std::atomic stop_thread; + bool stop_thread; std::mutex m; std::condition_variable cond_var; diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 970a4929d28..1790c14dbaa 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -554,7 +554,6 @@ bool QueryStatus::checkTimeLimit() void QueryStatus::throwQueryWasCancelled() const { - std::lock_guard lock{cancellation_exception_mutex}; if (cancellation_exception) std::rethrow_exception(cancellation_exception); else diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 1d4bcf8b758..d46603cd73e 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -116,10 +116,10 @@ protected: bool is_cancelling { false }; /// KILL was send to the query std::atomic is_killed { false }; - CancelReason cancel_reason { CancelReason::UNDEFINED }; - std::exception_ptr cancellation_exception; - mutable std::mutex cancellation_exception_mutex; + mutable std::mutex cancel_mutex; + CancelReason cancel_reason { CancelReason::UNDEFINED }; + std::exception_ptr cancellation_exception TSA_GUARDED_BY(cancel_mutex); /// All data to the client already had been sent. /// Including EndOfStream or Exception. @@ -139,10 +139,9 @@ protected: /// A weak pointer is used here because it's a ProcessListEntry which owns this QueryStatus, and not vice versa. void setProcessListEntry(std::weak_ptr process_list_entry_); - [[noreturn]] void throwQueryWasCancelled() const; + [[noreturn]] void throwQueryWasCancelled() const TSA_REQUIRES(cancel_mutex); mutable std::mutex executors_mutex; - mutable std::mutex cancel_mutex; struct ExecutorHolder { From 969bc6adba77899c4fa566b40fcc1706aea1055b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Tue, 26 Nov 2024 16:05:27 +0000 Subject: [PATCH 186/502] Address review comments --- .../Passes/LogicalExpressionOptimizerPass.cpp | 112 ++++++-------- .../Passes/LogicalExpressionOptimizerPass.h | 2 +- ...2_common_expression_optimization.reference | 140 +++++++++++++++++- .../03262_common_expression_optimization.sql | 44 +++--- 4 files changed, 211 insertions(+), 87 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index aa8c0f0cd75..c600055ef7b 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -180,55 +180,50 @@ void insertIfNotPresentInSet(QueryTreeNodePtrWithHashSet& set, QueryTreeNodes &n } // Returns the flattened AND/OR node if the passed-in node can be flattened. Doesn't modify the passed-in node. -std::shared_ptr getFlattenedAndOrOr(const FunctionNode & node, const ContextPtr & context) +std::shared_ptr getFlattenedLogicalExpression(const FunctionNode & node, const ContextPtr & context) { const auto & function_name = node.getFunctionName(); if (function_name != "or" && function_name != "and") return nullptr; const auto & arguments = node.getArguments().getNodes(); - QueryTreeNodePtrWithHashMap arguments_to_replace; + QueryTreeNodes new_arguments; + + bool flattened_anything = false; for (const auto & argument : arguments) { auto * maybe_function = argument->as(); + // If the nested function is not the same, just use it as is if (!maybe_function || maybe_function->getFunctionName() != function_name) + { + new_arguments.push_back(argument); continue; + } - auto it = arguments_to_replace.find(argument); - if (it != arguments_to_replace.end()) - continue; + flattened_anything = true; - auto maybe_flattened = getFlattenedAndOrOr(*maybe_function, context); + // If the nested function is the same, just lift the its or its flattened form's arguments + auto maybe_flattened = getFlattenedLogicalExpression(*maybe_function, context); if (maybe_flattened) - arguments_to_replace.emplace(argument, std::move(maybe_flattened->getArguments().getNodes())); + { + for (auto & flattened_argument : maybe_flattened->getArguments().getNodes()) + new_arguments.emplace_back(std::move(flattened_argument)); + } else - arguments_to_replace.emplace(argument, maybe_function->getArguments().getNodes()); + { + for (auto & nested_argument : maybe_function->getArguments().getNodes()) + new_arguments.push_back(nested_argument); + } } - if (arguments_to_replace.empty()) - return nullptr; // nothing to flatten + // Nothing has changed, let's no create a flattened node + if (!flattened_anything && new_arguments.size() == arguments.size()) + return {}; auto flattened = std::make_shared(function_name); - auto & new_arguments = flattened->getArguments().getNodes(); - new_arguments.reserve(arguments.size()); - for (const auto & argument : arguments) - { - if (auto it = arguments_to_replace.find(argument); it != arguments_to_replace.end()) - { - if (it->second.empty()) - continue; - - new_arguments.insert( - new_arguments.end(), std::make_move_iterator(it->second.begin()), std::make_move_iterator(it->second.end())); - it->second.clear(); - } - else - { - new_arguments.push_back(argument); - } - } + flattened->getArguments().getNodes() = std::move(new_arguments); auto function_resolver = FunctionFactory::instance().get(function_name, context); flattened->resolveAsFunction(function_resolver); @@ -264,7 +259,7 @@ std::optional tryExtractCommonExpressions(cons if (!or_node || or_node->getFunctionName() != "or") return {}; // the optimization can only be done on or nodes - auto flattened_or_node = getFlattenedAndOrOr(*or_node, context); + auto flattened_or_node = getFlattenedLogicalExpression(*or_node, context); if (flattened_or_node) or_node = flattened_or_node.get(); @@ -277,18 +272,13 @@ std::optional tryExtractCommonExpressions(cons QueryTreeNodes common_exprs; QueryTreeNodePtrWithHashMap flattened_ands; - auto insert_possible_new_common_expr = [&common_exprs_set, &common_exprs](QueryTreeNodePtr node_to_insert) - { - insertIfNotPresentInSet(common_exprs_set, common_exprs, std::move(node_to_insert)); - }; - for (auto & maybe_and_node : or_argument_nodes) { auto * and_node = maybe_and_node->as(); if (!and_node || and_node->getFunctionName() != "and") return {}; // one of the nodes is not an "AND", thus there is no common expression to extract - auto flattened_and_node = getFlattenedAndOrOr(*and_node, context); + auto flattened_and_node = getFlattenedLogicalExpression(*and_node, context); if (flattened_and_node) { flattened_ands.emplace(maybe_and_node, flattened_and_node); @@ -300,8 +290,9 @@ std::optional tryExtractCommonExpressions(cons auto & current_arguments = and_node->getArguments().getNodes(); common_exprs.reserve(current_arguments.size()); - for (auto & or_argument : current_arguments) - insert_possible_new_common_expr(or_argument); + for (auto & and_argument : current_arguments) + insertIfNotPresentInSet(common_exprs_set, common_exprs, and_argument); + first_argument = false; } else @@ -309,14 +300,10 @@ std::optional tryExtractCommonExpressions(cons QueryTreeNodePtrWithHashSet new_common_exprs_set; QueryTreeNodes new_common_exprs; - - for (auto & or_argument : and_node->getArguments()) + for (auto & and_argument : and_node->getArguments()) { - if (auto ptr_with_hash = QueryTreeNodePtrWithHash{or_argument}; common_exprs_set.contains(ptr_with_hash)) - { - new_common_exprs.push_back(or_argument); - new_common_exprs_set.insert(std::move(ptr_with_hash)); - } + if (common_exprs_set.contains(and_argument)) + insertIfNotPresentInSet(new_common_exprs_set, new_common_exprs, and_argument); } common_exprs_set = std::move(new_common_exprs_set); @@ -344,11 +331,7 @@ std::optional tryExtractCommonExpressions(cons std::remove_if( and_arguments.begin(), and_arguments.end(), - [&common_exprs_set](const QueryTreeNodePtr & ptr) - { - QueryTreeNodeWithHash ptr_with_hash{ptr}; - return common_exprs_set.contains(ptr_with_hash); - }), + [&common_exprs_set](const QueryTreeNodePtr & ptr) { return common_exprs_set.contains(ptr); }), and_arguments.end()); if (and_arguments.empty()) @@ -359,18 +342,14 @@ std::optional tryExtractCommonExpressions(cons } else if (and_arguments.size() == 1) { - const auto [_, inserted] = new_or_arguments_set.insert(and_arguments.front()); - if (inserted) - new_or_arguments.push_back(std::move(and_arguments.front())); - + insertIfNotPresentInSet(new_or_arguments_set, new_or_arguments, and_arguments.front()); } else { auto and_function_resolver = FunctionFactory::instance().get("and", context); and_node.resolveAsFunction(and_function_resolver); - const auto [_, inserted] = new_or_arguments_set.insert(or_argument); - if (inserted) - new_or_arguments.push_back(std::move(or_argument)); + + insertIfNotPresentInSet(new_or_arguments_set, new_or_arguments, or_argument); } } @@ -771,15 +750,20 @@ public: if (!query_node) return; - auto & where_node = query_node->getWhere(); - if (!where_node) - return; + const auto try_optimize_if_function = [this](QueryTreeNodePtr & maybe_node) + { + if (!maybe_node) + return; + auto * function_node = maybe_node->as(); + if (!function_node) + return; + tryOptimizeCommonExpressions(maybe_node, *function_node, getContext()); + }; - auto * function_node = where_node->as(); - if (!function_node) - return; - - tryOptimizeCommonExpressions(where_node, *function_node, getContext()); + try_optimize_if_function(query_node->getWhere()); + try_optimize_if_function(query_node->getPrewhere()); + try_optimize_if_function(query_node->getHaving()); + try_optimize_if_function(query_node->getQualify()); } private: diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h index dac6ccdc880..76bb5e6b432 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h @@ -109,7 +109,7 @@ namespace DB * SELECT * FROM t1 WHERE NOT a IN (n) * ------------------------------- * - * 8. Extract common expressions from AND expressions of a single OR expression only in WHERE and ON expressions. + * 8. Extract common subexpressions from AND expressions of a single OR expression only in WHERE and ON expressions. * If possible, AND and OR expressions will be flattened during performing this. * This might break some lazily evaluated expressions, but this optimization can be turned off by optimize_extract_common_expressions = 0. * ------------------------------- diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index fa42fd8f56c..8b8dd849915 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -21,6 +21,11 @@ QUERY id: 0 COLUMN id: 8, column_name: A, result_type: UInt8, source_id: 3 COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 3 SETTINGS optimize_extract_common_expressions=0 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE (__table1.A AND __table1.B) OR (__table1.A AND __table1.C) +SETTINGS optimize_extract_common_expressions = 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -40,6 +45,11 @@ QUERY id: 0 COLUMN id: 9, column_name: B, result_type: UInt8, source_id: 3 COLUMN id: 10, column_name: C, result_type: UInt8, source_id: 3 SETTINGS optimize_extract_common_expressions=1 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.A AND (__table1.B OR __table1.C) +SETTINGS optimize_extract_common_expressions = 1 -4890149726841642150 1 1 1 0 0 0 6142641079032712283 1 1 1 0 0 0 -558050477524086008 1 1 1 0 0 0 @@ -75,6 +85,10 @@ QUERY id: 0 COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 COLUMN id: 8, column_name: C, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.A AND __table1.B AND __table1.C -1417696215412225994 1 1 1 0 0 1 -7491222816216432223 1 1 1 0 0 1 -7667054131998434193 1 1 1 0 0 1 @@ -115,6 +129,10 @@ QUERY id: 0 COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: B, result_type: UInt8, source_id: 3 COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.A AND (__table1.E OR __table1.F) AND __table1.B AND __table1.C -1417696215412225994 1 1 1 0 0 1 -7491222816216432223 1 1 1 0 0 1 -7667054131998434193 1 1 1 0 0 1 @@ -155,6 +173,10 @@ QUERY id: 0 COLUMN id: 10, column_name: F, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: B, result_type: UInt8, source_id: 3 COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.A AND (__table1.E OR __table1.F) AND __table1.B AND __table1.C -7064227703501672459 1 1 0 0 1 0 -5989383837337135212 1 1 0 0 1 0 -2960971598243161531 1 1 0 0 1 0 @@ -195,6 +217,10 @@ QUERY id: 0 COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 12, column_name: B, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.A AND (__table1.C OR __table1.D OR __table1.E) AND __table1.B -7064227703501672459 1 1 0 0 1 0 -5989383837337135212 1 1 0 0 1 0 -2960971598243161531 1 1 0 0 1 0 @@ -235,6 +261,10 @@ QUERY id: 0 COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 12, column_name: B, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.A AND (__table1.C OR __table1.D OR __table1.E) AND __table1.B 7046366705027993992 0 1 1 0 0 0 5876398964123690873 0 1 1 0 0 0 -4849840134697008981 0 1 1 0 0 0 @@ -269,6 +299,10 @@ QUERY id: 0 LIST id: 5, nodes: 2 COLUMN id: 6, column_name: B, result_type: UInt8, source_id: 3 COLUMN id: 7, column_name: C, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.B AND __table1.C 331058316081182921 0 1 1 0 0 1 -2878539180531591201 0 1 1 0 0 1 5634271855584736521 0 1 1 0 0 1 @@ -308,6 +342,10 @@ QUERY id: 0 LIST id: 9, nodes: 2 COLUMN id: 10, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: F, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.B AND __table1.C AND (__table1.E OR __table1.F) 331058316081182921 0 1 1 0 0 1 -2878539180531591201 0 1 1 0 0 1 5634271855584736521 0 1 1 0 0 1 @@ -347,6 +385,10 @@ QUERY id: 0 LIST id: 9, nodes: 2 COLUMN id: 10, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: F, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.B AND __table1.C AND (__table1.E OR __table1.F) -8350064335402276481 0 1 0 0 1 0 2766908592531260293 0 1 0 0 1 0 -8500464668406226757 0 1 0 0 1 0 @@ -386,6 +428,10 @@ QUERY id: 0 COLUMN id: 9, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.B AND (__table1.C OR __table1.D OR __table1.E) -8350064335402276481 0 1 0 0 1 0 2766908592531260293 0 1 0 0 1 0 -8500464668406226757 0 1 0 0 1 0 @@ -425,6 +471,10 @@ QUERY id: 0 COLUMN id: 9, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 10, column_name: D, result_type: UInt8, source_id: 3 COLUMN id: 11, column_name: E, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.B AND (__table1.C OR __table1.D OR __table1.E) 7296788055532769937 0 1 0 0 0 0 -8085083078325036349 0 1 0 0 0 0 8056746393406743698 0 1 0 0 0 0 @@ -473,6 +523,10 @@ QUERY id: 0 LIST id: 15, nodes: 2 COLUMN id: 16, column_name: A, result_type: UInt8, source_id: 3 COLUMN id: 17, column_name: B, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE (sipHash64(__table1.C) = sipHash64(__table1.D)) AND (__table1.A OR __table1.B) 2589067852420946093 0 0 0 0 1 1 566510304360553038 0 0 0 0 1 1 4712156778925374545 0 0 0 0 1 1 @@ -523,6 +577,10 @@ QUERY id: 0 LIST id: 17, nodes: 2 COLUMN id: 18, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 19, column_name: F, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE (__table1.A AND __table1.B) OR ((__table1.C AND __table1.D) OR (__table1.E AND __table1.F)) 813655996633899418 0 0 0 0 1 1 6513794688600930866 0 0 0 0 1 1 2151442296722776334 0 0 0 0 1 1 @@ -573,6 +631,10 @@ QUERY id: 0 LIST id: 16, nodes: 2 COLUMN id: 17, column_name: E, result_type: UInt8, source_id: 3 COLUMN id: 18, column_name: F, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE (__table1.A AND __table1.B) OR ((__table1.B AND __table1.D) OR (__table1.E AND __table1.F)) 6015136002503835948 1 1 0 0 0 1 -3883799183493487473 1 1 0 0 0 1 -3002427897361111856 1 1 0 0 0 1 @@ -604,10 +666,9 @@ QUERY id: 0 WHERE FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 ARGUMENTS - LIST id: 5, nodes: 4 + LIST id: 5, nodes: 3 COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 - COLUMN id: 7, column_name: B, result_type: UInt8, source_id: 3 FUNCTION id: 8, function_name: or, function_type: ordinary, result_type: UInt8 ARGUMENTS LIST id: 9, nodes: 3 @@ -622,6 +683,58 @@ QUERY id: 0 LIST id: 15, nodes: 2 COLUMN id: 16, column_name: F, result_type: UInt8, source_id: 3 COLUMN id: 16, column_name: F, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.A AND __table1.B AND (__table1.C OR (__table1.E AND __table1.E) OR (__table1.F AND __table1.F)) +-7079903752397632178 1 1 0 1 0 1 +7410531019670261947 1 1 0 1 0 1 +233866682657554364 1 1 0 1 0 1 +704993795727957061 1 1 0 1 0 1 +4554773761331986827 1 1 0 1 0 1 +5016240913683434733 1 1 0 1 0 1 +21469839474705446 1 1 0 1 0 1 +2093705149861909566 1 1 0 1 0 1 +2363609859499966990 1 1 0 1 0 1 +5595451911922654649 1 1 0 1 0 1 +-7079903752397632178 1 1 0 1 0 1 +7410531019670261947 1 1 0 1 0 1 +233866682657554364 1 1 0 1 0 1 +704993795727957061 1 1 0 1 0 1 +4554773761331986827 1 1 0 1 0 1 +5016240913683434733 1 1 0 1 0 1 +21469839474705446 1 1 0 1 0 1 +2093705149861909566 1 1 0 1 0 1 +2363609859499966990 1 1 0 1 0 1 +5595451911922654649 1 1 0 1 0 1 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 4 + FUNCTION id: 6, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: C, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: D, result_type: UInt8, source_id: 3 + COLUMN id: 10, column_name: A, result_type: UInt8, source_id: 3 + COLUMN id: 11, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 12, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 2 + COLUMN id: 14, column_name: E, result_type: UInt8, source_id: 3 + COLUMN id: 15, column_name: F, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE (__table1.C OR __table1.D) AND __table1.A AND __table1.B AND (__table1.E OR __table1.F) QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -652,6 +765,10 @@ QUERY id: 0 LIST id: 17, nodes: 2 COLUMN id: 14, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 18, column_name: E, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.A OR (__table1.B AND ((__table1.C AND __table1.D) OR (__table1.C AND __table1.E))) -9220342792229035180 1 1 0 1 0 1 -9218747728814747926 1 1 0 0 0 1 -9220342792229035180 1 1 0 1 0 1 -9217081065560798881 1 0 0 1 0 1 -9220342792229035180 1 1 0 1 0 1 -9211856624818647543 1 1 1 0 0 0 @@ -706,6 +823,10 @@ QUERY id: 0 LIST id: 19, nodes: 2 COLUMN id: 20, column_name: C, result_type: UInt8, source_id: 5 CONSTANT id: 21, constant_value: UInt64_1, constant_value_type: UInt8 + +SELECT count() AS `count()` +FROM default.x AS __table1 +ALL INNER JOIN default.y AS __table2 ON (__table1.A = __table2.A) AND ((__table1.B = 1) OR (__table2.C = 1)) QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -764,6 +885,10 @@ QUERY id: 0 LIST id: 37, nodes: 2 COLUMN id: 38, column_name: E, result_type: UInt8, source_id: 4 COLUMN id: 39, column_name: E, result_type: UInt8, source_id: 5 + +SELECT count() AS `count()` +FROM default.x AS __table1 +ALL INNER JOIN default.y AS __table2 ON (__table1.A = __table2.A) OR ((__table1.B = __table2.B) AND (((__table1.C = __table2.C) AND (__table1.D = __table2.D)) OR ((__table1.C = __table2.C) AND (__table1.E = __table2.E)))) -9220342792229035180 1 1 0 1 0 1 -9215479184005563028 0 1 1 1 0 1 -9211524111550600160 0 0 1 1 1 0 @@ -796,3 +921,14 @@ QUERY id: 0 LIST id: 11, nodes: 2 COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 3 COLUMN id: 13, column_name: E, result_type: UInt8, source_id: 3 + +SELECT + __table1.x AS x, + __table1.A AS A, + __table1.B AS B, + __table1.C AS C, + __table1.D AS D, + __table1.E AS E, + __table1.F AS F +FROM default.x AS __table1 +WHERE __table1.C AND __table1.E diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index ada294d74b9..0473a367bf6 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -6,72 +6,76 @@ CREATE TABLE x (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) E INSERT INTO x SELECT x, A%2 AS A, B%2 AS B, C%2 AS C, D%2 AS D, E%2 AS E, F%2 AS F FROM generateRandom('x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8', 42) LIMIT 2000; -- Verify that optimization optimization setting works as expected -EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND B) OR (A AND C) SETTINGS optimize_extract_common_expressions = 0; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND B) OR (A AND C) SETTINGS optimize_extract_common_expressions = 1; +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE (A AND B) OR (A AND C) SETTINGS optimize_extract_common_expressions = 0; +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE (A AND B) OR (A AND C) SETTINGS optimize_extract_common_expressions = 1; -- Test multiple cases SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND C AND F)); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND C AND F)); SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)); SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)); SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)); SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))); -- Without AND as a root SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR (B AND C AND F)); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND C) OR (B AND C AND F)); SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C AND E) OR (B AND C AND F)); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND C AND E) OR (B AND C AND F)); SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)); SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)); SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))); -- Complex expression SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))); -- Flattening is only happening if something can be extracted SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))); SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))); -- Duplicates SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; -EXPLAIN QUERY TREE SELECT count() FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)); + +SELECT * FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) OR (B AND A AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) OR (B AND A AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) OR (B AND A AND F)); -- Check that optimization only happen on top level, (C AND D) OR (C AND E) shouldn't be optimized -EXPLAIN QUERY TREE SELECT count() FROM x WHERE A OR (B AND ((C AND D) OR (C AND E))); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A OR (B AND ((C AND D) OR (C AND E))); DROP TABLE IF EXISTS y; @@ -82,12 +86,12 @@ INSERT INTO y SELECT x, A%2 AS A, B%2 AS B, C%2 AS C, D%2 AS D, E%2 AS E, F%2 AS -- As the optimization code is shared between ON and WHERE, it is enough to test that the optimization is done also in ON SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, optimize_extract_common_expressions = 0; SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, optimize_extract_common_expressions = 1; -EXPLAIN QUERY TREE SELECT count() FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1); -- Check that optimization only happen on top level, (x.C = y.C AND x.D = y.D) OR (x.C = y.C AND x.E = y.E) shouldn't be optimized -EXPLAIN QUERY TREE SELECT count() FROM x INNER JOIN y ON (x.A = y.A) OR ((x.B = y.B) AND ((x.C = y.C AND x.D = y.D) OR (x.C = y.C AND x.E = y.E))); +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x INNER JOIN y ON (x.A = y.A) OR ((x.B = y.B) AND ((x.C = y.C AND x.D = y.D) OR (x.C = y.C AND x.E = y.E))); -- Duplicated subexpressions, found by fuzzer SELECT * FROM x WHERE (D AND 5) OR ((C AND E) AND (C AND E)) ORDER BY ALL LIMIT 3 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE (D AND 5) OR ((C AND E) AND (C AND E)) ORDER BY ALL LIMIT 3 SETTINGS optimize_extract_common_expressions = 1; -EXPLAIN QUERY TREE SELECT * FROM x WHERE (C AND E) OR ((C AND E) AND (C AND E)); +EXPLAIN QUERY TREE dump_ast = 1 SELECT * FROM x WHERE (C AND E) OR ((C AND E) AND (C AND E)); From a2d7539de76fdc0ff6dd1efdef6b88c9ac93443c Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 26 Nov 2024 16:23:28 +0000 Subject: [PATCH 187/502] Update tests --- ...to_json_alter_compact_merge_tree.reference | 44 +++++++++++++++++++ ...bject_to_json_alter_compact_merge_tree.sql | 27 ++++++++++++ ...3270_object_to_json_alter_memory.reference | 44 +++++++++++++++++++ .../03270_object_to_json_alter_memory.sql | 27 ++++++++++++ ...ct_to_json_alter_wide_merge_tree.reference | 44 +++++++++++++++++++ ...0_object_to_json_alter_wide_merge_tree.sql | 26 +++++++++++ 6 files changed, 212 insertions(+) create mode 100644 tests/queries/0_stateless/03270_object_to_json_alter_compact_merge_tree.reference create mode 100644 tests/queries/0_stateless/03270_object_to_json_alter_compact_merge_tree.sql create mode 100644 tests/queries/0_stateless/03270_object_to_json_alter_memory.reference create mode 100644 tests/queries/0_stateless/03270_object_to_json_alter_memory.sql create mode 100644 tests/queries/0_stateless/03270_object_to_json_alter_wide_merge_tree.reference create mode 100644 tests/queries/0_stateless/03270_object_to_json_alter_wide_merge_tree.sql diff --git a/tests/queries/0_stateless/03270_object_to_json_alter_compact_merge_tree.reference b/tests/queries/0_stateless/03270_object_to_json_alter_compact_merge_tree.reference new file mode 100644 index 00000000000..0a2125e815c --- /dev/null +++ b/tests/queries/0_stateless/03270_object_to_json_alter_compact_merge_tree.reference @@ -0,0 +1,44 @@ +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 diff --git a/tests/queries/0_stateless/03270_object_to_json_alter_compact_merge_tree.sql b/tests/queries/0_stateless/03270_object_to_json_alter_compact_merge_tree.sql new file mode 100644 index 00000000000..0792de1d1e7 --- /dev/null +++ b/tests/queries/0_stateless/03270_object_to_json_alter_compact_merge_tree.sql @@ -0,0 +1,27 @@ +-- Tags: long + +set allow_experimental_object_type = 1; +set allow_experimental_json_type = 1; +set max_block_size = 100; +set max_insert_block_size = 100; +set min_insert_block_size_rows = 100; +set output_format_json_quote_64bit_integers = 0; + +drop table if exists test; + +create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=10000000, min_bytes_for_wide_part=100000000; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON; +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; + +create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=10000000, min_bytes_for_wide_part=100000000; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON(max_dynamic_paths=10); +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; + diff --git a/tests/queries/0_stateless/03270_object_to_json_alter_memory.reference b/tests/queries/0_stateless/03270_object_to_json_alter_memory.reference new file mode 100644 index 00000000000..0a2125e815c --- /dev/null +++ b/tests/queries/0_stateless/03270_object_to_json_alter_memory.reference @@ -0,0 +1,44 @@ +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 diff --git a/tests/queries/0_stateless/03270_object_to_json_alter_memory.sql b/tests/queries/0_stateless/03270_object_to_json_alter_memory.sql new file mode 100644 index 00000000000..6d903b4bfbe --- /dev/null +++ b/tests/queries/0_stateless/03270_object_to_json_alter_memory.sql @@ -0,0 +1,27 @@ +-- Tags: long + +set allow_experimental_object_type = 1; +set allow_experimental_json_type = 1; +set max_block_size = 100; +set max_insert_block_size = 100; +set min_insert_block_size_rows = 100; +set output_format_json_quote_64bit_integers = 0; + +drop table if exists test; + +create table test (json Object('json')) engine=Memory; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON; +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; + +create table test (json Object('json')) engine=Memory; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON(max_dynamic_paths=10); +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; + diff --git a/tests/queries/0_stateless/03270_object_to_json_alter_wide_merge_tree.reference b/tests/queries/0_stateless/03270_object_to_json_alter_wide_merge_tree.reference new file mode 100644 index 00000000000..0a2125e815c --- /dev/null +++ b/tests/queries/0_stateless/03270_object_to_json_alter_wide_merge_tree.reference @@ -0,0 +1,44 @@ +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 +{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} +0 +100 +200 +300 +400 +500 +600 +700 +800 +900 +0 +99 +199 +299 +399 +499 +599 +699 +799 +899 +999 diff --git a/tests/queries/0_stateless/03270_object_to_json_alter_wide_merge_tree.sql b/tests/queries/0_stateless/03270_object_to_json_alter_wide_merge_tree.sql new file mode 100644 index 00000000000..780a543ea81 --- /dev/null +++ b/tests/queries/0_stateless/03270_object_to_json_alter_wide_merge_tree.sql @@ -0,0 +1,26 @@ +-- Tags: long + +set allow_experimental_object_type = 1; +set allow_experimental_json_type = 1; +set max_block_size = 100; +set max_insert_block_size = 100; +set min_insert_block_size_rows = 100; +set output_format_json_quote_64bit_integers = 0; + +drop table if exists test; + +create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON(); +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; + +create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1; +insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); +alter table test modify column json JSON(max_dynamic_paths=10); +select distinctJSONPathsAndTypes(json) from test; +select distinct json.a0 from test order by json.a0.:Int64; +select distinct json.a99 from test order by json.a99.:Int64; +drop table test; From a1650e053e530a02038127c4e82522176702d881 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 26 Nov 2024 17:08:19 +0000 Subject: [PATCH 188/502] Try to fix build --- src/TableFunctions/TableFunctionURL.cpp | 39 ----------------------- src/TableFunctions/TableFunctionURL.h | 42 ++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 5de3a302772..cad5cac3e31 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -77,45 +77,6 @@ void TableFunctionURL::parseArgumentsImpl(ASTs & args, const ContextPtr & contex } } -void TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context, bool with_structure) -{ - if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) - { - /// In case of named collection, just add key-value pairs "format='...', structure='...'" - /// at the end of arguments to override existed format and structure with "auto" values. - if (collection->getOrDefault("format", "auto") == "auto") - { - ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; - auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); - args.push_back(format_equal_func); - } - if (with_structure && collection->getOrDefault("structure", "auto") == "auto") - { - ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; - auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); - args.push_back(structure_equal_func); - } - } - else - { - /// If arguments contain headers, just remove it and add to the end of arguments later. - HTTPHeaderEntries tmp_headers; - size_t count = StorageURL::evalArgsAndCollectHeaders(args, tmp_headers, context); - ASTPtr headers_ast; - if (count != args.size()) - { - chassert(count + 1 == args.size()); - headers_ast = args.back(); - args.pop_back(); - } - - ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context, with_structure); - - if (headers_ast) - args.push_back(headers_ast); - } -} - StoragePtr TableFunctionURL::getStorage( const String & source, const String & format_, const ColumnsDescription & columns, ContextPtr global_context, const std::string & table_name, const String & compression_method_) const diff --git a/src/TableFunctions/TableFunctionURL.h b/src/TableFunctions/TableFunctionURL.h index d2c9d4d9ddf..d76fb9abea6 100644 --- a/src/TableFunctions/TableFunctionURL.h +++ b/src/TableFunctions/TableFunctionURL.h @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include namespace DB @@ -34,7 +37,44 @@ public: ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; - static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context, bool with_structure); + static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context, bool with_structure) + { + if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) + { + /// In case of named collection, just add key-value pairs "format='...', structure='...'" + /// at the end of arguments to override existed format and structure with "auto" values. + if (collection->getOrDefault("format", "auto") == "auto") + { + ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; + auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); + args.push_back(format_equal_func); + } + if (with_structure && collection->getOrDefault("structure", "auto") == "auto") + { + ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; + auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); + args.push_back(structure_equal_func); + } + } + else + { + /// If arguments contain headers, just remove it and add to the end of arguments later. + HTTPHeaderEntries tmp_headers; + size_t count = StorageURL::evalArgsAndCollectHeaders(args, tmp_headers, context); + ASTPtr headers_ast; + if (count != args.size()) + { + chassert(count + 1 == args.size()); + headers_ast = args.back(); + args.pop_back(); + } + + ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context, with_structure); + + if (headers_ast) + args.push_back(headers_ast); + } + } protected: void parseArguments(const ASTPtr & ast, ContextPtr context) override; From 9cc8ae2d38a60b1c1091b23e256df947d416d5af Mon Sep 17 00:00:00 2001 From: erickurbanov Date: Wed, 27 Nov 2024 01:58:15 +0300 Subject: [PATCH 189/502] Add indexOfAssumeSorted Function --- src/Functions/array/arrayIndex.h | 138 +++++++++++++++----- src/Functions/array/indexOfAssumeSorted.cpp | 14 ++ 2 files changed, 117 insertions(+), 35 deletions(-) create mode 100644 src/Functions/array/indexOfAssumeSorted.cpp diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index e6ea9f4c4ee..4f0acfebd7e 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -53,6 +53,10 @@ struct IndexOfAction static constexpr void apply(ResultType& current, size_t j) noexcept { current = j + 1; } }; +struct IndexOfAssumeSorted : public IndexOfAction +{ +}; + struct CountEqualAction { using ResultType = UInt64; @@ -111,13 +115,110 @@ private: return 0 == left.compareAt(i, RightArgIsConstant ? 0 : j, right, 1); } + static constexpr bool less(const PaddedPODArray & left, const Result & right, size_t i, size_t) noexcept + { + return left[i] >= right; + } + + static constexpr bool less(const IColumn & left, const Result & right, size_t i, size_t) noexcept { return left[i] >= right; } + #pragma clang diagnostic pop + /** Assuming that the array is sorted, use a binary search */ + template + static constexpr ResultType lowerBound(const Data & data, const Target & target, size_t array_size, ArrOffset current_offset) + { + ResultType current = 0; + size_t low = 0, high = array_size; + while (high - low > 0) + { + auto middle = low + ((high - low) >> 1); + auto compare_result = less(data, target, current_offset + middle, 0); + /// avoid conditional branching + high = compare_result ? middle : high; + low = compare_result ? low : middle + 1; + } + if (low < array_size && compare(data, target, current_offset + low, 0)) { + ConcreteAction::apply(current, low); + } + return current; + } + + template + static constexpr ResultType linearSearch( + const Data & data, + const Target & target, + size_t array_size, + const NullMap * const null_map_data, + const NullMap * const null_map_item, + size_t row_index, + ArrOffset current_offset) + { + ResultType current = 0; + for (size_t j = 0; j < array_size; ++j) + { + if constexpr (Case == 2) /// Right arg is Nullable + if (hasNull(null_map_item, row_index)) + continue; + + if constexpr (Case == 3) /// Left arg is an array of Nullables + if (hasNull(null_map_data, current_offset + j)) + continue; + + if constexpr (Case == 4) /// Both args are nullable + { + const bool right_is_null = hasNull(null_map_data, current_offset + j); + const bool left_is_null = hasNull(null_map_item, row_index); + + if (right_is_null != left_is_null) + continue; + + if (!right_is_null && !compare(data, target, current_offset + j, row_index)) + continue; + } + else if (!compare(data, target, current_offset + j, row_index)) + continue; + + ConcreteAction::apply(current, j); + + if constexpr (!ConcreteAction::resume_execution) + break; + } + return current; + } + + /** Looking for the target element index in the data (array) */ + template + static constexpr ResultType getIndex( + const Data & data, + const Target & target, + size_t array_size, + const NullMap * const null_map_data, + const NullMap * const null_map_item, + size_t row_index, + ArrOffset current_offset) + { + /** Use binary search if the following conditions are met. + * 1. The array type is not nullable. (Case = 1) + * 2. Target is not a column or an array. + */ + if constexpr ( + std::is_same_v && !std::is_same_v> + && !std::is_same_v && Case == 1) + { + return lowerBound(data, target, array_size, current_offset); + } + return linearSearch(data, target, array_size, null_map_data, null_map_item, row_index, current_offset); + } + static constexpr bool hasNull(const NullMap * const null_map, size_t i) noexcept { return (*null_map)[i]; } template static void process( - const Data & data, const ArrOffsets & offsets, const Target & target, ResultArr & result, + const Data & data, + const ArrOffsets & offsets, + const Target & target, + ResultArr & result, [[maybe_unused]] const NullMap * const null_map_data, [[maybe_unused]] const NullMap * const null_map_item) { @@ -129,7 +230,6 @@ private: } const size_t size = offsets.size(); - result.resize(size); ArrOffset current_offset = 0; @@ -137,39 +237,7 @@ private: for (size_t i = 0; i < size; ++i) { const size_t array_size = offsets[i] - current_offset; - ResultType current = 0; - - for (size_t j = 0; j < array_size; ++j) - { - if constexpr (Case == 2) /// Right arg is Nullable - if (hasNull(null_map_item, i)) - continue; - - if constexpr (Case == 3) /// Left arg is an array of Nullables - if (hasNull(null_map_data, current_offset + j)) - continue; - - if constexpr (Case == 4) /// Both args are nullable - { - const bool right_is_null = hasNull(null_map_data, current_offset + j); - const bool left_is_null = hasNull(null_map_item, i); - - if (right_is_null != left_is_null) - continue; - - if (!right_is_null && !compare(data, target, current_offset + j, i)) - continue; - } - else if (!compare(data, target, current_offset + j, i)) - continue; - - ConcreteAction::apply(current, j); - - if constexpr (!ConcreteAction::resume_execution) - break; - } - - result[i] = current; + result[i] = getIndex(data, target, array_size, null_map_data, null_map_item, i, current_offset); current_offset = offsets[i]; } } diff --git a/src/Functions/array/indexOfAssumeSorted.cpp b/src/Functions/array/indexOfAssumeSorted.cpp new file mode 100644 index 00000000000..0a43eca8739 --- /dev/null +++ b/src/Functions/array/indexOfAssumeSorted.cpp @@ -0,0 +1,14 @@ +#include "arrayIndex.h" +#include +#include + +namespace DB +{ +struct NameIndexOfAssumeSorted { static constexpr auto name = "indexOfAssumeSorted"; }; + +/// indexOfAssumeSorted(arr, x) - returns the index of the element x (starting with 1), if it exists in the array, or 0 if it +/// should be used when the array is sorted (applies binary search to array) +using FunctionIndexOfAssumeSorted = FunctionArrayIndex; + +REGISTER_FUNCTION(IndexOfAssumeSorted) { factory.registerFunction(); } +} From 59d3478ae29cf21b2897becabf8774287102af1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 27 Nov 2024 11:08:20 +0000 Subject: [PATCH 190/502] Respect result nullability --- .../Passes/LogicalExpressionOptimizerPass.cpp | 75 ++++++++++++------- ...2_common_expression_optimization.reference | 30 ++++++++ .../03262_common_expression_optimization.sql | 4 + 3 files changed, 81 insertions(+), 28 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index c600055ef7b..38e1fe9dd74 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -325,31 +325,36 @@ std::optional tryExtractCommonExpressions(cons if (auto it = flattened_ands.find(or_argument); it != flattened_ands.end()) or_argument = it->second; - auto & and_node = or_argument->as(); - auto & and_arguments = and_node.getArguments().getNodes(); - and_arguments.erase( - std::remove_if( - and_arguments.begin(), - and_arguments.end(), - [&common_exprs_set](const QueryTreeNodePtr & ptr) { return common_exprs_set.contains(ptr); }), - and_arguments.end()); + // Avoid changing the original tree, it might be used later + const auto & and_node = or_argument->as(); + const auto & and_arguments = and_node.getArguments().getNodes(); - if (and_arguments.empty()) + QueryTreeNodes filtered_and_arguments; + filtered_and_arguments.reserve(and_arguments.size()); + std::copy_if( + and_arguments.begin(), + and_arguments.end(), + std::back_inserter(filtered_and_arguments), + [&common_exprs_set](const QueryTreeNodePtr & ptr) { return !common_exprs_set.contains(ptr); }); + + if (filtered_and_arguments.empty()) { has_completely_extracted_and_expression = true; // As we will discard new_or_arguments, no need for further processing break; } - else if (and_arguments.size() == 1) + else if (filtered_and_arguments.size() == 1) { - insertIfNotPresentInSet(new_or_arguments_set, new_or_arguments, and_arguments.front()); + insertIfNotPresentInSet(new_or_arguments_set, new_or_arguments, std::move( filtered_and_arguments.front())); } else { + auto new_and_node = std::make_shared("and"); + new_and_node->getArguments().getNodes() = std::move(filtered_and_arguments); auto and_function_resolver = FunctionFactory::instance().get("and", context); - and_node.resolveAsFunction(and_function_resolver); + new_and_node->resolveAsFunction(and_function_resolver); - insertIfNotPresentInSet(new_or_arguments_set, new_or_arguments, or_argument); + insertIfNotPresentInSet(new_or_arguments_set, new_or_arguments, std::move(new_and_node)); } } @@ -375,25 +380,35 @@ void tryOptimizeCommonExpressionsInOr(QueryTreeNodePtr & node, const ContextPtr auto * root_node = node->as(); chassert(root_node && root_node->getFunctionName() == "or"); + QueryTreeNodePtr new_root_node{}; + if (auto maybe_result = tryExtractCommonExpressions(node, context); maybe_result.has_value()) { auto & result = *maybe_result; - auto & root_arguments = root_node->getArguments().getNodes(); - root_arguments = std::move(result.common_expressions); + QueryTreeNodes new_root_arguments = std::move(result.common_expressions); if (result.new_node != nullptr) - root_arguments.push_back(std::move(result.new_node)); + new_root_arguments.push_back(std::move(result.new_node)); - if (root_arguments.size() == 1) + if (new_root_arguments.size() == 1) { - node = std::move(root_arguments[0]); - return; + new_root_node = std::move(new_root_arguments.front()); + } + else + { + // The OR expression must be replaced by and AND expression that will contain the common expressions + // and the new_node, if it is not nullptr. + auto new_function_node = std::make_shared("and"); + new_function_node->getArguments().getNodes() = std::move(new_root_arguments); + auto and_function_resolver = FunctionFactory::instance().get("and", context); + new_function_node->resolveAsFunction(and_function_resolver); + new_root_node = std::move(new_function_node); } - - // The OR expression must be replaced by and AND expression that will contain the common expressions - // and the new_node, if it is not nullptr. - auto and_function_resolver = FunctionFactory::instance().get("and", context); - root_node->resolveAsFunction(and_function_resolver); } + + // As only equivalent transformations are done on logical expressions (deduplication/flattening/elimination) (in other words we don't modify the values in (in)equalities), + // we can be sure the result type always remains convertible to UInt8 or Nullable(UInt8). We shouldn't change the nullability of the expression though. + if (new_root_node != nullptr && new_root_node->getResultType()->isNullable() == node->getResultType()->isNullable()) + node = std::move(new_root_node); } void tryOptimizeCommonExpressionsInAnd(QueryTreeNodePtr & node, const ContextPtr & context) @@ -409,7 +424,6 @@ void tryOptimizeCommonExpressionsInAnd(QueryTreeNodePtr & node, const ContextPtr insertIfNotPresentInSet(new_top_level_arguments_set, new_top_level_arguments, std::move(node_to_insert)); }; auto extracted_something = false; - auto & root_arguments = root_node->getArguments(); for (const auto & argument : root_node->getArguments()) { @@ -431,10 +445,15 @@ void tryOptimizeCommonExpressionsInAnd(QueryTreeNodePtr & node, const ContextPtr if (!extracted_something) return; - root_arguments.getNodes() = std::move(new_top_level_arguments); - + auto new_root_node = std::make_shared("and"); + new_root_node->getArguments().getNodes() = std::move(new_top_level_arguments); auto and_function_resolver = FunctionFactory::instance().get("and", context); - root_node->resolveAsFunction(and_function_resolver); + new_root_node->resolveAsFunction(and_function_resolver); + + // As only equivalent transformations are done on logical expressions (deduplication/flattening/elimination) (in other words we don't modify the values in (in)equalities), + // we can be sure the result type always remains convertible to UInt8 or Nullable(UInt8). We shouldn't change the nullability of the expression though. + if (new_root_node != nullptr && new_root_node->getResultType()->isNullable() == node->getResultType()->isNullable()) + node = std::move(new_root_node); } void tryOptimizeCommonExpressions(QueryTreeNodePtr & node, FunctionNode& function_node, const ContextPtr & context) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index 8b8dd849915..2aab69f9fed 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -735,6 +735,36 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE (__table1.C OR __table1.D) AND __table1.A AND __table1.B AND (__table1.E OR __table1.F) +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: or, function_type: ordinary, result_type: Nullable(UInt8) + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: C, result_type: UInt8, source_id: 3 + FUNCTION id: 10, function_name: and, function_type: ordinary, result_type: Nullable(UInt8) + ARGUMENTS + LIST id: 11, nodes: 3 + COLUMN id: 8, column_name: B, result_type: UInt8, source_id: 3 + COLUMN id: 9, column_name: C, result_type: UInt8, source_id: 3 + FUNCTION id: 12, function_name: toNullable, function_type: ordinary, result_type: Nullable(UInt8) + ARGUMENTS + LIST id: 13, nodes: 1 + COLUMN id: 14, column_name: F, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE (__table1.B AND __table1.C) OR (__table1.B AND __table1.C AND toNullable(__table1.F)) QUERY id: 0 PROJECTION COLUMNS count() UInt64 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index 0473a367bf6..fa691908526 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -74,6 +74,10 @@ SELECT * FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) SELECT * FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) OR (B AND A AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) OR (B AND A AND F)); + +-- Optimization is not applied if the result type would change in nullability, thus `toNullable(F)` cannot be eliminated +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND C) OR (B AND C AND toNullable(F))); + -- Check that optimization only happen on top level, (C AND D) OR (C AND E) shouldn't be optimized EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A OR (B AND ((C AND D) OR (C AND E))); From c2f74fa4aa6c6aa48dfdfb351d79cbbe19341306 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 27 Nov 2024 12:33:01 +0100 Subject: [PATCH 191/502] Init --- .../reference/groupconcat.md | 8 ++- .../AggregateFunctionGroupConcat.cpp | 56 +++++++++++++++---- .../0_stateless/03156_group_concat.reference | 4 ++ .../0_stateless/03156_group_concat.sql | 6 ++ 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md index de2f4a0a44b..7f22e4125a6 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md +++ b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md @@ -15,12 +15,18 @@ groupConcat[(delimiter [, limit])](expression); **Arguments** -- `expression` — The expression or column name that outputs strings to be concatenated.. +- `expression` — The expression or column name that outputs strings to be concatenated. +- `delimiter` — A [string](../../../sql-reference/data-types/string.md) that will be used to separate concatenated values. This parameter is optional and defaults to an empty string if not specified. + +**Parameters** + - `delimiter` — A [string](../../../sql-reference/data-types/string.md) that will be used to separate concatenated values. This parameter is optional and defaults to an empty string if not specified. - `limit` — A positive [integer](../../../sql-reference/data-types/int-uint.md) specifying the maximum number of elements to concatenate. If more elements are present, excess elements are ignored. This parameter is optional. :::note If delimiter is specified without limit, it must be the first parameter. If both delimiter and limit are specified, delimiter must precede limit. + +Also, if different delimiters are specified as parameters and arguments, the delimiter from arguments will be used only. ::: **Returned value** diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp index 8cf5ec5705a..f27c99db0b0 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp @@ -31,16 +31,24 @@ namespace ErrorCodes extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } namespace { +enum GroupConcatOverload +{ + ONE_ARGUMENT, + TWO_ARGUMENTS +}; + struct GroupConcatDataBase { UInt64 data_size = 0; UInt64 allocated_size = 0; char * data = nullptr; + String data_delimiter; void checkAndUpdateSize(UInt64 add, Arena * arena) { @@ -117,16 +125,18 @@ class GroupConcatImpl final UInt64 limit; const String delimiter; const DataTypePtr type; + GroupConcatOverload overload = ONE_ARGUMENT; public: - GroupConcatImpl(const DataTypePtr & data_type_, const Array & parameters_, UInt64 limit_, const String & delimiter_) + GroupConcatImpl(const DataTypes & data_types, const Array & parameters_, UInt64 limit_, const String & delimiter_) : IAggregateFunctionDataHelper, GroupConcatImpl>( - {data_type_}, parameters_, std::make_shared()) + {data_types}, parameters_, std::make_shared()) , limit(limit_) , delimiter(delimiter_) - , type(data_type_) + , type(data_types[0]) { serialization = isFixedString(type) ? std::make_shared()->getDefaultSerialization() : this->argument_types[0]->getDefaultSerialization(); + overload = data_types.size() > 1 ? TWO_ARGUMENTS : ONE_ARGUMENT; } String getName() const override { return name; } @@ -134,13 +144,20 @@ public: void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { auto & cur_data = this->data(place); + if (cur_data.data_delimiter.empty()) + { + cur_data.data_delimiter = delimiter; + if (overload == GroupConcatOverload::TWO_ARGUMENTS) + cur_data.data_delimiter = columns[1]->getDataAt(0).toString(); + } + const String & delim = cur_data.data_delimiter; if constexpr (has_limit) if (cur_data.num_rows >= limit) return; if (cur_data.data_size != 0) - cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); + cur_data.insertChar(delim.c_str(), delim.size(), arena); if (isFixedString(type)) { @@ -160,13 +177,15 @@ public: if (rhs_data.data_size == 0) return; + const String & delim = cur_data.data_delimiter; + if constexpr (has_limit) { UInt64 new_elems_count = std::min(rhs_data.num_rows, limit - cur_data.num_rows); for (UInt64 i = 0; i < new_elems_count; ++i) { if (cur_data.data_size != 0) - cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); + cur_data.insertChar(delim.c_str(), delim.size(), arena); cur_data.offsets.push_back(cur_data.data_size, arena); cur_data.insertChar(rhs_data.data + rhs_data.getString(i), rhs_data.getSize(i), arena); @@ -177,7 +196,7 @@ public: else { if (cur_data.data_size != 0) - cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); + cur_data.insertChar(delim.c_str(), delim.size(), arena); cur_data.insertChar(rhs_data.data, rhs_data.data_size, arena); } @@ -238,9 +257,16 @@ public: }; AggregateFunctionPtr createAggregateFunctionGroupConcat( - const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) + const std::string & name, + const DataTypes & argument_types, + const Array & parameters, + const Settings * /* settings */ +) { - assertUnary(name, argument_types); + // Ensure the number of arguments is between 1 and 2 + if (argument_types.empty() || argument_types.size() > 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Incorrect number of arguments for function {}, expected 1 to 2, got {}", name, argument_types.size()); bool has_limit = false; UInt64 limit = 0; @@ -273,9 +299,19 @@ AggregateFunctionPtr createAggregateFunctionGroupConcat( limit = parameters[1].safeGet(); } + // Handle additional arguments if provided (delimiter and limit as arguments) + if (argument_types.size() == 2) + { + // Second argument should be delimiter (string) + const DataTypePtr & delimiter_type = argument_types[1]; + if (!isString(delimiter_type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Second argument for function {} must be a string", name); + } + if (has_limit) - return std::make_shared>(argument_types[0], parameters, limit, delimiter); - return std::make_shared>(argument_types[0], parameters, limit, delimiter); + return std::make_shared>(argument_types, parameters, limit, delimiter); + return std::make_shared>(argument_types, parameters, limit, delimiter); } } diff --git a/tests/queries/0_stateless/03156_group_concat.reference b/tests/queries/0_stateless/03156_group_concat.reference index c1ab35e96c0..23c5d43bd60 100644 --- a/tests/queries/0_stateless/03156_group_concat.reference +++ b/tests/queries/0_stateless/03156_group_concat.reference @@ -16,4 +16,8 @@ abc,a,makson95 abc,a,makson95,abc,a,makson95,abc,a,makson95 [1,2,3][993,986,979,972][][1,2,3][993,986,979,972][][1,2,3][993,986,979,972][] 488890 +TESTING GroupConcat second argument overload +95,123 +abc.a.makson95 +[1,2,3]/[993,986,979,972]/[] 488890 diff --git a/tests/queries/0_stateless/03156_group_concat.sql b/tests/queries/0_stateless/03156_group_concat.sql index 0d561c69f0a..8989bb20df9 100644 --- a/tests/queries/0_stateless/03156_group_concat.sql +++ b/tests/queries/0_stateless/03156_group_concat.sql @@ -42,6 +42,12 @@ SELECT groupConcat(',', 3, 3)(number) FROM numbers(10); -- { serverError TOO_MAN SELECT length(groupConcat(number)) FROM numbers(100000); +SELECT 'TESTING GroupConcat second argument overload'; + +SELECT groupConcat(p_int, ',') FROM test_groupConcat; +SELECT groupConcat('.')(p_string) FROM test_groupConcat; +SELECT groupConcat(p_array, '/') FROM test_groupConcat; + DROP TABLE IF EXISTS test_groupConcat; CREATE TABLE test_groupConcat From 32cbd7d009d337f9a51b2473c34bf791f4163a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 27 Nov 2024 11:45:50 +0000 Subject: [PATCH 192/502] Add more tests --- ...2_common_expression_optimization.reference | 21 +++++++++++++++++++ .../03262_common_expression_optimization.sql | 4 ++++ 2 files changed, 25 insertions(+) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index 2aab69f9fed..9e32e864286 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -765,6 +765,27 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE (__table1.B AND __table1.C) OR (__table1.B AND __table1.C AND toNullable(__table1.F)) +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: and, function_type: ordinary, result_type: Nullable(UInt8) + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 7, function_name: toNullable, function_type: ordinary, result_type: Nullable(UInt8) + ARGUMENTS + LIST id: 8, nodes: 1 + COLUMN id: 9, column_name: C, result_type: UInt8, source_id: 3 + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE __table1.B AND toNullable(__table1.C) QUERY id: 0 PROJECTION COLUMNS count() UInt64 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index fa691908526..36add67bba4 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -77,6 +77,8 @@ EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((A AND B AND C) OR -- Optimization is not applied if the result type would change in nullability, thus `toNullable(F)` cannot be eliminated EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND C) OR (B AND C AND toNullable(F))); +-- Here the result type stays nullable because of `toNullable(C)`, so optimization will be applied +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND toNullable(C)) OR (B AND toNullable(C) AND toNullable(F))); -- Check that optimization only happen on top level, (C AND D) OR (C AND E) shouldn't be optimized EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A OR (B AND ((C AND D) OR (C AND E))); @@ -99,3 +101,5 @@ EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x INNER JOIN y ON (x.A = y.A SELECT * FROM x WHERE (D AND 5) OR ((C AND E) AND (C AND E)) ORDER BY ALL LIMIT 3 SETTINGS optimize_extract_common_expressions = 0; SELECT * FROM x WHERE (D AND 5) OR ((C AND E) AND (C AND E)) ORDER BY ALL LIMIT 3 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE dump_ast = 1 SELECT * FROM x WHERE (C AND E) OR ((C AND E) AND (C AND E)); + +-- TODO(antaljanosbenjamin): Add test for HAVING and QUALIFY From 852cb3705a78de535a8e3a23ac8fbdca3021ca91 Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Wed, 27 Nov 2024 09:08:00 -0300 Subject: [PATCH 193/502] remove trailing whitespace --- src/Functions/array/arrayPrAUC.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Functions/array/arrayPrAUC.cpp b/src/Functions/array/arrayPrAUC.cpp index d8ea470b541..4724bc3cfef 100644 --- a/src/Functions/array/arrayPrAUC.cpp +++ b/src/Functions/array/arrayPrAUC.cpp @@ -29,7 +29,7 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; * f(object) = score * predicted_label = score > threshold * - * This way classifier may predict positive or negative value correctly - true positive (tp) or true negative (tn) + * This way classifier may predict positive or negative value correctly - true positive (tp) or true negative (tn) * or have false positive (fp) or false negative (fn) result. * Varying the threshold we can get different probabilities of false positive or false negatives or true positives, etc... * @@ -44,7 +44,7 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; * Recall = P(score > threshold | label = positive) * * We can draw a curve of values of Precision and Recall with different threshold on [0..1] x [0..1] unit square. - * This curve is named "Precision Recall curve" (PR). + * This curve is named "Precision Recall curve" (PR). * * For the curve we can calculate, literally, Area Under the Curve, that will be in the range of [0..1]. * @@ -64,10 +64,10 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; * threshold = 0, TP = 2, FP = 2, FN = 0, Recall = 1.0, Precision = 0.5 * * This implementation uses the right Riemann sum (see https://en.wikipedia.org/wiki/Riemann_sum) to calculate the AUC. - * That is, each increment in area is calculated using `(R_n - R_{n-1}) * P_n`, + * That is, each increment in area is calculated using `(R_n - R_{n-1}) * P_n`, * where `R_n` is the Recall at the `n`-th point and `P_n` is the Precision at the `n`-th point. * - * This implementation is not interpolated and is different from computing the AUC with the trapezoidal rule, + * This implementation is not interpolated and is different from computing the AUC with the trapezoidal rule, * which uses linear interpolation and can be too optimistic for the Precision Recall AUC metric. */ @@ -112,15 +112,15 @@ private: { if (sorted_labels[i].score != prev_score) { - /* Precision = TP / (TP + FP) + /* Precision = TP / (TP + FP) * Recall = TP / (TP + FN) * * Instead of calculating - * d_Area = Precision_n * (Recall_n - Recall_{n-1}), - * we can just calculate - * d_Area = Precision_n * (TP_n - TP_{n-1}) - * and later divide it by (TP + FN). - * + * d_Area = Precision_n * (Recall_n - Recall_{n-1}), + * we can just calculate + * d_Area = Precision_n * (TP_n - TP_{n-1}) + * and later divide it by (TP + FN). + * * This can be done because (TP + FN) is constant and equal to total positive labels. */ curr_precision = static_cast(curr_tp) / curr_p; From b6599316b3d5e35eb3af46fe40dad5fc8e87406c Mon Sep 17 00:00:00 2001 From: erickurbanov Date: Wed, 27 Nov 2024 15:15:14 +0300 Subject: [PATCH 194/502] add function for const column --- src/Functions/array/arrayIndex.h | 50 +++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index 4f0acfebd7e..b69fda95113 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include "Common/FieldVisitors.h" #include "Common/Logger.h" #include "Common/logger_useful.h" #include @@ -115,15 +117,25 @@ private: return 0 == left.compareAt(i, RightArgIsConstant ? 0 : j, right, 1); } - static constexpr bool less(const PaddedPODArray & left, const Result & right, size_t i, size_t) noexcept + static bool compare(const Array & arr, const Field& rhs, size_t pos, size_t) + { + return applyVisitor(FieldVisitorAccurateEquals(), arr[pos], rhs); + } + + static constexpr bool lessOrEqual(const PaddedPODArray & left, const Result & right, size_t i, size_t) noexcept { return left[i] >= right; } - static constexpr bool less(const IColumn & left, const Result & right, size_t i, size_t) noexcept { return left[i] >= right; } + static constexpr bool lessOrEqual(const IColumn & left, const Result & right, size_t i, size_t) noexcept { return left[i] >= right; } + + static constexpr bool lessOrEqual(const Array& arr, const Field& rhs, size_t pos, size_t) noexcept { + return applyVisitor(FieldVisitorAccurateLessOrEqual(), rhs, arr[pos]); + } #pragma clang diagnostic pop +public: /** Assuming that the array is sorted, use a binary search */ template static constexpr ResultType lowerBound(const Data & data, const Target & target, size_t array_size, ArrOffset current_offset) @@ -133,7 +145,7 @@ private: while (high - low > 0) { auto middle = low + ((high - low) >> 1); - auto compare_result = less(data, target, current_offset + middle, 0); + auto compare_result = lessOrEqual(data, target, current_offset + middle, 0); /// avoid conditional branching high = compare_result ? middle : high; low = compare_result ? low : middle + 1; @@ -187,6 +199,22 @@ private: return current; } + static ResultType linearSearchConst(const Array & arr, const Field& value) { + ResultType current = 0; + for (size_t i = 0, size = arr.size(); i < size; ++i) + { + if (!applyVisitor(FieldVisitorAccurateEquals(), arr[i], value)) + continue; + + ConcreteAction::apply(current, i); + + if constexpr (!ConcreteAction::resume_execution) + break; + } + return current; + } + +private: /** Looking for the target element index in the data (array) */ template static constexpr ResultType getIndex( @@ -920,18 +948,12 @@ private: if (isColumnConst(*item_arg)) { - ResultType current = 0; + ResultType current; const auto & value = (*item_arg)[0]; - - for (size_t i = 0, size = arr.size(); i < size; ++i) - { - if (!applyVisitor(FieldVisitorAccurateEquals(), arr[i], value)) - continue; - - ConcreteAction::apply(current, i); - - if constexpr (!ConcreteAction::resume_execution) - break; + if constexpr (std::is_same_v) { + current = Impl::Main::lowerBound(arr, value, arr.size(), 0); + } else { + current = Impl::Main::linearSearchConst(arr, value); } return result_type->createColumnConst(item_arg->size(), current); From 2a095248d8764efada7c2933e29b337003c638c0 Mon Sep 17 00:00:00 2001 From: erickurbanov Date: Wed, 27 Nov 2024 15:17:25 +0300 Subject: [PATCH 195/502] fix initialization --- src/Functions/array/arrayIndex.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index b69fda95113..9be06861709 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -948,7 +948,7 @@ private: if (isColumnConst(*item_arg)) { - ResultType current; + ResultType current = 0; const auto & value = (*item_arg)[0]; if constexpr (std::is_same_v) { current = Impl::Main::lowerBound(arr, value, arr.size(), 0); From d650f756dd4226349cf5d89606a2ff0dfe021ed1 Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Wed, 27 Nov 2024 09:39:01 -0300 Subject: [PATCH 196/502] fix documented functions --- .../02415_all_new_functions_must_be_documented.reference | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index dea41174c65..5f374689614 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -124,6 +124,7 @@ arrayMax arrayMin arrayPopBack arrayPopFront +arrayPrAUC arrayProduct arrayPushBack arrayPushFront From fe4d0dc360ef84e6d075776cfc9d02873039fa34 Mon Sep 17 00:00:00 2001 From: erickurbanov Date: Wed, 27 Nov 2024 15:43:08 +0300 Subject: [PATCH 197/502] add stateless test --- .../03276_index_of_assume_sorted.reference | 8 ++++++ .../03276_index_of_assume_sorted.sql | 26 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/queries/0_stateless/03276_index_of_assume_sorted.reference create mode 100644 tests/queries/0_stateless/03276_index_of_assume_sorted.sql diff --git a/tests/queries/0_stateless/03276_index_of_assume_sorted.reference b/tests/queries/0_stateless/03276_index_of_assume_sorted.reference new file mode 100644 index 00000000000..400d55b21ea --- /dev/null +++ b/tests/queries/0_stateless/03276_index_of_assume_sorted.reference @@ -0,0 +1,8 @@ +7 +5 +0 +0 +0 +8 +0 +0 diff --git a/tests/queries/0_stateless/03276_index_of_assume_sorted.sql b/tests/queries/0_stateless/03276_index_of_assume_sorted.sql new file mode 100644 index 00000000000..b0784664619 --- /dev/null +++ b/tests/queries/0_stateless/03276_index_of_assume_sorted.sql @@ -0,0 +1,26 @@ +DROP TABLE IF EXISTS test; + +CREATE TABLE test( + id UInt64, + numbers Array(Int64) +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test VALUES(1, [1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 6, 7]); +INSERT INTO test VALUES (2, [1, 2, 3, 4, 5, 6, 7, 8]); +INSERT INTO test VALUES(3, [1, 3, 7, 10]); +INSERT INTO test VALUES(4, [0, 0, 0]); +INSERT INTO test VALUES(5, [10, 10, 10]); + +SELECT indexOfAssumeSorted(numbers, 4) FROM test WHERE id = 1; +SELECT indexOfAssumeSorted(numbers, 5) FROM test WHERE id = 2; +SELECT indexOfAssumeSorted(numbers, 5) FROM test WHERE id = 3; +SELECT indexOfAssumeSorted(numbers, 1) FROM test WHERE id = 4; +SELECT indexOfAssumeSorted(numbers, 1) FROM test WHERE id = 5; + +SELECT indexOfAssumeSorted([1, 2, 2, 2, 3, 3, 3, 4, 4], 4); +SELECT indexOfAssumeSorted([10, 10, 10], 1); +SELECT indexOfAssumeSorted([1, 1, 1], 10); + +DROP TABLE IF EXISTS test; From 0def39831509e9cb10564c044e4a269e43a41225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 27 Nov 2024 12:47:21 +0000 Subject: [PATCH 198/502] Fix formatting --- src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 38e1fe9dd74..35b25ad01de 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -345,7 +345,7 @@ std::optional tryExtractCommonExpressions(cons } else if (filtered_and_arguments.size() == 1) { - insertIfNotPresentInSet(new_or_arguments_set, new_or_arguments, std::move( filtered_and_arguments.front())); + insertIfNotPresentInSet(new_or_arguments_set, new_or_arguments, std::move(filtered_and_arguments.front())); } else { From 94d77d74a67806f98ab46502ae35669ed1caebcf Mon Sep 17 00:00:00 2001 From: erickurbanov Date: Wed, 27 Nov 2024 17:07:28 +0300 Subject: [PATCH 199/502] fix format --- src/Functions/array/arrayIndex.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index 9be06861709..f7addad2add 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -150,7 +150,8 @@ public: high = compare_result ? middle : high; low = compare_result ? low : middle + 1; } - if (low < array_size && compare(data, target, current_offset + low, 0)) { + if (low < array_size && compare(data, target, current_offset + low, 0)) + { ConcreteAction::apply(current, low); } return current; @@ -199,7 +200,8 @@ public: return current; } - static ResultType linearSearchConst(const Array & arr, const Field& value) { + static ResultType linearSearchConst(const Array & arr, const Field & value) + { ResultType current = 0; for (size_t i = 0, size = arr.size(); i < size; ++i) { @@ -950,9 +952,12 @@ private: { ResultType current = 0; const auto & value = (*item_arg)[0]; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) + { current = Impl::Main::lowerBound(arr, value, arr.size(), 0); - } else { + } + else + { current = Impl::Main::linearSearchConst(arr, value); } From f4fb1f89c7f82fc700f1bee4ce3363f0f6eec115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 27 Nov 2024 14:31:34 +0000 Subject: [PATCH 200/502] Add tests for `QUALIFY` and `HAVING` --- ...2_common_expression_optimization.reference | 198 ++++++++++++++++++ .../03262_common_expression_optimization.sql | 36 +++- 2 files changed, 233 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index 9e32e864286..53765308968 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -983,3 +983,201 @@ SELECT __table1.F AS F FROM default.x AS __table1 WHERE __table1.C AND __table1.E +-9220342792229035180 1 1 0 +-9185424447834220126 1 0 1 +-9147221708390825551 1 1 1 +-9114850274891890829 1 1 1 +-9114000995382454016 1 0 1 +-9099767307959300756 1 1 0 +-9077645148896772804 1 1 1 +-9076550218784816665 1 1 0 +-9074857609628938868 1 1 0 +-9066338222341664310 1 1 1 +-9220342792229035180 1 1 0 +-9185424447834220126 1 0 1 +-9147221708390825551 1 1 1 +-9114850274891890829 1 1 1 +-9114000995382454016 1 0 1 +-9099767307959300756 1 1 0 +-9077645148896772804 1 1 1 +-9076550218784816665 1 1 0 +-9074857609628938868 1 1 0 +-9066338222341664310 1 1 1 +QUERY id: 0 + PROJECTION COLUMNS + x Int64 + mA UInt8 + mB UInt8 + mC UInt8 + PROJECTION + LIST id: 1, nodes: 4 + COLUMN id: 2, column_name: x, result_type: Int64, source_id: 3 + FUNCTION id: 4, function_name: max, function_type: aggregate, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 1 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + FUNCTION id: 7, function_name: max, function_type: aggregate, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 1 + COLUMN id: 9, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 10, function_name: max, function_type: aggregate, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 1 + COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 3 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + GROUP BY + LIST id: 13, nodes: 1 + COLUMN id: 2, column_name: x, result_type: Int64, source_id: 3 + HAVING + FUNCTION id: 14, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 15, nodes: 2 + FUNCTION id: 4, function_name: max, function_type: aggregate, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 1 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + FUNCTION id: 16, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 17, nodes: 2 + FUNCTION id: 7, function_name: max, function_type: aggregate, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 1 + COLUMN id: 9, column_name: B, result_type: UInt8, source_id: 3 + FUNCTION id: 10, function_name: max, function_type: aggregate, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 1 + COLUMN id: 12, column_name: C, result_type: UInt8, source_id: 3 + +SELECT + __table1.x AS x, + max(__table1.A) AS mA, + max(__table1.B) AS mB, + max(__table1.C) AS mC +FROM default.x AS __table1 +GROUP BY __table1.x +HAVING max(__table1.A) AND (max(__table1.B) OR max(__table1.C)) +-9220342792229035180 1 1 0 +-9215479184005563028 1 1 1 +-9185424447834220126 1 1 1 +-9173414302980908256 1 1 1 +-9152339068839888086 1 1 1 +-9147221708390825551 1 1 1 +-9120736630719629705 1 0 1 +-9114850274891890829 1 1 1 +-9114000995382454016 1 1 1 +-9099767307959300756 1 1 0 +-9220342792229035180 1 1 0 +-9215479184005563028 1 1 1 +-9185424447834220126 1 1 1 +-9173414302980908256 1 1 1 +-9152339068839888086 1 1 1 +-9147221708390825551 1 1 1 +-9120736630719629705 1 0 1 +-9114850274891890829 1 1 1 +-9114000995382454016 1 1 1 +-9099767307959300756 1 1 0 +QUERY id: 0 + PROJECTION COLUMNS + x Int64 + mA UInt8 + mB UInt8 + mC UInt8 + PROJECTION + LIST id: 1, nodes: 4 + COLUMN id: 2, column_name: x, result_type: Int64, source_id: 3 + FUNCTION id: 4, function_name: max, function_type: window, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 1 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + WINDOW + WINDOW id: 7, frame_type: RANGE, frame_begin_type: unbounded preceding, frame_end_type: current + PARTITION BY + LIST id: 8, nodes: 1 + FUNCTION id: 9, function_name: modulo, function_type: ordinary, result_type: Int32 + ARGUMENTS + LIST id: 10, nodes: 2 + COLUMN id: 2, column_name: x, result_type: Int64, source_id: 3 + CONSTANT id: 11, constant_value: UInt64_1000, constant_value_type: UInt16 + FUNCTION id: 12, function_name: max, function_type: window, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 1 + COLUMN id: 14, column_name: B, result_type: UInt8, source_id: 3 + WINDOW + WINDOW id: 15, frame_type: RANGE, frame_begin_type: unbounded preceding, frame_end_type: current + PARTITION BY + LIST id: 16, nodes: 1 + FUNCTION id: 17, function_name: modulo, function_type: ordinary, result_type: Int32 + ARGUMENTS + LIST id: 18, nodes: 2 + COLUMN id: 2, column_name: x, result_type: Int64, source_id: 3 + CONSTANT id: 19, constant_value: UInt64_1000, constant_value_type: UInt16 + FUNCTION id: 20, function_name: max, function_type: window, result_type: UInt8 + ARGUMENTS + LIST id: 21, nodes: 1 + COLUMN id: 22, column_name: C, result_type: UInt8, source_id: 3 + WINDOW + WINDOW id: 23, frame_type: RANGE, frame_begin_type: unbounded preceding, frame_end_type: current + PARTITION BY + LIST id: 24, nodes: 1 + FUNCTION id: 25, function_name: modulo, function_type: ordinary, result_type: Int32 + ARGUMENTS + LIST id: 26, nodes: 2 + COLUMN id: 2, column_name: x, result_type: Int64, source_id: 3 + CONSTANT id: 27, constant_value: UInt64_1000, constant_value_type: UInt16 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + QUALIFY + FUNCTION id: 28, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 29, nodes: 2 + FUNCTION id: 4, function_name: max, function_type: window, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 1 + COLUMN id: 6, column_name: A, result_type: UInt8, source_id: 3 + WINDOW + WINDOW id: 7, frame_type: RANGE, frame_begin_type: unbounded preceding, frame_end_type: current + PARTITION BY + LIST id: 8, nodes: 1 + FUNCTION id: 9, function_name: modulo, function_type: ordinary, result_type: Int32 + ARGUMENTS + LIST id: 10, nodes: 2 + COLUMN id: 2, column_name: x, result_type: Int64, source_id: 3 + CONSTANT id: 11, constant_value: UInt64_1000, constant_value_type: UInt16 + FUNCTION id: 30, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 31, nodes: 2 + FUNCTION id: 12, function_name: max, function_type: window, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 1 + COLUMN id: 14, column_name: B, result_type: UInt8, source_id: 3 + WINDOW + WINDOW id: 15, frame_type: RANGE, frame_begin_type: unbounded preceding, frame_end_type: current + PARTITION BY + LIST id: 16, nodes: 1 + FUNCTION id: 17, function_name: modulo, function_type: ordinary, result_type: Int32 + ARGUMENTS + LIST id: 18, nodes: 2 + COLUMN id: 2, column_name: x, result_type: Int64, source_id: 3 + CONSTANT id: 19, constant_value: UInt64_1000, constant_value_type: UInt16 + FUNCTION id: 20, function_name: max, function_type: window, result_type: UInt8 + ARGUMENTS + LIST id: 21, nodes: 1 + COLUMN id: 22, column_name: C, result_type: UInt8, source_id: 3 + WINDOW + WINDOW id: 23, frame_type: RANGE, frame_begin_type: unbounded preceding, frame_end_type: current + PARTITION BY + LIST id: 24, nodes: 1 + FUNCTION id: 25, function_name: modulo, function_type: ordinary, result_type: Int32 + ARGUMENTS + LIST id: 26, nodes: 2 + COLUMN id: 2, column_name: x, result_type: Int64, source_id: 3 + CONSTANT id: 27, constant_value: UInt64_1000, constant_value_type: UInt16 + +SELECT + __table1.x AS x, + max(__table1.A) OVER (PARTITION BY __table1.x % 1000) AS mA, + max(__table1.B) OVER (PARTITION BY __table1.x % 1000) AS mB, + max(__table1.C) OVER (PARTITION BY __table1.x % 1000) AS mC +FROM default.x AS __table1 +QUALIFY max(__table1.A) OVER (PARTITION BY (__table1.x % 1000)) AND (max(__table1.B) OVER (PARTITION BY (__table1.x % 1000)) OR max(__table1.C) OVER (PARTITION BY (__table1.x % 1000))) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index 36add67bba4..202185022af 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -102,4 +102,38 @@ SELECT * FROM x WHERE (D AND 5) OR ((C AND E) AND (C AND E)) ORDER BY ALL LIMIT SELECT * FROM x WHERE (D AND 5) OR ((C AND E) AND (C AND E)) ORDER BY ALL LIMIT 3 SETTINGS optimize_extract_common_expressions = 1; EXPLAIN QUERY TREE dump_ast = 1 SELECT * FROM x WHERE (C AND E) OR ((C AND E) AND (C AND E)); --- TODO(antaljanosbenjamin): Add test for HAVING and QUALIFY +-- HAVING +SELECT x, max(A) AS mA, max(B) AS mB, max(C) AS mC FROM x GROUP BY x HAVING (mA AND mB) OR (mA AND mC) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT x, max(A) AS mA, max(B) AS mB, max(C) AS mC FROM x GROUP BY x HAVING (mA AND mB) OR (mA AND mC) ORDER BY x LIMIT 10; +EXPLAIN QUERY TREE dump_ast = 1 SELECT x, max(A) AS mA, max(B) AS mB, max(C) AS mC FROM x GROUP BY x HAVING (mA AND mB) OR (mA AND mC); + +-- QUALIFY +SELECT + x, + max(A) OVER (PARTITION BY x % 1000) AS mA, + max(B) OVER (PARTITION BY x % 1000) AS mB, + max(C) OVER (PARTITION BY x % 1000) AS mC +FROM x +QUALIFY (mA AND mB) OR (mA AND mC) +ORDER BY x +LIMIT 10 +SETTINGS optimize_extract_common_expressions = 0; + +SELECT + x, + max(A) OVER (PARTITION BY x % 1000) AS mA, + max(B) OVER (PARTITION BY x % 1000) AS mB, + max(C) OVER (PARTITION BY x % 1000) AS mC +FROM x +QUALIFY (mA AND mB) OR (mA AND mC) +ORDER BY x +LIMIT 10; + +EXPLAIN QUERY TREE dump_ast = 1 +SELECT + x, + max(A) OVER (PARTITION BY x % 1000) AS mA, + max(B) OVER (PARTITION BY x % 1000) AS mB, + max(C) OVER (PARTITION BY x % 1000) AS mC +FROM x +QUALIFY (mA AND mB) OR (mA AND mC); From 7c0c68ce1fd72cfd3dd7538045b79dddfd60a13f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:42:08 +0100 Subject: [PATCH 201/502] Add mutex at termination. --- src/Interpreters/CancellationChecker.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/CancellationChecker.cpp b/src/Interpreters/CancellationChecker.cpp index e0c1f88f89d..b6114905bbf 100644 --- a/src/Interpreters/CancellationChecker.cpp +++ b/src/Interpreters/CancellationChecker.cpp @@ -37,6 +37,7 @@ CancellationChecker& CancellationChecker::getInstance() void CancellationChecker::terminateThread() { + std::unique_lock lock(m); LOG_TRACE(getLogger("CancellationChecker"), "Stopping CancellationChecker"); stop_thread = true; cond_var.notify_all(); From 349668be96d4966beb0491e128dddd71dd4a4880 Mon Sep 17 00:00:00 2001 From: erickurbanov Date: Wed, 27 Nov 2024 21:24:32 +0300 Subject: [PATCH 202/502] add documentation for indexOfAssumeSorted --- src/Functions/array/indexOfAssumeSorted.cpp | 16 +++++++++++++++- ...ll_new_functions_must_be_documented.reference | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Functions/array/indexOfAssumeSorted.cpp b/src/Functions/array/indexOfAssumeSorted.cpp index 0a43eca8739..c4aeec665dd 100644 --- a/src/Functions/array/indexOfAssumeSorted.cpp +++ b/src/Functions/array/indexOfAssumeSorted.cpp @@ -1,6 +1,7 @@ #include "arrayIndex.h" #include #include +#include "Common/FunctionDocumentation.h" namespace DB { @@ -10,5 +11,18 @@ struct NameIndexOfAssumeSorted { static constexpr auto name = "indexOfAssumeSort /// should be used when the array is sorted (applies binary search to array) using FunctionIndexOfAssumeSorted = FunctionArrayIndex; -REGISTER_FUNCTION(IndexOfAssumeSorted) { factory.registerFunction(); } +REGISTER_FUNCTION(IndexOfAssumeSorted) +{ + factory.registerFunction(FunctionDocumentation{ + .description = R"( +The function finds the position of the first occurrence of element X in the array. +Indexing from one. +The function can be used when the internal array type is not Nullable and the array is sorted in non-decreasing order. +If the array type is Nullable, the 'indexOf' function will be used. +The binary search algorithm is used for the search. +For more details, see [https://en.wikipedia.org/wiki/Binary_search] +For an unsorted array, the behavior is undefined. +)", + .examples = {{.name = "", .query = "SELECT indexOfAssumeSorted([1, 2, 2, 2, 3, 3, 3, 4], 3) FROM test_table;"}}}); +} } diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index b97394742ea..89ae668331e 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -364,6 +364,7 @@ in inIgnoreSet indexHint indexOf +indexOfAssumeSorted initcap initcapUTF8 initialQueryID From c6634981c53d07cab8c60568c296465caeb13217 Mon Sep 17 00:00:00 2001 From: erickurbanov Date: Wed, 27 Nov 2024 21:40:38 +0300 Subject: [PATCH 203/502] fix -Werror --- src/Functions/array/indexOfAssumeSorted.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/array/indexOfAssumeSorted.cpp b/src/Functions/array/indexOfAssumeSorted.cpp index c4aeec665dd..8e36a157721 100644 --- a/src/Functions/array/indexOfAssumeSorted.cpp +++ b/src/Functions/array/indexOfAssumeSorted.cpp @@ -23,6 +23,6 @@ The binary search algorithm is used for the search. For more details, see [https://en.wikipedia.org/wiki/Binary_search] For an unsorted array, the behavior is undefined. )", - .examples = {{.name = "", .query = "SELECT indexOfAssumeSorted([1, 2, 2, 2, 3, 3, 3, 4], 3) FROM test_table;"}}}); + .examples = {{.name = "", .query = "SELECT indexOfAssumeSorted([1, 2, 2, 2, 3, 3, 3, 4], 3) FROM test_table;", .result=""}}}); } } From bb77b0f0c2291fe7eaef60e01cc39b9d6fc8e528 Mon Sep 17 00:00:00 2001 From: erickurbanov Date: Thu, 28 Nov 2024 00:59:49 +0300 Subject: [PATCH 204/502] add documentation --- .../sql-reference/functions/array-functions.md | 18 ++++++++++++++++++ .../sql-reference/functions/array-functions.md | 18 ++++++++++++++++++ .../sql-reference/functions/array-functions.md | 18 ++++++++++++++++++ .../sql-reference/functions/array-functions.md | 18 ++++++++++++++++++ 4 files changed, 72 insertions(+) diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index 5957b45a881..1b7e2d56455 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -786,6 +786,24 @@ SELECT indexOf([1, 3, NULL, NULL], NULL) Elements set to `NULL` are handled as normal values. +## indexOfAssumeSorted(arr, x) + +Returns the index of the first ‘x’ element (starting from 1) if it is in the array, or 0 if it is not. +The function should be used for an array sorted not in descending order since binary search is used for the search. +If the internal array type is Nullable, the ‘indexOf‘ function will be used. + +Example: + +``` sql +SELECT indexOfAssumeSorted([1, 3, 3, 3, 4, 4, 5], 4) +``` + +``` text +┌─indexOf([1, 3, 3, 3, 4, 4, 5], NULL)─┐ +│ 5 │ +└──────────────────────────────────--─-┘ +``` + ## arrayCount(\[func,\] arr1, ...) Returns the number of elements for which `func(arr1[i], ..., arrN[i])` returns something other than 0. If `func` is not specified, it returns the number of non-zero elements in the array. diff --git a/docs/ja/sql-reference/functions/array-functions.md b/docs/ja/sql-reference/functions/array-functions.md index bc4c9bef05c..4a900c5a8dc 100644 --- a/docs/ja/sql-reference/functions/array-functions.md +++ b/docs/ja/sql-reference/functions/array-functions.md @@ -785,6 +785,24 @@ SELECT indexOf([1, 3, NULL, NULL], NULL) `NULL` に設定された要素は通常の値として扱われます。 +# indexOfAssumeSorted(arr, x) + +配列内にある場合は最初の'x'要素(1から始まる)のインデックスを返し、そうでない場合は0を返します。 +この関数は、バイナリ検索が検索に使用されるため、降順ではなくソートされた配列に使用する必要があります。 +内部配列型がNull許容の場合は、‘indexOf‘関数が使用されます + +例: + +``` sql +SELECT indexOfAssumeSorted([1, 3, 3, 3, 4, 4, 5], 4) +``` + +``` text +┌─indexOf([1, 3, 3, 3, 4, 4, 5], NULL)─┐ +│ 5 │ +└──────────────────────────────────--─-┘ +``` + ## arrayCount(\[func,\] arr1, ...) `func(arr1[i], ..., arrN[i])`が0以外の値を返す要素の数を返します。`func` が指定されていない場合、配列内の0以外の要素の数を返します。 diff --git a/docs/ru/sql-reference/functions/array-functions.md b/docs/ru/sql-reference/functions/array-functions.md index 825e3f06be2..63d2595dcc8 100644 --- a/docs/ru/sql-reference/functions/array-functions.md +++ b/docs/ru/sql-reference/functions/array-functions.md @@ -306,6 +306,24 @@ SELECT indexOf([1, 3, NULL, NULL], NULL) └───────────────────────────────────┘ ``` +## indexOfAssumeSorted(arr, x) + +Возвращает индекс первого элемента x (начиная с 1), если он есть в массиве, или 0, если его нет. +Функция должна использоваться, если массив отсортирован в неубывающем порядке, так как используется бинарный поиск. +Если внутренний тип Nullable, то будет использована функция ‘indexOf‘. + +Пример: + +``` sql +SELECT indexOfAssumeSorted([1, 3, 3, 3, 4, 4, 5], 4) +``` + +``` text +┌─indexOf([1, 3, 3, 3, 4, 4, 5], NULL)─┐ +│ 5 │ +└──────────────────────────────────--─-┘ +``` + Элементы, равные `NULL`, обрабатываются как обычные значения. ## arrayCount(\[func,\] arr1, ...) {#array-count} diff --git a/docs/zh/sql-reference/functions/array-functions.md b/docs/zh/sql-reference/functions/array-functions.md index 69db34e4a36..83f93f8490a 100644 --- a/docs/zh/sql-reference/functions/array-functions.md +++ b/docs/zh/sql-reference/functions/array-functions.md @@ -337,6 +337,24 @@ SELECT indexOf([1, 3, NULL, NULL], NULL) 设置为«NULL»的元素将作为普通的元素值处理。 +## indexOfAssumeSorted(arr, x) + +返回数组中第一个’x’元素的索引（从1开始），如果’x’元素不存在在数组中，则返回0. +该函数应用于不按降序排序的数组，因为二进制搜索用于搜索。 +如果内部数组类型为空，则将使用’indexOf’函数。 + +示例: + +``` sql +SELECT indexOfAssumeSorted([1, 3, 3, 3, 4, 4, 5], 4) +``` + +``` text +┌─indexOf([1, 3, 3, 3, 4, 4, 5], NULL)─┐ +│ 5 │ +└──────────────────────────────────--─-┘ +``` + ## arrayCount(\[func,\] arr1, ...) {#array-count} `func`将arr数组作为参数，其返回结果为非零值的数量。如果未指定“func”，则返回数组中非零元素的数量。 From e9abd28ed12c46f143b51d140284294860201aba Mon Sep 17 00:00:00 2001 From: erickurbanov Date: Thu, 28 Nov 2024 01:13:32 +0300 Subject: [PATCH 205/502] add function`s name to aspell-dict --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 26358f83ce6..6726cfe7c61 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1953,6 +1953,7 @@ ilike incrementing indexHint indexOf +indexOfAssumeSorted inequal infi inflight From 83adb29584fdf7ae482b3ad5cfa00c8996da93d2 Mon Sep 17 00:00:00 2001 From: erickurbanov Date: Thu, 28 Nov 2024 02:00:24 +0300 Subject: [PATCH 206/502] Typos --- .../02415_all_new_functions_must_be_documented.reference | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index 89ae668331e..b97394742ea 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -364,7 +364,6 @@ in inIgnoreSet indexHint indexOf -indexOfAssumeSorted initcap initcapUTF8 initialQueryID From ffc33fb7c138b4e67f96b9ee2d21facc723e7090 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Thu, 28 Nov 2024 11:39:43 +0800 Subject: [PATCH 207/502] delete toUnixTimestampEx --- .../functions/date-time-functions.md | 38 ------------------- ...new_functions_must_be_documented.reference | 1 - .../03246_to_timestampex.reference | 9 ----- .../0_stateless/03246_to_timestampex.sql | 9 ----- 4 files changed, 57 deletions(-) delete mode 100644 tests/queries/0_stateless/03246_to_timestampex.reference delete mode 100644 tests/queries/0_stateless/03246_to_timestampex.sql diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index ed03ed46d76..2c563495777 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -868,44 +868,6 @@ Behavior for * Functions `toStartOfDay`, `toStartOfHour`, `toStartOfFifteenMinutes`, `toStartOfTenMinutes`, `toStartOfFiveMinutes`, `toStartOfMinute`, `timeSlot` return `DateTime` if their argument is a `Date` or `DateTime`, and they return `DateTime64` if their argument is a `Date32` or `DateTime64`. ::: -## toUnixTimestampEx - -Like `toUnixTimestamp`, converts a string, a date or a date with time to the [Unix Timestamp](https://en.wikipedia.org/wiki/Unix_time) in `Int64` representation. - -If the function is called with a string, it accepts an optional timezone argument. - -**Syntax** - -``` sql -toUnixTimestampEx(date) -toUnixTimestampEx(str, [timezone]) -``` - -**Returned value** - -- Returns the unix timestamp. [UInt32](../data-types/int-uint.md). - -**Example** - -``` sql -SELECT - '1969-01-01 00:00:00' AS dt_str, - toUnixTimestamp(dt_str) AS from_str, - toUnixTimestampEx(dt_str) AS ex_str, - toUnixTimestamp(dt_str, 'Asia/Tokyo') AS from_str_tokyo, - toUnixTimestampEx(dt_str, 'Asia/Tokyo') AS ex_str_tokyo, - toUnixTimestamp(toDateTime(dt_str)) AS from_datetime, - toUnixTimestampEx(toDateTime64(dt_str, 0)) AS ex_datetime64 -``` - -Result: - -``` -┌─dt_str──────────────┬─from_str─┬────ex_str─┬─from_str_tokyo─┬─ex_str_tokyo─┬─from_datetime─┬─ex_datetime64─┐ -│ 1969-01-01 00:00:00 │ 0 │ -31564800 │ 0 │ -31568400 │ 0 │ -31564800 │ -└─────────────────────┴──────────┴───────────┴────────────────┴──────────────┴───────────────┴───────────────┘ -``` - ## toStartOfYear Rounds down a date or date with time to the first day of the year. Returns the date as a `Date` object. diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index 5d047f6333e..39fdf042930 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -858,7 +858,6 @@ toUnixTimestamp64Micro toUnixTimestamp64Milli toUnixTimestamp64Nano toUnixTimestamp64Second -toUnixTimestampEx toValidUTF8 toWeek toYYYYMM diff --git a/tests/queries/0_stateless/03246_to_timestampex.reference b/tests/queries/0_stateless/03246_to_timestampex.reference deleted file mode 100644 index 23b6c78b943..00000000000 --- a/tests/queries/0_stateless/03246_to_timestampex.reference +++ /dev/null @@ -1,9 +0,0 @@ -1683676800 -1683676800 -1683676800 -1683676800 --1293882467 -0 -0 --28800 --28800 diff --git a/tests/queries/0_stateless/03246_to_timestampex.sql b/tests/queries/0_stateless/03246_to_timestampex.sql deleted file mode 100644 index 861a23c0bb1..00000000000 --- a/tests/queries/0_stateless/03246_to_timestampex.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT toUnixTimestampEx(makeDate(2023, 5, 10)); -SELECT toUnixTimestampEx(makeDate32(2023, 5, 10)); -SELECT toUnixTimestampEx(makeDate(2023, 5, 10), 'Pacific/Auckland'); -SELECT toUnixTimestampEx(makeDate32(2023, 5, 10), 'Pacific/Auckland'); -SELECT toUnixTimestampEx(toDateTime64('1928-12-31 12:12:12.123', 3, 'UTC')); -SELECT toUnixTimestampEx('1970-01-01 00:00:00', 'UTC'); -SELECT toUnixTimestampEx(materialize('1970-01-01 00:00:00'), 'UTC'); -SELECT toUnixTimestampEx('1970-01-01 00:00:00', 'Asia/Shanghai'); -SELECT toUnixTimestampEx(materialize('1970-01-01 00:00:00'), 'Asia/Shanghai'); From 128080c24914517e4d197302d606efe89c92eb76 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Thu, 28 Nov 2024 11:42:59 +0800 Subject: [PATCH 208/502] delete toUnixTimestampEx --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 23f202d4997..0383642385d 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -3165,5 +3165,4 @@ znode znodes zookeeperSessionUptime zstd -toUnixTimestampEx BFloat From bd26a06f237ecec606365e546a5ea5bd60f522ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 28 Nov 2024 09:12:39 +0000 Subject: [PATCH 209/502] Move setting changes to next release --- src/Core/SettingsChangesHistory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 66dbd943f9f..7c320724e4f 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -60,6 +60,7 @@ static std::initializer_list Date: Thu, 28 Nov 2024 12:46:57 +0100 Subject: [PATCH 210/502] fix pipeline cacelation --- src/Interpreters/AsynchronousInsertQueue.cpp | 1 + .../0_stateless/03277_async_insert.reference | 0 .../queries/0_stateless/03277_async_insert.sh | 40 +++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 tests/queries/0_stateless/03277_async_insert.reference create mode 100755 tests/queries/0_stateless/03277_async_insert.sh diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index ceda7adbcd4..f07548ded57 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -978,6 +978,7 @@ try if (chunk.getNumRows() == 0) { finish_entries(/*num_rows=*/ 0, /*num_bytes=*/ 0); + pipeline.cancel(); return; } diff --git a/tests/queries/0_stateless/03277_async_insert.reference b/tests/queries/0_stateless/03277_async_insert.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03277_async_insert.sh b/tests/queries/0_stateless/03277_async_insert.sh new file mode 100755 index 00000000000..d2ae2ac9a07 --- /dev/null +++ b/tests/queries/0_stateless/03277_async_insert.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -e + + +rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE}" +mkdir -p "${CLICKHOUSE_USER_FILES_UNIQUE}" + + +FILE_CSV="${CLICKHOUSE_USER_FILES_UNIQUE}/03277.csv" +FILE_ARROW="${CLICKHOUSE_USER_FILES_UNIQUE}/03277.arrow" + + +$CLICKHOUSE_CLIENT -q " + SET async_insert = 1; + SELECT '${FILE_CSV}'; + CREATE TABLE t0 (c0 Int) ENGINE = File(CSV); + INSERT INTO TABLE t0 (c0) VALUES (1); + INSERT INTO TABLE FUNCTION file('$FILE_CSV', 'CSV', 'c0 Int') SELECT c0 FROM t0; + INSERT INTO TABLE t0 (c0) FROM INFILE '$FILE_CSV' FORMAT CSV; +" +$CLICKHOUSE_CLIENT -q "SELECT * from t0" +$CLICKHOUSE_CLIENT -q "DROP TABLE t0" + +$CLICKHOUSE_CLIENT -q " + SET async_insert = 1; + SELECT '${FILE_ARROW}'; + CREATE TABLE t0 (c0 Int) ENGINE = Join(ANY, INNER, c0); + INSERT INTO TABLE FUNCTION file('$FILE_ARROW', 'Arrow', 'c0 Int') SELECT c0 FROM t0; + INSERT INTO TABLE t0 (c0) FROM INFILE '$FILE_ARROW' FORMAT Arrow; +" +$CLICKHOUSE_CLIENT -q "SELECT * from t0" +$CLICKHOUSE_CLIENT -q "DROP TABLE t0" + +rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE}" \ No newline at end of file From 285e27cf6f99f6633cc6177188fe4cec958571c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 28 Nov 2024 15:02:33 +0000 Subject: [PATCH 211/502] Cast the result type instead of discarding optimization result --- .../Passes/LogicalExpressionOptimizerPass.cpp | 22 ++++++------- ...2_common_expression_optimization.reference | 32 ++++++++++++------- .../03262_common_expression_optimization.sql | 9 +++--- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 35b25ad01de..92ba3e64626 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -11,6 +11,7 @@ #include #include #include +#include "Analyzer/IQueryTreeNode.h" namespace DB { @@ -403,12 +404,11 @@ void tryOptimizeCommonExpressionsInOr(QueryTreeNodePtr & node, const ContextPtr new_function_node->resolveAsFunction(and_function_resolver); new_root_node = std::move(new_function_node); } - } - // As only equivalent transformations are done on logical expressions (deduplication/flattening/elimination) (in other words we don't modify the values in (in)equalities), - // we can be sure the result type always remains convertible to UInt8 or Nullable(UInt8). We shouldn't change the nullability of the expression though. - if (new_root_node != nullptr && new_root_node->getResultType()->isNullable() == node->getResultType()->isNullable()) + if (!new_root_node->getResultType()->equals(*node->getResultType())) + new_root_node = buildCastFunction(new_root_node, node->getResultType(), context); node = std::move(new_root_node); + } } void tryOptimizeCommonExpressionsInAnd(QueryTreeNodePtr & node, const ContextPtr & context) @@ -445,15 +445,15 @@ void tryOptimizeCommonExpressionsInAnd(QueryTreeNodePtr & node, const ContextPtr if (!extracted_something) return; - auto new_root_node = std::make_shared("and"); - new_root_node->getArguments().getNodes() = std::move(new_top_level_arguments); + auto and_function_node = std::make_shared("and"); + and_function_node->getArguments().getNodes() = std::move(new_top_level_arguments); auto and_function_resolver = FunctionFactory::instance().get("and", context); - new_root_node->resolveAsFunction(and_function_resolver); + and_function_node->resolveAsFunction(and_function_resolver); + QueryTreeNodePtr new_root_node = and_function_node; - // As only equivalent transformations are done on logical expressions (deduplication/flattening/elimination) (in other words we don't modify the values in (in)equalities), - // we can be sure the result type always remains convertible to UInt8 or Nullable(UInt8). We shouldn't change the nullability of the expression though. - if (new_root_node != nullptr && new_root_node->getResultType()->isNullable() == node->getResultType()->isNullable()) - node = std::move(new_root_node); + if (!new_root_node->getResultType()->equals(*node->getResultType())) + new_root_node = buildCastFunction(new_root_node, node->getResultType(), context); + node = std::move(new_root_node); } void tryOptimizeCommonExpressions(QueryTreeNodePtr & node, FunctionNode& function_node, const ContextPtr & context) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index 53765308968..07bdcb33c40 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -744,7 +744,7 @@ QUERY id: 0 JOIN TREE TABLE id: 3, alias: __table1, table_name: default.x WHERE - FUNCTION id: 4, function_name: or, function_type: ordinary, result_type: Nullable(UInt8) + FUNCTION id: 4, function_name: _CAST, function_type: ordinary, result_type: Nullable(UInt8) ARGUMENTS LIST id: 5, nodes: 2 FUNCTION id: 6, function_name: and, function_type: ordinary, result_type: UInt8 @@ -752,19 +752,29 @@ QUERY id: 0 LIST id: 7, nodes: 2 COLUMN id: 8, column_name: B, result_type: UInt8, source_id: 3 COLUMN id: 9, column_name: C, result_type: UInt8, source_id: 3 - FUNCTION id: 10, function_name: and, function_type: ordinary, result_type: Nullable(UInt8) - ARGUMENTS - LIST id: 11, nodes: 3 - COLUMN id: 8, column_name: B, result_type: UInt8, source_id: 3 - COLUMN id: 9, column_name: C, result_type: UInt8, source_id: 3 - FUNCTION id: 12, function_name: toNullable, function_type: ordinary, result_type: Nullable(UInt8) - ARGUMENTS - LIST id: 13, nodes: 1 - COLUMN id: 14, column_name: F, result_type: UInt8, source_id: 3 + CONSTANT id: 10, constant_value: \'Nullable(UInt8)\', constant_value_type: String SELECT count() AS `count()` FROM default.x AS __table1 -WHERE (__table1.B AND __table1.C) OR (__table1.B AND __table1.C AND toNullable(__table1.F)) +WHERE _CAST(__table1.B AND __table1.C, \'Nullable(UInt8)\') +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.x + WHERE + FUNCTION id: 4, function_name: _CAST, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: x, result_type: Int64, source_id: 3 + CONSTANT id: 7, constant_value: \'UInt8\', constant_value_type: String + +SELECT count() AS `count()` +FROM default.x AS __table1 +WHERE _CAST(__table1.x, \'UInt8\') QUERY id: 0 PROJECTION COLUMNS count() UInt64 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index 202185022af..705824b01e8 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -75,9 +75,10 @@ SELECT * FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) OR (B AND A AND F)); --- Optimization is not applied if the result type would change in nullability, thus `toNullable(F)` cannot be eliminated +-- _CAST function has to be used to maintain the same result type EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND C) OR (B AND C AND toNullable(F))); --- Here the result type stays nullable because of `toNullable(C)`, so optimization will be applied +EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE (x AND x) OR (x AND x); +-- Here the result type stays nullable because of `toNullable(C)`, so no cast is needed EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND toNullable(C)) OR (B AND toNullable(C) AND toNullable(F))); -- Check that optimization only happen on top level, (C AND D) OR (C AND E) shouldn't be optimized @@ -91,7 +92,7 @@ INSERT INTO y SELECT x, A%2 AS A, B%2 AS B, C%2 AS C, D%2 AS D, E%2 AS E, F%2 AS -- JOIN expressions -- As the optimization code is shared between ON and WHERE, it is enough to test that the optimization is done also in ON SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, optimize_extract_common_expressions = 0; -SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1, optimize_extract_common_expressions = 1; +SELECT * FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1) ORDER BY ALL LIMIT 10 SETTINGS allow_experimental_join_condition = 1; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x INNER JOIN y ON ((x.A = y.A ) AND x.B = 1) OR ((x.A = y.A) AND y.C = 1); -- Check that optimization only happen on top level, (x.C = y.C AND x.D = y.D) OR (x.C = y.C AND x.E = y.E) shouldn't be optimized @@ -99,7 +100,7 @@ EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x INNER JOIN y ON (x.A = y.A -- Duplicated subexpressions, found by fuzzer SELECT * FROM x WHERE (D AND 5) OR ((C AND E) AND (C AND E)) ORDER BY ALL LIMIT 3 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE (D AND 5) OR ((C AND E) AND (C AND E)) ORDER BY ALL LIMIT 3 SETTINGS optimize_extract_common_expressions = 1; +SELECT * FROM x WHERE (D AND 5) OR ((C AND E) AND (C AND E)) ORDER BY ALL LIMIT 3; EXPLAIN QUERY TREE dump_ast = 1 SELECT * FROM x WHERE (C AND E) OR ((C AND E) AND (C AND E)); -- HAVING From 43d8a65b1027dd8a7274ff678c364a30b288953b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 28 Nov 2024 15:04:04 +0000 Subject: [PATCH 212/502] Update setting description with new functionality --- src/Core/Settings.cpp | 2 +- src/Core/SettingsChangesHistory.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 41ab1f8d89a..d61d0fca544 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -5736,7 +5736,7 @@ Allow writing simple SELECT queries without the leading SELECT keyword, which ma In `clickhouse-local` it is enabled by default and can be explicitly disabled. )", 0) \ DECLARE(Bool, optimize_extract_common_expressions, false, R"( -Allow extracting common expressions from disjunctions in WHERE and ON expressions. A logical expression like `(A AND B) OR (A AND C)` can be rewritten to `A AND (B OR C)`, which might help to utilize: +Allow extracting common expressions from disjunctions in WHERE, PREWHERE, ON, HAVING and QUALIFY expressions. A logical expression like `(A AND B) OR (A AND C)` can be rewritten to `A AND (B OR C)`, which might help to utilize: - indices in simple filtering expressions - cross to inner join optimization )", 0) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 8b420c468be..4edbd63cfb1 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -60,7 +60,7 @@ static std::initializer_list Date: Fri, 29 Nov 2024 10:51:43 +0000 Subject: [PATCH 213/502] Make values deterministic on all platforms and builds --- ...2_common_expression_optimization.reference | 732 +++++++++--------- .../03262_common_expression_optimization.sql | 35 +- 2 files changed, 388 insertions(+), 379 deletions(-) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index 07bdcb33c40..38a86cb421f 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -50,26 +50,26 @@ SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND (__table1.B OR __table1.C) SETTINGS optimize_extract_common_expressions = 1 --4890149726841642150 1 1 1 0 0 0 -6142641079032712283 1 1 1 0 0 0 --558050477524086008 1 1 1 0 0 0 -670122215481070522 1 1 1 0 0 0 --2818642828283588318 1 1 1 0 0 0 --6769625230840672398 1 1 1 0 0 0 -2360327909276087486 1 1 1 0 0 0 -5422332327868912849 1 1 1 0 0 0 --6185482623805481021 1 1 1 0 0 0 --6276639858940903557 1 1 1 0 0 0 --4890149726841642150 1 1 1 0 0 0 -6142641079032712283 1 1 1 0 0 0 --558050477524086008 1 1 1 0 0 0 -670122215481070522 1 1 1 0 0 0 --2818642828283588318 1 1 1 0 0 0 --6769625230840672398 1 1 1 0 0 0 -2360327909276087486 1 1 1 0 0 0 -5422332327868912849 1 1 1 0 0 0 --6185482623805481021 1 1 1 0 0 0 --6276639858940903557 1 1 1 0 0 0 +-9137658153367139416 1 1 1 0 0 0 +5551918026146320404 1 1 1 0 0 0 +-2996395377931803345 1 1 1 0 0 0 +5025358099778248663 1 1 1 0 0 0 +9185779160712471769 1 1 1 0 0 0 +95474718890257185 1 1 1 0 0 0 +2248520110901842660 1 1 1 0 0 0 +4269743235373690057 1 1 1 0 0 0 +-3254171257796361610 1 1 1 0 0 0 +-2057347811709491782 1 1 1 0 0 0 +-9137658153367139416 1 1 1 0 0 0 +5551918026146320404 1 1 1 0 0 0 +-2996395377931803345 1 1 1 0 0 0 +5025358099778248663 1 1 1 0 0 0 +9185779160712471769 1 1 1 0 0 0 +95474718890257185 1 1 1 0 0 0 +2248520110901842660 1 1 1 0 0 0 +4269743235373690057 1 1 1 0 0 0 +-3254171257796361610 1 1 1 0 0 0 +-2057347811709491782 1 1 1 0 0 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -89,26 +89,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND __table1.B AND __table1.C --1417696215412225994 1 1 1 0 0 1 --7491222816216432223 1 1 1 0 0 1 --7667054131998434193 1 1 1 0 0 1 -1150290670655138617 1 1 1 0 0 1 --3651160188931679372 1 1 1 0 0 1 -7727375535152756381 1 1 1 0 0 1 -1248518317831711188 1 1 1 0 0 1 --7224086688461492380 1 1 1 0 0 1 -9032006395921235708 1 1 1 0 0 1 -7044288989182144315 1 1 1 0 0 1 --1417696215412225994 1 1 1 0 0 1 --7491222816216432223 1 1 1 0 0 1 --7667054131998434193 1 1 1 0 0 1 -1150290670655138617 1 1 1 0 0 1 --3651160188931679372 1 1 1 0 0 1 -7727375535152756381 1 1 1 0 0 1 -1248518317831711188 1 1 1 0 0 1 --7224086688461492380 1 1 1 0 0 1 -9032006395921235708 1 1 1 0 0 1 -7044288989182144315 1 1 1 0 0 1 +-1718308154946016060 1 1 1 0 0 1 +-909098580002376266 1 1 1 0 0 1 +-3361624987358969173 1 1 1 0 0 1 +-3655654558316512012 1 1 1 0 0 1 +-8699036683208290351 1 1 1 0 0 1 +4783429523421833175 1 1 1 0 0 1 +7828110511858076458 1 1 1 0 0 1 +-1265166225143531838 1 1 1 0 0 1 +-7556142413115723841 1 1 1 0 0 1 +7754782253328504134 1 1 1 0 0 1 +-1718308154946016060 1 1 1 0 0 1 +-909098580002376266 1 1 1 0 0 1 +-3361624987358969173 1 1 1 0 0 1 +-3655654558316512012 1 1 1 0 0 1 +-8699036683208290351 1 1 1 0 0 1 +4783429523421833175 1 1 1 0 0 1 +7828110511858076458 1 1 1 0 0 1 +-1265166225143531838 1 1 1 0 0 1 +-7556142413115723841 1 1 1 0 0 1 +7754782253328504134 1 1 1 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -133,26 +133,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND (__table1.E OR __table1.F) AND __table1.B AND __table1.C --1417696215412225994 1 1 1 0 0 1 --7491222816216432223 1 1 1 0 0 1 --7667054131998434193 1 1 1 0 0 1 -1150290670655138617 1 1 1 0 0 1 --3651160188931679372 1 1 1 0 0 1 -7727375535152756381 1 1 1 0 0 1 -1248518317831711188 1 1 1 0 0 1 --7224086688461492380 1 1 1 0 0 1 -9032006395921235708 1 1 1 0 0 1 -7044288989182144315 1 1 1 0 0 1 --1417696215412225994 1 1 1 0 0 1 --7491222816216432223 1 1 1 0 0 1 --7667054131998434193 1 1 1 0 0 1 -1150290670655138617 1 1 1 0 0 1 --3651160188931679372 1 1 1 0 0 1 -7727375535152756381 1 1 1 0 0 1 -1248518317831711188 1 1 1 0 0 1 --7224086688461492380 1 1 1 0 0 1 -9032006395921235708 1 1 1 0 0 1 -7044288989182144315 1 1 1 0 0 1 +-1718308154946016060 1 1 1 0 0 1 +-909098580002376266 1 1 1 0 0 1 +-3361624987358969173 1 1 1 0 0 1 +-3655654558316512012 1 1 1 0 0 1 +-8699036683208290351 1 1 1 0 0 1 +4783429523421833175 1 1 1 0 0 1 +7828110511858076458 1 1 1 0 0 1 +-1265166225143531838 1 1 1 0 0 1 +-7556142413115723841 1 1 1 0 0 1 +7754782253328504134 1 1 1 0 0 1 +-1718308154946016060 1 1 1 0 0 1 +-909098580002376266 1 1 1 0 0 1 +-3361624987358969173 1 1 1 0 0 1 +-3655654558316512012 1 1 1 0 0 1 +-8699036683208290351 1 1 1 0 0 1 +4783429523421833175 1 1 1 0 0 1 +7828110511858076458 1 1 1 0 0 1 +-1265166225143531838 1 1 1 0 0 1 +-7556142413115723841 1 1 1 0 0 1 +7754782253328504134 1 1 1 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -177,26 +177,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND (__table1.E OR __table1.F) AND __table1.B AND __table1.C --7064227703501672459 1 1 0 0 1 0 --5989383837337135212 1 1 0 0 1 0 --2960971598243161531 1 1 0 0 1 0 --2091785646518281290 1 1 0 0 1 0 --2587761388937978501 1 1 0 0 1 0 -4189632917510416150 1 1 0 0 1 0 -329410740548269838 1 1 0 0 1 0 --966942443397313698 1 1 0 0 1 0 -8454473642815416800 1 1 0 0 1 0 -8268045340657794461 1 1 0 0 1 0 --7064227703501672459 1 1 0 0 1 0 --5989383837337135212 1 1 0 0 1 0 --2960971598243161531 1 1 0 0 1 0 --2091785646518281290 1 1 0 0 1 0 --2587761388937978501 1 1 0 0 1 0 -4189632917510416150 1 1 0 0 1 0 -329410740548269838 1 1 0 0 1 0 --966942443397313698 1 1 0 0 1 0 -8454473642815416800 1 1 0 0 1 0 -8268045340657794461 1 1 0 0 1 0 +3574384104825527969 1 1 0 0 1 0 +-2211493827666596761 1 1 0 0 1 0 +8958830522204821629 1 1 0 0 1 0 +1346739481280160211 1 1 0 0 1 0 +3279338559385463408 1 1 0 0 1 0 +3312987885102520931 1 1 0 0 1 0 +1164803530066053002 1 1 0 0 1 0 +-5278928290423082776 1 1 0 0 1 0 +8491129310781185476 1 1 0 0 1 0 +-7239011331174116716 1 1 0 0 1 0 +3574384104825527969 1 1 0 0 1 0 +-2211493827666596761 1 1 0 0 1 0 +8958830522204821629 1 1 0 0 1 0 +1346739481280160211 1 1 0 0 1 0 +3279338559385463408 1 1 0 0 1 0 +3312987885102520931 1 1 0 0 1 0 +1164803530066053002 1 1 0 0 1 0 +-5278928290423082776 1 1 0 0 1 0 +8491129310781185476 1 1 0 0 1 0 +-7239011331174116716 1 1 0 0 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -221,26 +221,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND (__table1.C OR __table1.D OR __table1.E) AND __table1.B --7064227703501672459 1 1 0 0 1 0 --5989383837337135212 1 1 0 0 1 0 --2960971598243161531 1 1 0 0 1 0 --2091785646518281290 1 1 0 0 1 0 --2587761388937978501 1 1 0 0 1 0 -4189632917510416150 1 1 0 0 1 0 -329410740548269838 1 1 0 0 1 0 --966942443397313698 1 1 0 0 1 0 -8454473642815416800 1 1 0 0 1 0 -8268045340657794461 1 1 0 0 1 0 --7064227703501672459 1 1 0 0 1 0 --5989383837337135212 1 1 0 0 1 0 --2960971598243161531 1 1 0 0 1 0 --2091785646518281290 1 1 0 0 1 0 --2587761388937978501 1 1 0 0 1 0 -4189632917510416150 1 1 0 0 1 0 -329410740548269838 1 1 0 0 1 0 --966942443397313698 1 1 0 0 1 0 -8454473642815416800 1 1 0 0 1 0 -8268045340657794461 1 1 0 0 1 0 +3574384104825527969 1 1 0 0 1 0 +-2211493827666596761 1 1 0 0 1 0 +8958830522204821629 1 1 0 0 1 0 +1346739481280160211 1 1 0 0 1 0 +3279338559385463408 1 1 0 0 1 0 +3312987885102520931 1 1 0 0 1 0 +1164803530066053002 1 1 0 0 1 0 +-5278928290423082776 1 1 0 0 1 0 +8491129310781185476 1 1 0 0 1 0 +-7239011331174116716 1 1 0 0 1 0 +3574384104825527969 1 1 0 0 1 0 +-2211493827666596761 1 1 0 0 1 0 +8958830522204821629 1 1 0 0 1 0 +1346739481280160211 1 1 0 0 1 0 +3279338559385463408 1 1 0 0 1 0 +3312987885102520931 1 1 0 0 1 0 +1164803530066053002 1 1 0 0 1 0 +-5278928290423082776 1 1 0 0 1 0 +8491129310781185476 1 1 0 0 1 0 +-7239011331174116716 1 1 0 0 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -265,26 +265,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND (__table1.C OR __table1.D OR __table1.E) AND __table1.B -7046366705027993992 0 1 1 0 0 0 -5876398964123690873 0 1 1 0 0 0 --4849840134697008981 0 1 1 0 0 0 -8219785106025342771 0 1 1 0 0 0 -7106055360641449065 0 1 1 0 0 0 -7782502397032484090 0 1 1 0 0 0 -1587022221789919998 0 1 1 0 0 0 --5488510425784433218 0 1 1 0 0 0 --3751993301879274716 0 1 1 0 0 0 -3995320070609749801 0 1 1 0 0 0 -7046366705027993992 0 1 1 0 0 0 -5876398964123690873 0 1 1 0 0 0 --4849840134697008981 0 1 1 0 0 0 -8219785106025342771 0 1 1 0 0 0 -7106055360641449065 0 1 1 0 0 0 -7782502397032484090 0 1 1 0 0 0 -1587022221789919998 0 1 1 0 0 0 --5488510425784433218 0 1 1 0 0 0 --3751993301879274716 0 1 1 0 0 0 -3995320070609749801 0 1 1 0 0 0 +910439267853176941 0 1 1 0 0 0 +-835764896252397456 0 1 1 0 0 0 +36802202306659819 0 1 1 0 0 0 +-21405676422526023 0 1 1 0 0 0 +-186684117184539357 0 1 1 0 0 0 +2294399721801435121 0 1 1 0 0 0 +271677944114046880 0 1 1 0 0 0 +-4341212657373873318 0 1 1 0 0 0 +-4556789282933643330 0 1 1 0 0 0 +4042734258081112561 0 1 1 0 0 0 +910439267853176941 0 1 1 0 0 0 +-835764896252397456 0 1 1 0 0 0 +36802202306659819 0 1 1 0 0 0 +-21405676422526023 0 1 1 0 0 0 +-186684117184539357 0 1 1 0 0 0 +2294399721801435121 0 1 1 0 0 0 +271677944114046880 0 1 1 0 0 0 +-4341212657373873318 0 1 1 0 0 0 +-4556789282933643330 0 1 1 0 0 0 +4042734258081112561 0 1 1 0 0 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -303,26 +303,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.B AND __table1.C -331058316081182921 0 1 1 0 0 1 --2878539180531591201 0 1 1 0 0 1 -5634271855584736521 0 1 1 0 0 1 -602708131544673121 0 1 1 0 0 1 --7538506014351088906 0 1 1 0 0 1 --7767907992783882398 0 1 1 0 0 1 -3427035445395945973 0 1 1 0 0 1 -1130846464892565445 0 1 1 0 0 1 --7229259894257180861 0 1 1 0 0 1 --9156504923995237243 0 1 1 0 0 1 -331058316081182921 0 1 1 0 0 1 --2878539180531591201 0 1 1 0 0 1 -5634271855584736521 0 1 1 0 0 1 -602708131544673121 0 1 1 0 0 1 --7538506014351088906 0 1 1 0 0 1 --7767907992783882398 0 1 1 0 0 1 -3427035445395945973 0 1 1 0 0 1 -1130846464892565445 0 1 1 0 0 1 --7229259894257180861 0 1 1 0 0 1 --9156504923995237243 0 1 1 0 0 1 +5604528747227576251 0 1 1 0 0 1 +-3227180127647371250 0 1 1 0 0 1 +-3777435661756238740 0 1 1 0 0 1 +-4105312414544198416 0 1 1 0 0 1 +3624151743867502404 0 1 1 0 0 1 +4327451746496881795 0 1 1 0 0 1 +-7076772903747348221 0 1 1 0 0 1 +520798717162063931 0 1 1 0 0 1 +-8449852538623664136 0 1 1 0 0 1 +1104848393205607021 0 1 1 0 0 1 +5604528747227576251 0 1 1 0 0 1 +-3227180127647371250 0 1 1 0 0 1 +-3777435661756238740 0 1 1 0 0 1 +-4105312414544198416 0 1 1 0 0 1 +3624151743867502404 0 1 1 0 0 1 +4327451746496881795 0 1 1 0 0 1 +-7076772903747348221 0 1 1 0 0 1 +520798717162063931 0 1 1 0 0 1 +-8449852538623664136 0 1 1 0 0 1 +1104848393205607021 0 1 1 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -346,26 +346,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.B AND __table1.C AND (__table1.E OR __table1.F) -331058316081182921 0 1 1 0 0 1 --2878539180531591201 0 1 1 0 0 1 -5634271855584736521 0 1 1 0 0 1 -602708131544673121 0 1 1 0 0 1 --7538506014351088906 0 1 1 0 0 1 --7767907992783882398 0 1 1 0 0 1 -3427035445395945973 0 1 1 0 0 1 -1130846464892565445 0 1 1 0 0 1 --7229259894257180861 0 1 1 0 0 1 --9156504923995237243 0 1 1 0 0 1 -331058316081182921 0 1 1 0 0 1 --2878539180531591201 0 1 1 0 0 1 -5634271855584736521 0 1 1 0 0 1 -602708131544673121 0 1 1 0 0 1 --7538506014351088906 0 1 1 0 0 1 --7767907992783882398 0 1 1 0 0 1 -3427035445395945973 0 1 1 0 0 1 -1130846464892565445 0 1 1 0 0 1 --7229259894257180861 0 1 1 0 0 1 --9156504923995237243 0 1 1 0 0 1 +5604528747227576251 0 1 1 0 0 1 +-3227180127647371250 0 1 1 0 0 1 +-3777435661756238740 0 1 1 0 0 1 +-4105312414544198416 0 1 1 0 0 1 +3624151743867502404 0 1 1 0 0 1 +4327451746496881795 0 1 1 0 0 1 +-7076772903747348221 0 1 1 0 0 1 +520798717162063931 0 1 1 0 0 1 +-8449852538623664136 0 1 1 0 0 1 +1104848393205607021 0 1 1 0 0 1 +5604528747227576251 0 1 1 0 0 1 +-3227180127647371250 0 1 1 0 0 1 +-3777435661756238740 0 1 1 0 0 1 +-4105312414544198416 0 1 1 0 0 1 +3624151743867502404 0 1 1 0 0 1 +4327451746496881795 0 1 1 0 0 1 +-7076772903747348221 0 1 1 0 0 1 +520798717162063931 0 1 1 0 0 1 +-8449852538623664136 0 1 1 0 0 1 +1104848393205607021 0 1 1 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -389,26 +389,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.B AND __table1.C AND (__table1.E OR __table1.F) --8350064335402276481 0 1 0 0 1 0 -2766908592531260293 0 1 0 0 1 0 --8500464668406226757 0 1 0 0 1 0 --4469615932557529717 0 1 0 0 1 0 --2676464305117434926 0 1 0 0 1 0 --3455317119139662830 0 1 0 0 1 0 -5588544998641306169 0 1 0 0 1 0 --9173414302980908256 0 1 0 0 1 0 -7212851845568196641 0 1 0 0 1 0 --3704547074519116929 0 1 0 0 1 0 --8350064335402276481 0 1 0 0 1 0 -2766908592531260293 0 1 0 0 1 0 --8500464668406226757 0 1 0 0 1 0 --4469615932557529717 0 1 0 0 1 0 --2676464305117434926 0 1 0 0 1 0 --3455317119139662830 0 1 0 0 1 0 -5588544998641306169 0 1 0 0 1 0 --9173414302980908256 0 1 0 0 1 0 -7212851845568196641 0 1 0 0 1 0 --3704547074519116929 0 1 0 0 1 0 +-761481560319605917 0 1 0 0 1 0 +-5533222425689494786 0 1 0 0 1 0 +989832766219156063 0 1 0 0 1 0 +-3972983892221867704 0 1 0 0 1 0 +-8548855706734087530 0 1 0 0 1 0 +5284071364923999442 0 1 0 0 1 0 +5223032192244036365 0 1 0 0 1 0 +8288659361821935470 0 1 0 0 1 0 +3669979250564809316 0 1 0 0 1 0 +-3068712899003736471 0 1 0 0 1 0 +-761481560319605917 0 1 0 0 1 0 +-5533222425689494786 0 1 0 0 1 0 +989832766219156063 0 1 0 0 1 0 +-3972983892221867704 0 1 0 0 1 0 +-8548855706734087530 0 1 0 0 1 0 +5284071364923999442 0 1 0 0 1 0 +5223032192244036365 0 1 0 0 1 0 +8288659361821935470 0 1 0 0 1 0 +3669979250564809316 0 1 0 0 1 0 +-3068712899003736471 0 1 0 0 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -432,26 +432,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.B AND (__table1.C OR __table1.D OR __table1.E) --8350064335402276481 0 1 0 0 1 0 -2766908592531260293 0 1 0 0 1 0 --8500464668406226757 0 1 0 0 1 0 --4469615932557529717 0 1 0 0 1 0 --2676464305117434926 0 1 0 0 1 0 --3455317119139662830 0 1 0 0 1 0 -5588544998641306169 0 1 0 0 1 0 --9173414302980908256 0 1 0 0 1 0 -7212851845568196641 0 1 0 0 1 0 --3704547074519116929 0 1 0 0 1 0 --8350064335402276481 0 1 0 0 1 0 -2766908592531260293 0 1 0 0 1 0 --8500464668406226757 0 1 0 0 1 0 --4469615932557529717 0 1 0 0 1 0 --2676464305117434926 0 1 0 0 1 0 --3455317119139662830 0 1 0 0 1 0 -5588544998641306169 0 1 0 0 1 0 --9173414302980908256 0 1 0 0 1 0 -7212851845568196641 0 1 0 0 1 0 --3704547074519116929 0 1 0 0 1 0 +-761481560319605917 0 1 0 0 1 0 +-5533222425689494786 0 1 0 0 1 0 +989832766219156063 0 1 0 0 1 0 +-3972983892221867704 0 1 0 0 1 0 +-8548855706734087530 0 1 0 0 1 0 +5284071364923999442 0 1 0 0 1 0 +5223032192244036365 0 1 0 0 1 0 +8288659361821935470 0 1 0 0 1 0 +3669979250564809316 0 1 0 0 1 0 +-3068712899003736471 0 1 0 0 1 0 +-761481560319605917 0 1 0 0 1 0 +-5533222425689494786 0 1 0 0 1 0 +989832766219156063 0 1 0 0 1 0 +-3972983892221867704 0 1 0 0 1 0 +-8548855706734087530 0 1 0 0 1 0 +5284071364923999442 0 1 0 0 1 0 +5223032192244036365 0 1 0 0 1 0 +8288659361821935470 0 1 0 0 1 0 +3669979250564809316 0 1 0 0 1 0 +-3068712899003736471 0 1 0 0 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -475,26 +475,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.B AND (__table1.C OR __table1.D OR __table1.E) -7296788055532769937 0 1 0 0 0 0 --8085083078325036349 0 1 0 0 0 0 -8056746393406743698 0 1 0 0 0 0 --5618901525230504315 0 1 0 0 0 0 --6790762979674887120 0 1 0 0 0 0 -7609629462704427873 0 1 0 0 0 0 -7776384315538625189 0 1 0 0 0 0 -1381341868259371991 0 1 0 0 0 0 -2483552736214921240 0 1 0 0 0 0 -5286533408672438546 0 1 0 0 0 0 -7296788055532769937 0 1 0 0 0 0 --8085083078325036349 0 1 0 0 0 0 -8056746393406743698 0 1 0 0 0 0 --5618901525230504315 0 1 0 0 0 0 --6790762979674887120 0 1 0 0 0 0 -7609629462704427873 0 1 0 0 0 0 -7776384315538625189 0 1 0 0 0 0 -1381341868259371991 0 1 0 0 0 0 -2483552736214921240 0 1 0 0 0 0 -5286533408672438546 0 1 0 0 0 0 +-608986750099316624 0 1 0 0 0 0 +4634030441303755773 0 1 0 0 0 0 +1594908885273767697 0 1 0 0 0 0 +4525560033677464079 0 1 0 0 0 0 +2306683870575600851 0 1 0 0 0 0 +-4812729937086184087 0 1 0 0 0 0 +-8185772668638940850 0 1 0 0 0 0 +-8315367594051711131 0 1 0 0 0 0 +5705028529783052191 0 1 0 0 0 0 +-8545685458500363239 0 1 0 0 0 0 +-608986750099316624 0 1 0 0 0 0 +4634030441303755773 0 1 0 0 0 0 +1594908885273767697 0 1 0 0 0 0 +4525560033677464079 0 1 0 0 0 0 +2306683870575600851 0 1 0 0 0 0 +-4812729937086184087 0 1 0 0 0 0 +-8185772668638940850 0 1 0 0 0 0 +-8315367594051711131 0 1 0 0 0 0 +5705028529783052191 0 1 0 0 0 0 +-8545685458500363239 0 1 0 0 0 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -527,26 +527,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE (sipHash64(__table1.C) = sipHash64(__table1.D)) AND (__table1.A OR __table1.B) -2589067852420946093 0 0 0 0 1 1 -566510304360553038 0 0 0 0 1 1 -4712156778925374545 0 0 0 0 1 1 -6513794688600930866 0 0 0 0 1 1 -8420009359281429633 0 0 0 0 1 1 -2151442296722776334 0 0 0 0 1 1 --7155015417725275505 0 0 0 0 1 1 -8063264694611659226 0 0 0 0 1 1 -7596093258110041712 0 0 0 0 1 1 --5463634588980750408 0 0 0 0 1 1 -2589067852420946093 0 0 0 0 1 1 -566510304360553038 0 0 0 0 1 1 -4712156778925374545 0 0 0 0 1 1 -6513794688600930866 0 0 0 0 1 1 -8420009359281429633 0 0 0 0 1 1 -2151442296722776334 0 0 0 0 1 1 --7155015417725275505 0 0 0 0 1 1 -8063264694611659226 0 0 0 0 1 1 -7596093258110041712 0 0 0 0 1 1 --5463634588980750408 0 0 0 0 1 1 +7762400598731351030 0 0 0 0 1 1 +4383923020933692842 0 0 0 0 1 1 +-8485383028044727808 0 0 0 0 1 1 +-8417770467973836608 0 0 0 0 1 1 +2299883611232961048 0 0 0 0 1 1 +1390628346580495568 0 0 0 0 1 1 +9080544688861627695 0 0 0 0 1 1 +-7721736184933939401 0 0 0 0 1 1 +4299643423346067577 0 0 0 0 1 1 +-4360831390913279988 0 0 0 0 1 1 +7762400598731351030 0 0 0 0 1 1 +4383923020933692842 0 0 0 0 1 1 +-8485383028044727808 0 0 0 0 1 1 +-8417770467973836608 0 0 0 0 1 1 +2299883611232961048 0 0 0 0 1 1 +1390628346580495568 0 0 0 0 1 1 +9080544688861627695 0 0 0 0 1 1 +-7721736184933939401 0 0 0 0 1 1 +4299643423346067577 0 0 0 0 1 1 +-4360831390913279988 0 0 0 0 1 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -581,26 +581,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE (__table1.A AND __table1.B) OR ((__table1.C AND __table1.D) OR (__table1.E AND __table1.F)) -813655996633899418 0 0 0 0 1 1 -6513794688600930866 0 0 0 0 1 1 -2151442296722776334 0 0 0 0 1 1 --2316167413850459561 0 0 0 0 1 1 --5864912137959118127 0 0 0 0 1 1 --5326152364639478952 0 0 0 0 1 1 --5463634588980750408 0 0 0 0 1 1 -6704351461338887300 0 0 0 0 1 1 --973750167321101279 0 0 0 0 1 1 -8063264694611659226 0 0 0 0 1 1 -813655996633899418 0 0 0 0 1 1 -6513794688600930866 0 0 0 0 1 1 -2151442296722776334 0 0 0 0 1 1 --2316167413850459561 0 0 0 0 1 1 --5864912137959118127 0 0 0 0 1 1 --5326152364639478952 0 0 0 0 1 1 --5463634588980750408 0 0 0 0 1 1 -6704351461338887300 0 0 0 0 1 1 --973750167321101279 0 0 0 0 1 1 -8063264694611659226 0 0 0 0 1 1 +2061801445197034299 0 0 0 0 1 1 +-4692710336366551784 0 0 0 0 1 1 +8827599087436950777 0 0 0 0 1 1 +-882303417518114752 0 0 0 0 1 1 +3625807179116440482 0 0 0 0 1 1 +-8417770467973836608 0 0 0 0 1 1 +2813303165430768949 0 0 0 0 1 1 +-8353024809925920489 0 0 0 0 1 1 +-6723016927829163686 0 0 0 0 1 1 +2766695568743886462 0 0 0 0 1 1 +2061801445197034299 0 0 0 0 1 1 +-4692710336366551784 0 0 0 0 1 1 +8827599087436950777 0 0 0 0 1 1 +-882303417518114752 0 0 0 0 1 1 +3625807179116440482 0 0 0 0 1 1 +-8417770467973836608 0 0 0 0 1 1 +2813303165430768949 0 0 0 0 1 1 +-8353024809925920489 0 0 0 0 1 1 +-6723016927829163686 0 0 0 0 1 1 +2766695568743886462 0 0 0 0 1 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -635,26 +635,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE (__table1.A AND __table1.B) OR ((__table1.B AND __table1.D) OR (__table1.E AND __table1.F)) -6015136002503835948 1 1 0 0 0 1 --3883799183493487473 1 1 0 0 0 1 --3002427897361111856 1 1 0 0 0 1 --2523952504416578832 1 1 0 0 0 1 --3187045566703922613 1 1 0 0 0 1 -4838814183373278674 1 1 0 0 0 1 -620980830413267931 1 1 0 0 0 1 --5907701125400227703 1 1 0 0 0 1 --1174991811984636509 1 1 0 0 0 1 -446104724568266173 1 1 0 0 0 1 -6015136002503835948 1 1 0 0 0 1 --3883799183493487473 1 1 0 0 0 1 --3002427897361111856 1 1 0 0 0 1 --2523952504416578832 1 1 0 0 0 1 --3187045566703922613 1 1 0 0 0 1 -4838814183373278674 1 1 0 0 0 1 -620980830413267931 1 1 0 0 0 1 --5907701125400227703 1 1 0 0 0 1 --1174991811984636509 1 1 0 0 0 1 -446104724568266173 1 1 0 0 0 1 +-5591479919192876859 1 1 0 0 0 1 +2342517051443094765 1 1 0 0 0 1 +-3440223316648760770 1 1 0 0 0 1 +-4242513015859548468 1 1 0 0 0 1 +-730351741811412687 1 1 0 0 0 1 +-2389210960120872170 1 1 0 0 0 1 +-4292066734783786923 1 1 0 0 0 1 +-4414169073687495257 1 1 0 0 0 1 +-6222496616288459687 1 1 0 0 0 1 +-506440052941725006 1 1 0 0 0 1 +-5591479919192876859 1 1 0 0 0 1 +2342517051443094765 1 1 0 0 0 1 +-3440223316648760770 1 1 0 0 0 1 +-4242513015859548468 1 1 0 0 0 1 +-730351741811412687 1 1 0 0 0 1 +-2389210960120872170 1 1 0 0 0 1 +-4292066734783786923 1 1 0 0 0 1 +-4414169073687495257 1 1 0 0 0 1 +-6222496616288459687 1 1 0 0 0 1 +-506440052941725006 1 1 0 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -687,26 +687,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND __table1.B AND (__table1.C OR (__table1.E AND __table1.E) OR (__table1.F AND __table1.F)) --7079903752397632178 1 1 0 1 0 1 -7410531019670261947 1 1 0 1 0 1 -233866682657554364 1 1 0 1 0 1 -704993795727957061 1 1 0 1 0 1 -4554773761331986827 1 1 0 1 0 1 -5016240913683434733 1 1 0 1 0 1 -21469839474705446 1 1 0 1 0 1 -2093705149861909566 1 1 0 1 0 1 -2363609859499966990 1 1 0 1 0 1 -5595451911922654649 1 1 0 1 0 1 --7079903752397632178 1 1 0 1 0 1 -7410531019670261947 1 1 0 1 0 1 -233866682657554364 1 1 0 1 0 1 -704993795727957061 1 1 0 1 0 1 -4554773761331986827 1 1 0 1 0 1 -5016240913683434733 1 1 0 1 0 1 -21469839474705446 1 1 0 1 0 1 -2093705149861909566 1 1 0 1 0 1 -2363609859499966990 1 1 0 1 0 1 -5595451911922654649 1 1 0 1 0 1 +-1198014781329290529 1 1 0 1 0 1 +3804289730569409059 1 1 0 1 0 1 +388942571848780949 1 1 0 1 0 1 +4303421928547547327 1 1 0 1 0 1 +5622057975514104466 1 1 0 1 0 1 +6003991070199658801 1 1 0 1 0 1 +6156030380809776164 1 1 0 1 0 1 +2991799832496995739 1 1 0 1 0 1 +3699238018341750080 1 1 0 1 0 1 +-8569013057605390975 1 1 0 1 0 1 +-1198014781329290529 1 1 0 1 0 1 +3804289730569409059 1 1 0 1 0 1 +388942571848780949 1 1 0 1 0 1 +4303421928547547327 1 1 0 1 0 1 +5622057975514104466 1 1 0 1 0 1 +6003991070199658801 1 1 0 1 0 1 +6156030380809776164 1 1 0 1 0 1 +2991799832496995739 1 1 0 1 0 1 +3699238018341750080 1 1 0 1 0 1 +-8569013057605390975 1 1 0 1 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -830,26 +830,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A OR (__table1.B AND ((__table1.C AND __table1.D) OR (__table1.C AND __table1.E))) --9220342792229035180 1 1 0 1 0 1 -9218747728814747926 1 1 0 0 0 1 --9220342792229035180 1 1 0 1 0 1 -9217081065560798881 1 0 0 1 0 1 --9220342792229035180 1 1 0 1 0 1 -9211856624818647543 1 1 1 0 0 0 --9220342792229035180 1 1 0 1 0 1 -9190487214870855200 1 1 1 1 1 1 --9220342792229035180 1 1 0 1 0 1 -9160211915115471089 1 0 0 0 1 1 --9220342792229035180 1 1 0 1 0 1 -9105451307219421605 1 0 1 1 1 1 --9220342792229035180 1 1 0 1 0 1 -9077371913032868764 1 1 0 0 1 1 --9220342792229035180 1 1 0 1 0 1 -9077295854519266595 1 1 0 0 1 0 --9220342792229035180 1 1 0 1 0 1 -9041842932373383548 1 1 0 0 0 1 --9220342792229035180 1 1 0 1 0 1 -9037094316313599791 1 0 0 1 0 1 --9220342792229035180 1 1 0 1 0 1 -9218747728814747926 1 1 0 0 0 1 --9220342792229035180 1 1 0 1 0 1 -9217081065560798881 1 0 0 1 0 1 --9220342792229035180 1 1 0 1 0 1 -9211856624818647543 1 1 1 0 0 0 --9220342792229035180 1 1 0 1 0 1 -9190487214870855200 1 1 1 1 1 1 --9220342792229035180 1 1 0 1 0 1 -9160211915115471089 1 0 0 0 1 1 --9220342792229035180 1 1 0 1 0 1 -9105451307219421605 1 0 1 1 1 1 --9220342792229035180 1 1 0 1 0 1 -9077371913032868764 1 1 0 0 1 1 --9220342792229035180 1 1 0 1 0 1 -9077295854519266595 1 1 0 0 1 0 --9220342792229035180 1 1 0 1 0 1 -9041842932373383548 1 1 0 0 0 1 --9220342792229035180 1 1 0 1 0 1 -9037094316313599791 1 0 0 1 0 1 +-9222995085389227671 0 0 0 0 0 1 -9180023040793172172 0 0 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9172372182410586193 0 1 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9140643988322994670 0 1 1 1 0 1 +-9222995085389227671 0 0 0 0 0 1 -9131908548041292850 0 0 1 0 0 0 +-9222995085389227671 0 0 0 0 0 1 -9125905281269664989 0 1 1 0 1 0 +-9222995085389227671 0 0 0 0 0 1 -9122596639456104649 0 0 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9072373031481714042 0 0 1 0 0 1 +-9222995085389227671 0 0 0 0 0 1 -9026061943293167962 0 1 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9004164398646425447 0 1 1 0 0 0 +-9222995085389227671 0 0 0 0 0 1 -8974795392131287614 0 1 1 0 1 0 +-9222995085389227671 0 0 0 0 0 1 -9180023040793172172 0 0 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9172372182410586193 0 1 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9140643988322994670 0 1 1 1 0 1 +-9222995085389227671 0 0 0 0 0 1 -9131908548041292850 0 0 1 0 0 0 +-9222995085389227671 0 0 0 0 0 1 -9125905281269664989 0 1 1 0 1 0 +-9222995085389227671 0 0 0 0 0 1 -9122596639456104649 0 0 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9072373031481714042 0 0 1 0 0 1 +-9222995085389227671 0 0 0 0 0 1 -9026061943293167962 0 1 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9004164398646425447 0 1 1 0 0 0 +-9222995085389227671 0 0 0 0 0 1 -8974795392131287614 0 1 1 0 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -950,12 +950,12 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 ALL INNER JOIN default.y AS __table2 ON (__table1.A = __table2.A) OR ((__table1.B = __table2.B) AND (((__table1.C = __table2.C) AND (__table1.D = __table2.D)) OR ((__table1.C = __table2.C) AND (__table1.E = __table2.E)))) --9220342792229035180 1 1 0 1 0 1 --9215479184005563028 0 1 1 1 0 1 --9211524111550600160 0 0 1 1 1 0 --9220342792229035180 1 1 0 1 0 1 --9215479184005563028 0 1 1 1 0 1 --9211524111550600160 0 0 1 1 1 0 +-9217261049539683905 0 1 1 0 1 1 +-9194298748734675155 0 0 0 1 0 0 +-9154303869107404275 1 0 1 1 1 1 +-9217261049539683905 0 1 1 0 1 1 +-9194298748734675155 0 0 0 1 0 0 +-9154303869107404275 1 0 1 1 1 1 QUERY id: 0 PROJECTION COLUMNS x Int64 @@ -993,26 +993,26 @@ SELECT __table1.F AS F FROM default.x AS __table1 WHERE __table1.C AND __table1.E --9220342792229035180 1 1 0 --9185424447834220126 1 0 1 --9147221708390825551 1 1 1 --9114850274891890829 1 1 1 --9114000995382454016 1 0 1 --9099767307959300756 1 1 0 --9077645148896772804 1 1 1 --9076550218784816665 1 1 0 --9074857609628938868 1 1 0 --9066338222341664310 1 1 1 --9220342792229035180 1 1 0 --9185424447834220126 1 0 1 --9147221708390825551 1 1 1 --9114850274891890829 1 1 1 --9114000995382454016 1 0 1 --9099767307959300756 1 1 0 --9077645148896772804 1 1 1 --9076550218784816665 1 1 0 --9074857609628938868 1 1 0 --9066338222341664310 1 1 1 +-9154303869107404275 1 0 1 +-9137658153367139416 1 1 1 +-9137309194079040284 1 1 1 +-9121738304623869295 1 1 0 +-9114056182077943575 1 1 1 +-9097647291775228882 1 0 1 +-9038891087920847933 1 0 1 +-9031027738484209035 1 0 1 +-9012321986923892113 1 1 0 +-9001504240784412840 1 1 0 +-9154303869107404275 1 0 1 +-9137658153367139416 1 1 1 +-9137309194079040284 1 1 1 +-9121738304623869295 1 1 0 +-9114056182077943575 1 1 1 +-9097647291775228882 1 0 1 +-9038891087920847933 1 0 1 +-9031027738484209035 1 0 1 +-9012321986923892113 1 1 0 +-9001504240784412840 1 1 0 QUERY id: 0 PROJECTION COLUMNS x Int64 @@ -1067,26 +1067,26 @@ SELECT FROM default.x AS __table1 GROUP BY __table1.x HAVING max(__table1.A) AND (max(__table1.B) OR max(__table1.C)) --9220342792229035180 1 1 0 --9215479184005563028 1 1 1 --9185424447834220126 1 1 1 --9173414302980908256 1 1 1 --9152339068839888086 1 1 1 --9147221708390825551 1 1 1 --9120736630719629705 1 0 1 --9114850274891890829 1 1 1 --9114000995382454016 1 1 1 --9099767307959300756 1 1 0 --9220342792229035180 1 1 0 --9215479184005563028 1 1 1 --9185424447834220126 1 1 1 --9173414302980908256 1 1 1 --9152339068839888086 1 1 1 --9147221708390825551 1 1 1 --9120736630719629705 1 0 1 --9114850274891890829 1 1 1 --9114000995382454016 1 1 1 --9099767307959300756 1 1 0 +-9220160771238933596 1 1 1 +-9154303869107404275 1 1 1 +-9137658153367139416 1 1 1 +-9137309194079040284 1 1 1 +-9136706315507142110 1 1 1 +-9136188450553078231 1 1 1 +-9121738304623869295 1 1 0 +-9119049435533988608 1 0 1 +-9114056182077943575 1 1 1 +-9097647291775228882 1 1 1 +-9220160771238933596 1 1 1 +-9154303869107404275 1 1 1 +-9137658153367139416 1 1 1 +-9137309194079040284 1 1 1 +-9136706315507142110 1 1 1 +-9136188450553078231 1 1 1 +-9121738304623869295 1 1 0 +-9119049435533988608 1 0 1 +-9114056182077943575 1 1 1 +-9097647291775228882 1 1 1 QUERY id: 0 PROJECTION COLUMNS x Int64 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index 705824b01e8..22a2f716afc 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -3,7 +3,16 @@ SET optimize_extract_common_expressions = 1; DROP TABLE IF EXISTS x; CREATE TABLE x (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) ENGINE = MergeTree ORDER BY x; -INSERT INTO x SELECT x, A%2 AS A, B%2 AS B, C%2 AS C, D%2 AS D, E%2 AS E, F%2 AS F FROM generateRandom('x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8', 42) LIMIT 2000; +INSERT INTO x + SELECT + cityHash64(number) AS x, + cityHash64(number + 1) % 2 AS A, + cityHash64(number + 2) % 2 AS B, + cityHash64(number + 3) % 2 AS C, + cityHash64(number + 4) % 2 AS D, + cityHash64(number + 5) % 2 AS E, + cityHash64(number + 6) % 2 AS F + FROM numbers(2000); -- Verify that optimization optimization setting works as expected EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE (A AND B) OR (A AND C) SETTINGS optimize_extract_common_expressions = 0; @@ -110,10 +119,10 @@ EXPLAIN QUERY TREE dump_ast = 1 SELECT x, max(A) AS mA, max(B) AS mB, max(C) AS -- QUALIFY SELECT - x, - max(A) OVER (PARTITION BY x % 1000) AS mA, - max(B) OVER (PARTITION BY x % 1000) AS mB, - max(C) OVER (PARTITION BY x % 1000) AS mC + x, + max(A) OVER (PARTITION BY x % 1000) AS mA, + max(B) OVER (PARTITION BY x % 1000) AS mB, + max(C) OVER (PARTITION BY x % 1000) AS mC FROM x QUALIFY (mA AND mB) OR (mA AND mC) ORDER BY x @@ -121,10 +130,10 @@ LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; SELECT - x, - max(A) OVER (PARTITION BY x % 1000) AS mA, - max(B) OVER (PARTITION BY x % 1000) AS mB, - max(C) OVER (PARTITION BY x % 1000) AS mC + x, + max(A) OVER (PARTITION BY x % 1000) AS mA, + max(B) OVER (PARTITION BY x % 1000) AS mB, + max(C) OVER (PARTITION BY x % 1000) AS mC FROM x QUALIFY (mA AND mB) OR (mA AND mC) ORDER BY x @@ -132,9 +141,9 @@ LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT - x, - max(A) OVER (PARTITION BY x % 1000) AS mA, - max(B) OVER (PARTITION BY x % 1000) AS mB, - max(C) OVER (PARTITION BY x % 1000) AS mC + x, + max(A) OVER (PARTITION BY x % 1000) AS mA, + max(B) OVER (PARTITION BY x % 1000) AS mB, + max(C) OVER (PARTITION BY x % 1000) AS mC FROM x QUALIFY (mA AND mB) OR (mA AND mC); From 997171a00e331ed0ee2dc2f69dbb40f5943a020f Mon Sep 17 00:00:00 2001 From: Michael Stetsyuk Date: Fri, 29 Nov 2024 15:29:07 +0000 Subject: [PATCH 214/502] add StartupScriptsExecutionState metric --- programs/server/Server.cpp | 12 ++++++++++++ src/Common/CurrentMetrics.cpp | 2 ++ .../03277_startup_scripts_state_ok.reference | 1 + .../0_stateless/03277_startup_scripts_state_ok.sql | 1 + 4 files changed, 16 insertions(+) create mode 100644 tests/queries/0_stateless/03277_startup_scripts_state_ok.reference create mode 100644 tests/queries/0_stateless/03277_startup_scripts_state_ok.sql diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 7d98178e096..a21658b9795 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -295,6 +295,7 @@ namespace CurrentMetrics extern const Metric MergesMutationsMemoryTracking; extern const Metric MaxDDLEntryID; extern const Metric MaxPushedDDLEntryID; + extern const Metric StartupScriptsExecutionState; } namespace ProfileEvents @@ -365,6 +366,14 @@ namespace ErrorCodes } +namespace StartupScriptsExecutionState +{ + [[maybe_unused]] const CurrentMetrics::Value NotFinished = 0; + const CurrentMetrics::Value Success = 1; + const CurrentMetrics::Value Failure = 2; +}; + + static std::string getCanonicalPath(std::string && path) { Poco::trimInPlace(path); @@ -781,9 +790,12 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, Contex startup_context->makeQueryContext(); executeQuery(read_buffer, write_buffer, true, startup_context, callback, QueryFlags{ .internal = true }, std::nullopt, {}); } + + CurrentMetrics::set(CurrentMetrics::StartupScriptsExecutionState, StartupScriptsExecutionState::Success); } catch (...) { + CurrentMetrics::set(CurrentMetrics::StartupScriptsExecutionState, StartupScriptsExecutionState::Failure); tryLogCurrentException(log, "Failed to parse startup scripts file"); } } diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index d073e477593..a29ebb2da4c 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -354,6 +354,8 @@ M(SharedCatalogDropZooKeeperThreadsScheduled, "Number of queued or active jobs in the threadpool for drop of object in ZooKeeper in Shared Catalog.") \ \ M(SharedDatabaseCatalogTablesInLocalDropDetachQueue, "Number of tables in the queue for local drop or detach in Shared Catalog.") \ + \ + M(StartupScriptsExecutionState, "State of startup scripts execution: 0 = not finished, 1 = success, 2 = failure.") \ #ifdef APPLY_FOR_EXTERNAL_METRICS #define APPLY_FOR_METRICS(M) APPLY_FOR_BUILTIN_METRICS(M) APPLY_FOR_EXTERNAL_METRICS(M) diff --git a/tests/queries/0_stateless/03277_startup_scripts_state_ok.reference b/tests/queries/0_stateless/03277_startup_scripts_state_ok.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/03277_startup_scripts_state_ok.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/03277_startup_scripts_state_ok.sql b/tests/queries/0_stateless/03277_startup_scripts_state_ok.sql new file mode 100644 index 00000000000..b32241d74ab --- /dev/null +++ b/tests/queries/0_stateless/03277_startup_scripts_state_ok.sql @@ -0,0 +1 @@ +SELECT value FROM system.metrics WHERE metric = 'StartupScriptsExecutionState' From b512997e50d60590b6897a84ecfa95632e72bb32 Mon Sep 17 00:00:00 2001 From: Michael Stetsyuk Date: Fri, 29 Nov 2024 17:45:55 +0000 Subject: [PATCH 215/502] better test --- .../__init__.py | 0 .../config/bad_script.xml | 10 ++++ .../config/good_script.xml | 11 ++++ .../config/users.xml | 8 +++ .../test.py | 56 +++++++++++++++++++ .../03277_startup_scripts_state_ok.reference | 1 - .../03277_startup_scripts_state_ok.sql | 1 - 7 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_startup_scripts_execution_state/__init__.py create mode 100644 tests/integration/test_startup_scripts_execution_state/config/bad_script.xml create mode 100644 tests/integration/test_startup_scripts_execution_state/config/good_script.xml create mode 100644 tests/integration/test_startup_scripts_execution_state/config/users.xml create mode 100644 tests/integration/test_startup_scripts_execution_state/test.py delete mode 100644 tests/queries/0_stateless/03277_startup_scripts_state_ok.reference delete mode 100644 tests/queries/0_stateless/03277_startup_scripts_state_ok.sql diff --git a/tests/integration/test_startup_scripts_execution_state/__init__.py b/tests/integration/test_startup_scripts_execution_state/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_startup_scripts_execution_state/config/bad_script.xml b/tests/integration/test_startup_scripts_execution_state/config/bad_script.xml new file mode 100644 index 00000000000..1306cc116c9 --- /dev/null +++ b/tests/integration/test_startup_scripts_execution_state/config/bad_script.xml @@ -0,0 +1,10 @@ + + + + SELECT 42; + + + SELECT * FROM non_existent_table; + + + diff --git a/tests/integration/test_startup_scripts_execution_state/config/good_script.xml b/tests/integration/test_startup_scripts_execution_state/config/good_script.xml new file mode 100644 index 00000000000..f16f800eb6c --- /dev/null +++ b/tests/integration/test_startup_scripts_execution_state/config/good_script.xml @@ -0,0 +1,11 @@ + + + + SELECT 0; + SELECT * FROM non_existent_table; + + + SELECT 42; + + + diff --git a/tests/integration/test_startup_scripts_execution_state/config/users.xml b/tests/integration/test_startup_scripts_execution_state/config/users.xml new file mode 100644 index 00000000000..c5de0b6819c --- /dev/null +++ b/tests/integration/test_startup_scripts_execution_state/config/users.xml @@ -0,0 +1,8 @@ + + + + default + + + + \ No newline at end of file diff --git a/tests/integration/test_startup_scripts_execution_state/test.py b/tests/integration/test_startup_scripts_execution_state/test.py new file mode 100644 index 00000000000..e379a7bf0aa --- /dev/null +++ b/tests/integration/test_startup_scripts_execution_state/test.py @@ -0,0 +1,56 @@ +from enum import Enum +import random +import string +import time + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +good = cluster.add_instance( + "good", + main_configs=[ + "config/users.xml", + "config/good_script.xml" + ], + stay_alive=True, +) +bad = cluster.add_instance( + "bad", + main_configs=[ + "config/users.xml", + "config/bad_script.xml" + ], + stay_alive=True, +) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_startup_execution_state(start_cluster): + """ + Making sure that the StartupScriptsExecutionState metric is set correctly. + """ + + STATE_SUCCESS = 1 + STATE_FAILURE = 2 + + assert int( + good.query( + "SELECT value FROM system.metrics WHERE metric = 'StartupScriptsExecutionState'" + ).strip() + ) == STATE_SUCCESS + + assert int( + bad.query( + "SELECT value FROM system.metrics WHERE metric = 'StartupScriptsExecutionState'" + ).strip() + ) == STATE_FAILURE diff --git a/tests/queries/0_stateless/03277_startup_scripts_state_ok.reference b/tests/queries/0_stateless/03277_startup_scripts_state_ok.reference deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/03277_startup_scripts_state_ok.reference +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/queries/0_stateless/03277_startup_scripts_state_ok.sql b/tests/queries/0_stateless/03277_startup_scripts_state_ok.sql deleted file mode 100644 index b32241d74ab..00000000000 --- a/tests/queries/0_stateless/03277_startup_scripts_state_ok.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT value FROM system.metrics WHERE metric = 'StartupScriptsExecutionState' From fad3f2ff0021262915a81faaf517dfe88ca03167 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 29 Nov 2024 17:53:59 +0000 Subject: [PATCH 216/502] Automatic style fix --- .../test.py | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_startup_scripts_execution_state/test.py b/tests/integration/test_startup_scripts_execution_state/test.py index e379a7bf0aa..f00026df2ff 100644 --- a/tests/integration/test_startup_scripts_execution_state/test.py +++ b/tests/integration/test_startup_scripts_execution_state/test.py @@ -1,7 +1,7 @@ -from enum import Enum import random import string import time +from enum import Enum import pytest @@ -10,18 +10,12 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) good = cluster.add_instance( "good", - main_configs=[ - "config/users.xml", - "config/good_script.xml" - ], + main_configs=["config/users.xml", "config/good_script.xml"], stay_alive=True, ) bad = cluster.add_instance( "bad", - main_configs=[ - "config/users.xml", - "config/bad_script.xml" - ], + main_configs=["config/users.xml", "config/bad_script.xml"], stay_alive=True, ) @@ -43,14 +37,20 @@ def test_startup_execution_state(start_cluster): STATE_SUCCESS = 1 STATE_FAILURE = 2 - assert int( - good.query( - "SELECT value FROM system.metrics WHERE metric = 'StartupScriptsExecutionState'" - ).strip() - ) == STATE_SUCCESS + assert ( + int( + good.query( + "SELECT value FROM system.metrics WHERE metric = 'StartupScriptsExecutionState'" + ).strip() + ) + == STATE_SUCCESS + ) - assert int( - bad.query( - "SELECT value FROM system.metrics WHERE metric = 'StartupScriptsExecutionState'" - ).strip() - ) == STATE_FAILURE + assert ( + int( + bad.query( + "SELECT value FROM system.metrics WHERE metric = 'StartupScriptsExecutionState'" + ).strip() + ) + == STATE_FAILURE + ) From 049fa54c4ddfc920311fc420f0fa9dbc058199ce Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Sat, 30 Nov 2024 12:31:24 +0100 Subject: [PATCH 217/502] Revert "Revert "make d-tor Finalizer more obvious"" --- src/Storages/MergeTree/MergeTreeSink.cpp | 2 ++ .../MergeTree/MergedBlockOutputStream.cpp | 17 +++++------------ .../MergeTree/MergedBlockOutputStream.h | 2 +- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index d65d1f3212f..bc7277d9123 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -44,6 +44,8 @@ MergeTreeSink::~MergeTreeSink() if (!delayed_chunk) return; + chassert(isCancelled()); + for (auto & partition : delayed_chunk->partitions) { partition.temp_part.cancel(); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index f965857a61d..a9fd1fad6c5 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -118,7 +118,7 @@ struct MergedBlockOutputStream::Finalizer::Impl } void finish(); - void cancel(); + void cancel() noexcept; }; void MergedBlockOutputStream::Finalizer::finish() @@ -129,7 +129,7 @@ void MergedBlockOutputStream::Finalizer::finish() to_finish->finish(); } -void MergedBlockOutputStream::Finalizer::cancel() +void MergedBlockOutputStream::Finalizer::cancel() noexcept { std::unique_ptr to_cancel = std::move(impl); impl.reset(); @@ -166,7 +166,7 @@ void MergedBlockOutputStream::Finalizer::Impl::finish() part->getDataPartStorage().removeFile(file_name); } -void MergedBlockOutputStream::Finalizer::Impl::cancel() +void MergedBlockOutputStream::Finalizer::Impl::cancel() noexcept { writer.cancel(); @@ -182,15 +182,8 @@ MergedBlockOutputStream::Finalizer::Finalizer(std::unique_ptr impl_) : imp MergedBlockOutputStream::Finalizer::~Finalizer() { - try - { - if (impl) - finish(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } + if (impl) + cancel(); } diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.h b/src/Storages/MergeTree/MergedBlockOutputStream.h index 7b5eda0a93a..5e06b040425 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -54,7 +54,7 @@ public: ~Finalizer(); void finish(); - void cancel(); + void cancel() noexcept; }; /// Finalize writing part and fill inner structures From a95d19806b2f0d5bfa8cd953400b797e97fa7997 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Sat, 30 Nov 2024 12:33:42 +0100 Subject: [PATCH 218/502] fix chassert in d-tor MergeTreeSink --- src/Storages/MergeTree/MergeTreeSink.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index bc7277d9123..582a722d678 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -44,7 +45,7 @@ MergeTreeSink::~MergeTreeSink() if (!delayed_chunk) return; - chassert(isCancelled()); + chassert(isCancelled() || std::uncaught_exceptions()); for (auto & partition : delayed_chunk->partitions) { From 0a9ac03671f933840d9331d428a519c8230c61ce Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Sat, 30 Nov 2024 12:37:18 +0100 Subject: [PATCH 219/502] add assert in d-tor ReplicatedMergeTreeSinkImpl --- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 19a69eb46be..507b4bfd214 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -179,6 +179,8 @@ ReplicatedMergeTreeSinkImpl::~ReplicatedMergeTreeSinkImpl() if (!delayed_chunk) return; + chassert(isCancelled() || std::uncaught_exceptions()); + for (auto & partition : delayed_chunk->partitions) { partition.temp_part.cancel(); From 777dd21640d0f8d330bfd60fa55303cb28eebe72 Mon Sep 17 00:00:00 2001 From: Xavier Leune Date: Sat, 30 Nov 2024 23:40:12 +0100 Subject: [PATCH 220/502] (docs) Fix examples for replication --- docs/en/engines/table-engines/mergetree-family/replication.md | 2 +- docs/ja/engines/table-engines/mergetree-family/replication.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 000422ab4d5..6373cd31b4f 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -177,7 +177,7 @@ CREATE TABLE table_name CounterID UInt32, UserID UInt32, ver UInt16 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/table_name', '{replica}', ver) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', ver) PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID); diff --git a/docs/ja/engines/table-engines/mergetree-family/replication.md b/docs/ja/engines/table-engines/mergetree-family/replication.md index 4527ab0fc42..5bcfc4cedf9 100644 --- a/docs/ja/engines/table-engines/mergetree-family/replication.md +++ b/docs/ja/engines/table-engines/mergetree-family/replication.md @@ -175,7 +175,7 @@ CREATE TABLE table_name CounterID UInt32, UserID UInt32, ver UInt16 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/table_name', '{replica}', ver) +) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', ver) PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID); From 5c4da392593876391e1e6a4e7b60b9f0f61c911e Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 30 Nov 2024 22:36:19 +0100 Subject: [PATCH 221/502] Add retries while creating a replicated table. --- src/Backups/RestorerFromBackup.cpp | 13 + src/Backups/RestorerFromBackup.h | 3 +- src/Databases/DatabaseOrdinary.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 36 +-- src/Interpreters/InterpreterSystemQuery.cpp | 11 +- .../ReplicatedMergeTreeAttachThread.cpp | 11 +- .../MergeTree/registerStorageMergeTree.cpp | 15 +- src/Storages/StorageReplicatedMergeTree.cpp | 244 ++++++++++++++---- src/Storages/StorageReplicatedMergeTree.h | 40 ++- 9 files changed, 291 insertions(+), 84 deletions(-) diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index f907d80a64a..9b3b2408706 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,9 @@ namespace DB { namespace Setting { + extern const SettingsUInt64 backup_restore_keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 backup_restore_keeper_retry_max_backoff_ms; + extern const SettingsUInt64 backup_restore_keeper_max_retries; extern const SettingsSeconds lock_acquire_timeout; } @@ -102,6 +106,10 @@ RestorerFromBackup::RestorerFromBackup( , after_task_callback(after_task_callback_) , create_table_timeout(context->getConfigRef().getUInt64("backups.create_table_timeout", 300000)) , log(getLogger("RestorerFromBackup")) + , zookeeper_retries_info( + context->getSettingsRef()[Setting::backup_restore_keeper_max_retries], + context->getSettingsRef()[Setting::backup_restore_keeper_retry_initial_backoff_ms], + context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms]) , tables_dependencies("RestorerFromBackup") , thread_pool(thread_pool_) { @@ -976,6 +984,11 @@ void RestorerFromBackup::createTable(const QualifiedTableName & table_name) query_context->setSetting("database_replicated_allow_explicit_uuid", 3); query_context->setSetting("database_replicated_allow_replicated_engine_arguments", 3); + /// Creating of replicated tables may need retries. + query_context->setSetting("keeper_max_retries", zookeeper_retries_info.max_retries); + query_context->setSetting("keeper_initial_backoff_ms", zookeeper_retries_info.initial_backoff_ms); + query_context->setSetting("keeper_max_backoff_ms", zookeeper_retries_info.max_backoff_ms); + /// Execute CREATE TABLE query (we call IDatabase::createTableRestoredFromBackup() to allow the database to do some /// database-specific things). database->createTableRestoredFromBackup( diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index aa5288643a0..3fb314e5c42 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -83,6 +84,7 @@ private: std::chrono::milliseconds create_table_timeout; LoggerPtr log; + const ZooKeeperRetriesInfo zookeeper_retries_info; Mode mode = Mode::RESTORE; Strings all_hosts; DDLRenamingMap renaming_map; @@ -170,7 +172,6 @@ private: TablesDependencyGraph tables_dependencies TSA_GUARDED_BY(mutex); std::vector data_restore_tasks TSA_GUARDED_BY(mutex); std::unique_ptr access_restorer TSA_GUARDED_BY(mutex); - bool access_restored TSA_GUARDED_BY(mutex) = false; std::vector> futures TSA_GUARDED_BY(mutex); std::atomic exception_caught = false; diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 3dbfd5f222d..eed29c0d821 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -408,7 +408,7 @@ void DatabaseOrdinary::restoreMetadataAfterConvertingToReplicated(StoragePtr tab } else { - rmt->restoreMetadataInZooKeeper(); + rmt->restoreMetadataInZooKeeper(/* query_status = */ nullptr, /* zookeeper_retries_info = */ {}); LOG_INFO ( log, diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 9e81871579d..e8add48a4c2 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1633,29 +1633,29 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (isReplicated(*inner_table_engine)) is_storage_replicated = true; } - } + } - bool allow_heavy_populate = getContext()->getSettingsRef()[Setting::database_replicated_allow_heavy_create] && create.is_populate; - if (!allow_heavy_populate && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate)) + bool allow_heavy_populate = getContext()->getSettingsRef()[Setting::database_replicated_allow_heavy_create] && create.is_populate; + if (!allow_heavy_populate && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate)) + { + const bool allow_create_select_for_replicated + = (create.isView() && !create.is_populate) || create.is_create_empty || !is_storage_replicated; + if (!allow_create_select_for_replicated) { - const bool allow_create_select_for_replicated - = (create.isView() && !create.is_populate) || create.is_create_empty || !is_storage_replicated; - if (!allow_create_select_for_replicated) - { - /// POPULATE can be enabled with setting, provide hint in error message - if (create.is_populate) - throw Exception( - ErrorCodes::SUPPORT_IS_DISABLED, - "CREATE with POPULATE is not supported with Replicated databases. Consider using separate CREATE and INSERT " - "queries. " - "Alternatively, you can enable 'database_replicated_allow_heavy_create' setting to allow this operation, use with " - "caution"); - + /// POPULATE can be enabled with setting, provide hint in error message + if (create.is_populate) throw Exception( ErrorCodes::SUPPORT_IS_DISABLED, - "CREATE AS SELECT is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); - } + "CREATE with POPULATE is not supported with Replicated databases. Consider using separate CREATE and INSERT " + "queries. " + "Alternatively, you can enable 'database_replicated_allow_heavy_create' setting to allow this operation, use with " + "caution"); + + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "CREATE AS SELECT is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); } + } if (create.is_clone_as) { diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 9aec16a3fb7..ca96ee3245f 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -98,6 +98,9 @@ namespace DB { namespace Setting { + extern const SettingsUInt64 keeper_max_retries; + extern const SettingsUInt64 keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 keeper_retry_max_backoff_ms; extern const SettingsSeconds lock_acquire_timeout; extern const SettingsSeconds receive_timeout; extern const SettingsMaxThreads max_threads; @@ -878,7 +881,13 @@ void InterpreterSystemQuery::restoreReplica() if (table_replicated_ptr == nullptr) throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs()); - table_replicated_ptr->restoreMetadataInZooKeeper(); + const auto & settings = getContext()->getSettingsRef(); + + table_replicated_ptr->restoreMetadataInZooKeeper( + getContext()->getProcessListElementSafe(), + ZooKeeperRetriesInfo{settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms]}); } StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, ContextMutablePtr system_context) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index c258048354e..c654b459c24 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -166,23 +166,24 @@ void ReplicatedMergeTreeAttachThread::runImpl() /// Just in case it was not removed earlier due to connection loss zookeeper->tryRemove(replica_path + "/flags/force_restore_data"); - storage.checkTableStructure(replica_path, metadata_snapshot); + /// Here `zookeeper_retries_info = {}` because the attach thread has its own retries (see ReplicatedMergeTreeAttachThread::run()). + storage.checkTableStructure(replica_path, metadata_snapshot, /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); storage.checkParts(skip_sanity_checks); /// Temporary directories contain uninitialized results of Merges or Fetches (after forced restart), /// don't allow to reinitialize them, delete each of them immediately. storage.clearOldTemporaryDirectories(0, {"tmp_", "delete_tmp_", "tmp-fetch_"}); - storage.createNewZooKeeperNodes(); - storage.syncPinnedPartUUIDs(); + storage.createNewZooKeeperNodes(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + storage.syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); std::lock_guard lock(storage.table_shared_id_mutex); - storage.createTableSharedID(); + storage.createTableSharedID(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); }; void ReplicatedMergeTreeAttachThread::finalizeInitialization() TSA_NO_THREAD_SAFETY_ANALYSIS { - storage.startupImpl(/* from_attach_thread */ true); + storage.startupImpl(/* from_attach_thread */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); storage.initialization_done = true; LOG_INFO(log, "Table is initialized"); } diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 9f66a079998..12b5d115903 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,9 @@ namespace Setting extern const SettingsBool allow_suspicious_ttl_expressions; extern const SettingsBool create_table_empty_primary_key_by_default; extern const SettingsUInt64 database_replicated_allow_replicated_engine_arguments; + extern const SettingsUInt64 keeper_max_retries; + extern const SettingsUInt64 keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 keeper_retry_max_backoff_ms; } namespace MergeTreeSetting @@ -831,6 +835,12 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (auto txn = args.getLocalContext()->getZooKeeperMetadataTransaction()) need_check_table_structure = txn->isInitialQuery(); + ZooKeeperRetriesInfo create_query_zk_retries_info; + create_query_zk_retries_info.max_retries = local_settings[Setting::keeper_max_retries]; + create_query_zk_retries_info.initial_backoff_ms = local_settings[Setting::keeper_retry_initial_backoff_ms]; + create_query_zk_retries_info.max_backoff_ms = local_settings[Setting::keeper_retry_max_backoff_ms]; + auto create_query_status = args.getLocalContext()->getProcessListElementSafe(); + return std::make_shared( zookeeper_info, args.mode, @@ -841,8 +851,11 @@ static StoragePtr create(const StorageFactory::Arguments & args) date_column_name, merging_params, std::move(storage_settings), - need_check_table_structure); + need_check_table_structure, + create_query_zk_retries_info, + create_query_status); } + return std::make_shared( args.table_id, args.relative_data_path, diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index bd476625081..ac1b37ecbdb 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -366,7 +366,9 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, - bool need_check_structure) + bool need_check_structure, + const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_, + QueryStatusPtr create_query_status_) : MergeTreeData(table_id_, metadata_, context_, @@ -380,6 +382,8 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( , zookeeper_path(zookeeper_info.path) , replica_name(zookeeper_info.replica_name) , replica_path(fs::path(zookeeper_path) / "replicas" / replica_name) + , create_query_zookeeper_retries_info(create_query_zookeeper_retries_info_) + , create_query_status(create_query_status_) , reader(*this) , writer(*this) , merger_mutator(*this) @@ -569,7 +573,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( try { - bool is_first_replica = createTableIfNotExists(metadata_snapshot); + bool is_first_replica = createTableIfNotExists(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); try { @@ -578,24 +582,22 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( /// We have to check granularity on other replicas. If it's fixed we /// must create our new replica with fixed granularity and store this /// information in /replica/metadata. - other_replicas_fixed_granularity = checkFixedGranularityInZookeeper(); + other_replicas_fixed_granularity = checkFixedGranularityInZookeeper(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); /// Allow structure mismatch for secondary queries from Replicated database. /// It may happen if the table was altered just after creation. /// Metadata will be updated in cloneMetadataIfNeeded(...), metadata_version will be 0 for a while. - bool same_structure = checkTableStructure(zookeeper_path, metadata_snapshot, need_check_structure); + int32_t metadata_version; + bool same_structure = checkTableStructure(zookeeper_path, metadata_snapshot, &metadata_version, need_check_structure, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); if (same_structure) { - Coordination::Stat metadata_stat; - current_zookeeper->get(fs::path(zookeeper_path) / "metadata", &metadata_stat); - /** We change metadata_snapshot so that `createReplica` method will create `metadata_version` node in ZooKeeper * with version of table '/metadata' node in Zookeeper. * * Otherwise `metadata_version` for not first replica will be initialized with 0 by default. */ - setInMemoryMetadata(metadata_snapshot->withMetadataVersion(metadata_stat.version)); + setInMemoryMetadata(metadata_snapshot->withMetadataVersion(metadata_version)); metadata_snapshot = getInMemoryMetadataPtr(); } } @@ -607,15 +609,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } if (!is_first_replica) - createReplica(metadata_snapshot); + createReplica(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); - createNewZooKeeperNodes(); - syncPinnedPartUUIDs(); + createNewZooKeeperNodes(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + syncPinnedPartUUIDs(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); if (!has_metadata_in_zookeeper.has_value() || *has_metadata_in_zookeeper) - createTableSharedID(); - - + createTableSharedID(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); } catch (...) { @@ -628,12 +628,29 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } -bool StorageReplicatedMergeTree::checkFixedGranularityInZookeeper() +bool StorageReplicatedMergeTree::checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const { - auto zookeeper = getZooKeeper(); - String metadata_str = zookeeper->get(zookeeper_path + "/metadata"); - auto metadata_from_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); - return metadata_from_zk.index_granularity_bytes == 0; + bool fixed_granularity = false; + + auto check_fixed_granularity = [&] + { + auto zookeeper = getZooKeeper(); + String metadata_str = zookeeper->get(zookeeper_path + "/metadata"); + auto metadata_from_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); + fixed_granularity = (metadata_from_zk.index_granularity_bytes == 0); + }; + + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkFixedGranularityInZookeeper", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { check_fixed_granularity(); }); + } + else + { + check_fixed_granularity(); + } + + return fixed_granularity; } @@ -808,7 +825,20 @@ std::vector getAncestors(const String & path) } -void StorageReplicatedMergeTree::createNewZooKeeperNodes() +void StorageReplicatedMergeTree::createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +{ + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createNewZooKeeperNodes", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { createNewZooKeeperNodesAttempt(); }); + } + else + { + createNewZooKeeperNodesAttempt(); + } +} + +void StorageReplicatedMergeTree::createNewZooKeeperNodesAttempt() const { auto zookeeper = getZooKeeper(); @@ -873,14 +903,34 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() } } +bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, + const ZooKeeperRetriesInfo & zookeeper_retries_info, + QueryStatusPtr process_list_element) const +{ + bool table_created = false; + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableIfNotExists", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { table_created = createTableIfNotExistsAttempt(metadata_snapshot, process_list_element); }); + } + else + { + table_created = createTableIfNotExistsAttempt(metadata_snapshot, process_list_element); + } + return table_created; +} -bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot) +bool StorageReplicatedMergeTree::createTableIfNotExistsAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const { auto zookeeper = getZooKeeper(); zookeeper->createAncestors(zookeeper_path); for (size_t i = 0; i < 1000; ++i) { + /// Check if the query was cancelled. + if (process_list_element) + process_list_element->checkTimeLimit(); + /// Invariant: "replicas" does not exist if there is no table or if there are leftovers from incompletely dropped table. if (zookeeper->exists(zookeeper_path + "/replicas")) { @@ -1019,7 +1069,22 @@ bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr "of wrong zookeeper_path or because of logical error"); } -void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metadata_snapshot) +void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metadata_snapshot, + const ZooKeeperRetriesInfo & zookeeper_retries_info, + QueryStatusPtr process_list_element) const +{ + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createReplica", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { createReplicaAttempt(metadata_snapshot, process_list_element); }); + } + else + { + createReplicaAttempt(metadata_snapshot, process_list_element); + } +} + +void StorageReplicatedMergeTree::createReplicaAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const { auto zookeeper = getZooKeeper(); @@ -1103,6 +1168,10 @@ void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metada do { + /// Check if the query was cancelled. + if (process_list_element) + process_list_element->checkTimeLimit(); + Coordination::Stat replicas_stat; String replicas_value; @@ -1169,6 +1238,25 @@ void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metada } while (code == Coordination::Error::ZBADVERSION); } +ZooKeeperRetriesInfo StorageReplicatedMergeTree::getCreateQueryZooKeeperRetriesInfo() const +{ + std::lock_guard lock{create_query_zookeeper_retries_info_mutex}; + return create_query_zookeeper_retries_info; +} + +QueryStatusPtr StorageReplicatedMergeTree::getCreateQueryStatus() const +{ + std::lock_guard lock{create_query_zookeeper_retries_info_mutex}; + return create_query_status; +} + +void StorageReplicatedMergeTree::clearCreateQueryZooKeeperRetriesInfo() +{ + std::lock_guard lock{create_query_zookeeper_retries_info_mutex}; + create_query_zookeeper_retries_info = {}; + create_query_status = {}; +} + zkutil::ZooKeeperPtr StorageReplicatedMergeTree::getZooKeeperIfTableShutDown() const { @@ -1530,7 +1618,26 @@ bool StorageReplicatedMergeTree::removeTableNodesFromZooKeeper(zkutil::ZooKeeper /** Verify that list of columns and table storage_settings_ptr match those specified in ZK (/metadata). * If not, throw an exception. */ -bool StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, bool strict_check) +bool StorageReplicatedMergeTree::checkTableStructure( + const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check, + const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +{ + bool same_structure = false; + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkTableStructure", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { same_structure = checkTableStructureAttempt(zookeeper_prefix, metadata_snapshot, metadata_version, strict_check); }); + } + else + { + same_structure = checkTableStructureAttempt(zookeeper_prefix, metadata_snapshot, metadata_version, strict_check); + } + return same_structure; +} + + +bool StorageReplicatedMergeTree::checkTableStructureAttempt( + const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check) const { auto zookeeper = getZooKeeper(); @@ -1541,6 +1648,9 @@ bool StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr auto metadata_from_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); old_metadata.checkEquals(metadata_from_zk, metadata_snapshot->getColumns(), getContext()); + if (metadata_version) + *metadata_version = metadata_stat.version; + Coordination::Stat columns_stat; auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(fs::path(zookeeper_prefix) / "columns", &columns_stat)); @@ -1860,21 +1970,35 @@ bool StorageReplicatedMergeTree::checkPartsImpl(bool skip_sanity_checks) } -void StorageReplicatedMergeTree::syncPinnedPartUUIDs() +void StorageReplicatedMergeTree::syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) { - auto zookeeper = getZooKeeper(); + String new_pinned_part_uuids_str; + Coordination::Stat new_stat; - Coordination::Stat stat; - String s = zookeeper->get(zookeeper_path + "/pinned_part_uuids", &stat); + auto read_pinned_part_uuids = [&] + { + auto zookeeper = getZooKeeper(); + new_pinned_part_uuids_str = zookeeper->get(zookeeper_path + "/pinned_part_uuids", &new_stat); + }; + + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::syncPinnedPartUUIDs", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { read_pinned_part_uuids(); }); + } + else + { + read_pinned_part_uuids(); + } std::lock_guard lock(pinned_part_uuids_mutex); /// Unsure whether or not this can be called concurrently. - if (pinned_part_uuids->stat.version < stat.version) + if (pinned_part_uuids->stat.version < new_stat.version) { auto new_pinned_part_uuids = std::make_shared(); - new_pinned_part_uuids->fromString(s); - new_pinned_part_uuids->stat = stat; + new_pinned_part_uuids->fromString(new_pinned_part_uuids_str); + new_pinned_part_uuids->stat = new_stat; pinned_part_uuids = new_pinned_part_uuids; } @@ -2228,7 +2352,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) case LogEntry::ALTER_METADATA: return executeMetadataAlter(entry); case LogEntry::SYNC_PINNED_PART_UUIDS: - syncPinnedPartUUIDs(); + syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); return true; case LogEntry::CLONE_PART_FROM_SHARD: executeClonePartFromShard(entry); @@ -4377,17 +4501,29 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n } -void StorageReplicatedMergeTree::startBeingLeader() +void StorageReplicatedMergeTree::startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) { - auto zookeeper = getZooKeeper(); - if (!(*getSettings())[MergeTreeSetting::replicated_can_become_leader]) { LOG_INFO(log, "Will not enter leader election because replicated_can_become_leader=0"); return; } - zkutil::checkNoOldLeaders(log.load(), *zookeeper, fs::path(zookeeper_path) / "leader_election"); + auto start_being_leader = [&] + { + auto zookeeper = getZooKeeper(); + zkutil::checkNoOldLeaders(log.load(), *zookeeper, fs::path(zookeeper_path) / "leader_election"); + }; + + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::startBeingLeader", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { start_being_leader(); }); + } + else + { + start_being_leader(); + } LOG_INFO(log, "Became leader"); is_leader = true; @@ -5260,10 +5396,10 @@ void StorageReplicatedMergeTree::startup() return; } - startupImpl(/* from_attach_thread */ false); + startupImpl(/* from_attach_thread */ false, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); } -void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread) +void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) { /// Do not start replication if ZooKeeper is not configured or there is no metadata in zookeeper if (!has_metadata_in_zookeeper.has_value() || !*has_metadata_in_zookeeper) @@ -5292,7 +5428,7 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread) getContext()->getInterserverIOHandler().addEndpoint( data_parts_exchange_ptr->getId(getEndpointName()), data_parts_exchange_ptr); - startBeingLeader(); + startBeingLeader(zookeeper_retries_info, process_list_element); if (from_attach_thread) { @@ -5330,6 +5466,9 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread) startBackgroundMovesIfNeeded(); part_moves_between_shards_orchestrator.start(); + + /// After finishing startup() create_query_zk_retries_info won't be used anymore. + clearCreateQueryZooKeeperRetriesInfo(); } catch (...) { @@ -6544,7 +6683,7 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition( return true; } -void StorageReplicatedMergeTree::restoreMetadataInZooKeeper() +void StorageReplicatedMergeTree::restoreMetadataInZooKeeper(QueryStatusPtr query_status, const ZooKeeperRetriesInfo & zookeeper_retries_info) { LOG_INFO(log, "Restoring replica metadata"); @@ -6587,14 +6726,14 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper() LOG_INFO(log, "Moved all parts to detached/"); - const bool is_first_replica = createTableIfNotExists(metadata_snapshot); + const bool is_first_replica = createTableIfNotExists(metadata_snapshot, zookeeper_retries_info, query_status); LOG_INFO(log, "Created initial ZK nodes, replica is first: {}", is_first_replica); if (!is_first_replica) - createReplica(metadata_snapshot); + createReplica(metadata_snapshot, zookeeper_retries_info, query_status); - createNewZooKeeperNodes(); + createNewZooKeeperNodes(zookeeper_retries_info, query_status); LOG_INFO(log, "Created ZK nodes for table"); @@ -6606,7 +6745,7 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper() LOG_INFO(log, "Attached all partitions, starting table"); - startupImpl(/* from_attach_thread */ false); + startupImpl(/* from_attach_thread */ false, zookeeper_retries_info, query_status); } void StorageReplicatedMergeTree::dropPartNoWaitNoThrow(const String & part_name) @@ -8785,7 +8924,7 @@ void StorageReplicatedMergeTree::movePartitionToShard( { /// Optimistic check that for compatible destination table structure. - checkTableStructure(to, getInMemoryMetadataPtr()); + checkTableStructure(to, getInMemoryMetadataPtr(), /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); } PinnedPartUUIDs src_pins; @@ -9388,7 +9527,7 @@ String StorageReplicatedMergeTree::getTableSharedID() const { /// Can happen if table was partially initialized before drop by DatabaseCatalog if (table_shared_id == UUIDHelpers::Nil) - createTableSharedID(); + createTableSharedID(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); } else { @@ -9403,7 +9542,20 @@ std::map StorageReplicatedMergeTree::getUnfinishe return queue.getUnfinishedMutations(); } -void StorageReplicatedMergeTree::createTableSharedID() const +void StorageReplicatedMergeTree::createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +{ + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableSharedID", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { createTableSharedIDAttempt(); }); + } + else + { + createTableSharedIDAttempt(); + } +} + +void StorageReplicatedMergeTree::createTableSharedIDAttempt() const { LOG_DEBUG(log, "Creating shared ID for table {}", getStorageID().getNameForLogs()); // can be set by the call to getTableSharedID diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index a790e548645..c6081d45bf4 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -108,7 +109,9 @@ public: const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, - bool need_check_structure); + bool need_check_structure, + const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_, + QueryStatusPtr create_query_status_); void startup() override; @@ -313,7 +316,7 @@ public: /// Restores table metadata if ZooKeeper lost it. /// Used only on restarted readonly replicas (not checked). All active (Active) parts are moved to detached/ /// folder and attached. Parts in all other states are just moved to detached/ folder. - void restoreMetadataInZooKeeper(); + void restoreMetadataInZooKeeper(QueryStatusPtr query_status, const ZooKeeperRetriesInfo & zookeeper_retries_info); /// Get throttler for replicated fetches ThrottlerPtr getFetchesThrottler() const @@ -424,6 +427,10 @@ private: const String replica_name; // shorthand for zookeeper_info.replica_name const String replica_path; + ZooKeeperRetriesInfo create_query_zookeeper_retries_info TSA_GUARDED_BY(create_query_zookeeper_retries_info_mutex); + QueryStatusPtr create_query_status TSA_GUARDED_BY(create_query_zookeeper_retries_info_mutex); + mutable std::mutex create_query_zookeeper_retries_info_mutex; + /** /replicas/me/is_active. */ zkutil::EphemeralNodeHolderPtr replica_is_active_node; @@ -572,18 +579,28 @@ private: /** Creates the minimum set of nodes in ZooKeeper and create first replica. * Returns true if was created, false if exists. */ - bool createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot); + bool createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + bool createTableIfNotExistsAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const; /** * Creates a replica in ZooKeeper and adds to the queue all that it takes to catch up with the rest of the replicas. */ - void createReplica(const StorageMetadataPtr & metadata_snapshot); + void createReplica(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createReplicaAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const; /** Create nodes in the ZK, which must always be, but which might not exist when older versions of the server are running. */ - void createNewZooKeeperNodes(); + void createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createNewZooKeeperNodesAttempt() const; - bool checkTableStructure(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, bool strict_check = true); + /// Returns the ZooKeeper retries info specified for the CREATE TABLE query which is creating and starting this table right now. + ZooKeeperRetriesInfo getCreateQueryZooKeeperRetriesInfo() const; + QueryStatusPtr getCreateQueryStatus() const; + void clearCreateQueryZooKeeperRetriesInfo(); + + bool checkTableStructure(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check, + const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + bool checkTableStructureAttempt(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check) const; /// A part of ALTER: apply metadata changes only (data parts are altered separately). /// Must be called under IStorage::lockForAlter() lock. @@ -602,7 +619,7 @@ private: /// Synchronize the list of part uuids which are currently pinned. These should be sent to root query executor /// to be used for deduplication. - void syncPinnedPartUUIDs(); + void syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); /** Check that the part's checksum is the same as the checksum of the same part on some other replica. * If no one has such a part, nothing checks. @@ -705,7 +722,7 @@ private: /// Start being leader (if not disabled by setting). /// Since multi-leaders are allowed, it just sets is_leader flag. - void startBeingLeader(); + void startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); void stopBeingLeader(); /** Selects the parts to merge and writes to the log. @@ -922,7 +939,7 @@ private: /// Check granularity of already existing replicated table in zookeeper if it exists /// return true if it's fixed - bool checkFixedGranularityInZookeeper(); + bool checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; /// Wait for timeout seconds mutation is finished on replicas void waitMutationToFinishOnReplicas( @@ -960,7 +977,8 @@ private: void createAndStoreFreezeMetadata(DiskPtr disk, DataPartPtr part, String backup_part_path) const override; // Create table id if needed - void createTableSharedID() const; + void createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createTableSharedIDAttempt() const; bool checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk, String & lock_replica); void watchZeroCopyLock(const String & part_name, const DiskPtr & disk); @@ -976,7 +994,7 @@ private: /// Or if node actually disappeared. bool waitZeroCopyLockToDisappear(const ZeroCopyLock & lock, size_t milliseconds_to_wait) override; - void startupImpl(bool from_attach_thread); + void startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); std::vector getZookeeperZeroCopyLockPaths() const; static void dropZookeeperZeroCopyLockPaths(zkutil::ZooKeeperPtr zookeeper, From 136c5de31c79a41a9a2f006b00b94ce6de8af976 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 1 Dec 2024 23:21:10 +0100 Subject: [PATCH 222/502] Add field query_status to the ZooKeeperRetriesInfo structure. --- src/Backups/BackupCoordinationOnCluster.cpp | 4 +- src/Backups/BackupCoordinationOnCluster.h | 3 +- src/Backups/BackupEntriesCollector.cpp | 8 +- src/Backups/BackupEntriesCollector.h | 4 +- src/Backups/RestoreCoordinationOnCluster.cpp | 4 +- src/Backups/RestoreCoordinationOnCluster.h | 3 +- src/Backups/RestorerFromBackup.cpp | 3 +- src/Backups/WithRetries.cpp | 5 +- src/Common/ZooKeeper/ZooKeeperRetries.h | 24 ++--- src/Databases/DatabaseOrdinary.cpp | 2 +- src/Databases/DatabaseReplicatedWorker.cpp | 2 +- src/Databases/DatabaseReplicatedWorker.h | 2 +- src/Interpreters/DDLWorker.cpp | 4 +- src/Interpreters/DDLWorker.h | 2 +- .../DistributedQueryStatusSource.cpp | 11 +-- .../DistributedQueryStatusSource.h | 2 +- src/Interpreters/InterpreterSystemQuery.cpp | 4 +- src/Interpreters/executeDDLQueryOnCluster.cpp | 2 +- .../ReplicatedMergeTreeAttachThread.cpp | 10 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 8 +- .../MergeTree/registerStorageMergeTree.cpp | 13 ++- src/Storages/StorageKeeperMap.cpp | 40 +++++--- src/Storages/StorageReplicatedMergeTree.cpp | 99 ++++++++----------- src/Storages/StorageReplicatedMergeTree.h | 25 +++-- .../System/StorageSystemZooKeeper.cpp | 7 +- 25 files changed, 148 insertions(+), 143 deletions(-) diff --git a/src/Backups/BackupCoordinationOnCluster.cpp b/src/Backups/BackupCoordinationOnCluster.cpp index 1b14f226eff..4fee3bb952e 100644 --- a/src/Backups/BackupCoordinationOnCluster.cpp +++ b/src/Backups/BackupCoordinationOnCluster.cpp @@ -182,6 +182,7 @@ BackupCoordinationOnCluster::BackupCoordinationOnCluster( , current_host(current_host_) , current_host_index(findCurrentHostIndex(current_host, all_hosts)) , plain_backup(is_plain_backup_) + , process_list_element(process_list_element_) , log(getLogger("BackupCoordinationOnCluster")) , with_retries(log, get_zookeeper_, keeper_settings, process_list_element_, [root_zookeeper_path_](Coordination::ZooKeeperWithFaultInjection::Ptr zk) { zk->sync(root_zookeeper_path_); }) , cleaner(/* is_restore = */ false, zookeeper_path, with_retries, log) @@ -273,7 +274,8 @@ ZooKeeperRetriesInfo BackupCoordinationOnCluster::getOnClusterInitializationKeep { return ZooKeeperRetriesInfo{keeper_settings.max_retries_while_initializing, static_cast(keeper_settings.retry_initial_backoff_ms.count()), - static_cast(keeper_settings.retry_max_backoff_ms.count())}; + static_cast(keeper_settings.retry_max_backoff_ms.count()), + process_list_element}; } void BackupCoordinationOnCluster::serializeToMultipleZooKeeperNodes(const String & path, const String & value, const String & logging_name) diff --git a/src/Backups/BackupCoordinationOnCluster.h b/src/Backups/BackupCoordinationOnCluster.h index b439ab619d8..9bfc33d9fbf 100644 --- a/src/Backups/BackupCoordinationOnCluster.h +++ b/src/Backups/BackupCoordinationOnCluster.h @@ -107,7 +107,8 @@ private: const String current_host; const size_t current_host_index; const bool plain_backup; - LoggerPtr const log; + const QueryStatusPtr process_list_element; + const LoggerPtr log; /// The order is important: `stage_sync` must be initialized after `with_retries` and `cleaner`. const WithRetries with_retries; diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 00a4471d994..1ff190c171d 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -111,10 +111,11 @@ BackupEntriesCollector::BackupEntriesCollector( context->getConfigRef().getUInt64("backups.max_sleep_before_next_attempt_to_collect_metadata", 5000)) , compare_collected_metadata(context->getConfigRef().getBool("backups.compare_collected_metadata", true)) , log(getLogger("BackupEntriesCollector")) - , global_zookeeper_retries_info( + , zookeeper_retries_info( context->getSettingsRef()[Setting::backup_restore_keeper_max_retries], context->getSettingsRef()[Setting::backup_restore_keeper_retry_initial_backoff_ms], - context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms]) + context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms], + context->getProcessListElementSafe()) , threadpool(threadpool_) { } @@ -582,8 +583,7 @@ std::vector> BackupEntriesCollector::findTablesInD try { /// Database or table could be replicated - so may use ZooKeeper. We need to retry. - auto zookeeper_retries_info = global_zookeeper_retries_info; - ZooKeeperRetriesControl retries_ctl("getTablesForBackup", log, zookeeper_retries_info, nullptr); + ZooKeeperRetriesControl retries_ctl("getTablesForBackup", log, zookeeper_retries_info); retries_ctl.retryLoop([&](){ db_tables = database->getTablesForBackup(filter_by_table_name, context); }); } catch (Exception & e) diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 504489cce6b..d8b2dfd38e7 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -48,7 +48,7 @@ public: std::shared_ptr getBackupCoordination() const { return backup_coordination; } const ReadSettings & getReadSettings() const { return read_settings; } ContextPtr getContext() const { return context; } - const ZooKeeperRetriesInfo & getZooKeeperRetriesInfo() const { return global_zookeeper_retries_info; } + const ZooKeeperRetriesInfo & getZooKeeperRetriesInfo() const { return zookeeper_retries_info; } /// Returns all access entities which can be put into a backup. std::unordered_map getAllAccessEntities(); @@ -129,7 +129,7 @@ private: LoggerPtr log; /// Unfortunately we can use ZooKeeper for collecting information for backup /// and we need to retry... - ZooKeeperRetriesInfo global_zookeeper_retries_info; + ZooKeeperRetriesInfo zookeeper_retries_info; Strings all_hosts; DDLRenamingMap renaming_map; diff --git a/src/Backups/RestoreCoordinationOnCluster.cpp b/src/Backups/RestoreCoordinationOnCluster.cpp index fad7341c044..c85cca45fa7 100644 --- a/src/Backups/RestoreCoordinationOnCluster.cpp +++ b/src/Backups/RestoreCoordinationOnCluster.cpp @@ -33,6 +33,7 @@ RestoreCoordinationOnCluster::RestoreCoordinationOnCluster( , all_hosts_without_initiator(BackupCoordinationOnCluster::excludeInitiator(all_hosts)) , current_host(current_host_) , current_host_index(BackupCoordinationOnCluster::findCurrentHostIndex(current_host, all_hosts)) + , process_list_element(process_list_element_) , log(getLogger("RestoreCoordinationOnCluster")) , with_retries(log, get_zookeeper_, keeper_settings, process_list_element_, [root_zookeeper_path_](Coordination::ZooKeeperWithFaultInjection::Ptr zk) { zk->sync(root_zookeeper_path_); }) , cleaner(/* is_restore = */ true, zookeeper_path, with_retries, log) @@ -122,7 +123,8 @@ ZooKeeperRetriesInfo RestoreCoordinationOnCluster::getOnClusterInitializationKee { return ZooKeeperRetriesInfo{keeper_settings.max_retries_while_initializing, static_cast(keeper_settings.retry_initial_backoff_ms.count()), - static_cast(keeper_settings.retry_max_backoff_ms.count())}; + static_cast(keeper_settings.retry_max_backoff_ms.count()), + process_list_element}; } bool RestoreCoordinationOnCluster::acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) diff --git a/src/Backups/RestoreCoordinationOnCluster.h b/src/Backups/RestoreCoordinationOnCluster.h index 99929cbdac3..890117e63e8 100644 --- a/src/Backups/RestoreCoordinationOnCluster.h +++ b/src/Backups/RestoreCoordinationOnCluster.h @@ -75,7 +75,8 @@ private: const Strings all_hosts_without_initiator; const String current_host; const size_t current_host_index; - LoggerPtr const log; + const QueryStatusPtr process_list_element; + const LoggerPtr log; /// The order is important: `stage_sync` must be initialized after `with_retries` and `cleaner`. const WithRetries with_retries; diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 9b3b2408706..249e4353296 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -109,7 +109,8 @@ RestorerFromBackup::RestorerFromBackup( , zookeeper_retries_info( context->getSettingsRef()[Setting::backup_restore_keeper_max_retries], context->getSettingsRef()[Setting::backup_restore_keeper_retry_initial_backoff_ms], - context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms]) + context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms], + context->getProcessListElementSafe()) , tables_dependencies("RestorerFromBackup") , thread_pool(thread_pool_) { diff --git a/src/Backups/WithRetries.cpp b/src/Backups/WithRetries.cpp index 9c18be3ca9e..53613b2149e 100644 --- a/src/Backups/WithRetries.cpp +++ b/src/Backups/WithRetries.cpp @@ -20,9 +20,10 @@ WithRetries::RetriesControlHolder::RetriesControlHolder(const WithRetries * pare : (kind == kErrorHandling) ? parent->settings.max_retries_while_handling_error : parent->settings.max_retries, parent->settings.retry_initial_backoff_ms.count(), - parent->settings.retry_max_backoff_ms.count()) + parent->settings.retry_max_backoff_ms.count(), + (kind == kErrorHandling) ? nullptr : parent->process_list_element) /// We don't use process_list_element while handling an error because the error handling can't be cancellable. - , retries_ctl(name, parent->log, info, (kind == kErrorHandling) ? nullptr : parent->process_list_element) + , retries_ctl(name, parent->log, info) , faulty_zookeeper(parent->getFaultyZooKeeper()) {} diff --git a/src/Common/ZooKeeper/ZooKeeperRetries.h b/src/Common/ZooKeeper/ZooKeeperRetries.h index acea521a7ce..9482f72cba0 100644 --- a/src/Common/ZooKeeper/ZooKeeperRetries.h +++ b/src/Common/ZooKeeper/ZooKeeperRetries.h @@ -16,21 +16,25 @@ namespace ErrorCodes struct ZooKeeperRetriesInfo { ZooKeeperRetriesInfo() = default; - ZooKeeperRetriesInfo(UInt64 max_retries_, UInt64 initial_backoff_ms_, UInt64 max_backoff_ms_) + + ZooKeeperRetriesInfo(UInt64 max_retries_, UInt64 initial_backoff_ms_, UInt64 max_backoff_ms_, QueryStatusPtr query_status_) : max_retries(max_retries_), initial_backoff_ms(std::min(initial_backoff_ms_, max_backoff_ms_)), max_backoff_ms(max_backoff_ms_) + , query_status(query_status_) { } UInt64 max_retries = 0; /// "max_retries = 0" means only one attempt. - UInt64 initial_backoff_ms = 100; - UInt64 max_backoff_ms = 5000; + UInt64 initial_backoff_ms = 0; + UInt64 max_backoff_ms = 0; + + QueryStatusPtr query_status; /// can be nullptr }; class ZooKeeperRetriesControl { public: - ZooKeeperRetriesControl(std::string name_, LoggerPtr logger_, ZooKeeperRetriesInfo retries_info_, QueryStatusPtr elem) - : name(std::move(name_)), logger(logger_), retries_info(retries_info_), process_list_element(elem) + ZooKeeperRetriesControl(std::string name_, LoggerPtr logger_, ZooKeeperRetriesInfo retries_info_) + : name(std::move(name_)), logger(logger_), retries_info(retries_info_) { } @@ -39,7 +43,6 @@ public: , logger(other.logger) , retries_info(other.retries_info) , total_failures(other.total_failures) - , process_list_element(other.process_list_element) , current_backoff_ms(other.current_backoff_ms) { } @@ -222,8 +225,8 @@ private: } /// Check if the query was cancelled. - if (process_list_element) - process_list_element->checkTimeLimit(); + if (retries_info.query_status) + retries_info.query_status->checkTimeLimit(); /// retries logLastError("will retry due to error"); @@ -231,8 +234,8 @@ private: current_backoff_ms = std::min(current_backoff_ms * 2, retries_info.max_backoff_ms); /// Check if the query was cancelled again after sleeping. - if (process_list_element) - process_list_element->checkTimeLimit(); + if (retries_info.query_status) + retries_info.query_status->checkTimeLimit(); return true; } @@ -288,7 +291,6 @@ private: std::function action_after_last_failed_retry = []() {}; bool iteration_succeeded = true; bool stop_retries = false; - QueryStatusPtr process_list_element; UInt64 current_iteration = 0; UInt64 current_backoff_ms = 0; diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index eed29c0d821..6c9f5b82116 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -408,7 +408,7 @@ void DatabaseOrdinary::restoreMetadataAfterConvertingToReplicated(StoragePtr tab } else { - rmt->restoreMetadataInZooKeeper(/* query_status = */ nullptr, /* zookeeper_retries_info = */ {}); + rmt->restoreMetadataInZooKeeper(/* zookeeper_retries_info = */ {}); LOG_INFO ( log, diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 6a711c92332..0000096c1c1 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -199,7 +199,7 @@ void DatabaseReplicatedDDLWorker::initializeReplication() active_node_holder = zkutil::EphemeralNodeHolder::existing(active_path, *active_node_holder_zookeeper); } -String DatabaseReplicatedDDLWorker::enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo &, QueryStatusPtr) +String DatabaseReplicatedDDLWorker::enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo &) { auto zookeeper = getAndSetZooKeeper(); return enqueueQueryImpl(zookeeper, entry, database); diff --git a/src/Databases/DatabaseReplicatedWorker.h b/src/Databases/DatabaseReplicatedWorker.h index d2385cbdba3..2bb603354ac 100644 --- a/src/Databases/DatabaseReplicatedWorker.h +++ b/src/Databases/DatabaseReplicatedWorker.h @@ -24,7 +24,7 @@ class DatabaseReplicatedDDLWorker : public DDLWorker public: DatabaseReplicatedDDLWorker(DatabaseReplicated * db, ContextPtr context_); - String enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo &, QueryStatusPtr) override; + String enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo &) override; String tryEnqueueAndExecuteEntry(DDLLogEntry & entry, ContextPtr query_context); diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index b1c7635b62b..3b4d31996eb 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -1054,12 +1054,12 @@ void DDLWorker::createStatusDirs(const std::string & node_path, const ZooKeeperP } -String DDLWorker::enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo & retries_info, QueryStatusPtr process_list_element) +String DDLWorker::enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo & retries_info) { String node_path; if (retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"DDLWorker::enqueueQuery", log, retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"DDLWorker::enqueueQuery", log, retries_info}; retries_ctl.retryLoop([&]{ node_path = enqueueQueryAttempt(entry); }); diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index a5f47a51bb3..ec697046d03 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -68,7 +68,7 @@ public: virtual ~DDLWorker(); /// Pushes query into DDL queue, returns path to created node - virtual String enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo & retries_info, QueryStatusPtr process_list_element); + virtual String enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo & retries_info); /// Host ID (name:port) for logging purposes /// Note that in each task hosts are identified individually by name:port from initiator server cluster config diff --git a/src/Interpreters/DistributedQueryStatusSource.cpp b/src/Interpreters/DistributedQueryStatusSource.cpp index 83701d41c57..aaddb1bd4e2 100644 --- a/src/Interpreters/DistributedQueryStatusSource.cpp +++ b/src/Interpreters/DistributedQueryStatusSource.cpp @@ -133,8 +133,7 @@ ExecutionStatus DistributedQueryStatusSource::getExecutionStatus(const fs::path String status_data; bool finished_exists = false; - auto retries_ctl = ZooKeeperRetriesControl( - "executeDDLQueryOnCluster", getLogger("DDLQueryStatusSource"), getRetriesInfo(), context->getProcessListElement()); + auto retries_ctl = ZooKeeperRetriesControl("executeDDLQueryOnCluster", getLogger("DDLQueryStatusSource"), getRetriesInfo()); retries_ctl.retryLoop([&]() { finished_exists = context->getZooKeeper()->tryGet(status_path, status_data); }); if (finished_exists) status.tryDeserializeText(status_data); @@ -142,13 +141,14 @@ ExecutionStatus DistributedQueryStatusSource::getExecutionStatus(const fs::path return status; } -ZooKeeperRetriesInfo DistributedQueryStatusSource::getRetriesInfo() +ZooKeeperRetriesInfo DistributedQueryStatusSource::getRetriesInfo() const { const auto & config_ref = Context::getGlobalContextInstance()->getConfigRef(); return ZooKeeperRetriesInfo( config_ref.getInt("distributed_ddl_keeper_max_retries", 5), config_ref.getInt("distributed_ddl_keeper_initial_backoff_ms", 100), - config_ref.getInt("distributed_ddl_keeper_max_backoff_ms", 5000)); + config_ref.getInt("distributed_ddl_keeper_max_backoff_ms", 5000), + context->getProcessListElement()); } std::pair DistributedQueryStatusSource::parseHostAndPort(const String & host_id) @@ -194,8 +194,7 @@ Chunk DistributedQueryStatusSource::generate() Strings tmp_active_hosts; { - auto retries_ctl = ZooKeeperRetriesControl( - "executeDistributedQueryOnCluster", getLogger(getName()), getRetriesInfo(), context->getProcessListElement()); + auto retries_ctl = ZooKeeperRetriesControl("executeDistributedQueryOnCluster", getLogger(getName()), getRetriesInfo()); retries_ctl.retryLoop( [&]() { diff --git a/src/Interpreters/DistributedQueryStatusSource.h b/src/Interpreters/DistributedQueryStatusSource.h index 4f58085a1f0..71315c5cd74 100644 --- a/src/Interpreters/DistributedQueryStatusSource.h +++ b/src/Interpreters/DistributedQueryStatusSource.h @@ -38,7 +38,7 @@ protected: Strings getNewAndUpdate(const Strings & current_finished_hosts); ExecutionStatus getExecutionStatus(const fs::path & status_path); - static ZooKeeperRetriesInfo getRetriesInfo(); + ZooKeeperRetriesInfo getRetriesInfo() const; static std::pair parseHostAndPort(const String & host_id); String node_path; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index ca96ee3245f..ac5f03e3b24 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -884,10 +884,10 @@ void InterpreterSystemQuery::restoreReplica() const auto & settings = getContext()->getSettingsRef(); table_replicated_ptr->restoreMetadataInZooKeeper( - getContext()->getProcessListElementSafe(), ZooKeeperRetriesInfo{settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], - settings[Setting::keeper_retry_max_backoff_ms]}); + settings[Setting::keeper_retry_max_backoff_ms], + getContext()->getProcessListElementSafe()}); } StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, ContextMutablePtr system_context) diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 0b88d07148c..43ffa946a57 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -189,7 +189,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context, entry.setSettingsIfRequired(context); entry.tracing_context = OpenTelemetry::CurrentContext(); entry.initial_query_id = context->getClientInfo().initial_query_id; - String node_path = ddl_worker.enqueueQuery(entry, params.retries_info, context->getProcessListElement()); + String node_path = ddl_worker.enqueueQuery(entry, params.retries_info); return getDDLOnClusterStatus(node_path, ddl_worker.getReplicasDir(), entry, context); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index c654b459c24..db5b1d5d0c9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -167,23 +167,23 @@ void ReplicatedMergeTreeAttachThread::runImpl() zookeeper->tryRemove(replica_path + "/flags/force_restore_data"); /// Here `zookeeper_retries_info = {}` because the attach thread has its own retries (see ReplicatedMergeTreeAttachThread::run()). - storage.checkTableStructure(replica_path, metadata_snapshot, /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + storage.checkTableStructure(replica_path, metadata_snapshot, /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}); storage.checkParts(skip_sanity_checks); /// Temporary directories contain uninitialized results of Merges or Fetches (after forced restart), /// don't allow to reinitialize them, delete each of them immediately. storage.clearOldTemporaryDirectories(0, {"tmp_", "delete_tmp_", "tmp-fetch_"}); - storage.createNewZooKeeperNodes(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); - storage.syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + storage.createNewZooKeeperNodes(/* zookeeper_retries_info = */ {}); + storage.syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}); std::lock_guard lock(storage.table_shared_id_mutex); - storage.createTableSharedID(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + storage.createTableSharedID(/* zookeeper_retries_info = */ {}); }; void ReplicatedMergeTreeAttachThread::finalizeInitialization() TSA_NO_THREAD_SAFETY_ANALYSIS { - storage.startupImpl(/* from_attach_thread */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + storage.startupImpl(/* from_attach_thread */ true, /* zookeeper_retries_info = */ {}); storage.initialization_done = true; LOG_INFO(log, "Table is initialized"); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 19a69eb46be..3422e534f7d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -201,8 +201,8 @@ size_t ReplicatedMergeTreeSinkImpl::checkQuorumPrecondition(const log, {settings[Setting::insert_keeper_max_retries], settings[Setting::insert_keeper_retry_initial_backoff_ms], - settings[Setting::insert_keeper_retry_max_backoff_ms]}, - context->getProcessListElement()); + settings[Setting::insert_keeper_retry_max_backoff_ms], + context->getProcessListElement()}); quorum_retries_ctl.retryLoop( [&]() { @@ -725,8 +725,8 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: log, {settings[Setting::insert_keeper_max_retries], settings[Setting::insert_keeper_retry_initial_backoff_ms], - settings[Setting::insert_keeper_retry_max_backoff_ms]}, - context->getProcessListElement()); + settings[Setting::insert_keeper_retry_max_backoff_ms], + context->getProcessListElement()}); auto resolve_duplicate_stage = [&] () -> CommitRetryContext::Stages { diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 12b5d115903..709f019a005 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -835,11 +835,11 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (auto txn = args.getLocalContext()->getZooKeeperMetadataTransaction()) need_check_table_structure = txn->isInitialQuery(); - ZooKeeperRetriesInfo create_query_zk_retries_info; - create_query_zk_retries_info.max_retries = local_settings[Setting::keeper_max_retries]; - create_query_zk_retries_info.initial_backoff_ms = local_settings[Setting::keeper_retry_initial_backoff_ms]; - create_query_zk_retries_info.max_backoff_ms = local_settings[Setting::keeper_retry_max_backoff_ms]; - auto create_query_status = args.getLocalContext()->getProcessListElementSafe(); + ZooKeeperRetriesInfo create_query_zk_retries_info{ + local_settings[Setting::keeper_max_retries], + local_settings[Setting::keeper_retry_initial_backoff_ms], + local_settings[Setting::keeper_retry_max_backoff_ms], + args.getLocalContext()->getProcessListElementSafe()}; return std::make_shared( zookeeper_info, @@ -852,8 +852,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) merging_params, std::move(storage_settings), need_check_table_structure, - create_query_zk_retries_info, - create_query_status); + create_query_zk_retries_info); } return std::make_shared( diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 2a4a5f3370f..1504656bce5 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -189,8 +189,8 @@ public: ZooKeeperRetriesInfo{ settings[Setting::insert_keeper_max_retries], settings[Setting::insert_keeper_retry_initial_backoff_ms], - settings[Setting::insert_keeper_retry_max_backoff_ms]}, - context->getProcessListElement()}; + settings[Setting::insert_keeper_retry_max_backoff_ms], + context->getProcessListElement()}}; zk_retry.retryLoop([&]() { @@ -425,8 +425,10 @@ StorageKeeperMap::StorageKeeperMap( getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - context_->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + context_->getProcessListElement()}}; zk_retry.retryLoop( [&] @@ -670,8 +672,10 @@ Pipe StorageKeeperMap::read( getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - context_->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + context_->getProcessListElement()}}; std::vector children; zk_retry.retryLoop([&] @@ -699,8 +703,10 @@ void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - local_context->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + local_context->getProcessListElement()}}; zk_retry.retryLoop([&] { @@ -1136,8 +1142,10 @@ StorageKeeperMap::TableStatus StorageKeeperMap::getTableStatus(const ContextPtr getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - local_context->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + local_context->getProcessListElement()}}; zk_retry.retryLoop([&] { @@ -1248,8 +1256,10 @@ Chunk StorageKeeperMap::getBySerializedKeys( getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - local_context->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + local_context->getProcessListElement()}}; zkutil::ZooKeeper::MultiTryGetResponse values; zk_retry.retryLoop([&]{ @@ -1394,8 +1404,10 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - local_context->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + local_context->getProcessListElement()}}; Coordination::Error status; zk_retry.retryLoop([&] diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ac1b37ecbdb..b57dcd144f2 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -367,8 +367,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const MergingParams & merging_params_, std::unique_ptr settings_, bool need_check_structure, - const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_, - QueryStatusPtr create_query_status_) + const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_) : MergeTreeData(table_id_, metadata_, context_, @@ -383,7 +382,6 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( , replica_name(zookeeper_info.replica_name) , replica_path(fs::path(zookeeper_path) / "replicas" / replica_name) , create_query_zookeeper_retries_info(create_query_zookeeper_retries_info_) - , create_query_status(create_query_status_) , reader(*this) , writer(*this) , merger_mutator(*this) @@ -573,7 +571,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( try { - bool is_first_replica = createTableIfNotExists(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + bool is_first_replica = createTableIfNotExists(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo()); try { @@ -582,13 +580,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( /// We have to check granularity on other replicas. If it's fixed we /// must create our new replica with fixed granularity and store this /// information in /replica/metadata. - other_replicas_fixed_granularity = checkFixedGranularityInZookeeper(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + other_replicas_fixed_granularity = checkFixedGranularityInZookeeper(getCreateQueryZooKeeperRetriesInfo()); /// Allow structure mismatch for secondary queries from Replicated database. /// It may happen if the table was altered just after creation. /// Metadata will be updated in cloneMetadataIfNeeded(...), metadata_version will be 0 for a while. int32_t metadata_version; - bool same_structure = checkTableStructure(zookeeper_path, metadata_snapshot, &metadata_version, need_check_structure, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + bool same_structure = checkTableStructure(zookeeper_path, metadata_snapshot, &metadata_version, need_check_structure, getCreateQueryZooKeeperRetriesInfo()); if (same_structure) { @@ -609,13 +607,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } if (!is_first_replica) - createReplica(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + createReplica(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo()); - createNewZooKeeperNodes(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); - syncPinnedPartUUIDs(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + createNewZooKeeperNodes(getCreateQueryZooKeeperRetriesInfo()); + syncPinnedPartUUIDs(getCreateQueryZooKeeperRetriesInfo()); if (!has_metadata_in_zookeeper.has_value() || *has_metadata_in_zookeeper) - createTableSharedID(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + createTableSharedID(getCreateQueryZooKeeperRetriesInfo()); } catch (...) { @@ -628,7 +626,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } -bool StorageReplicatedMergeTree::checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +bool StorageReplicatedMergeTree::checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info) const { bool fixed_granularity = false; @@ -642,7 +640,7 @@ bool StorageReplicatedMergeTree::checkFixedGranularityInZookeeper(const ZooKeepe if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkFixedGranularityInZookeeper", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkFixedGranularityInZookeeper", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { check_fixed_granularity(); }); } else @@ -825,11 +823,11 @@ std::vector getAncestors(const String & path) } -void StorageReplicatedMergeTree::createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +void StorageReplicatedMergeTree::createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info) const { if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createNewZooKeeperNodes", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createNewZooKeeperNodes", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { createNewZooKeeperNodesAttempt(); }); } else @@ -903,19 +901,17 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodesAttempt() const } } -bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, - const ZooKeeperRetriesInfo & zookeeper_retries_info, - QueryStatusPtr process_list_element) const +bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info) const { bool table_created = false; if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableIfNotExists", log.load(), zookeeper_retries_info, process_list_element}; - retries_ctl.retryLoop([&] { table_created = createTableIfNotExistsAttempt(metadata_snapshot, process_list_element); }); + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableIfNotExists", log.load(), zookeeper_retries_info}; + retries_ctl.retryLoop([&] { table_created = createTableIfNotExistsAttempt(metadata_snapshot, zookeeper_retries_info.query_status); }); } else { - table_created = createTableIfNotExistsAttempt(metadata_snapshot, process_list_element); + table_created = createTableIfNotExistsAttempt(metadata_snapshot, zookeeper_retries_info.query_status); } return table_created; } @@ -1069,18 +1065,16 @@ bool StorageReplicatedMergeTree::createTableIfNotExistsAttempt(const StorageMeta "of wrong zookeeper_path or because of logical error"); } -void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metadata_snapshot, - const ZooKeeperRetriesInfo & zookeeper_retries_info, - QueryStatusPtr process_list_element) const +void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info) const { if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createReplica", log.load(), zookeeper_retries_info, process_list_element}; - retries_ctl.retryLoop([&] { createReplicaAttempt(metadata_snapshot, process_list_element); }); + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createReplica", log.load(), zookeeper_retries_info}; + retries_ctl.retryLoop([&] { createReplicaAttempt(metadata_snapshot, zookeeper_retries_info.query_status); }); } else { - createReplicaAttempt(metadata_snapshot, process_list_element); + createReplicaAttempt(metadata_snapshot, zookeeper_retries_info.query_status); } } @@ -1244,17 +1238,10 @@ ZooKeeperRetriesInfo StorageReplicatedMergeTree::getCreateQueryZooKeeperRetriesI return create_query_zookeeper_retries_info; } -QueryStatusPtr StorageReplicatedMergeTree::getCreateQueryStatus() const -{ - std::lock_guard lock{create_query_zookeeper_retries_info_mutex}; - return create_query_status; -} - void StorageReplicatedMergeTree::clearCreateQueryZooKeeperRetriesInfo() { std::lock_guard lock{create_query_zookeeper_retries_info_mutex}; create_query_zookeeper_retries_info = {}; - create_query_status = {}; } @@ -1620,12 +1607,12 @@ bool StorageReplicatedMergeTree::removeTableNodesFromZooKeeper(zkutil::ZooKeeper */ bool StorageReplicatedMergeTree::checkTableStructure( const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check, - const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const + const ZooKeeperRetriesInfo & zookeeper_retries_info) const { bool same_structure = false; if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkTableStructure", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkTableStructure", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { same_structure = checkTableStructureAttempt(zookeeper_prefix, metadata_snapshot, metadata_version, strict_check); }); } else @@ -1970,7 +1957,7 @@ bool StorageReplicatedMergeTree::checkPartsImpl(bool skip_sanity_checks) } -void StorageReplicatedMergeTree::syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) +void StorageReplicatedMergeTree::syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info) { String new_pinned_part_uuids_str; Coordination::Stat new_stat; @@ -1983,7 +1970,7 @@ void StorageReplicatedMergeTree::syncPinnedPartUUIDs(const ZooKeeperRetriesInfo if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::syncPinnedPartUUIDs", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::syncPinnedPartUUIDs", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { read_pinned_part_uuids(); }); } else @@ -2352,7 +2339,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) case LogEntry::ALTER_METADATA: return executeMetadataAlter(entry); case LogEntry::SYNC_PINNED_PART_UUIDS: - syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}); return true; case LogEntry::CLONE_PART_FROM_SHARD: executeClonePartFromShard(entry); @@ -4501,7 +4488,7 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n } -void StorageReplicatedMergeTree::startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) +void StorageReplicatedMergeTree::startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info) { if (!(*getSettings())[MergeTreeSetting::replicated_can_become_leader]) { @@ -4517,7 +4504,7 @@ void StorageReplicatedMergeTree::startBeingLeader(const ZooKeeperRetriesInfo & z if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::startBeingLeader", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::startBeingLeader", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { start_being_leader(); }); } else @@ -5396,10 +5383,10 @@ void StorageReplicatedMergeTree::startup() return; } - startupImpl(/* from_attach_thread */ false, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + startupImpl(/* from_attach_thread */ false, getCreateQueryZooKeeperRetriesInfo()); } -void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) +void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info) { /// Do not start replication if ZooKeeper is not configured or there is no metadata in zookeeper if (!has_metadata_in_zookeeper.has_value() || !*has_metadata_in_zookeeper) @@ -5428,7 +5415,7 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread, const ZooK getContext()->getInterserverIOHandler().addEndpoint( data_parts_exchange_ptr->getId(getEndpointName()), data_parts_exchange_ptr); - startBeingLeader(zookeeper_retries_info, process_list_element); + startBeingLeader(zookeeper_retries_info); if (from_attach_thread) { @@ -6683,7 +6670,7 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition( return true; } -void StorageReplicatedMergeTree::restoreMetadataInZooKeeper(QueryStatusPtr query_status, const ZooKeeperRetriesInfo & zookeeper_retries_info) +void StorageReplicatedMergeTree::restoreMetadataInZooKeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info) { LOG_INFO(log, "Restoring replica metadata"); @@ -6726,14 +6713,14 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper(QueryStatusPtr query LOG_INFO(log, "Moved all parts to detached/"); - const bool is_first_replica = createTableIfNotExists(metadata_snapshot, zookeeper_retries_info, query_status); + const bool is_first_replica = createTableIfNotExists(metadata_snapshot, zookeeper_retries_info); LOG_INFO(log, "Created initial ZK nodes, replica is first: {}", is_first_replica); if (!is_first_replica) - createReplica(metadata_snapshot, zookeeper_retries_info, query_status); + createReplica(metadata_snapshot, zookeeper_retries_info); - createNewZooKeeperNodes(zookeeper_retries_info, query_status); + createNewZooKeeperNodes(zookeeper_retries_info); LOG_INFO(log, "Created ZK nodes for table"); @@ -6745,7 +6732,7 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper(QueryStatusPtr query LOG_INFO(log, "Attached all partitions, starting table"); - startupImpl(/* from_attach_thread */ false, zookeeper_retries_info, query_status); + startupImpl(/* from_attach_thread */ false, zookeeper_retries_info); } void StorageReplicatedMergeTree::dropPartNoWaitNoThrow(const String & part_name) @@ -8034,8 +8021,8 @@ void StorageReplicatedMergeTree::forcefullyRemoveBrokenOutdatedPartFromZooKeeper String part_path = replica_path + "/parts/" + part_name; const auto & settings = getContext()->getSettingsRef(); ZooKeeperRetriesInfo retries_info{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}; - ZooKeeperRetriesControl retries_ctl("outdatedPartExists", log.load(), retries_info, nullptr); + settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms], nullptr}; + ZooKeeperRetriesControl retries_ctl("outdatedPartExists", log.load(), retries_info); retries_ctl.retryLoop([&]() { exists = getZooKeeper()->exists(part_path); }); if (!exists) @@ -8924,7 +8911,7 @@ void StorageReplicatedMergeTree::movePartitionToShard( { /// Optimistic check that for compatible destination table structure. - checkTableStructure(to, getInMemoryMetadataPtr(), /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + checkTableStructure(to, getInMemoryMetadataPtr(), /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}); } PinnedPartUUIDs src_pins; @@ -9527,7 +9514,7 @@ String StorageReplicatedMergeTree::getTableSharedID() const { /// Can happen if table was partially initialized before drop by DatabaseCatalog if (table_shared_id == UUIDHelpers::Nil) - createTableSharedID(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + createTableSharedID(/* zookeeper_retries_info = */ {}); } else { @@ -9542,11 +9529,11 @@ std::map StorageReplicatedMergeTree::getUnfinishe return queue.getUnfinishedMutations(); } -void StorageReplicatedMergeTree::createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +void StorageReplicatedMergeTree::createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info) const { if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableSharedID", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableSharedID", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { createTableSharedIDAttempt(); }); } else @@ -10836,7 +10823,7 @@ void StorageReplicatedMergeTree::backupData( bool exists = false; Strings mutation_ids; { - ZooKeeperRetriesControl retries_ctl("getMutations", log.load(), zookeeper_retries_info, nullptr); + ZooKeeperRetriesControl retries_ctl("getMutations", log.load(), zookeeper_retries_info); retries_ctl.retryLoop([&]() { if (!zookeeper || zookeeper->expired()) @@ -10855,7 +10842,7 @@ void StorageReplicatedMergeTree::backupData( bool mutation_id_exists = false; String mutation; - ZooKeeperRetriesControl retries_ctl("getMutation", log.load(), zookeeper_retries_info, nullptr); + ZooKeeperRetriesControl retries_ctl("getMutation", log.load(), zookeeper_retries_info); retries_ctl.retryLoop([&]() { if (!zookeeper || zookeeper->expired()) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index c6081d45bf4..f153ae91361 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -110,8 +110,7 @@ public: const MergingParams & merging_params_, std::unique_ptr settings_, bool need_check_structure, - const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_, - QueryStatusPtr create_query_status_); + const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_); void startup() override; @@ -316,7 +315,7 @@ public: /// Restores table metadata if ZooKeeper lost it. /// Used only on restarted readonly replicas (not checked). All active (Active) parts are moved to detached/ /// folder and attached. Parts in all other states are just moved to detached/ folder. - void restoreMetadataInZooKeeper(QueryStatusPtr query_status, const ZooKeeperRetriesInfo & zookeeper_retries_info); + void restoreMetadataInZooKeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info); /// Get throttler for replicated fetches ThrottlerPtr getFetchesThrottler() const @@ -428,7 +427,6 @@ private: const String replica_path; ZooKeeperRetriesInfo create_query_zookeeper_retries_info TSA_GUARDED_BY(create_query_zookeeper_retries_info_mutex); - QueryStatusPtr create_query_status TSA_GUARDED_BY(create_query_zookeeper_retries_info_mutex); mutable std::mutex create_query_zookeeper_retries_info_mutex; /** /replicas/me/is_active. @@ -579,27 +577,26 @@ private: /** Creates the minimum set of nodes in ZooKeeper and create first replica. * Returns true if was created, false if exists. */ - bool createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + bool createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info) const; bool createTableIfNotExistsAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const; /** * Creates a replica in ZooKeeper and adds to the queue all that it takes to catch up with the rest of the replicas. */ - void createReplica(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createReplica(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info) const; void createReplicaAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const; /** Create nodes in the ZK, which must always be, but which might not exist when older versions of the server are running. */ - void createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info) const; void createNewZooKeeperNodesAttempt() const; /// Returns the ZooKeeper retries info specified for the CREATE TABLE query which is creating and starting this table right now. ZooKeeperRetriesInfo getCreateQueryZooKeeperRetriesInfo() const; - QueryStatusPtr getCreateQueryStatus() const; void clearCreateQueryZooKeeperRetriesInfo(); bool checkTableStructure(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check, - const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + const ZooKeeperRetriesInfo & zookeeper_retries_info) const; bool checkTableStructureAttempt(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check) const; /// A part of ALTER: apply metadata changes only (data parts are altered separately). @@ -619,7 +616,7 @@ private: /// Synchronize the list of part uuids which are currently pinned. These should be sent to root query executor /// to be used for deduplication. - void syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); + void syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info); /** Check that the part's checksum is the same as the checksum of the same part on some other replica. * If no one has such a part, nothing checks. @@ -722,7 +719,7 @@ private: /// Start being leader (if not disabled by setting). /// Since multi-leaders are allowed, it just sets is_leader flag. - void startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); + void startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info); void stopBeingLeader(); /** Selects the parts to merge and writes to the log. @@ -939,7 +936,7 @@ private: /// Check granularity of already existing replicated table in zookeeper if it exists /// return true if it's fixed - bool checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + bool checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info) const; /// Wait for timeout seconds mutation is finished on replicas void waitMutationToFinishOnReplicas( @@ -977,7 +974,7 @@ private: void createAndStoreFreezeMetadata(DiskPtr disk, DataPartPtr part, String backup_part_path) const override; // Create table id if needed - void createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info) const; void createTableSharedIDAttempt() const; bool checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk, String & lock_replica); @@ -994,7 +991,7 @@ private: /// Or if node actually disappeared. bool waitZeroCopyLockToDisappear(const ZeroCopyLock & lock, size_t milliseconds_to_wait) override; - void startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); + void startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info); std::vector getZookeeperZeroCopyLockPaths() const; static void dropZookeeperZeroCopyLockPaths(zkutil::ZooKeeperPtr zookeeper, diff --git a/src/Storages/System/StorageSystemZooKeeper.cpp b/src/Storages/System/StorageSystemZooKeeper.cpp index 000098af80d..1cd17cc0cd3 100644 --- a/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/src/Storages/System/StorageSystemZooKeeper.cpp @@ -518,7 +518,8 @@ Chunk SystemZooKeeperSource::generate() ZooKeeperRetriesInfo retries_seetings( settings[Setting::insert_keeper_max_retries], settings[Setting::insert_keeper_retry_initial_backoff_ms], - settings[Setting::insert_keeper_retry_max_backoff_ms]); + settings[Setting::insert_keeper_retry_max_backoff_ms], + query_status); /// Handles reconnects when needed auto get_zookeeper = [&] () @@ -586,7 +587,7 @@ Chunk SystemZooKeeperSource::generate() } zkutil::ZooKeeper::MultiTryGetChildrenResponse list_responses; - ZooKeeperRetriesControl("", nullptr, retries_seetings, query_status).retryLoop( + ZooKeeperRetriesControl("", nullptr, retries_seetings).retryLoop( [&]() { list_responses = get_zookeeper()->tryGetChildren(paths_to_list); }); struct GetTask @@ -632,7 +633,7 @@ Chunk SystemZooKeeperSource::generate() } zkutil::ZooKeeper::MultiTryGetResponse get_responses; - ZooKeeperRetriesControl("", nullptr, retries_seetings, query_status).retryLoop( + ZooKeeperRetriesControl("", nullptr, retries_seetings).retryLoop( [&]() { get_responses = get_zookeeper()->tryGet(paths_to_get); }); /// Add children count to query total rows. We can not get total rows in advance, From 098e20d9b9bb68ec0281ac1f1a423c1823f972af Mon Sep 17 00:00:00 2001 From: shuai-xu Date: Tue, 5 Nov 2024 11:26:17 +0800 Subject: [PATCH 223/502] translate support second arg longer than the third --- .../functions/string-replace-functions.md | 2 +- src/Functions/translate.cpp | 86 ++++++++++++++----- .../0_stateless/02353_translate.reference | 4 + tests/queries/0_stateless/02353_translate.sql | 7 +- 4 files changed, 74 insertions(+), 25 deletions(-) diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index 3f50cd24f93..9e2f063dfd7 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -253,7 +253,7 @@ SELECT format('{} {}', 'Hello', 'World') ## translate -Replaces characters in the string `s` using a one-to-one character mapping defined by `from` and `to` strings. `from` and `to` must be constant ASCII strings of the same size. Non-ASCII characters in the original string are not modified. +Replaces characters in the string `s` using a one-to-one character mapping defined by `from` and `to` strings. `from` and `to` must be constant ASCII strings. Non-ASCII characters in the original string are not modified. **Syntax** diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp index f7077f99629..97e16020869 100644 --- a/src/Functions/translate.cpp +++ b/src/Functions/translate.cpp @@ -32,18 +32,25 @@ struct TranslateImpl const std::string & map_from, const std::string & map_to) { - if (map_from.size() != map_to.size()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second and third arguments must be the same length"); + if (map_from.size() < map_to.size()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second arguments must be equal or longer than the third"); iota(map.data(), map.size(), UInt8(0)); - for (size_t i = 0; i < map_from.size(); ++i) + for (size_t i = 0; i < map_to.size(); ++i) { if (!isASCII(map_from[i]) || !isASCII(map_to[i])) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second and third arguments must be ASCII strings"); map[map_from[i]] = map_to[i]; } + for (size_t i = map_to.size(); i < map_from.size(); ++i) + { + if (!isASCII(map_from[i])) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second arguments must be ASCII strings"); + + map[map_from[i]] = ascii_upper_bound + 1; + } } static void vector( @@ -70,13 +77,18 @@ struct TranslateImpl while (src < src_end) { - if (*src <= ascii_upper_bound) + if (*src <= ascii_upper_bound && map[*src] != ascii_upper_bound + 1) + { *dst = map[*src]; - else + ++dst; + } + else if (*src > ascii_upper_bound) + { *dst = *src; + ++dst; + } ++src; - ++dst; } /// Technically '\0' can be mapped into other character, @@ -103,13 +115,18 @@ struct TranslateImpl while (src < src_end) { - if (*src <= ascii_upper_bound) + if (*src <= ascii_upper_bound && map[*src] != ascii_upper_bound + 1) + { *dst = map[*src]; - else + ++dst; + } + else if (*src > ascii_upper_bound) + { *dst = *src; + ++dst; + } ++src; - ++dst; } } @@ -131,8 +148,8 @@ struct TranslateUTF8Impl auto map_from_size = UTF8::countCodePoints(reinterpret_cast(map_from.data()), map_from.size()); auto map_to_size = UTF8::countCodePoints(reinterpret_cast(map_to.data()), map_to.size()); - if (map_from_size != map_to_size) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second and third arguments must be the same length"); + if (map_from_size < map_to_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second arguments must be equal or longer than third"); iota(map_ascii.data(), map_ascii.size(), UInt32(0)); @@ -141,32 +158,44 @@ struct TranslateUTF8Impl const UInt8 * map_to_ptr = reinterpret_cast(map_to.data()); const UInt8 * map_to_end = map_to_ptr + map_to.size(); - while (map_from_ptr < map_from_end && map_to_ptr < map_to_end) + while (map_from_ptr < map_from_end) { size_t len_from = UTF8::seqLength(*map_from_ptr); - size_t len_to = UTF8::seqLength(*map_to_ptr); std::optional res_from, res_to; if (map_from_ptr + len_from <= map_from_end) res_from = UTF8::convertUTF8ToCodePoint(map_from_ptr, len_from); - if (map_to_ptr + len_to <= map_to_end) - res_to = UTF8::convertUTF8ToCodePoint(map_to_ptr, len_to); - if (!res_from) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second argument must be a valid UTF-8 string"); - if (!res_to) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Third argument must be a valid UTF-8 string"); + if (map_to_ptr < map_to_end) + { + size_t len_to = UTF8::seqLength(*map_to_ptr); - if (*map_from_ptr <= ascii_upper_bound) - map_ascii[*map_from_ptr] = *res_to; + if (map_to_ptr + len_to <= map_to_end) + res_to = UTF8::convertUTF8ToCodePoint(map_to_ptr, len_to); + + if (!res_to) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Third argument must be a valid UTF-8 string"); + + if (*map_from_ptr <= ascii_upper_bound) + map_ascii[*map_from_ptr] = *res_to; + else + map[*res_from] = *res_to; + + map_to_ptr += len_to; + } else - map[*res_from] = *res_to; + { + if (*map_from_ptr <= ascii_upper_bound) + map_ascii[*map_from_ptr] = max_uint32; + else + map[*res_from] = max_uint32; + } map_from_ptr += len_from; - map_to_ptr += len_to; } } @@ -205,6 +234,12 @@ struct TranslateUTF8Impl if (*src <= ascii_upper_bound) { + if (map_ascii[*src] == max_uint32) + { + src += 1; + continue; + } + size_t dst_len = UTF8::convertCodePointToUTF8(map_ascii[*src], dst, 4); assert(0 < dst_len && dst_len <= 4); @@ -226,6 +261,12 @@ struct TranslateUTF8Impl auto * it = map.find(*src_code_point); if (it != map.end()) { + if (it->getMapped() == max_uint32) + { + src += src_len; + continue; + } + size_t dst_len = UTF8::convertCodePointToUTF8(it->getMapped(), dst, 4); assert(0 < dst_len && dst_len <= 4); @@ -270,6 +311,7 @@ struct TranslateUTF8Impl private: static constexpr auto ascii_upper_bound = '\x7f'; + static constexpr auto max_uint32 = 0xffffffff; }; diff --git a/tests/queries/0_stateless/02353_translate.reference b/tests/queries/0_stateless/02353_translate.reference index 557b5182127..7c4d3c50db0 100644 --- a/tests/queries/0_stateless/02353_translate.reference +++ b/tests/queries/0_stateless/02353_translate.reference @@ -14,3 +14,7 @@ HotelGenev ¿йðՅনй abc abc +abc +abc +内码 +1A2BC diff --git a/tests/queries/0_stateless/02353_translate.sql b/tests/queries/0_stateless/02353_translate.sql index f6f40c4265d..5a261696ebc 100644 --- a/tests/queries/0_stateless/02353_translate.sql +++ b/tests/queries/0_stateless/02353_translate.sql @@ -9,5 +9,8 @@ SELECT translateUTF8(toString(number), '1234567890', 'ዩय𐑿𐐏নՅðй¿ SELECT translate('abc', '', ''); SELECT translateUTF8('abc', '', ''); -SELECT translate('abc', 'Ááéíóúôè', 'aaeiouoe'); -- { serverError BAD_ARGUMENTS } -SELECT translateUTF8('abc', 'efg', ''); -- { serverError BAD_ARGUMENTS } +SELECT translate('abc', 'Ááéíóúôè', 'aaeiouoe'); +SELECT translateUTF8('abc', 'efg', ''); + +SELECT translateUTF8('中文内码', '中文', ''); +SELECT translate('aAbBcC', 'abc', '12'); From e1011929e782c879851a63b6744a1da569cde025 Mon Sep 17 00:00:00 2001 From: shuai-xu Date: Thu, 7 Nov 2024 11:29:22 +0800 Subject: [PATCH 224/502] fix test failure --- tests/queries/0_stateless/02353_translate.reference | 1 - tests/queries/0_stateless/02353_translate.sql | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02353_translate.reference b/tests/queries/0_stateless/02353_translate.reference index 7c4d3c50db0..5c36a029b81 100644 --- a/tests/queries/0_stateless/02353_translate.reference +++ b/tests/queries/0_stateless/02353_translate.reference @@ -15,6 +15,5 @@ HotelGenev abc abc abc -abc 内码 1A2BC diff --git a/tests/queries/0_stateless/02353_translate.sql b/tests/queries/0_stateless/02353_translate.sql index 5a261696ebc..777cc873fd2 100644 --- a/tests/queries/0_stateless/02353_translate.sql +++ b/tests/queries/0_stateless/02353_translate.sql @@ -9,7 +9,7 @@ SELECT translateUTF8(toString(number), '1234567890', 'ዩय𐑿𐐏নՅðй¿ SELECT translate('abc', '', ''); SELECT translateUTF8('abc', '', ''); -SELECT translate('abc', 'Ááéíóúôè', 'aaeiouoe'); +SELECT translate('abc', 'Ááéíóúôè', 'aaeiouoe'); -- { serverError BAD_ARGUMENTS } SELECT translateUTF8('abc', 'efg', ''); SELECT translateUTF8('中文内码', '中文', ''); From ee152f8c534575369a50b93e53fc365747433b9e Mon Sep 17 00:00:00 2001 From: shuai-xu Date: Thu, 7 Nov 2024 16:51:06 +0800 Subject: [PATCH 225/502] fix tests --- src/Functions/translate.cpp | 44 ++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp index 97e16020869..9524bd2b787 100644 --- a/src/Functions/translate.cpp +++ b/src/Functions/translate.cpp @@ -32,24 +32,25 @@ struct TranslateImpl const std::string & map_from, const std::string & map_to) { - if (map_from.size() < map_to.size()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second arguments must be equal or longer than the third"); - iota(map.data(), map.size(), UInt8(0)); - for (size_t i = 0; i < map_to.size(); ++i) + for (size_t i = 0; i < map_from.size(); ++i) { - if (!isASCII(map_from[i]) || !isASCII(map_to[i])) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second and third arguments must be ASCII strings"); + if (i < map_to.size()) + { + if (!isASCII(map_from[i]) || !isASCII(map_to[i])) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second and third arguments must be ASCII strings"); - map[map_from[i]] = map_to[i]; + map[map_from[i]] = map_to[i]; + } + else + map[map_from[i]] = ascii_upper_bound + 1; } - for (size_t i = map_to.size(); i < map_from.size(); ++i) + + for (size_t i = map_from.size(); i < map_to.size(); ++i) { if (!isASCII(map_from[i])) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second arguments must be ASCII strings"); - - map[map_from[i]] = ascii_upper_bound + 1; + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Third arguments must be ASCII strings"); } } @@ -66,10 +67,11 @@ struct TranslateImpl fillMapWithValues(map, map_from, map_to); res_data.resize(data.size()); - res_offsets.assign(offsets); + res_offsets.resize(input_rows_count); UInt8 * dst = res_data.data(); + UInt64 data_size = 0; for (UInt64 i = 0; i < input_rows_count; ++i) { const UInt8 * src = data.data() + offsets[i - 1]; @@ -81,11 +83,13 @@ struct TranslateImpl { *dst = map[*src]; ++dst; + ++data_size; } else if (*src > ascii_upper_bound) { *dst = *src; ++dst; + ++data_size; } ++src; @@ -94,6 +98,8 @@ struct TranslateImpl /// Technically '\0' can be mapped into other character, /// so we need to process '\0' delimiter separately *dst++ = 0; + ++data_size; + res_offsets[i] = data_size; } } @@ -104,6 +110,9 @@ struct TranslateImpl const std::string & map_to, ColumnString::Chars & res_data) { + if (map_from.size() != map_to.size()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second and third arguments must be the same length"); + std::array map; fillMapWithValues(map, map_from, map_to); @@ -115,18 +124,13 @@ struct TranslateImpl while (src < src_end) { - if (*src <= ascii_upper_bound && map[*src] != ascii_upper_bound + 1) - { + if (*src <= ascii_upper_bound) *dst = map[*src]; - ++dst; - } - else if (*src > ascii_upper_bound) - { + else *dst = *src; - ++dst; - } ++src; + ++dst; } } From be117a1da3caa68c91da6813fe117012e4963406 Mon Sep 17 00:00:00 2001 From: shuai-xu Date: Thu, 7 Nov 2024 17:25:31 +0800 Subject: [PATCH 226/502] fix confilic --- src/Functions/translate.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp index 9524bd2b787..43236555e7d 100644 --- a/src/Functions/translate.cpp +++ b/src/Functions/translate.cpp @@ -149,12 +149,6 @@ struct TranslateUTF8Impl const std::string & map_from, const std::string & map_to) { - auto map_from_size = UTF8::countCodePoints(reinterpret_cast(map_from.data()), map_from.size()); - auto map_to_size = UTF8::countCodePoints(reinterpret_cast(map_to.data()), map_to.size()); - - if (map_from_size < map_to_size) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second arguments must be equal or longer than third"); - iota(map_ascii.data(), map_ascii.size(), UInt32(0)); const UInt8 * map_from_ptr = reinterpret_cast(map_from.data()); From eaa31d04c1f84eda89b5fa2222b6cb852dc2ea92 Mon Sep 17 00:00:00 2001 From: shuai-xu Date: Fri, 8 Nov 2024 11:10:06 +0800 Subject: [PATCH 227/502] fix test --- src/Functions/translate.cpp | 9 +++++++-- tests/queries/0_stateless/02353_translate.reference | 1 + tests/queries/0_stateless/02353_translate.sql | 3 +++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp index 43236555e7d..0d434acc840 100644 --- a/src/Functions/translate.cpp +++ b/src/Functions/translate.cpp @@ -44,13 +44,18 @@ struct TranslateImpl map[map_from[i]] = map_to[i]; } else + { + if (!isASCII(map_from[i])) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second argument must be ASCII strings"); + map[map_from[i]] = ascii_upper_bound + 1; + } } for (size_t i = map_from.size(); i < map_to.size(); ++i) { - if (!isASCII(map_from[i])) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Third arguments must be ASCII strings"); + if (!isASCII(map_to[i])) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Third argument must be ASCII strings"); } } diff --git a/tests/queries/0_stateless/02353_translate.reference b/tests/queries/0_stateless/02353_translate.reference index 5c36a029b81..4106bf9c077 100644 --- a/tests/queries/0_stateless/02353_translate.reference +++ b/tests/queries/0_stateless/02353_translate.reference @@ -17,3 +17,4 @@ abc abc 内码 1A2BC +1A2B3C diff --git a/tests/queries/0_stateless/02353_translate.sql b/tests/queries/0_stateless/02353_translate.sql index 777cc873fd2..99f50c777b0 100644 --- a/tests/queries/0_stateless/02353_translate.sql +++ b/tests/queries/0_stateless/02353_translate.sql @@ -14,3 +14,6 @@ SELECT translateUTF8('abc', 'efg', ''); SELECT translateUTF8('中文内码', '中文', ''); SELECT translate('aAbBcC', 'abc', '12'); + +SELECT translate('aAbBcC', 'abc', '1235'); +SELECT translate('aAbBcC', '中文内码', '12'); -- { serverError BAD_ARGUMENTS } From 39740d2ed95e9527c172452e7dd36af34fbf44d7 Mon Sep 17 00:00:00 2001 From: shuai-xu Date: Mon, 11 Nov 2024 10:15:27 +0800 Subject: [PATCH 228/502] some refines --- src/Functions/translate.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp index 0d434acc840..5d8d3f52793 100644 --- a/src/Functions/translate.cpp +++ b/src/Functions/translate.cpp @@ -103,8 +103,7 @@ struct TranslateImpl /// Technically '\0' can be mapped into other character, /// so we need to process '\0' delimiter separately *dst++ = 0; - ++data_size; - res_offsets[i] = data_size; + res_offsets[i] = ++data_size; } } From d2b4448fb1f988fea99f69a92e1d702e8d0e1c55 Mon Sep 17 00:00:00 2001 From: shuai-xu Date: Fri, 22 Nov 2024 11:21:05 +0800 Subject: [PATCH 229/502] some refines --- .../functions/string-replace-functions.md | 2 +- src/Functions/translate.cpp | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index 9e2f063dfd7..f8d46cacbdc 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -253,7 +253,7 @@ SELECT format('{} {}', 'Hello', 'World') ## translate -Replaces characters in the string `s` using a one-to-one character mapping defined by `from` and `to` strings. `from` and `to` must be constant ASCII strings. Non-ASCII characters in the original string are not modified. +Replaces characters in the string `s` using a one-to-one character mapping defined by `from` and `to` strings. `from` and `to` must be constant ASCII strings. Non-ASCII characters in the original string are not modified. If the number of characters in `from` list is larger than the `to` list, non overlapping characters will be deleted from the input string. **Syntax** diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp index 5d8d3f52793..208232e0aca 100644 --- a/src/Functions/translate.cpp +++ b/src/Functions/translate.cpp @@ -45,10 +45,15 @@ struct TranslateImpl } else { - if (!isASCII(map_from[i])) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second argument must be ASCII strings"); + while (i < map_from.size()) + { + if (!isASCII(map_from[i])) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second argument must be ASCII strings"); - map[map_from[i]] = ascii_upper_bound + 1; + map[map_from[i]] = ascii_upper_bound + 1; + ++i; + } + return; } } From f201119575eef4041dc987d59ab244ec5b6b6e40 Mon Sep 17 00:00:00 2001 From: shuai-xu Date: Mon, 2 Dec 2024 12:23:00 +0800 Subject: [PATCH 230/502] refine codes --- src/Functions/translate.cpp | 49 ++++++++----------- .../0_stateless/02353_translate.reference | 1 + tests/queries/0_stateless/02353_translate.sql | 1 + 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp index 208232e0aca..1f4ff147a6a 100644 --- a/src/Functions/translate.cpp +++ b/src/Functions/translate.cpp @@ -34,30 +34,26 @@ struct TranslateImpl { iota(map.data(), map.size(), UInt8(0)); - for (size_t i = 0; i < map_from.size(); ++i) + size_t min_size = std::min(map_from.size(), map_to.size()); + + // Map characters from map_from to map_to for the overlapping range + for (size_t i = 0; i < min_size; ++i) { - if (i < map_to.size()) - { - if (!isASCII(map_from[i]) || !isASCII(map_to[i])) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second and third arguments must be ASCII strings"); - - map[map_from[i]] = map_to[i]; - } - else - { - while (i < map_from.size()) - { - if (!isASCII(map_from[i])) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second argument must be ASCII strings"); - - map[map_from[i]] = ascii_upper_bound + 1; - ++i; - } - return; - } + if (!isASCII(map_from[i]) || !isASCII(map_to[i])) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second and third arguments must be ASCII strings"); + map[static_cast(map_from[i])] = static_cast(map_to[i]); } - for (size_t i = map_from.size(); i < map_to.size(); ++i) + // Handle any remaining characters in map_from by assigning a default value + for (size_t i = min_size; i < map_from.size(); ++i) + { + if (!isASCII(map_from[i])) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second argument must be ASCII strings"); + map[static_cast(map_from[i])] = ascii_upper_bound + 1; + } + + // Validate any extra characters in map_to to ensure they are ASCII + for (size_t i = min_size; i < map_to.size(); ++i) { if (!isASCII(map_to[i])) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Third argument must be ASCII strings"); @@ -91,14 +87,12 @@ struct TranslateImpl { if (*src <= ascii_upper_bound && map[*src] != ascii_upper_bound + 1) { - *dst = map[*src]; - ++dst; + *dst++ = map[*src]; ++data_size; } else if (*src > ascii_upper_bound) { - *dst = *src; - ++dst; + *dst++ = *src; ++data_size; } @@ -268,16 +262,13 @@ struct TranslateUTF8Impl auto * it = map.find(*src_code_point); if (it != map.end()) { + src += src_len; if (it->getMapped() == max_uint32) - { - src += src_len; continue; - } size_t dst_len = UTF8::convertCodePointToUTF8(it->getMapped(), dst, 4); assert(0 < dst_len && dst_len <= 4); - src += src_len; dst += dst_len; data_size += dst_len; continue; diff --git a/tests/queries/0_stateless/02353_translate.reference b/tests/queries/0_stateless/02353_translate.reference index 4106bf9c077..3c448d59ea2 100644 --- a/tests/queries/0_stateless/02353_translate.reference +++ b/tests/queries/0_stateless/02353_translate.reference @@ -18,3 +18,4 @@ abc 内码 1A2BC 1A2B3C +ABC diff --git a/tests/queries/0_stateless/02353_translate.sql b/tests/queries/0_stateless/02353_translate.sql index 99f50c777b0..fd3f48f854d 100644 --- a/tests/queries/0_stateless/02353_translate.sql +++ b/tests/queries/0_stateless/02353_translate.sql @@ -16,4 +16,5 @@ SELECT translateUTF8('中文内码', '中文', ''); SELECT translate('aAbBcC', 'abc', '12'); SELECT translate('aAbBcC', 'abc', '1235'); +SELECT translate('aAbBcC', 'abc', ''); SELECT translate('aAbBcC', '中文内码', '12'); -- { serverError BAD_ARGUMENTS } From ca49fcfc388f97487c01dfb71c1aed23872adbcc Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Mon, 2 Dec 2024 18:00:43 +0800 Subject: [PATCH 231/502] fix enum (not)in exception --- src/DataTypes/Serializations/SerializationEnum.cpp | 4 +++- src/Interpreters/convertFieldToType.cpp | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/DataTypes/Serializations/SerializationEnum.cpp b/src/DataTypes/Serializations/SerializationEnum.cpp index 847d4fa5201..e9e62380df3 100644 --- a/src/DataTypes/Serializations/SerializationEnum.cpp +++ b/src/DataTypes/Serializations/SerializationEnum.cpp @@ -96,7 +96,9 @@ void SerializationEnum::deserializeWholeText(IColumn & column, ReadBuffer { std::string field_name; readStringUntilEOF(field_name, istr); - assert_cast(column).getData().push_back(ref_enum_values.getValue(StringRef(field_name))); + Type t; + if (ref_enum_values.tryGetValue(t, StringRef(field_name))) + assert_cast(column).getData().push_back(t); } } diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index b4cfa9e8603..0a54ccbd5d0 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -660,7 +660,15 @@ static bool decimalEqualsFloat(Field field, Float64 float_value) std::optional convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type, const FormatSettings & format_settings) { - Field result_value = convertFieldToType(from_value, to_type, &from_type, format_settings); + Field result_value; + try + { + result_value = convertFieldToType(from_value, to_type, &from_type, format_settings); + } + catch(const std::exception &) + { + return {}; + } if (Field::isDecimal(from_value.getType()) && Field::isDecimal(result_value.getType())) { From 12ac21f10f9006078aa33a4a4b6160e2d626d4a0 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 2 Dec 2024 11:08:43 +0100 Subject: [PATCH 232/502] fix flatten_nested when recovering a Replicated db --- src/Databases/DatabaseReplicated.cpp | 2 ++ .../test_replicated_database/test.py | 22 ++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 9ea6ec27df1..e3e4735c0d7 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -1226,6 +1226,8 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep query_context->setSetting("database_replicated_allow_explicit_uuid", 3); query_context->setSetting("database_replicated_allow_replicated_engine_arguments", 3); + query_context->setSetting("flatten_nested", false); + auto txn = std::make_shared(current_zookeeper, zookeeper_path, false, ""); query_context->initZooKeeperMetadataTransaction(txn); return query_context; diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 6fd337cf214..36739c4ff34 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -104,11 +104,27 @@ def test_flatten_nested(started_cluster): "CREATE MATERIALIZED VIEW create_replicated_table.mv ENGINE=ReplicatedMergeTree ORDER BY tuple() AS select d, cast([(k, toString(i32))] as Nested(a UInt64, b String)) from create_replicated_table.replicated_table" ) - assert main_node.query( - "show create create_replicated_table.mv" - ) == dummy_node.query("show create create_replicated_table.mv") + main_node.query( + "CREATE TABLE create_replicated_table.no_flatten (n Nested(a UInt64, b String)) ENGINE=ReplicatedMergeTree ORDER BY tuple();", + settings={"flatten_nested": 0}, + ) + + snapshot_recovering_node.query( + "CREATE DATABASE create_replicated_table ENGINE = Replicated('/test/create_replicated_table', 'shard1', 'replica3');" + ) + snapshot_recovering_node.query( + "SYSTEM SYNC DATABASE REPLICA create_replicated_table" + ) + + for node in [dummy_node, snapshot_recovering_node]: + for table in ["replicated_table", "mv", "no_flatten"]: + assert main_node.query( + f"show create create_replicated_table.{table}" + ) == node.query(f"show create create_replicated_table.{table}") + main_node.query("DROP DATABASE create_replicated_table SYNC") dummy_node.query("DROP DATABASE create_replicated_table SYNC") + snapshot_recovering_node.query("DROP DATABASE create_replicated_table SYNC") def test_create_replicated_table(started_cluster): From ccce7e8ba40e982de3efdb29e5592491983b03fd Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Mon, 2 Dec 2024 18:15:11 +0800 Subject: [PATCH 233/502] add test --- .../03278_enum_in_unknown_value.reference | 6 +++++ .../03278_enum_in_unknown_value.sql | 22 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/queries/0_stateless/03278_enum_in_unknown_value.reference create mode 100644 tests/queries/0_stateless/03278_enum_in_unknown_value.sql diff --git a/tests/queries/0_stateless/03278_enum_in_unknown_value.reference b/tests/queries/0_stateless/03278_enum_in_unknown_value.reference new file mode 100644 index 00000000000..786e4a78984 --- /dev/null +++ b/tests/queries/0_stateless/03278_enum_in_unknown_value.reference @@ -0,0 +1,6 @@ +a +a +a +a +a +a diff --git a/tests/queries/0_stateless/03278_enum_in_unknown_value.sql b/tests/queries/0_stateless/03278_enum_in_unknown_value.sql new file mode 100644 index 00000000000..25072547c24 --- /dev/null +++ b/tests/queries/0_stateless/03278_enum_in_unknown_value.sql @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS t_enum_in_unknown_value; + +CREATE TABLE t_enum_in_unknown_value (e Enum('a'=1, 'b'=2)) ENGINE=Memory; + +INSERT INTO t_enum_in_unknown_value VALUES ('a'); + +SELECT * FROM t_enum_in_unknown_value; + +SELECT * FROM t_enum_in_unknown_value WHERE e IN ('a'); +SELECT * FROM t_enum_in_unknown_value WHERE e NOT IN ('a'); + +SELECT * FROM t_enum_in_unknown_value WHERE e IN ('a', 'b'); +SELECT * FROM t_enum_in_unknown_value WHERE e NOT IN ('a', 'b'); + +SELECT * FROM t_enum_in_unknown_value WHERE e IN ('a', 'c'); +SELECT * FROM t_enum_in_unknown_value WHERE e NOT IN ('a', 'c'); + +SELECT * FROM t_enum_in_unknown_value WHERE e IN ('a', 'b', 'c'); +SELECT * FROM t_enum_in_unknown_value WHERE e NOT IN ('a', 'b', 'c'); + +SELECT * FROM t_enum_in_unknown_value WHERE e IN ('c'); +SELECT * FROM t_enum_in_unknown_value WHERE e NOT IN ('c'); \ No newline at end of file From 08aae4db74339a0b4ad29219ffdf721d1e5d556e Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Mon, 2 Dec 2024 18:23:18 +0800 Subject: [PATCH 234/502] add blank line --- tests/queries/0_stateless/03278_enum_in_unknown_value.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03278_enum_in_unknown_value.sql b/tests/queries/0_stateless/03278_enum_in_unknown_value.sql index 25072547c24..12aa585d0f9 100644 --- a/tests/queries/0_stateless/03278_enum_in_unknown_value.sql +++ b/tests/queries/0_stateless/03278_enum_in_unknown_value.sql @@ -19,4 +19,4 @@ SELECT * FROM t_enum_in_unknown_value WHERE e IN ('a', 'b', 'c'); SELECT * FROM t_enum_in_unknown_value WHERE e NOT IN ('a', 'b', 'c'); SELECT * FROM t_enum_in_unknown_value WHERE e IN ('c'); -SELECT * FROM t_enum_in_unknown_value WHERE e NOT IN ('c'); \ No newline at end of file +SELECT * FROM t_enum_in_unknown_value WHERE e NOT IN ('c'); From c3c8f9d2ff3957311164ac410f2ae1e6d999d2d0 Mon Sep 17 00:00:00 2001 From: nauu Date: Mon, 2 Dec 2024 19:05:45 +0800 Subject: [PATCH 235/502] Add total_bytes_with_inactive to system.tables to count the byte size of inactive parts --- src/Storages/IStorage.h | 9 +++++++++ src/Storages/StorageMergeTree.cpp | 10 ++++++++++ src/Storages/StorageMergeTree.h | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 9 +++++++++ src/Storages/StorageReplicatedMergeTree.h | 1 + src/Storages/System/StorageSystemTables.cpp | 13 +++++++++++++ ..._system_columns_and_system_tables_long.reference | 9 +++++---- .../00753_system_columns_and_system_tables_long.sql | 9 +++++++-- .../02117_show_create_table_system.reference | 1 + 9 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 0dc48634282..684b6b6dd2c 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -718,6 +718,15 @@ public: /// Does not take underlying Storage (if any) into account virtual std::optional totalBytesUncompressed(const Settings &) const { return {}; } + /// If it is possible to quickly determine exact number of bytes for the table on storage: + /// - disk (compressed) + /// + /// Used for: + /// - For total_bytes_with_inactive column in system.tables + // + /// Does not takes underlying Storage (if any) into account + virtual std::optional totalBytesWithInactive(const Settings &) const { return {}; } + /// Number of rows INSERTed since server start. /// /// Does not take the underlying Storage (if any) into account. diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 1ba0617d8ae..c7c42c343d0 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -317,6 +317,16 @@ std::optional StorageMergeTree::totalBytesUncompressed(const Settings &) return res; } +std::optional StorageMergeTree::totalBytesWithInactive(const Settings &) const +{ + UInt64 res = 0; + auto outdated_parts = getDataPartsVectorForInternalUsage({DataPartState::Outdated}); + for (const auto & part : outdated_parts) + res += part->getBytesOnDisk(); + return res; +} + + SinkToStoragePtr StorageMergeTree::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context, bool /*async_insert*/) { diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 7bc070b12b4..f61d5cb1e9a 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -68,6 +68,7 @@ public: std::optional totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr) const override; std::optional totalBytes(const Settings &) const override; std::optional totalBytesUncompressed(const Settings &) const override; + std::optional totalBytesWithInactive(const Settings &) const override; SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context, bool async_insert) override; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 15341cca976..1a69dd75e8b 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5711,6 +5711,15 @@ std::optional StorageReplicatedMergeTree::totalBytesUncompressed(const S return res; } +std::optional StorageReplicatedMergeTree::totalBytesWithInactive(const Settings &) const +{ + UInt64 res = 0; + auto outdated_parts = getDataPartsStateRange(DataPartState::Outdated); + for (const auto & part : outdated_parts) + res += part->getBytesOnDisk(); + return res; +} + void StorageReplicatedMergeTree::assertNotReadonly() const { if (is_readonly) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index a790e548645..972dc7572e5 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -162,6 +162,7 @@ public: std::optional totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr context) const override; std::optional totalBytes(const Settings & settings) const override; std::optional totalBytesUncompressed(const Settings & settings) const override; + std::optional totalBytesWithInactive(const Settings & settings) const override; SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context, bool async_insert) override; diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 9d5c68c261f..c414891ddb7 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -179,6 +179,10 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_) "Total number of uncompressed bytes, if it's possible to quickly determine the exact number " "of bytes from the part checksums for the table on storage, otherwise NULL (does not take underlying storage (if any) into account)." }, + {"total_bytes_with_inactive", std::make_shared(std::make_shared()), + "Total number of bytes with inactive parts, if it is possible to quickly determine exact number " + "of bytes for the table on storage, otherwise NULL (does not includes any underlying storage). " + }, {"parts", std::make_shared(std::make_shared()), "The total number of parts in this table."}, {"active_parts", std::make_shared(std::make_shared()), "The number of active parts in this table."}, {"total_marks", std::make_shared(std::make_shared()), "The total number of marks in all parts in this table."}, @@ -597,6 +601,15 @@ protected: res_columns[res_index++]->insertDefault(); } + if (columns_mask[src_index++]) + { + auto total_bytes_with_inactive = table->totalBytesWithInactive(settings); + if (total_bytes_with_inactive) + res_columns[res_index++]->insert(*total_bytes_with_inactive); + else + res_columns[res_index++]->insertDefault(); + } + auto table_merge_tree = std::dynamic_pointer_cast(table); if (columns_mask[src_index++]) { diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference index dd5860ae491..26946cd55b4 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference @@ -1,12 +1,13 @@ -┌─name────────────────┬─partition_key─┬─sorting_key─┬─primary_key─┬─sampling_key─┬─storage_policy─┬─total_rows─┐ -│ check_system_tables │ name2 │ name1 │ name1 │ name1 │ default │ 0 │ -└─────────────────────┴───────────────┴─────────────┴─────────────┴──────────────┴────────────────┴────────────┘ +┌─name────────────────┬─partition_key─┬─sorting_key─┬─primary_key─┬─sampling_key─┬─storage_policy─┬─total_rows─┬─total_bytes_with_inactive─┐ +│ check_system_tables │ name2 │ name1 │ name1 │ name1 │ default │ 0 │ 0 │ +└─────────────────────┴───────────────┴─────────────┴─────────────┴──────────────┴────────────────┴────────────┴───────────────────────────┘ ┌─name──┬─is_in_partition_key─┬─is_in_sorting_key─┬─is_in_primary_key─┬─is_in_sampling_key─┐ │ name1 │ 0 │ 1 │ 1 │ 1 │ │ name2 │ 1 │ 0 │ 0 │ 0 │ │ name3 │ 0 │ 0 │ 0 │ 0 │ └───────┴─────────────────────┴───────────────────┴───────────────────┴────────────────────┘ -3 231 1 +1 1 3 231 1 0 +3 1 6 234 2 462 ┌─name────────────────┬─partition_key─┬─sorting_key───┬─primary_key─┬─sampling_key─┐ │ check_system_tables │ date │ date, version │ date │ │ └─────────────────────┴───────────────┴───────────────┴─────────────┴──────────────┘ diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql index 009fc0bbb9f..4c6a7b63a31 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql @@ -15,7 +15,7 @@ CREATE TABLE check_system_tables SAMPLE BY name1 SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key = false, ratio_of_defaults_for_sparse_serialization = 1; -SELECT name, partition_key, sorting_key, primary_key, sampling_key, storage_policy, total_rows +SELECT name, partition_key, sorting_key, primary_key, sampling_key, storage_policy, total_rows, total_bytes_with_inactive FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase() FORMAT PrettyCompactNoEscapes; @@ -24,7 +24,12 @@ FROM system.columns WHERE table = 'check_system_tables' AND database = currentDa FORMAT PrettyCompactNoEscapes; INSERT INTO check_system_tables VALUES (1, 1, 1); -SELECT total_bytes_uncompressed, total_bytes, total_rows FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase(); +SELECT parts, active_parts, total_bytes_uncompressed, total_bytes, total_rows, total_bytes_with_inactive FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase(); + +INSERT INTO check_system_tables VALUES (2, 1, 3); +OPTIMIZE TABLE check_system_tables; +SELECT parts, active_parts, total_bytes_uncompressed, total_bytes, total_rows, total_bytes_with_inactive FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase(); + DROP TABLE IF EXISTS check_system_tables; diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 2ea62444cff..60d9ed334f5 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -1113,6 +1113,7 @@ CREATE TABLE system.tables `total_rows` Nullable(UInt64), `total_bytes` Nullable(UInt64), `total_bytes_uncompressed` Nullable(UInt64), + `total_bytes_with_inactive` Nullable(UInt64), `parts` Nullable(UInt64), `active_parts` Nullable(UInt64), `total_marks` Nullable(UInt64), From dad7e4aa9efaeebbcc0aef179bc8f9b48bfe91d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 12:10:01 +0100 Subject: [PATCH 236/502] Run setting history checks with sanitizers too --- tests/queries/0_stateless/02995_new_settings_history.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02995_new_settings_history.sh b/tests/queries/0_stateless/02995_new_settings_history.sh index 7fb21f88fae..50fa2a0e325 100755 --- a/tests/queries/0_stateless/02995_new_settings_history.sh +++ b/tests/queries/0_stateless/02995_new_settings_history.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-cpu-aarch64, no-random-settings +# Tags: no-cpu-aarch64, no-random-settings # Some settings can be different for builds with sanitizers or aarch64 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) From f64c004059466bd571aa495cb9da01df03062f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 12:17:33 +0100 Subject: [PATCH 237/502] Update baseline to 24.11 --- ..._24_7_2.tsv => 02995_baseline_24_11_2.tsv} | 122 +++++++++++++++--- .../0_stateless/02995_new_settings_history.sh | 10 +- 2 files changed, 109 insertions(+), 23 deletions(-) rename tests/queries/0_stateless/{02995_baseline_24_7_2.tsv => 02995_baseline_24_11_2.tsv} (90%) diff --git a/tests/queries/0_stateless/02995_baseline_24_7_2.tsv b/tests/queries/0_stateless/02995_baseline_24_11_2.tsv similarity index 90% rename from tests/queries/0_stateless/02995_baseline_24_7_2.tsv rename to tests/queries/0_stateless/02995_baseline_24_11_2.tsv index 10b392f3e04..95119f69b32 100644 --- a/tests/queries/0_stateless/02995_baseline_24_7_2.tsv +++ b/tests/queries/0_stateless/02995_baseline_24_11_2.tsv @@ -5,6 +5,7 @@ aggregate_functions_null_for_empty 0 aggregation_in_order_max_block_bytes 50000000 aggregation_memory_efficient_merge_threads 0 allow_aggregate_partitions_independently 0 +allow_archive_path_syntax 1 allow_asynchronous_read_from_io_pool_for_merge_tree 0 allow_changing_replica_until_first_data_packet 0 allow_create_index_without_type 0 @@ -20,6 +21,7 @@ allow_execute_multiif_columnar 1 allow_experimental_alter_materialized_view_structure 1 allow_experimental_analyzer 1 allow_experimental_annoy_index 0 +allow_experimental_bfloat16_type 0 allow_experimental_bigint_types 1 allow_experimental_codecs 0 allow_experimental_database_atomic 1 @@ -33,6 +35,9 @@ allow_experimental_geo_types 1 allow_experimental_hash_functions 0 allow_experimental_inverted_index 0 allow_experimental_join_condition 0 +allow_experimental_join_right_table_sorting 0 +allow_experimental_json_type 0 +allow_experimental_kafka_offsets_storage_in_keeper 0 allow_experimental_lightweight_delete 1 allow_experimental_live_view 0 allow_experimental_map_type 1 @@ -43,19 +48,23 @@ allow_experimental_parallel_reading_from_replicas 0 allow_experimental_projection_optimization 1 allow_experimental_query_cache 1 allow_experimental_query_deduplication 0 -allow_experimental_refreshable_materialized_view 0 +allow_experimental_refreshable_materialized_view 1 allow_experimental_s3queue 1 allow_experimental_shared_merge_tree 1 +allow_experimental_shared_set_join 0 allow_experimental_statistic 0 allow_experimental_statistics 0 +allow_experimental_time_series_table 0 allow_experimental_undrop_table_query 1 allow_experimental_usearch_index 0 allow_experimental_variant_type 0 +allow_experimental_vector_similarity_index 0 allow_experimental_window_functions 1 allow_experimental_window_view 0 allow_get_client_http_header 0 allow_hyperscan 1 allow_introspection_functions 0 +allow_materialized_view_with_bad_select 1 allow_named_collection_override_by_default 1 allow_non_metadata_alters 1 allow_nonconst_timezone_arguments 0 @@ -64,6 +73,7 @@ allow_nondeterministic_optimize_skip_unused_shards 0 allow_prefetched_read_pool_for_local_filesystem 0 allow_prefetched_read_pool_for_remote_filesystem 1 allow_push_predicate_when_subquery_contains_with 1 +allow_reorder_prewhere_conditions 1 allow_settings_after_format_in_insert 0 allow_simdjson 1 allow_statistic_optimize 0 @@ -74,6 +84,8 @@ allow_suspicious_indices 0 allow_suspicious_low_cardinality_types 0 allow_suspicious_primary_key 0 allow_suspicious_ttl_expressions 0 +allow_suspicious_types_in_group_by 0 +allow_suspicious_types_in_order_by 0 allow_suspicious_variant_types 0 allow_unrestricted_reads_from_keeper 0 alter_move_to_space_execute_async 0 @@ -104,6 +116,7 @@ async_insert_use_adaptive_busy_timeout 1 async_query_sending_for_remote 1 async_socket_for_remote 1 azure_allow_parallel_part_upload 1 +azure_check_objects_after_upload 0 azure_create_new_file_on_insert 0 azure_ignore_file_doesnt_exist 0 azure_list_object_keys_size 1000 @@ -135,9 +148,13 @@ background_pool_size 16 background_schedule_pool_size 128 backup_restore_batch_size_for_keeper_multi 1000 backup_restore_batch_size_for_keeper_multiread 10000 +backup_restore_failure_after_host_disconnected_for_seconds 3600 +backup_restore_finish_timeout_after_error_sec 180 backup_restore_keeper_fault_injection_probability 0 backup_restore_keeper_fault_injection_seed 0 -backup_restore_keeper_max_retries 20 +backup_restore_keeper_max_retries 1000 +backup_restore_keeper_max_retries_while_handling_error 20 +backup_restore_keeper_max_retries_while_initializing 20 backup_restore_keeper_retry_initial_backoff_ms 100 backup_restore_keeper_retry_max_backoff_ms 5000 backup_restore_keeper_value_max_size 1048576 @@ -156,6 +173,7 @@ check_referential_table_dependencies 0 check_table_dependencies 1 checksum_on_read 1 cloud_mode 0 +cloud_mode_database_engine 1 cloud_mode_engine 1 cluster_for_parallel_replicas collect_hash_table_stats_during_aggregation 1 @@ -175,6 +193,7 @@ connections_with_failover_max_tries 3 convert_query_to_cnf 0 count_distinct_implementation uniqExact count_distinct_optimization 0 +create_if_not_exists 0 create_index_ignore_unique 0 create_replicated_merge_tree_fault_injection_probability 0 create_table_empty_primary_key_by_default 0 @@ -183,13 +202,15 @@ cross_join_min_rows_to_compress 10000000 cross_to_inner_join_rewrite 1 data_type_default_nullable 0 database_atomic_wait_for_drop_and_detach_synchronously 0 +database_replicated_allow_explicit_uuid 0 database_replicated_allow_heavy_create 0 database_replicated_allow_only_replicated_engine 0 -database_replicated_allow_replicated_engine_arguments 1 +database_replicated_allow_replicated_engine_arguments 0 database_replicated_always_detach_permanently 0 database_replicated_ddl_output 1 database_replicated_enforce_synchronous_settings 0 database_replicated_initial_query_timeout_sec 300 +date_time_64_output_format_cut_trailing_zeros_align_to_groups_of_thousands 0 date_time_input_format basic date_time_output_format simple date_time_overflow_behavior ignore @@ -216,6 +237,19 @@ distributed_background_insert_max_sleep_time_ms 30000 distributed_background_insert_sleep_time_ms 100 distributed_background_insert_split_batch_on_failure 0 distributed_background_insert_timeout 0 +distributed_cache_bypass_connection_pool 0 +distributed_cache_connect_max_tries 100 +distributed_cache_data_packet_ack_window 5 +distributed_cache_discard_connection_if_unread_data 1 +distributed_cache_fetch_metrics_only_from_current_az 1 +distributed_cache_log_mode on_error +distributed_cache_max_unacked_inflight_packets 10 +distributed_cache_pool_behaviour_on_limit allocate_bypassing_pool +distributed_cache_read_alignment 0 +distributed_cache_receive_response_wait_milliseconds 60000 +distributed_cache_receive_timeout_milliseconds 10000 +distributed_cache_throw_on_error 0 +distributed_cache_wait_connection_from_pool_milliseconds 100 distributed_connections_pool_size 1024 distributed_ddl_entry_format_version 5 distributed_ddl_output_mode throw @@ -236,9 +270,11 @@ do_not_merge_across_partitions_select_final 0 drain_timeout 3 empty_result_for_aggregation_by_constant_keys_on_empty_set 1 empty_result_for_aggregation_by_empty_set 0 +enable_analyzer 1 enable_blob_storage_log 1 enable_debug_queries 0 enable_deflate_qpl_codec 0 +enable_dynamic_type 0 enable_early_constant_folding 1 enable_extended_results_for_datetime_functions 0 enable_filesystem_cache 1 @@ -247,14 +283,17 @@ enable_filesystem_cache_on_write_operations 0 enable_filesystem_read_prefetches_log 0 enable_global_with_statement 1 enable_http_compression 0 -enable_job_stack_trace 0 +enable_job_stack_trace 1 +enable_json_type 0 enable_lightweight_delete 1 enable_memory_bound_merging_of_aggregation_results 1 enable_multiple_prewhere_read_steps 1 -enable_named_columns_in_function_tuple 1 +enable_named_columns_in_function_tuple 0 enable_optimize_predicate_expression 1 enable_optimize_predicate_expression_to_final_subquery 1 enable_order_by_all 1 +enable_parallel_replicas 0 +enable_parsing_to_custom_serialization 1 enable_positional_arguments 1 enable_reads_from_query_cache 1 enable_s3_requests_logging 0 @@ -263,9 +302,11 @@ enable_sharing_sets_for_mutations 1 enable_software_prefetch_in_aggregation 1 enable_unaligned_array_join 0 enable_url_encoding 1 +enable_variant_type 0 enable_vertical_final 1 enable_writes_to_query_cache 1 enable_zstd_qat_codec 0 +enforce_strict_identifier_format 0 engine_file_allow_create_multiple_files 0 engine_file_empty_if_not_exists 0 engine_file_skip_empty_files 0 @@ -284,9 +325,15 @@ extract_key_value_pairs_max_pairs_per_row 1000 extract_kvp_max_pairs_per_row 1000 extremes 0 fallback_to_stale_replicas_for_distributed_queries 1 +filesystem_cache_boundary_alignment 0 +filesystem_cache_enable_background_download_during_fetch 1 +filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage 1 filesystem_cache_max_download_size 137438953472 +filesystem_cache_name +filesystem_cache_prefer_bigger_buffer_size 1 filesystem_cache_reserve_space_wait_lock_timeout_milliseconds 1000 filesystem_cache_segments_batch_size 20 +filesystem_cache_skip_download_if_exceeds_per_query_cache_write_limit 1 filesystem_prefetch_max_memory_usage 1073741824 filesystem_prefetch_min_bytes_for_single_read_task 2097152 filesystem_prefetch_step_bytes 0 @@ -362,6 +409,7 @@ hdfs_skip_empty_files 0 hdfs_throw_on_zero_files_match 0 hdfs_truncate_on_insert 0 hedged_connection_timeout_ms 50 +hnsw_candidate_list_size_for_search 256 hsts_max_age 0 http_connection_timeout 1 http_headers_progress_interval_ms 100 @@ -393,6 +441,7 @@ ignore_materialized_views_with_dropped_target_table 0 ignore_on_cluster_for_replicated_access_entities_queries 0 ignore_on_cluster_for_replicated_named_collections_queries 0 ignore_on_cluster_for_replicated_udf_queries 0 +implicit_select 0 implicit_transaction 0 input_format_allow_errors_num 0 input_format_allow_errors_ratio 0 @@ -404,6 +453,7 @@ input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference 0 input_format_avro_allow_missing_fields 0 input_format_avro_null_as_default 0 input_format_binary_decode_types_in_binary_format 0 +input_format_binary_read_json_as_string 0 input_format_bson_skip_fields_with_unsupported_types_in_schema_inference 0 input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference 0 input_format_csv_allow_cr_end_of_line 0 @@ -433,12 +483,13 @@ input_format_hive_text_map_keys_delimiter  input_format_import_nested_json 0 input_format_ipv4_default_on_conversion_error 0 input_format_ipv6_default_on_conversion_error 0 -input_format_json_case_insensitive_column_matching 0 input_format_json_compact_allow_variable_number_of_columns 0 input_format_json_defaults_for_missing_elements_in_named_tuple 1 +input_format_json_empty_as_default 0 input_format_json_ignore_unknown_keys_in_named_tuple 1 input_format_json_ignore_unnecessary_fields 1 input_format_json_infer_incomplete_types_as_strings 1 +input_format_json_max_depth 1000 input_format_json_named_tuples_as_objects 1 input_format_json_read_arrays_as_strings 1 input_format_json_read_bools_as_numbers 1 @@ -460,6 +511,7 @@ input_format_native_decode_types_in_binary_format 0 input_format_null_as_default 1 input_format_orc_allow_missing_columns 1 input_format_orc_case_insensitive_column_matching 0 +input_format_orc_dictionary_as_low_cardinality 1 input_format_orc_filter_push_down 1 input_format_orc_import_nested 0 input_format_orc_reader_time_zone_name GMT @@ -468,7 +520,9 @@ input_format_orc_skip_columns_with_unsupported_types_in_schema_inference 0 input_format_orc_use_fast_decoder 1 input_format_parallel_parsing 1 input_format_parquet_allow_missing_columns 1 +input_format_parquet_bloom_filter_push_down 0 input_format_parquet_case_insensitive_column_matching 0 +input_format_parquet_enable_row_group_prefetch 1 input_format_parquet_filter_push_down 1 input_format_parquet_import_nested 0 input_format_parquet_local_file_min_bytes_for_seek 8192 @@ -483,8 +537,10 @@ input_format_record_errors_file_path input_format_skip_unknown_fields 1 input_format_try_infer_dates 1 input_format_try_infer_datetimes 1 +input_format_try_infer_datetimes_only_datetime64 0 input_format_try_infer_exponent_floats 0 input_format_try_infer_integers 1 +input_format_try_infer_variants 0 input_format_tsv_allow_variable_number_of_columns 0 input_format_tsv_crlf_end_of_line 0 input_format_tsv_detect_header 1 @@ -494,7 +550,6 @@ input_format_tsv_skip_first_lines 0 input_format_tsv_skip_trailing_empty_lines 0 input_format_tsv_use_best_effort_in_schema_inference 1 input_format_values_accurate_types_of_literals 1 -input_format_values_allow_data_after_semicolon 0 input_format_values_deduce_templates_of_expressions 1 input_format_values_interpret_expressions 1 input_format_with_names_use_header 1 @@ -522,7 +577,10 @@ join_algorithm default join_any_take_last_row 0 join_default_strictness ALL join_on_disk_max_files_to_merge 64 +join_output_by_rowlist_perkey_rows_threshold 5 join_overflow_mode throw +join_to_sort_maximum_table_rows 10000 +join_to_sort_minimum_perkey_rows 40 join_use_nulls 0 joined_subquery_requires_alias 1 kafka_disable_num_consumers_limit 0 @@ -561,7 +619,7 @@ materialize_skip_indexes_on_insert 1 materialize_statistics_on_insert 1 materialize_ttl_after_modify 1 materialized_views_ignore_errors 0 -max_alter_threads \'auto(16)\' +max_alter_threads \'auto(32)\' max_analyze_depth 5000 max_ast_depth 1000 max_ast_elements 50000 @@ -593,7 +651,7 @@ max_execution_time 0 max_execution_time_leaf 0 max_expanded_ast_elements 500000 max_fetch_partition_retries_count 5 -max_final_threads \'auto(16)\' +max_final_threads \'auto(32)\' max_http_get_redirects 0 max_hyperscan_regexp_length 0 max_hyperscan_regexp_total_length 0 @@ -616,10 +674,11 @@ max_number_of_partitions_for_independent_aggregation 128 max_parallel_replicas 1 max_parser_backtracks 1000000 max_parser_depth 1000 -max_parsing_threads \'auto(16)\' +max_parsing_threads \'auto(32)\' max_partition_size_to_drop 50000000000 max_partitions_per_insert_block 100 max_partitions_to_read -1 +max_parts_to_move 1000 max_pipeline_depth 0 max_query_size 262144 max_read_buffer_size 1048576 @@ -656,7 +715,7 @@ max_temporary_columns 0 max_temporary_data_on_disk_size_for_query 0 max_temporary_data_on_disk_size_for_user 0 max_temporary_non_const_columns 0 -max_threads \'auto(16)\' +max_threads \'auto(32)\' max_threads_for_annoy_index_creation 4 max_threads_for_indexes 0 max_untracked_memory 4194304 @@ -676,14 +735,16 @@ merge_tree_determine_task_size_by_prewhere_columns 1 merge_tree_max_bytes_to_use_cache 2013265920 merge_tree_max_rows_to_use_cache 1048576 merge_tree_min_bytes_for_concurrent_read 251658240 -merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem 251658240 +merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem 0 merge_tree_min_bytes_for_seek 0 -merge_tree_min_bytes_per_task_for_remote_reading 4194304 +merge_tree_min_bytes_per_task_for_remote_reading 2097152 +merge_tree_min_read_task_size 8 merge_tree_min_rows_for_concurrent_read 163840 -merge_tree_min_rows_for_concurrent_read_for_remote_filesystem 163840 +merge_tree_min_rows_for_concurrent_read_for_remote_filesystem 0 merge_tree_min_rows_for_seek 0 merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability 0 merge_tree_use_const_size_tasks_for_remote_reading 1 +merge_tree_use_v1_object_and_dynamic_serialization 0 metrics_perf_events_enabled 0 metrics_perf_events_list min_bytes_to_use_direct_io 0 @@ -697,12 +758,16 @@ min_execution_speed 0 min_execution_speed_bytes 0 min_external_table_block_size_bytes 268402944 min_external_table_block_size_rows 1048449 +min_free_disk_bytes_to_perform_insert 0 +min_free_disk_ratio_to_perform_insert 0 min_free_disk_space_for_temporary_data 0 min_hit_rate_to_use_consecutive_keys_optimization 0.5 min_insert_block_size_bytes 268402944 min_insert_block_size_bytes_for_materialized_views 0 min_insert_block_size_rows 1048449 min_insert_block_size_rows_for_materialized_views 0 +min_joined_block_size_bytes 524288 +mongodb_throw_on_unsupported_query 1 move_all_conditions_to_prewhere 1 move_primary_key_columns_to_end_of_prewhere 1 multiple_joins_rewriter_version 0 @@ -734,7 +799,7 @@ optimize_count_from_files 1 optimize_distinct_in_order 1 optimize_distributed_group_by_sharding_key 1 optimize_duplicate_order_by_and_distinct 0 -optimize_functions_to_subcolumns 0 +optimize_functions_to_subcolumns 1 optimize_fuse_sum_count_avg 0 optimize_group_by_constant_keys 1 optimize_group_by_function_keys 1 @@ -788,6 +853,7 @@ output_format_avro_rows_in_file 1 output_format_avro_string_column_pattern output_format_avro_sync_interval 16384 output_format_binary_encode_types_in_binary_format 0 +output_format_binary_write_json_as_string 0 output_format_bson_string_as_string 0 output_format_compression_level 3 output_format_compression_zstd_window_log 0 @@ -807,7 +873,9 @@ output_format_json_validate_utf8 0 output_format_markdown_escape_special_characters 0 output_format_msgpack_uuid_representation ext output_format_native_encode_types_in_binary_format 0 +output_format_native_write_json_as_string 0 output_format_orc_compression_method zstd +output_format_orc_dictionary_key_size_threshold 0 output_format_orc_row_index_stride 10000 output_format_orc_string_as_string 1 output_format_parallel_formatting 1 @@ -854,9 +922,11 @@ parallel_replicas_custom_key_filter_type default parallel_replicas_custom_key_range_lower 0 parallel_replicas_custom_key_range_upper 0 parallel_replicas_for_non_replicated_merge_tree 0 -parallel_replicas_mark_segment_size 128 +parallel_replicas_local_plan 1 +parallel_replicas_mark_segment_size 0 parallel_replicas_min_number_of_granules_to_enable 0 parallel_replicas_min_number_of_rows_per_replica 0 +parallel_replicas_mode read_tasks parallel_replicas_prefer_local_join 1 parallel_replicas_single_task_marks_count_multiplier 2 parallel_view_processing 0 @@ -887,6 +957,7 @@ preferred_optimize_projection_name prefetch_buffer_size 1048576 print_pretty_type_names 1 priority 0 +push_external_roles_in_interserver_queries 1 query_cache_compress_entries 1 query_cache_max_entries 0 query_cache_max_size_in_bytes 0 @@ -897,7 +968,9 @@ query_cache_share_between_users 0 query_cache_squash_partial_results 1 query_cache_store_results_of_queries_with_nondeterministic_functions 0 query_cache_system_table_handling throw +query_cache_tag query_cache_ttl 60 +query_metric_log_interval -1 query_plan_aggregation_in_order 1 query_plan_convert_outer_join_to_inner_join 1 query_plan_enable_multithreading_after_window_functions 1 @@ -908,7 +981,7 @@ query_plan_lift_up_array_join 1 query_plan_lift_up_union 1 query_plan_max_optimizations_to_apply 10000 query_plan_merge_expressions 1 -query_plan_merge_filters 0 +query_plan_merge_filters 1 query_plan_optimize_prewhere 1 query_plan_optimize_primary_key 1 query_plan_optimize_projection 1 @@ -931,9 +1004,11 @@ read_from_filesystem_cache_if_exists_otherwise_bypass_cache 0 read_from_page_cache_if_exists_otherwise_bypass_cache 0 read_in_order_two_level_merge_threshold 100 read_in_order_use_buffering 1 +read_in_order_use_virtual_row 0 read_overflow_mode throw read_overflow_mode_leaf throw read_priority 0 +read_through_distributed_cache 0 readonly 0 receive_data_timeout_ms 2000 receive_timeout 300 @@ -954,9 +1029,13 @@ replace_running_query_max_wait_ms 5000 replication_alter_columns_timeout 60 replication_alter_partitions_sync 1 replication_wait_for_inactive_replica_timeout 120 +restore_replace_external_dictionary_source_to_null 0 +restore_replace_external_engines_to_null 0 +restore_replace_external_table_functions_to_null 0 restore_threads 16 result_overflow_mode throw rewrite_count_distinct_if_with_count_distinct_implementation 0 +rows_before_aggregation 0 s3_allow_parallel_part_upload 1 s3_check_objects_after_upload 0 s3_connect_timeout_ms 1000 @@ -980,7 +1059,7 @@ s3_max_upload_part_size 5368709120 s3_min_upload_part_size 16777216 s3_request_timeout_ms 30000 s3_retry_attempts 100 -s3_skip_empty_files 0 +s3_skip_empty_files 1 s3_strict_upload_part_size 0 s3_throw_on_zero_files_match 0 s3_truncate_on_insert 0 @@ -1008,6 +1087,8 @@ send_timeout 300 session_timezone set_overflow_mode throw short_circuit_function_evaluation enable +show_create_query_identifier_quoting_rule when_necessary +show_create_query_identifier_quoting_style Backticks show_table_uuid_in_table_create_query_if_not_nil 0 single_join_prefer_left_table 1 skip_download_if_exceeds_query_cache 1 @@ -1046,6 +1127,7 @@ trace_profile_events 0 transfer_overflow_mode throw transform_null_in 0 traverse_shadow_remote_data_paths 0 +type_json_skip_duplicated_paths 0 union_default_mode unknown_packet_in_send_data 0 update_insert_deduplication_token_in_dependent_materialized_views 0 @@ -1054,8 +1136,10 @@ use_client_time_zone 0 use_compact_format_in_distributed_parts_names 1 use_concurrency_control 1 use_hedged_requests 1 +use_hive_partitioning 0 use_index_for_in_with_subqueries 1 use_index_for_in_with_subqueries_max_values 0 +use_json_alias_for_old_object_type 0 use_local_cache_for_remote_storage 1 use_mysql_types_in_show_columns 0 use_page_cache_for_disks_without_file_cache 0 @@ -1067,6 +1151,7 @@ use_uncompressed_cache 0 use_variant_as_common_type 0 use_with_fill_by_sorting_prefix 1 validate_experimental_and_suspicious_types_inside_nested_types 1 +validate_mutation_query 1 validate_polygons 1 wait_changes_become_visible_after_commit_mode wait_unknown wait_for_async_insert 1 @@ -1075,4 +1160,5 @@ wait_for_window_view_fire_signal_timeout 10 window_view_clean_interval 60 window_view_heartbeat_interval 15 workload default +write_through_distributed_cache 0 zstd_window_log_max 0 diff --git a/tests/queries/0_stateless/02995_new_settings_history.sh b/tests/queries/0_stateless/02995_new_settings_history.sh index 50fa2a0e325..eb5e7415aee 100755 --- a/tests/queries/0_stateless/02995_new_settings_history.sh +++ b/tests/queries/0_stateless/02995_new_settings_history.sh @@ -7,12 +7,12 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh # Note that this is a broad check. A per version check is done in the upgrade test -# Baseline generated with 24.7.2 -# clickhouse local --query "select name, default from system.settings order by name format TSV" > 02995_baseline_24_7_2.tsv +# Baseline generated with 24.11.2 +# clickhouse local --query "select name, default from system.settings order by name format TSV" > 02995_settings_24_11_2.tsv $CLICKHOUSE_LOCAL --query " WITH old_settings AS ( - SELECT * FROM file('${CUR_DIR}/02995_baseline_24_7_2.tsv', 'TSV', 'name String, default String') + SELECT * FROM file('${CUR_DIR}/02995_settings_24_11_2.tsv', 'TSV', 'name String, default String') ), new_settings AS ( @@ -29,7 +29,7 @@ $CLICKHOUSE_LOCAL --query " )) AND (name NOT IN ( SELECT arrayJoin(tupleElement(changes, 'name')) FROM system.settings_changes - WHERE splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 7 + WHERE splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 11 )) UNION ALL ( @@ -39,7 +39,7 @@ $CLICKHOUSE_LOCAL --query " WHERE (new_settings.default != old_settings.default) AND (name NOT IN ( SELECT arrayJoin(tupleElement(changes, 'name')) FROM system.settings_changes - WHERE splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 7 + WHERE splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 11 )) ) ) From 389c31f2b884d148b63ee7a86eaaedafe3ee8e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 12:39:51 +0100 Subject: [PATCH 238/502] Add MergeTreeSettings to system.settings_changes --- .../System/StorageSystemSettingsChanges.cpp | 17 +- ...5_merge_tree_settings_settings_24_11_2.tsv | 211 ++++++++++++++++++ .../02995_mergetree_settings_24_11_2.tsv | 0 .../0_stateless/02995_new_settings_history.sh | 43 +++- ...24_11_2.tsv => 02995_settings_24_11_2.tsv} | 0 5 files changed, 264 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/02995_merge_tree_settings_settings_24_11_2.tsv create mode 100644 tests/queries/0_stateless/02995_mergetree_settings_24_11_2.tsv rename tests/queries/0_stateless/{02995_baseline_24_11_2.tsv => 02995_settings_24_11_2.tsv} (100%) diff --git a/src/Storages/System/StorageSystemSettingsChanges.cpp b/src/Storages/System/StorageSystemSettingsChanges.cpp index d6c83870741..34455316630 100644 --- a/src/Storages/System/StorageSystemSettingsChanges.cpp +++ b/src/Storages/System/StorageSystemSettingsChanges.cpp @@ -12,6 +12,7 @@ ColumnsDescription StorageSystemSettingsChanges::getColumnsDescription() /// TODO: Fill in all the comments return ColumnsDescription { + {"type", std::make_shared(), "The group of settings (Core, MergeTree...)"}, {"version", std::make_shared(), "The ClickHouse server version."}, {"changes", std::make_shared(std::make_shared( @@ -29,11 +30,23 @@ void StorageSystemSettingsChanges::fillData(MutableColumns & res_columns, Contex const auto & settings_changes_history = getSettingsChangesHistory(); for (auto it = settings_changes_history.rbegin(); it != settings_changes_history.rend(); ++it) { - res_columns[0]->insert(it->first.toString()); + res_columns[0]->insert("Core"); + res_columns[1]->insert(it->first.toString()); Array changes; for (const auto & change : it->second) changes.push_back(Tuple{change.name, toString(change.previous_value), toString(change.new_value), change.reason}); - res_columns[1]->insert(changes); + res_columns[2]->insert(changes); + } + + const auto & mergetree_settings_changes_history = getMergeTreeSettingsChangesHistory(); + for (auto it = mergetree_settings_changes_history.rbegin(); it != mergetree_settings_changes_history.rend(); ++it) + { + res_columns[0]->insert("MergeTree"); + res_columns[1]->insert(it->first.toString()); + Array changes; + for (const auto & change : it->second) + changes.push_back(Tuple{change.name, toString(change.previous_value), toString(change.new_value), change.reason}); + res_columns[2]->insert(changes); } } diff --git a/tests/queries/0_stateless/02995_merge_tree_settings_settings_24_11_2.tsv b/tests/queries/0_stateless/02995_merge_tree_settings_settings_24_11_2.tsv new file mode 100644 index 00000000000..204aeb2770f --- /dev/null +++ b/tests/queries/0_stateless/02995_merge_tree_settings_settings_24_11_2.tsv @@ -0,0 +1,211 @@ +adaptive_write_buffer_initial_size 16384 +add_implicit_sign_column_constraint_for_collapsing_engine 0 +allow_experimental_replacing_merge_with_cleanup 0 +allow_floating_point_partition_key 0 +allow_nullable_key 0 +allow_remote_fs_zero_copy_replication 0 +allow_suspicious_indices 0 +allow_vertical_merges_from_compact_to_wide_parts 1 +always_fetch_merged_part 0 +always_use_copy_instead_of_hardlinks 0 +assign_part_uuids 0 +async_block_ids_cache_update_wait_ms 100 +async_insert 0 +background_task_preferred_step_execution_time_ms 50 +cache_populated_by_fetch 0 +check_delay_period 60 +check_sample_column_is_correct 1 +clean_deleted_rows Never +cleanup_delay_period 30 +cleanup_delay_period_random_add 10 +cleanup_thread_preferred_points_per_iteration 150 +cleanup_threads 128 +columns_to_prewarm_mark_cache +compact_parts_max_bytes_to_buffer 134217728 +compact_parts_max_granules_to_buffer 128 +compact_parts_merge_max_bytes_to_prefetch_part 16777216 +compatibility_allow_sampling_expression_not_in_primary_key 0 +compress_marks 1 +compress_primary_key 1 +concurrent_part_removal_threshold 100 +deduplicate_merge_projection_mode throw +detach_not_byte_identical_parts 0 +detach_old_local_parts_when_cloning_replica 1 +disable_detach_partition_for_zero_copy_replication 1 +disable_fetch_partition_for_zero_copy_replication 1 +disable_freeze_partition_for_zero_copy_replication 1 +disk +enable_block_number_column 0 +enable_block_offset_column 0 +enable_index_granularity_compression 1 +enable_mixed_granularity_parts 1 +enable_the_endpoint_id_with_zookeeper_name_prefix 0 +enable_vertical_merge_algorithm 1 +exclude_deleted_rows_for_part_size_in_merge 0 +execute_merges_on_single_replica_time_threshold 0 +fault_probability_after_part_commit 0 +fault_probability_before_part_commit 0 +finished_mutations_to_keep 100 +force_read_through_cache_for_merges 0 +fsync_after_insert 0 +fsync_part_directory 0 +in_memory_parts_enable_wal 1 +in_memory_parts_insert_sync 0 +inactive_parts_to_delay_insert 0 +inactive_parts_to_throw_insert 0 +index_granularity 8192 +index_granularity_bytes 10485760 +initialization_retry_period 60 +kill_delay_period 30 +kill_delay_period_random_add 10 +kill_threads 128 +lightweight_mutation_projection_mode throw +load_existing_rows_count_for_old_parts 0 +lock_acquire_timeout_for_background_operations 120 +marks_compress_block_size 65536 +marks_compression_codec ZSTD(3) +materialize_ttl_recalculate_only 0 +max_avg_part_size_for_too_many_parts 1073741824 +max_bytes_to_merge_at_max_space_in_pool 161061273600 +max_bytes_to_merge_at_min_space_in_pool 1048576 +max_cleanup_delay_period 300 +max_compress_block_size 0 +max_concurrent_queries 0 +max_delay_to_insert 1 +max_delay_to_mutate_ms 1000 +max_digestion_size_per_segment 268435456 +max_file_name_length 127 +max_files_to_modify_in_alter_columns 75 +max_files_to_remove_in_alter_columns 50 +max_merge_selecting_sleep_ms 60000 +max_number_of_merges_with_ttl_in_pool 2 +max_number_of_mutations_for_replica 0 +max_part_loading_threads \'auto(32)\' +max_part_removal_threads \'auto(32)\' +max_partitions_to_read -1 +max_parts_in_total 100000 +max_parts_to_merge_at_once 100 +max_postpone_time_for_failed_mutations_ms 300000 +max_projections 25 +max_replicated_fetches_network_bandwidth 0 +max_replicated_logs_to_keep 1000 +max_replicated_merges_in_queue 1000 +max_replicated_merges_with_ttl_in_queue 1 +max_replicated_mutations_in_queue 8 +max_replicated_sends_network_bandwidth 0 +max_suspicious_broken_parts 100 +max_suspicious_broken_parts_bytes 1073741824 +merge_max_block_size 8192 +merge_max_block_size_bytes 10485760 +merge_selecting_sleep_ms 5000 +merge_selecting_sleep_slowdown_factor 1.2 +merge_selector_algorithm Simple +merge_selector_base 5 +merge_selector_blurry_base_scale_factor 0 +merge_selector_enable_heuristic_to_remove_small_parts_at_right 1 +merge_selector_window_size 1000 +merge_tree_clear_old_broken_detached_parts_ttl_timeout_seconds 2592000 +merge_tree_clear_old_parts_interval_seconds 1 +merge_tree_clear_old_temporary_directories_interval_seconds 60 +merge_tree_enable_clear_old_broken_detached 0 +merge_with_recompression_ttl_timeout 14400 +merge_with_ttl_timeout 14400 +merge_workload +min_absolute_delay_to_close 0 +min_age_to_force_merge_on_partition_only 0 +min_age_to_force_merge_seconds 0 +min_bytes_for_compact_part 0 +min_bytes_for_full_part_storage 0 +min_bytes_for_wide_part 10485760 +min_bytes_to_rebalance_partition_over_jbod 0 +min_compress_block_size 0 +min_compressed_bytes_to_fsync_after_fetch 0 +min_compressed_bytes_to_fsync_after_merge 0 +min_delay_to_insert_ms 10 +min_delay_to_mutate_ms 10 +min_free_disk_bytes_to_perform_insert 0 +min_free_disk_ratio_to_perform_insert 0 +min_index_granularity_bytes 1024 +min_marks_to_honor_max_concurrent_queries 0 +min_merge_bytes_to_use_direct_io 10737418240 +min_parts_to_merge_at_once 0 +min_relative_delay_to_close 300 +min_relative_delay_to_measure 120 +min_relative_delay_to_yield_leadership 120 +min_replicated_logs_to_keep 10 +min_rows_for_compact_part 0 +min_rows_for_full_part_storage 0 +min_rows_for_wide_part 0 +min_rows_to_fsync_after_merge 0 +mutation_workload +non_replicated_deduplication_window 0 +number_of_free_entries_in_pool_to_execute_mutation 20 +number_of_free_entries_in_pool_to_execute_optimize_entire_partition 25 +number_of_free_entries_in_pool_to_lower_max_size_of_merge 8 +number_of_mutations_to_delay 500 +number_of_mutations_to_throw 1000 +old_parts_lifetime 480 +optimize_row_order 0 +part_moves_between_shards_delay_seconds 30 +part_moves_between_shards_enable 0 +parts_to_delay_insert 1000 +parts_to_throw_insert 3000 +prefer_fetch_merged_part_size_threshold 10737418240 +prefer_fetch_merged_part_time_threshold 3600 +prewarm_mark_cache 0 +primary_key_compress_block_size 65536 +primary_key_compression_codec ZSTD(3) +primary_key_lazy_load 1 +primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns 0.9 +ratio_of_defaults_for_sparse_serialization 0.9375 +remote_fs_execute_merges_on_single_replica_time_threshold 10800 +remote_fs_zero_copy_path_compatible_mode 0 +remote_fs_zero_copy_zookeeper_path /clickhouse/zero_copy +remove_empty_parts 1 +remove_rolled_back_parts_immediately 1 +replace_long_file_name_to_hash 1 +replicated_can_become_leader 1 +replicated_deduplication_window 1000 +replicated_deduplication_window_for_async_inserts 10000 +replicated_deduplication_window_seconds 604800 +replicated_deduplication_window_seconds_for_async_inserts 604800 +replicated_fetches_http_connection_timeout 0 +replicated_fetches_http_receive_timeout 0 +replicated_fetches_http_send_timeout 0 +replicated_max_mutations_in_one_entry 10000 +replicated_max_parallel_fetches 0 +replicated_max_parallel_fetches_for_host 15 +replicated_max_parallel_fetches_for_table 0 +replicated_max_parallel_sends 0 +replicated_max_parallel_sends_for_table 0 +replicated_max_ratio_of_wrong_parts 0.5 +shared_merge_tree_disable_merges_and_mutations_assignment 0 +shared_merge_tree_partitions_hint_ratio_to_reload_merge_pred_for_mutations 0.5 +shared_merge_tree_parts_load_batch_size 32 +simultaneous_parts_removal_limit 0 +sleep_before_commit_local_part_in_replicated_table_ms 0 +sleep_before_loading_outdated_parts_ms 0 +storage_policy default +temporary_directories_lifetime 86400 +try_fetch_recompressed_part_timeout 7200 +ttl_only_drop_parts 0 +use_adaptive_write_buffer_for_dynamic_subcolumns 1 +use_async_block_ids_cache 1 +use_compact_variant_discriminators_serialization 1 +use_const_adaptive_granularity 0 +use_metadata_cache 0 +use_minimalistic_checksums_in_zookeeper 1 +use_minimalistic_part_header_in_zookeeper 1 +vertical_merge_algorithm_min_bytes_to_activate 0 +vertical_merge_algorithm_min_columns_to_activate 11 +vertical_merge_algorithm_min_rows_to_activate 131072 +vertical_merge_remote_filesystem_prefetch 1 +wait_for_unique_parts_send_before_shutdown_ms 0 +write_ahead_log_bytes_to_fsync 104857600 +write_ahead_log_interval_ms_to_fsync 100 +write_ahead_log_max_bytes 1073741824 +write_final_mark 1 +zero_copy_concurrent_part_removal_max_postpone_ratio 0.05 +zero_copy_concurrent_part_removal_max_split_times 5 +zero_copy_merge_mutation_min_parts_size_sleep_before_lock 1073741824 +zookeeper_session_expiration_check_period 60 diff --git a/tests/queries/0_stateless/02995_mergetree_settings_24_11_2.tsv b/tests/queries/0_stateless/02995_mergetree_settings_24_11_2.tsv new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02995_new_settings_history.sh b/tests/queries/0_stateless/02995_new_settings_history.sh index eb5e7415aee..e3478d2e5fc 100755 --- a/tests/queries/0_stateless/02995_new_settings_history.sh +++ b/tests/queries/0_stateless/02995_new_settings_history.sh @@ -7,21 +7,30 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh # Note that this is a broad check. A per version check is done in the upgrade test -# Baseline generated with 24.11.2 +# Baselines generated with 24.11.2 # clickhouse local --query "select name, default from system.settings order by name format TSV" > 02995_settings_24_11_2.tsv +# clickhouse local --query "select name, value from system.merge_tree_settings order by name format TSV" > 02995_merge_tree_settings_settings_24_11_2.tsv $CLICKHOUSE_LOCAL --query " WITH old_settings AS ( SELECT * FROM file('${CUR_DIR}/02995_settings_24_11_2.tsv', 'TSV', 'name String, default String') ), + old_merge_tree_settings AS + ( + SELECT * FROM file('${CUR_DIR}/02995_merge_tree_settings_settings_24_11_2.tsv', 'TSV', 'name String, default String') + ), new_settings AS ( -- Ignore settings that depend on the machine config (max_threads and similar) SELECT name, default FROM system.settings WHERE default NOT LIKE '%auto(%' + ), + new_merge_tree_settings AS + ( + SELECT name, value as default FROM system.merge_tree_settings WHERE default NOT LIKE '%auto(%' ) SELECT * FROM ( - SELECT 'PLEASE ADD THE NEW SETTING TO SettingsChangesHistory.cpp: ' || name || ' WAS ADDED', + SELECT 'PLEASE ADD THE NEW SETTING TO SettingsChangesHistory.cpp: ' || name || ' WAS ADDED' FROM new_settings WHERE (name NOT IN ( SELECT name @@ -29,17 +38,41 @@ $CLICKHOUSE_LOCAL --query " )) AND (name NOT IN ( SELECT arrayJoin(tupleElement(changes, 'name')) FROM system.settings_changes - WHERE splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 11 + WHERE type = 'Core' AND splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 11 )) UNION ALL ( - SELECT 'PLEASE ADD THE SETTING VALUE CHANGE TO SettingsChangesHistory.cpp: ' || name || ' WAS CHANGED FROM ' || old_settings.default || ' TO ' || new_settings.default, + SELECT 'PLEASE ADD THE NEW MERGE_TREE_SETTING TO SettingsChangesHistory.cpp: ' || name || ' WAS ADDED' + FROM new_merge_tree_settings + WHERE (name NOT IN ( + SELECT name + FROM old_merge_tree_settings + )) AND (name NOT IN ( + SELECT arrayJoin(tupleElement(changes, 'name')) + FROM system.settings_changes + WHERE type = 'MergeTree' AND splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 11 + )) + ) + UNION ALL + ( + SELECT 'PLEASE ADD THE SETTING VALUE CHANGE TO SettingsChangesHistory.cpp: ' || name || ' WAS CHANGED FROM ' || old_settings.default || ' TO ' || new_settings.default FROM new_settings LEFT JOIN old_settings ON new_settings.name = old_settings.name WHERE (new_settings.default != old_settings.default) AND (name NOT IN ( SELECT arrayJoin(tupleElement(changes, 'name')) FROM system.settings_changes - WHERE splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 11 + WHERE type = 'Core' AND splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 11 + )) + ) + UNION ALL + ( + SELECT 'PLEASE ADD THE MERGE_TREE_SETTING VALUE CHANGE TO SettingsChangesHistory.cpp: ' || name || ' WAS CHANGED FROM ' || old_merge_tree_settings.default || ' TO ' || new_merge_tree_settings.default + FROM new_merge_tree_settings + LEFT JOIN old_merge_tree_settings ON new_merge_tree_settings.name = old_merge_tree_settings.name + WHERE (new_merge_tree_settings.default != old_merge_tree_settings.default) AND (name NOT IN ( + SELECT arrayJoin(tupleElement(changes, 'name')) + FROM system.settings_changes + WHERE type = 'MergeTree' AND splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 11 )) ) ) diff --git a/tests/queries/0_stateless/02995_baseline_24_11_2.tsv b/tests/queries/0_stateless/02995_settings_24_11_2.tsv similarity index 100% rename from tests/queries/0_stateless/02995_baseline_24_11_2.tsv rename to tests/queries/0_stateless/02995_settings_24_11_2.tsv From 3df482066b8c221fad52062da55e2622f0553181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 12:47:20 +0100 Subject: [PATCH 239/502] Fix MergeTreeSettings history for 24.12 --- src/Core/SettingsChangesHistory.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index aeeb9f0fdd4..f84e116e079 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -616,7 +616,9 @@ static std::initializer_list & settings_changes_history, std::once_flag & initialized_flag, - std::initializer_list> & initializer + std::initializer_list> const & initializer ) { std::call_once(initialized_flag, [&]() From b0d555b892351878999f1dafcc86e52f59953e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 13:02:41 +0100 Subject: [PATCH 240/502] Check in upgrade tests --- tests/docker_scripts/upgrade_runner.sh | 66 ++++++++++++++++++- .../0_stateless/02995_new_settings_history.sh | 2 +- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/tests/docker_scripts/upgrade_runner.sh b/tests/docker_scripts/upgrade_runner.sh index 15c5ab69521..c91019e7471 100755 --- a/tests/docker_scripts/upgrade_runner.sh +++ b/tests/docker_scripts/upgrade_runner.sh @@ -66,6 +66,12 @@ function save_settings_clean() script -q -c "clickhouse-local --implicit-select 0 -q \"select * from system.settings into outfile '$out'\"" --log-out /dev/null } +function save_mergetree_settings_clean() +{ + local out=$1 && shift + script -q -c "clickhouse-local --implicit-select 0 -q \"select * from system.merge_tree_settings into outfile '$out'\"" --log-out /dev/null +} + # We save the (numeric) version of the old server to compare setting changes between the 2 # We do this since we are testing against the latest release, not taking into account release candidates, so we might # be testing current master (24.6) against the latest stable release (24.4) @@ -76,6 +82,7 @@ function save_major_version() } save_settings_clean 'old_settings.native' +save_mergetree_settings_clean 'old_merge_tree_settings.native' save_major_version 'old_version.native' # Initial run without S3 to create system.*_log on local file system to make it @@ -135,10 +142,13 @@ IS_SANITIZED=$(clickhouse-local --query "SELECT value LIKE '%-fsanitize=%' FROM if [ "${IS_SANITIZED}" -eq "0" ] then save_settings_clean 'new_settings.native' + save_merge_tree_settings_clean 'new_merge_tree_settings.native' clickhouse-local -nmq " CREATE TABLE old_settings AS file('old_settings.native'); + CREATE TABLE old_merge_tree_settings AS file('old_merge_tree_settings.native'); CREATE TABLE old_version AS file('old_version.native'); CREATE TABLE new_settings AS file('new_settings.native'); + CREATE TABLE new_merge_tree_settings AS file('new_merge_tree_settings.native'); SELECT name, @@ -151,7 +161,7 @@ then SELECT arrayJoin(tupleElement(changes, 'name')) FROM ( - SELECT *, splitByChar('.', version) AS version_array FROM system.settings_changes + SELECT *, splitByChar('.', version) AS version_array FROM system.settings_changes WHERE type = 'Core' ) WHERE (version_array[1]::UInt64 * 100 + version_array[2]::UInt64) > (SELECT v FROM old_version LIMIT 1) )) @@ -159,6 +169,25 @@ then INTO OUTFILE 'changed_settings.txt' FORMAT PrettyCompactNoEscapes; + SELECT + name, + new_merge_tree_settings.value AS new_value, + old_merge_tree_settings.value AS old_value + FROM new_merge_tree_settings + LEFT JOIN old_merge_tree_settings ON new_merge_tree_settings.name = old_merge_tree_settings.name + WHERE (new_value != old_value) + AND (name NOT IN ( + SELECT arrayJoin(tupleElement(changes, 'name')) + FROM + ( + SELECT *, splitByChar('.', version) AS version_array FROM system.settings_changes WHERE type = 'MergeTree' + ) + WHERE (version_array[1]::UInt64 * 100 + version_array[2]::UInt64) > (SELECT v FROM old_version LIMIT 1) + )) + SETTINGS join_use_nulls = 1 + INTO OUTFILE 'changed_merge_tree_settings.txt' + FORMAT PrettyCompactNoEscapes; + SELECT name FROM new_settings WHERE (name NOT IN ( @@ -168,12 +197,28 @@ then SELECT arrayJoin(tupleElement(changes, 'name')) FROM ( - SELECT *, splitByChar('.', version) AS version_array FROM system.settings_changes + SELECT *, splitByChar('.', version) AS version_array FROM system.settings_changes WHERE type = 'Core' ) WHERE (version_array[1]::UInt64 * 100 + version_array[2]::UInt64) > (SELECT v FROM old_version LIMIT 1) )) INTO OUTFILE 'new_settings.txt' FORMAT PrettyCompactNoEscapes; + + SELECT name + FROM new_merge_tree_settings + WHERE (name NOT IN ( + SELECT name + FROM old_merge_tree_settings + )) AND (name NOT IN ( + SELECT arrayJoin(tupleElement(changes, 'name')) + FROM + ( + SELECT *, splitByChar('.', version) AS version_array FROM system.settings_changes WHERE type = 'MergeTree' + ) + WHERE (version_array[1]::UInt64 * 100 + version_array[2]::UInt64) > (SELECT v FROM old_version LIMIT 1) + )) + INTO OUTFILE 'new_merge_tree_settings.txt' + FORMAT PrettyCompactNoEscapes; " if [ -s changed_settings.txt ] @@ -184,6 +229,14 @@ then echo -e "There are no changed settings or they are reflected in settings changes history$OK" >> /test_output/test_results.tsv fi + if [ -s changed_merge_tree_settings.txt ] + then + mv changed_merge_tree_settings.txt /test_output/ + echo -e "Changed MergeTree settings are not reflected in the settings changes history (see changed_merge_tree_settings.txt)$FAIL$(head_escaped /test_output/changed_merge_tree_settings.txt)" >> /test_output/test_results.tsv + else + echo -e "There are no changed MergeTree settings or they are reflected in settings changes history$OK" >> /test_output/test_results.tsv + fi + if [ -s new_settings.txt ] then mv new_settings.txt /test_output/ @@ -191,6 +244,15 @@ then else echo -e "There are no new settings or they are reflected in settings changes history$OK" >> /test_output/test_results.tsv fi + + if [ -s new_merge_tree_settings.txt ] + then + mv new_merge_tree_settings.txt /test_output/ + echo -e "New MergeTree settings are not reflected in settings changes history (see new_merge_tree_settings.txt)$FAIL$(head_escaped /test_output/new_merge_tree_settings.txt)" >> /test_output/test_results.tsv + else + echo -e "There are no new MergeTree settings or they are reflected in settings changes history$OK" >> /test_output/test_results.tsv + fi + fi # Just in case previous version left some garbage in zk diff --git a/tests/queries/0_stateless/02995_new_settings_history.sh b/tests/queries/0_stateless/02995_new_settings_history.sh index e3478d2e5fc..6405f7f16ab 100755 --- a/tests/queries/0_stateless/02995_new_settings_history.sh +++ b/tests/queries/0_stateless/02995_new_settings_history.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-cpu-aarch64, no-random-settings +# Tags: no-cpu-aarch64, no-random-settings, no-random-merge-tree-settings # Some settings can be different for builds with sanitizers or aarch64 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) From e97ccb6b075eb4da0ae4ca5c7ca0f35b01239bb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 13:48:50 +0100 Subject: [PATCH 241/502] Adjust tests and docs --- docs/en/operations/system-tables/settings_changes.md | 2 ++ .../0_stateless/02326_settings_changes_system_table.reference | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/settings_changes.md b/docs/en/operations/system-tables/settings_changes.md index c097915d430..f380023c4fa 100644 --- a/docs/en/operations/system-tables/settings_changes.md +++ b/docs/en/operations/system-tables/settings_changes.md @@ -7,6 +7,7 @@ Contains information about setting changes in previous ClickHouse versions. Columns: +- `type` ([String](../../sql-reference/data-types/string.md)) - The group of settings (Core, MergeTree...) - `version` ([String](../../sql-reference/data-types/string.md)) — The ClickHouse version in which settings were changed - `changes` ([Array](../../sql-reference/data-types/array.md) of [Tuple](../../sql-reference/data-types/tuple.md)) — A description of the setting changes: (setting name, previous value, new value, reason for the change) @@ -22,6 +23,7 @@ FORMAT Vertical ``` text Row 1: ────── +type: Core version: 23.5 changes: [('input_format_parquet_preserve_order','1','0','Allow Parquet reader to reorder rows for better parallelism.'),('parallelize_output_from_storages','0','1','Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows.'),('use_with_fill_by_sorting_prefix','0','1','Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently'),('output_format_parquet_compliant_nested_types','0','1','Change an internal field name in output Parquet file schema.')] ``` diff --git a/tests/queries/0_stateless/02326_settings_changes_system_table.reference b/tests/queries/0_stateless/02326_settings_changes_system_table.reference index 946b2727d30..026b96cd16c 100644 --- a/tests/queries/0_stateless/02326_settings_changes_system_table.reference +++ b/tests/queries/0_stateless/02326_settings_changes_system_table.reference @@ -1,3 +1,4 @@ +type String The group of settings (Core, MergeTree...) version String The ClickHouse server version. changes Array(Tuple(\n name String,\n previous_value String,\n new_value String,\n reason String)) The list of changes in settings which changed the behaviour of ClickHouse. -22.5 [('memory_overcommit_ratio_denominator','0','1073741824','Enable memory overcommit feature by default'),('memory_overcommit_ratio_denominator_for_user','0','1073741824','Enable memory overcommit feature by default')] +Core 22.5 [('memory_overcommit_ratio_denominator','0','1073741824','Enable memory overcommit feature by default'),('memory_overcommit_ratio_denominator_for_user','0','1073741824','Enable memory overcommit feature by default')] From 4c5caa85036b5eded44f304ddbdff0499429e934 Mon Sep 17 00:00:00 2001 From: Michael Stetsyuk Date: Mon, 2 Dec 2024 13:26:53 +0000 Subject: [PATCH 242/502] empty From 0e30bd3b72fd00433f49ac5e67a2a6af03483de1 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Mon, 2 Dec 2024 21:57:54 +0800 Subject: [PATCH 243/502] fix style fail --- src/Interpreters/convertFieldToType.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 0a54ccbd5d0..8dd1eb56fbf 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -665,7 +665,7 @@ std::optional convertFieldToTypeStrict(const Field & from_value, const ID { result_value = convertFieldToType(from_value, to_type, &from_type, format_settings); } - catch(const std::exception &) + catch (...) { return {}; } From 72c63817e6d04a9464424afa2ecddeda4b8d1194 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Mon, 2 Dec 2024 15:01:27 +0100 Subject: [PATCH 244/502] fix test --- tests/queries/0_stateless/03277_async_insert.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/03277_async_insert.sh b/tests/queries/0_stateless/03277_async_insert.sh index d2ae2ac9a07..ee2f848e953 100755 --- a/tests/queries/0_stateless/03277_async_insert.sh +++ b/tests/queries/0_stateless/03277_async_insert.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -8,8 +7,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE}" mkdir -p "${CLICKHOUSE_USER_FILES_UNIQUE}" +rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE}"/* FILE_CSV="${CLICKHOUSE_USER_FILES_UNIQUE}/03277.csv" @@ -27,6 +26,7 @@ $CLICKHOUSE_CLIENT -q " $CLICKHOUSE_CLIENT -q "SELECT * from t0" $CLICKHOUSE_CLIENT -q "DROP TABLE t0" + $CLICKHOUSE_CLIENT -q " SET async_insert = 1; SELECT '${FILE_ARROW}'; @@ -37,4 +37,5 @@ $CLICKHOUSE_CLIENT -q " $CLICKHOUSE_CLIENT -q "SELECT * from t0" $CLICKHOUSE_CLIENT -q "DROP TABLE t0" -rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE}" \ No newline at end of file + +rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE}"/* \ No newline at end of file From c3bc7fd64b16bfd1de35986686b3c27ded99e375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Mon, 2 Dec 2024 14:38:17 +0000 Subject: [PATCH 245/502] Make query results deterministic in test --- ...2_common_expression_optimization.reference | 584 +++++++++--------- .../03262_common_expression_optimization.sql | 60 +- 2 files changed, 322 insertions(+), 322 deletions(-) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index 38a86cb421f..035769cce67 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -51,25 +51,25 @@ FROM default.x AS __table1 WHERE __table1.A AND (__table1.B OR __table1.C) SETTINGS optimize_extract_common_expressions = 1 -9137658153367139416 1 1 1 0 0 0 -5551918026146320404 1 1 1 0 0 0 --2996395377931803345 1 1 1 0 0 0 -5025358099778248663 1 1 1 0 0 0 -9185779160712471769 1 1 1 0 0 0 -95474718890257185 1 1 1 0 0 0 -2248520110901842660 1 1 1 0 0 0 -4269743235373690057 1 1 1 0 0 0 --3254171257796361610 1 1 1 0 0 0 --2057347811709491782 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9114056182077943575 1 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8751975391509985483 1 1 1 0 1 1 +-8699036683208290351 1 1 1 0 0 1 +-8647725213825146240 1 1 1 1 0 1 +-8525570993468372516 1 1 1 1 1 0 -9137658153367139416 1 1 1 0 0 0 -5551918026146320404 1 1 1 0 0 0 --2996395377931803345 1 1 1 0 0 0 -5025358099778248663 1 1 1 0 0 0 -9185779160712471769 1 1 1 0 0 0 -95474718890257185 1 1 1 0 0 0 -2248520110901842660 1 1 1 0 0 0 -4269743235373690057 1 1 1 0 0 0 --3254171257796361610 1 1 1 0 0 0 --2057347811709491782 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9114056182077943575 1 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8751975391509985483 1 1 1 0 1 1 +-8699036683208290351 1 1 1 0 0 1 +-8647725213825146240 1 1 1 1 0 1 +-8525570993468372516 1 1 1 1 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -89,26 +89,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND __table1.B AND __table1.C --1718308154946016060 1 1 1 0 0 1 --909098580002376266 1 1 1 0 0 1 --3361624987358969173 1 1 1 0 0 1 --3655654558316512012 1 1 1 0 0 1 +-9114056182077943575 1 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8751975391509985483 1 1 1 0 1 1 -8699036683208290351 1 1 1 0 0 1 -4783429523421833175 1 1 1 0 0 1 -7828110511858076458 1 1 1 0 0 1 --1265166225143531838 1 1 1 0 0 1 --7556142413115723841 1 1 1 0 0 1 -7754782253328504134 1 1 1 0 0 1 --1718308154946016060 1 1 1 0 0 1 --909098580002376266 1 1 1 0 0 1 --3361624987358969173 1 1 1 0 0 1 --3655654558316512012 1 1 1 0 0 1 +-8647725213825146240 1 1 1 1 0 1 +-8525570993468372516 1 1 1 1 1 0 +-8476148063717219538 1 1 1 1 1 1 +-8387418414841552371 1 1 1 1 0 1 +-9114056182077943575 1 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8751975391509985483 1 1 1 0 1 1 -8699036683208290351 1 1 1 0 0 1 -4783429523421833175 1 1 1 0 0 1 -7828110511858076458 1 1 1 0 0 1 --1265166225143531838 1 1 1 0 0 1 --7556142413115723841 1 1 1 0 0 1 -7754782253328504134 1 1 1 0 0 1 +-8647725213825146240 1 1 1 1 0 1 +-8525570993468372516 1 1 1 1 1 0 +-8476148063717219538 1 1 1 1 1 1 +-8387418414841552371 1 1 1 1 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -133,26 +133,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND (__table1.E OR __table1.F) AND __table1.B AND __table1.C --1718308154946016060 1 1 1 0 0 1 --909098580002376266 1 1 1 0 0 1 --3361624987358969173 1 1 1 0 0 1 --3655654558316512012 1 1 1 0 0 1 +-9114056182077943575 1 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8751975391509985483 1 1 1 0 1 1 -8699036683208290351 1 1 1 0 0 1 -4783429523421833175 1 1 1 0 0 1 -7828110511858076458 1 1 1 0 0 1 --1265166225143531838 1 1 1 0 0 1 --7556142413115723841 1 1 1 0 0 1 -7754782253328504134 1 1 1 0 0 1 --1718308154946016060 1 1 1 0 0 1 --909098580002376266 1 1 1 0 0 1 --3361624987358969173 1 1 1 0 0 1 --3655654558316512012 1 1 1 0 0 1 +-8647725213825146240 1 1 1 1 0 1 +-8525570993468372516 1 1 1 1 1 0 +-8476148063717219538 1 1 1 1 1 1 +-8387418414841552371 1 1 1 1 0 1 +-9114056182077943575 1 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8751975391509985483 1 1 1 0 1 1 -8699036683208290351 1 1 1 0 0 1 -4783429523421833175 1 1 1 0 0 1 -7828110511858076458 1 1 1 0 0 1 --1265166225143531838 1 1 1 0 0 1 --7556142413115723841 1 1 1 0 0 1 -7754782253328504134 1 1 1 0 0 1 +-8647725213825146240 1 1 1 1 0 1 +-8525570993468372516 1 1 1 1 1 0 +-8476148063717219538 1 1 1 1 1 1 +-8387418414841552371 1 1 1 1 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -177,26 +177,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND (__table1.E OR __table1.F) AND __table1.B AND __table1.C -3574384104825527969 1 1 0 0 1 0 --2211493827666596761 1 1 0 0 1 0 -8958830522204821629 1 1 0 0 1 0 -1346739481280160211 1 1 0 0 1 0 -3279338559385463408 1 1 0 0 1 0 -3312987885102520931 1 1 0 0 1 0 -1164803530066053002 1 1 0 0 1 0 --5278928290423082776 1 1 0 0 1 0 -8491129310781185476 1 1 0 0 1 0 --7239011331174116716 1 1 0 0 1 0 -3574384104825527969 1 1 0 0 1 0 --2211493827666596761 1 1 0 0 1 0 -8958830522204821629 1 1 0 0 1 0 -1346739481280160211 1 1 0 0 1 0 -3279338559385463408 1 1 0 0 1 0 -3312987885102520931 1 1 0 0 1 0 -1164803530066053002 1 1 0 0 1 0 --5278928290423082776 1 1 0 0 1 0 -8491129310781185476 1 1 0 0 1 0 --7239011331174116716 1 1 0 0 1 0 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9121738304623869295 1 1 0 0 1 0 +-9114056182077943575 1 1 1 1 0 1 +-9001504240784412840 1 1 0 0 1 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8757579268243194545 1 1 0 1 0 0 +-8753962234239378281 1 1 0 1 0 0 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9121738304623869295 1 1 0 0 1 0 +-9114056182077943575 1 1 1 1 0 1 +-9001504240784412840 1 1 0 0 1 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8757579268243194545 1 1 0 1 0 0 +-8753962234239378281 1 1 0 1 0 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -221,26 +221,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND (__table1.C OR __table1.D OR __table1.E) AND __table1.B -3574384104825527969 1 1 0 0 1 0 --2211493827666596761 1 1 0 0 1 0 -8958830522204821629 1 1 0 0 1 0 -1346739481280160211 1 1 0 0 1 0 -3279338559385463408 1 1 0 0 1 0 -3312987885102520931 1 1 0 0 1 0 -1164803530066053002 1 1 0 0 1 0 --5278928290423082776 1 1 0 0 1 0 -8491129310781185476 1 1 0 0 1 0 --7239011331174116716 1 1 0 0 1 0 -3574384104825527969 1 1 0 0 1 0 --2211493827666596761 1 1 0 0 1 0 -8958830522204821629 1 1 0 0 1 0 -1346739481280160211 1 1 0 0 1 0 -3279338559385463408 1 1 0 0 1 0 -3312987885102520931 1 1 0 0 1 0 -1164803530066053002 1 1 0 0 1 0 --5278928290423082776 1 1 0 0 1 0 -8491129310781185476 1 1 0 0 1 0 --7239011331174116716 1 1 0 0 1 0 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9121738304623869295 1 1 0 0 1 0 +-9114056182077943575 1 1 1 1 0 1 +-9001504240784412840 1 1 0 0 1 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8757579268243194545 1 1 0 1 0 0 +-8753962234239378281 1 1 0 1 0 0 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9121738304623869295 1 1 0 0 1 0 +-9114056182077943575 1 1 1 1 0 1 +-9001504240784412840 1 1 0 0 1 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8757579268243194545 1 1 0 1 0 0 +-8753962234239378281 1 1 0 1 0 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -265,26 +265,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND (__table1.C OR __table1.D OR __table1.E) AND __table1.B -910439267853176941 0 1 1 0 0 0 --835764896252397456 0 1 1 0 0 0 -36802202306659819 0 1 1 0 0 0 --21405676422526023 0 1 1 0 0 0 --186684117184539357 0 1 1 0 0 0 -2294399721801435121 0 1 1 0 0 0 -271677944114046880 0 1 1 0 0 0 --4341212657373873318 0 1 1 0 0 0 --4556789282933643330 0 1 1 0 0 0 -4042734258081112561 0 1 1 0 0 0 -910439267853176941 0 1 1 0 0 0 --835764896252397456 0 1 1 0 0 0 -36802202306659819 0 1 1 0 0 0 --21405676422526023 0 1 1 0 0 0 --186684117184539357 0 1 1 0 0 0 -2294399721801435121 0 1 1 0 0 0 -271677944114046880 0 1 1 0 0 0 --4341212657373873318 0 1 1 0 0 0 --4556789282933643330 0 1 1 0 0 0 -4042734258081112561 0 1 1 0 0 0 +-9217261049539683905 0 1 1 0 1 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9128210600225023826 0 1 1 0 1 1 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9027896542766292056 0 1 1 1 0 1 +-9016286694578688546 0 1 1 1 0 1 +-9007101892394668722 0 1 1 0 0 1 +-8971956622356473539 0 1 1 1 0 1 +-9217261049539683905 0 1 1 0 1 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9128210600225023826 0 1 1 0 1 1 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9027896542766292056 0 1 1 1 0 1 +-9016286694578688546 0 1 1 1 0 1 +-9007101892394668722 0 1 1 0 0 1 +-8971956622356473539 0 1 1 1 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -303,26 +303,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.B AND __table1.C -5604528747227576251 0 1 1 0 0 1 --3227180127647371250 0 1 1 0 0 1 --3777435661756238740 0 1 1 0 0 1 --4105312414544198416 0 1 1 0 0 1 -3624151743867502404 0 1 1 0 0 1 -4327451746496881795 0 1 1 0 0 1 --7076772903747348221 0 1 1 0 0 1 -520798717162063931 0 1 1 0 0 1 --8449852538623664136 0 1 1 0 0 1 -1104848393205607021 0 1 1 0 0 1 -5604528747227576251 0 1 1 0 0 1 --3227180127647371250 0 1 1 0 0 1 --3777435661756238740 0 1 1 0 0 1 --4105312414544198416 0 1 1 0 0 1 -3624151743867502404 0 1 1 0 0 1 -4327451746496881795 0 1 1 0 0 1 --7076772903747348221 0 1 1 0 0 1 -520798717162063931 0 1 1 0 0 1 --8449852538623664136 0 1 1 0 0 1 -1104848393205607021 0 1 1 0 0 1 +-9217261049539683905 0 1 1 0 1 1 +-9128210600225023826 0 1 1 0 1 1 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9027896542766292056 0 1 1 1 0 1 +-9016286694578688546 0 1 1 1 0 1 +-9007101892394668722 0 1 1 0 0 1 +-8971956622356473539 0 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8884339606857089357 0 1 1 0 1 1 +-9217261049539683905 0 1 1 0 1 1 +-9128210600225023826 0 1 1 0 1 1 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9027896542766292056 0 1 1 1 0 1 +-9016286694578688546 0 1 1 1 0 1 +-9007101892394668722 0 1 1 0 0 1 +-8971956622356473539 0 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8884339606857089357 0 1 1 0 1 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -346,26 +346,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.B AND __table1.C AND (__table1.E OR __table1.F) -5604528747227576251 0 1 1 0 0 1 --3227180127647371250 0 1 1 0 0 1 --3777435661756238740 0 1 1 0 0 1 --4105312414544198416 0 1 1 0 0 1 -3624151743867502404 0 1 1 0 0 1 -4327451746496881795 0 1 1 0 0 1 --7076772903747348221 0 1 1 0 0 1 -520798717162063931 0 1 1 0 0 1 --8449852538623664136 0 1 1 0 0 1 -1104848393205607021 0 1 1 0 0 1 -5604528747227576251 0 1 1 0 0 1 --3227180127647371250 0 1 1 0 0 1 --3777435661756238740 0 1 1 0 0 1 --4105312414544198416 0 1 1 0 0 1 -3624151743867502404 0 1 1 0 0 1 -4327451746496881795 0 1 1 0 0 1 --7076772903747348221 0 1 1 0 0 1 -520798717162063931 0 1 1 0 0 1 --8449852538623664136 0 1 1 0 0 1 -1104848393205607021 0 1 1 0 0 1 +-9217261049539683905 0 1 1 0 1 1 +-9128210600225023826 0 1 1 0 1 1 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9027896542766292056 0 1 1 1 0 1 +-9016286694578688546 0 1 1 1 0 1 +-9007101892394668722 0 1 1 0 0 1 +-8971956622356473539 0 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8884339606857089357 0 1 1 0 1 1 +-9217261049539683905 0 1 1 0 1 1 +-9128210600225023826 0 1 1 0 1 1 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9027896542766292056 0 1 1 1 0 1 +-9016286694578688546 0 1 1 1 0 1 +-9007101892394668722 0 1 1 0 0 1 +-8971956622356473539 0 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8884339606857089357 0 1 1 0 1 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -389,26 +389,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.B AND __table1.C AND (__table1.E OR __table1.F) --761481560319605917 0 1 0 0 1 0 --5533222425689494786 0 1 0 0 1 0 -989832766219156063 0 1 0 0 1 0 --3972983892221867704 0 1 0 0 1 0 --8548855706734087530 0 1 0 0 1 0 -5284071364923999442 0 1 0 0 1 0 -5223032192244036365 0 1 0 0 1 0 -8288659361821935470 0 1 0 0 1 0 -3669979250564809316 0 1 0 0 1 0 --3068712899003736471 0 1 0 0 1 0 --761481560319605917 0 1 0 0 1 0 --5533222425689494786 0 1 0 0 1 0 -989832766219156063 0 1 0 0 1 0 --3972983892221867704 0 1 0 0 1 0 --8548855706734087530 0 1 0 0 1 0 -5284071364923999442 0 1 0 0 1 0 -5223032192244036365 0 1 0 0 1 0 -8288659361821935470 0 1 0 0 1 0 -3669979250564809316 0 1 0 0 1 0 --3068712899003736471 0 1 0 0 1 0 +-9217261049539683905 0 1 1 0 1 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9128210600225023826 0 1 1 0 1 1 +-9121738304623869295 1 1 0 0 1 0 +-9120901636510510987 0 1 0 1 0 0 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9034436041280906643 0 1 0 1 0 0 +-9032227688076112856 0 1 0 1 0 0 +-9217261049539683905 0 1 1 0 1 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9128210600225023826 0 1 1 0 1 1 +-9121738304623869295 1 1 0 0 1 0 +-9120901636510510987 0 1 0 1 0 0 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9034436041280906643 0 1 0 1 0 0 +-9032227688076112856 0 1 0 1 0 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -432,26 +432,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.B AND (__table1.C OR __table1.D OR __table1.E) --761481560319605917 0 1 0 0 1 0 --5533222425689494786 0 1 0 0 1 0 -989832766219156063 0 1 0 0 1 0 --3972983892221867704 0 1 0 0 1 0 --8548855706734087530 0 1 0 0 1 0 -5284071364923999442 0 1 0 0 1 0 -5223032192244036365 0 1 0 0 1 0 -8288659361821935470 0 1 0 0 1 0 -3669979250564809316 0 1 0 0 1 0 --3068712899003736471 0 1 0 0 1 0 --761481560319605917 0 1 0 0 1 0 --5533222425689494786 0 1 0 0 1 0 -989832766219156063 0 1 0 0 1 0 --3972983892221867704 0 1 0 0 1 0 --8548855706734087530 0 1 0 0 1 0 -5284071364923999442 0 1 0 0 1 0 -5223032192244036365 0 1 0 0 1 0 -8288659361821935470 0 1 0 0 1 0 -3669979250564809316 0 1 0 0 1 0 --3068712899003736471 0 1 0 0 1 0 +-9217261049539683905 0 1 1 0 1 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9128210600225023826 0 1 1 0 1 1 +-9121738304623869295 1 1 0 0 1 0 +-9120901636510510987 0 1 0 1 0 0 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9034436041280906643 0 1 0 1 0 0 +-9032227688076112856 0 1 0 1 0 0 +-9217261049539683905 0 1 1 0 1 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9128210600225023826 0 1 1 0 1 1 +-9121738304623869295 1 1 0 0 1 0 +-9120901636510510987 0 1 0 1 0 0 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9034436041280906643 0 1 0 1 0 0 +-9032227688076112856 0 1 0 1 0 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -475,26 +475,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.B AND (__table1.C OR __table1.D OR __table1.E) --608986750099316624 0 1 0 0 0 0 -4634030441303755773 0 1 0 0 0 0 -1594908885273767697 0 1 0 0 0 0 -4525560033677464079 0 1 0 0 0 0 -2306683870575600851 0 1 0 0 0 0 --4812729937086184087 0 1 0 0 0 0 --8185772668638940850 0 1 0 0 0 0 --8315367594051711131 0 1 0 0 0 0 -5705028529783052191 0 1 0 0 0 0 --8545685458500363239 0 1 0 0 0 0 --608986750099316624 0 1 0 0 0 0 -4634030441303755773 0 1 0 0 0 0 -1594908885273767697 0 1 0 0 0 0 -4525560033677464079 0 1 0 0 0 0 -2306683870575600851 0 1 0 0 0 0 --4812729937086184087 0 1 0 0 0 0 --8185772668638940850 0 1 0 0 0 0 --8315367594051711131 0 1 0 0 0 0 -5705028529783052191 0 1 0 0 0 0 --8545685458500363239 0 1 0 0 0 0 +-9220160771238933596 1 0 0 0 1 1 +-9154303869107404275 1 0 1 1 1 1 +-9154240572955605594 1 0 0 0 0 1 +-9136706315507142110 1 0 0 0 1 0 +-9136188450553078231 0 1 0 0 0 1 +-9121738304623869295 1 1 0 0 1 0 +-9119049435533988608 1 0 0 0 1 1 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9054835338197246193 1 0 0 0 1 0 +-9220160771238933596 1 0 0 0 1 1 +-9154303869107404275 1 0 1 1 1 1 +-9154240572955605594 1 0 0 0 0 1 +-9136706315507142110 1 0 0 0 1 0 +-9136188450553078231 0 1 0 0 0 1 +-9121738304623869295 1 1 0 0 1 0 +-9119049435533988608 1 0 0 0 1 1 +-9114056182077943575 1 1 1 1 0 1 +-9092546166635132947 0 1 1 1 1 0 +-9054835338197246193 1 0 0 0 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -527,26 +527,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE (sipHash64(__table1.C) = sipHash64(__table1.D)) AND (__table1.A OR __table1.B) -7762400598731351030 0 0 0 0 1 1 -4383923020933692842 0 0 0 0 1 1 --8485383028044727808 0 0 0 0 1 1 --8417770467973836608 0 0 0 0 1 1 -2299883611232961048 0 0 0 0 1 1 -1390628346580495568 0 0 0 0 1 1 -9080544688861627695 0 0 0 0 1 1 --7721736184933939401 0 0 0 0 1 1 -4299643423346067577 0 0 0 0 1 1 --4360831390913279988 0 0 0 0 1 1 -7762400598731351030 0 0 0 0 1 1 -4383923020933692842 0 0 0 0 1 1 --8485383028044727808 0 0 0 0 1 1 --8417770467973836608 0 0 0 0 1 1 -2299883611232961048 0 0 0 0 1 1 -1390628346580495568 0 0 0 0 1 1 -9080544688861627695 0 0 0 0 1 1 --7721736184933939401 0 0 0 0 1 1 -4299643423346067577 0 0 0 0 1 1 --4360831390913279988 0 0 0 0 1 1 +-9220160771238933596 1 0 0 0 1 1 +-9217261049539683905 0 1 1 0 1 1 +-9154303869107404275 1 0 1 1 1 1 +-9146648809962620241 0 0 1 1 1 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9128210600225023826 0 1 1 0 1 1 +-9121738304623869295 1 1 0 0 1 0 +-9119049435533988608 1 0 0 0 1 1 +-9118603327247981370 0 0 1 1 0 1 +-9220160771238933596 1 0 0 0 1 1 +-9217261049539683905 0 1 1 0 1 1 +-9154303869107404275 1 0 1 1 1 1 +-9146648809962620241 0 0 1 1 1 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9128210600225023826 0 1 1 0 1 1 +-9121738304623869295 1 1 0 0 1 0 +-9119049435533988608 1 0 0 0 1 1 +-9118603327247981370 0 0 1 1 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -581,26 +581,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE (__table1.A AND __table1.B) OR ((__table1.C AND __table1.D) OR (__table1.E AND __table1.F)) -2061801445197034299 0 0 0 0 1 1 --4692710336366551784 0 0 0 0 1 1 -8827599087436950777 0 0 0 0 1 1 --882303417518114752 0 0 0 0 1 1 -3625807179116440482 0 0 0 0 1 1 --8417770467973836608 0 0 0 0 1 1 -2813303165430768949 0 0 0 0 1 1 --8353024809925920489 0 0 0 0 1 1 --6723016927829163686 0 0 0 0 1 1 -2766695568743886462 0 0 0 0 1 1 -2061801445197034299 0 0 0 0 1 1 --4692710336366551784 0 0 0 0 1 1 -8827599087436950777 0 0 0 0 1 1 --882303417518114752 0 0 0 0 1 1 -3625807179116440482 0 0 0 0 1 1 --8417770467973836608 0 0 0 0 1 1 -2813303165430768949 0 0 0 0 1 1 --8353024809925920489 0 0 0 0 1 1 --6723016927829163686 0 0 0 0 1 1 -2766695568743886462 0 0 0 0 1 1 +-9220160771238933596 1 0 0 0 1 1 +-9217261049539683905 0 1 1 0 1 1 +-9154303869107404275 1 0 1 1 1 1 +-9146648809962620241 0 0 1 1 1 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9128210600225023826 0 1 1 0 1 1 +-9121738304623869295 1 1 0 0 1 0 +-9120901636510510987 0 1 0 1 0 0 +-9119049435533988608 1 0 0 0 1 1 +-9220160771238933596 1 0 0 0 1 1 +-9217261049539683905 0 1 1 0 1 1 +-9154303869107404275 1 0 1 1 1 1 +-9146648809962620241 0 0 1 1 1 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9128210600225023826 0 1 1 0 1 1 +-9121738304623869295 1 1 0 0 1 0 +-9120901636510510987 0 1 0 1 0 0 +-9119049435533988608 1 0 0 0 1 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -635,26 +635,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE (__table1.A AND __table1.B) OR ((__table1.B AND __table1.D) OR (__table1.E AND __table1.F)) --5591479919192876859 1 1 0 0 0 1 -2342517051443094765 1 1 0 0 0 1 --3440223316648760770 1 1 0 0 0 1 --4242513015859548468 1 1 0 0 0 1 --730351741811412687 1 1 0 0 0 1 --2389210960120872170 1 1 0 0 0 1 --4292066734783786923 1 1 0 0 0 1 --4414169073687495257 1 1 0 0 0 1 --6222496616288459687 1 1 0 0 0 1 --506440052941725006 1 1 0 0 0 1 --5591479919192876859 1 1 0 0 0 1 -2342517051443094765 1 1 0 0 0 1 --3440223316648760770 1 1 0 0 0 1 --4242513015859548468 1 1 0 0 0 1 --730351741811412687 1 1 0 0 0 1 --2389210960120872170 1 1 0 0 0 1 --4292066734783786923 1 1 0 0 0 1 --4414169073687495257 1 1 0 0 0 1 --6222496616288459687 1 1 0 0 0 1 --506440052941725006 1 1 0 0 0 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9121738304623869295 1 1 0 0 1 0 +-9114056182077943575 1 1 1 1 0 1 +-9001504240784412840 1 1 0 0 1 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8751975391509985483 1 1 1 0 1 1 +-8699036683208290351 1 1 1 0 0 1 +-9137658153367139416 1 1 1 0 0 0 +-9137309194079040284 1 1 1 0 0 0 +-9121738304623869295 1 1 0 0 1 0 +-9114056182077943575 1 1 1 1 0 1 +-9001504240784412840 1 1 0 0 1 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8751975391509985483 1 1 1 0 1 1 +-8699036683208290351 1 1 1 0 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 @@ -687,26 +687,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A AND __table1.B AND (__table1.C OR (__table1.E AND __table1.E) OR (__table1.F AND __table1.F)) --1198014781329290529 1 1 0 1 0 1 -3804289730569409059 1 1 0 1 0 1 -388942571848780949 1 1 0 1 0 1 -4303421928547547327 1 1 0 1 0 1 -5622057975514104466 1 1 0 1 0 1 -6003991070199658801 1 1 0 1 0 1 -6156030380809776164 1 1 0 1 0 1 -2991799832496995739 1 1 0 1 0 1 -3699238018341750080 1 1 0 1 0 1 +-9114056182077943575 1 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8751975391509985483 1 1 1 0 1 1 +-8699036683208290351 1 1 1 0 0 1 +-8647725213825146240 1 1 1 1 0 1 -8569013057605390975 1 1 0 1 0 1 --1198014781329290529 1 1 0 1 0 1 -3804289730569409059 1 1 0 1 0 1 -388942571848780949 1 1 0 1 0 1 -4303421928547547327 1 1 0 1 0 1 -5622057975514104466 1 1 0 1 0 1 -6003991070199658801 1 1 0 1 0 1 -6156030380809776164 1 1 0 1 0 1 -2991799832496995739 1 1 0 1 0 1 -3699238018341750080 1 1 0 1 0 1 +-8548886235737406452 1 1 0 1 0 1 +-8525570993468372516 1 1 1 1 1 0 +-9114056182077943575 1 1 1 1 0 1 +-8941493636296671386 1 1 1 1 1 1 +-8816638533025328863 1 1 1 0 1 1 +-8778069835572290074 1 1 1 1 1 1 +-8751975391509985483 1 1 1 0 1 1 +-8699036683208290351 1 1 1 0 0 1 +-8647725213825146240 1 1 1 1 0 1 -8569013057605390975 1 1 0 1 0 1 +-8548886235737406452 1 1 0 1 0 1 +-8525570993468372516 1 1 1 1 1 0 QUERY id: 0 PROJECTION COLUMNS count() UInt64 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index 22a2f716afc..af8a4dd8e73 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -19,68 +19,68 @@ EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE (A AND B) OR (A AND EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE (A AND B) OR (A AND C) SETTINGS optimize_extract_common_expressions = 1; -- Test multiple cases -SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND C AND F)) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND C AND F)); -SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A AND ((B AND C AND E) OR (B AND C AND F)); -SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A AND ((B AND (C AND E)) OR (B AND C AND F)); -SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A AND ((B AND C) OR (B AND D) OR (B AND E)); -SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A AND ((B AND C) OR ((B AND D) OR (B AND E))); -- Without AND as a root -SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((B AND C) OR (B AND C AND F)) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND C) OR (B AND C AND F)); -SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((B AND C AND E) OR (B AND C AND F)) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND C AND E) OR (B AND C AND F)); -SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND (C AND E)) OR (B AND C AND F)); -SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND C) OR (B AND D) OR (B AND E)); -SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((B AND C) OR ((B AND D) OR (B AND E))); -- Complex expression -SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE (A AND (sipHash64(C) = sipHash64(D))) OR (B AND (sipHash64(C) = sipHash64(D))); -- Flattening is only happening if something can be extracted -SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((A AND B) OR ((C AND D) OR (E AND F))); -SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((A AND B) OR ((B AND D) OR (E AND F))); -- Duplicates -SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE (A AND B AND C) OR ((A AND A AND A AND B AND B AND E AND E) OR (A AND B AND B AND F AND F)); -SELECT * FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) OR (B AND A AND F)) ORDER BY A, B, C, D, E, F LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; -SELECT * FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) OR (B AND A AND F)) ORDER BY A, B, C, D, E, F LIMIT 10; +SELECT * FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) OR (B AND A AND F)) ORDER BY x LIMIT 10 SETTINGS optimize_extract_common_expressions = 0; +SELECT * FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) OR (B AND A AND F)) ORDER BY x LIMIT 10; EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE ((A AND B AND C) OR (A AND B AND D)) AND ((B AND A AND E) OR (B AND A AND F)); From 935a592a598e02f46fdfc74c2e0a9b2b8e097e3a Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Mon, 2 Dec 2024 15:53:17 +0100 Subject: [PATCH 246/502] fix var ussage --- tests/queries/0_stateless/03277_async_insert.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/03277_async_insert.sh b/tests/queries/0_stateless/03277_async_insert.sh index ee2f848e953..ce45a6df274 100755 --- a/tests/queries/0_stateless/03277_async_insert.sh +++ b/tests/queries/0_stateless/03277_async_insert.sh @@ -7,12 +7,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -mkdir -p "${CLICKHOUSE_USER_FILES_UNIQUE}" -rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE}"/* +mkdir -p "${CLICKHOUSE_USER_FILES_UNIQUE:?}" +rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE:?}"/* -FILE_CSV="${CLICKHOUSE_USER_FILES_UNIQUE}/03277.csv" -FILE_ARROW="${CLICKHOUSE_USER_FILES_UNIQUE}/03277.arrow" +FILE_CSV="${CLICKHOUSE_USER_FILES_UNIQUE:?}/03277.csv" +FILE_ARROW="${CLICKHOUSE_USER_FILES_UNIQUE:?}/03277.arrow" $CLICKHOUSE_CLIENT -q " @@ -38,4 +38,4 @@ $CLICKHOUSE_CLIENT -q "SELECT * from t0" $CLICKHOUSE_CLIENT -q "DROP TABLE t0" -rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE}"/* \ No newline at end of file +rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE:?}"/* \ No newline at end of file From dd330d9aeed9d9c0cf21144bec28cfb5357aeaa2 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Mon, 2 Dec 2024 16:20:16 +0100 Subject: [PATCH 247/502] fix tags in test --- tests/queries/0_stateless/03277_async_insert.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/03277_async_insert.sh b/tests/queries/0_stateless/03277_async_insert.sh index ce45a6df274..46cfcc38e6a 100755 --- a/tests/queries/0_stateless/03277_async_insert.sh +++ b/tests/queries/0_stateless/03277_async_insert.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 692926c9abbc6ec7262a845197fb7edb7262f5d5 Mon Sep 17 00:00:00 2001 From: nauu Date: Mon, 2 Dec 2024 23:34:03 +0800 Subject: [PATCH 248/502] Trigger CI From 4eba52e0d5e3bfcb4996f7b6de45f02b43e9be7c Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 2 Dec 2024 15:35:58 +0000 Subject: [PATCH 249/502] Align PR setting changes history with 24.10 --- src/Core/SettingsChangesHistory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 4c64e2fe6d5..a99d114f0d8 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -63,7 +63,6 @@ static std::initializer_list Date: Mon, 2 Dec 2024 18:05:28 +0100 Subject: [PATCH 250/502] add a comment --- src/Databases/DatabaseReplicated.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index e3e4735c0d7..e7ce3f0a047 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -1226,7 +1226,12 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep query_context->setSetting("database_replicated_allow_explicit_uuid", 3); query_context->setSetting("database_replicated_allow_replicated_engine_arguments", 3); + /// We apply the flatten_nested setting after writing the CREATE query to the DDL log, + /// but before writing metadata to ZooKeeper. So we have to apply the setting on secondary replicas, but not in recovery mode. + /// Set it to false, so it will do nothing on recovery. The metadata in ZooKeeper should be used as is. + /// Same for data_type_default_nullable. query_context->setSetting("flatten_nested", false); + query_context->setSetting("data_type_default_nullable", false); auto txn = std::make_shared(current_zookeeper, zookeeper_path, false, ""); query_context->initZooKeeperMetadataTransaction(txn); From 6a4a8926752210a75c33198645e6defada44baef Mon Sep 17 00:00:00 2001 From: Michael Stetsyuk Date: Mon, 2 Dec 2024 17:40:34 +0000 Subject: [PATCH 251/502] empty From e3e2db1db7a4854fc9226aacec3efadc17f57a69 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 2 Dec 2024 17:45:24 +0000 Subject: [PATCH 252/502] Remove old long test --- .../03270_object_to_json_alter.reference | 132 ------------------ .../03270_object_to_json_alter.sql | 58 -------- 2 files changed, 190 deletions(-) delete mode 100644 tests/queries/0_stateless/03270_object_to_json_alter.reference delete mode 100644 tests/queries/0_stateless/03270_object_to_json_alter.sql diff --git a/tests/queries/0_stateless/03270_object_to_json_alter.reference b/tests/queries/0_stateless/03270_object_to_json_alter.reference deleted file mode 100644 index a04884fb374..00000000000 --- a/tests/queries/0_stateless/03270_object_to_json_alter.reference +++ /dev/null @@ -1,132 +0,0 @@ -{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} -0 -100 -200 -300 -400 -500 -600 -700 -800 -900 -0 -99 -199 -299 -399 -499 -599 -699 -799 -899 -999 -{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} -0 -100 -200 -300 -400 -500 -600 -700 -800 -900 -0 -99 -199 -299 -399 -499 -599 -699 -799 -899 -999 -{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} -0 -100 -200 -300 -400 -500 -600 -700 -800 -900 -0 -99 -199 -299 -399 -499 -599 -699 -799 -899 -999 -{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} -0 -100 -200 -300 -400 -500 -600 -700 -800 -900 -0 -99 -199 -299 -399 -499 -599 -699 -799 -899 -999 -{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} -0 -100 -200 -300 -400 -500 -600 -700 -800 -900 -0 -99 -199 -299 -399 -499 -599 -699 -799 -899 -999 -{'a0':['Int64'],'a1':['Int64'],'a10':['Int64'],'a11':['Int64'],'a12':['Int64'],'a13':['Int64'],'a14':['Int64'],'a15':['Int64'],'a16':['Int64'],'a17':['Int64'],'a18':['Int64'],'a19':['Int64'],'a2':['Int64'],'a20':['Int64'],'a21':['Int64'],'a22':['Int64'],'a23':['Int64'],'a24':['Int64'],'a25':['Int64'],'a26':['Int64'],'a27':['Int64'],'a28':['Int64'],'a29':['Int64'],'a3':['Int64'],'a30':['Int64'],'a31':['Int64'],'a32':['Int64'],'a33':['Int64'],'a34':['Int64'],'a35':['Int64'],'a36':['Int64'],'a37':['Int64'],'a38':['Int64'],'a39':['Int64'],'a4':['Int64'],'a40':['Int64'],'a41':['Int64'],'a42':['Int64'],'a43':['Int64'],'a44':['Int64'],'a45':['Int64'],'a46':['Int64'],'a47':['Int64'],'a48':['Int64'],'a49':['Int64'],'a5':['Int64'],'a50':['Int64'],'a51':['Int64'],'a52':['Int64'],'a53':['Int64'],'a54':['Int64'],'a55':['Int64'],'a56':['Int64'],'a57':['Int64'],'a58':['Int64'],'a59':['Int64'],'a6':['Int64'],'a60':['Int64'],'a61':['Int64'],'a62':['Int64'],'a63':['Int64'],'a64':['Int64'],'a65':['Int64'],'a66':['Int64'],'a67':['Int64'],'a68':['Int64'],'a69':['Int64'],'a7':['Int64'],'a70':['Int64'],'a71':['Int64'],'a72':['Int64'],'a73':['Int64'],'a74':['Int64'],'a75':['Int64'],'a76':['Int64'],'a77':['Int64'],'a78':['Int64'],'a79':['Int64'],'a8':['Int64'],'a80':['Int64'],'a81':['Int64'],'a82':['Int64'],'a83':['Int64'],'a84':['Int64'],'a85':['Int64'],'a86':['Int64'],'a87':['Int64'],'a88':['Int64'],'a89':['Int64'],'a9':['Int64'],'a90':['Int64'],'a91':['Int64'],'a92':['Int64'],'a93':['Int64'],'a94':['Int64'],'a95':['Int64'],'a96':['Int64'],'a97':['Int64'],'a98':['Int64'],'a99':['Int64']} -0 -100 -200 -300 -400 -500 -600 -700 -800 -900 -0 -99 -199 -299 -399 -499 -599 -699 -799 -899 -999 diff --git a/tests/queries/0_stateless/03270_object_to_json_alter.sql b/tests/queries/0_stateless/03270_object_to_json_alter.sql deleted file mode 100644 index 1191857d484..00000000000 --- a/tests/queries/0_stateless/03270_object_to_json_alter.sql +++ /dev/null @@ -1,58 +0,0 @@ --- Tags: long - -set allow_experimental_object_type = 1; -set allow_experimental_json_type = 1; -set max_block_size = 100; -set max_insert_block_size = 100; -set min_insert_block_size_rows = 100; -set output_format_json_quote_64bit_integers = 0; - -drop table if exists test; - -create table test (json Object('json')) engine=Memory; -insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); -alter table test modify column json JSON; -select distinctJSONPathsAndTypes(json) from test; -select distinct json.a0 from test order by json.a0.:Int64; -select distinct json.a99 from test order by json.a99.:Int64; -drop table test; - -create table test (json Object('json')) engine=Memory; -insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); -alter table test modify column json JSON(max_dynamic_paths=10); -select distinctJSONPathsAndTypes(json) from test; -select distinct json.a0 from test order by json.a0.:Int64; -select distinct json.a99 from test order by json.a99.:Int64; -drop table test; - -create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=10000000, min_bytes_for_wide_part=100000000; -insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); -alter table test modify column json JSON; -select distinctJSONPathsAndTypes(json) from test; -select distinct json.a0 from test order by json.a0.:Int64; -select distinct json.a99 from test order by json.a99.:Int64; -drop table test; - -create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=10000000, min_bytes_for_wide_part=100000000; -insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); -alter table test modify column json JSON(max_dynamic_paths=10); -select distinctJSONPathsAndTypes(json) from test; -select distinct json.a0 from test order by json.a0.:Int64; -select distinct json.a99 from test order by json.a99.:Int64; -drop table test; - -create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1; -insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); -alter table test modify column json JSON(); -select distinctJSONPathsAndTypes(json) from test; -select distinct json.a0 from test order by json.a0.:Int64; -select distinct json.a99 from test order by json.a99.:Int64; -drop table test; - -create table test (json Object('json')) engine=MergeTree order by tuple() settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1; -insert into test select toJSONString(map('a' || number % 100, number)) from numbers(1000); -alter table test modify column json JSON(max_dynamic_paths=10); -select distinctJSONPathsAndTypes(json) from test; -select distinct json.a0 from test order by json.a0.:Int64; -select distinct json.a99 from test order by json.a99.:Int64; -drop table test; From d49e22cbae1eacbe0758f611a85b352a174563f3 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 2 Dec 2024 17:50:32 +0000 Subject: [PATCH 253/502] Fix builds --- src/TableFunctions/ITableFunctionFileLike.cpp | 36 ------------------- src/TableFunctions/ITableFunctionFileLike.h | 36 ++++++++++++++++++- 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index b591ea97e3d..159eb0a8f5d 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -88,42 +88,6 @@ void ITableFunctionFileLike::parseArgumentsImpl(ASTs & args, const ContextPtr & compression_method = checkAndGetLiteralArgument(args[3], "compression_method"); } -void ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr & context, bool with_structure) -{ - if (args.empty() || args.size() > getMaxNumberOfArguments()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function, got {}", getMaxNumberOfArguments(), args.size()); - - auto format_literal = std::make_shared(format); - auto structure_literal = std::make_shared(structure); - - for (auto & arg : args) - arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - - /// f(filename) - if (args.size() == 1) - { - args.push_back(format_literal); - if (with_structure) - args.push_back(structure_literal); - } - /// f(filename, format) - else if (args.size() == 2) - { - if (checkAndGetLiteralArgument(args[1], "format") == "auto") - args.back() = format_literal; - if (with_structure) - args.push_back(structure_literal); - } - /// f(filename, format, structure) or f(filename, format, structure, compression) or f(filename, format, compression) - else if (args.size() >= 3) - { - if (checkAndGetLiteralArgument(args[1], "format") == "auto") - args[1] = format_literal; - if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") - args[2] = structure_literal; - } -} - StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/, bool /*is_insert_query*/) const { ColumnsDescription columns; diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index f9b5e30e85e..ef8226ea62e 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -35,7 +35,41 @@ public: static size_t getMaxNumberOfArguments() { return max_number_of_arguments; } - static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &, bool with_structure); + static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &, bool with_structure) + { + if (args.empty() || args.size() > getMaxNumberOfArguments()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function, got {}", getMaxNumberOfArguments(), args.size()); + + auto format_literal = std::make_shared(format); + auto structure_literal = std::make_shared(structure); + + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); + + /// f(filename) + if (args.size() == 1) + { + args.push_back(format_literal); + if (with_structure) + args.push_back(structure_literal); + } + /// f(filename, format) + else if (args.size() == 2) + { + if (checkAndGetLiteralArgument(args[1], "format") == "auto") + args.back() = format_literal; + if (with_structure) + args.push_back(structure_literal); + } + /// f(filename, format, structure) or f(filename, format, structure, compression) or f(filename, format, compression) + else if (args.size() >= 3) + { + if (checkAndGetLiteralArgument(args[1], "format") == "auto") + args[1] = format_literal; + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") + args[2] = structure_literal; + } + } protected: From 6a31154ccdffd45f9c4c31ed4d173e5cf2c8c5c5 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 2 Dec 2024 18:17:25 +0000 Subject: [PATCH 254/502] Fix tests --- src/Functions/FunctionsConversion.cpp | 8 ++++++++ .../0_stateless/03276_empty_variant_type.reference | 0 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/03276_empty_variant_type.reference diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index bbe87b600a9..3fa695b39bf 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -4635,6 +4635,14 @@ private: if (const auto * variant_type = typeid_cast(from_type.get())) return createVariantToDynamicWrapper(*variant_type, dynamic_type); + if (from_type->onlyNull()) + return [](ColumnsWithTypeAndName &, const DataTypePtr & result_type, const ColumnNullable *, size_t input_rows_count) -> ColumnPtr + { + auto result = result_type->createColumn(); + result->insertManyDefaults(input_rows_count); + return result; + }; + if (context && context->getSettingsRef()[Setting::cast_string_to_dynamic_use_inference] && isStringOrFixedString(removeNullable(removeLowCardinality(from_type)))) return createStringToDynamicThroughParsingWrapper(); diff --git a/tests/queries/0_stateless/03276_empty_variant_type.reference b/tests/queries/0_stateless/03276_empty_variant_type.reference new file mode 100644 index 00000000000..e69de29bb2d From deddd1db312822460dcf163c7ccf1160862af6a1 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:22:04 +0100 Subject: [PATCH 255/502] Change the logic --- .../reference/groupconcat.md | 3 +- .../AggregateFunctionGroupConcat.cpp | 56 ++++--------------- src/Parsers/ASTFunction.cpp | 42 +++++++++++++- src/Parsers/ASTFunction.h | 2 + .../0_stateless/03156_group_concat.sql | 4 +- 5 files changed, 56 insertions(+), 51 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md index 7f22e4125a6..304676f1819 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md +++ b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md @@ -15,8 +15,9 @@ groupConcat[(delimiter [, limit])](expression); **Arguments** +- `delimiter` — A [string](../../../sql-reference/data-types/string.md) that will be used to separate concatenated values. This parameter is optional and defaults to an empty string or delimiter from parameters if not specified. - `expression` — The expression or column name that outputs strings to be concatenated. -- `delimiter` — A [string](../../../sql-reference/data-types/string.md) that will be used to separate concatenated values. This parameter is optional and defaults to an empty string if not specified. + **Parameters** diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp index f27c99db0b0..8cf5ec5705a 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp @@ -31,24 +31,16 @@ namespace ErrorCodes extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } namespace { -enum GroupConcatOverload -{ - ONE_ARGUMENT, - TWO_ARGUMENTS -}; - struct GroupConcatDataBase { UInt64 data_size = 0; UInt64 allocated_size = 0; char * data = nullptr; - String data_delimiter; void checkAndUpdateSize(UInt64 add, Arena * arena) { @@ -125,18 +117,16 @@ class GroupConcatImpl final UInt64 limit; const String delimiter; const DataTypePtr type; - GroupConcatOverload overload = ONE_ARGUMENT; public: - GroupConcatImpl(const DataTypes & data_types, const Array & parameters_, UInt64 limit_, const String & delimiter_) + GroupConcatImpl(const DataTypePtr & data_type_, const Array & parameters_, UInt64 limit_, const String & delimiter_) : IAggregateFunctionDataHelper, GroupConcatImpl>( - {data_types}, parameters_, std::make_shared()) + {data_type_}, parameters_, std::make_shared()) , limit(limit_) , delimiter(delimiter_) - , type(data_types[0]) + , type(data_type_) { serialization = isFixedString(type) ? std::make_shared()->getDefaultSerialization() : this->argument_types[0]->getDefaultSerialization(); - overload = data_types.size() > 1 ? TWO_ARGUMENTS : ONE_ARGUMENT; } String getName() const override { return name; } @@ -144,20 +134,13 @@ public: void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { auto & cur_data = this->data(place); - if (cur_data.data_delimiter.empty()) - { - cur_data.data_delimiter = delimiter; - if (overload == GroupConcatOverload::TWO_ARGUMENTS) - cur_data.data_delimiter = columns[1]->getDataAt(0).toString(); - } - const String & delim = cur_data.data_delimiter; if constexpr (has_limit) if (cur_data.num_rows >= limit) return; if (cur_data.data_size != 0) - cur_data.insertChar(delim.c_str(), delim.size(), arena); + cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); if (isFixedString(type)) { @@ -177,15 +160,13 @@ public: if (rhs_data.data_size == 0) return; - const String & delim = cur_data.data_delimiter; - if constexpr (has_limit) { UInt64 new_elems_count = std::min(rhs_data.num_rows, limit - cur_data.num_rows); for (UInt64 i = 0; i < new_elems_count; ++i) { if (cur_data.data_size != 0) - cur_data.insertChar(delim.c_str(), delim.size(), arena); + cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); cur_data.offsets.push_back(cur_data.data_size, arena); cur_data.insertChar(rhs_data.data + rhs_data.getString(i), rhs_data.getSize(i), arena); @@ -196,7 +177,7 @@ public: else { if (cur_data.data_size != 0) - cur_data.insertChar(delim.c_str(), delim.size(), arena); + cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); cur_data.insertChar(rhs_data.data, rhs_data.data_size, arena); } @@ -257,16 +238,9 @@ public: }; AggregateFunctionPtr createAggregateFunctionGroupConcat( - const std::string & name, - const DataTypes & argument_types, - const Array & parameters, - const Settings * /* settings */ -) + const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { - // Ensure the number of arguments is between 1 and 2 - if (argument_types.empty() || argument_types.size() > 2) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Incorrect number of arguments for function {}, expected 1 to 2, got {}", name, argument_types.size()); + assertUnary(name, argument_types); bool has_limit = false; UInt64 limit = 0; @@ -299,19 +273,9 @@ AggregateFunctionPtr createAggregateFunctionGroupConcat( limit = parameters[1].safeGet(); } - // Handle additional arguments if provided (delimiter and limit as arguments) - if (argument_types.size() == 2) - { - // Second argument should be delimiter (string) - const DataTypePtr & delimiter_type = argument_types[1]; - if (!isString(delimiter_type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Second argument for function {} must be a string", name); - } - if (has_limit) - return std::make_shared>(argument_types, parameters, limit, delimiter); - return std::make_shared>(argument_types, parameters, limit, delimiter); + return std::make_shared>(argument_types[0], parameters, limit, delimiter); + return std::make_shared>(argument_types[0], parameters, limit, delimiter); } } diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 11cfe2e584e..358b7b1e26b 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -130,13 +130,51 @@ String ASTFunction::getID(char delim) const return "Function" + (delim + name); } +void ASTFunction::groupConcatArgumentOverride(std::shared_ptr res) const +{ + // Clone the first argument to be used as a parameter + ASTPtr first_arg = arguments->children[0]->clone(); + + // Clone the second argument to remain as the function argument + ASTPtr second_arg = arguments->children[1]->clone(); + + // Initialize or clear parameters + if (!res->parameters) + res->parameters = std::make_shared(); + else + res->parameters->children.clear(); + + // Add the first argument as a parameter + res->parameters->children.emplace_back(first_arg); + res->children.emplace_back(res->parameters); + + // Initialize arguments with the second argument only + res->arguments = std::make_shared(); + res->arguments->children.emplace_back(second_arg); + res->children.emplace_back(res->arguments); +} + ASTPtr ASTFunction::clone() const { auto res = std::make_shared(*this); res->children.clear(); - if (arguments) { res->arguments = arguments->clone(); res->children.push_back(res->arguments); } - if (parameters) { res->parameters = parameters->clone(); res->children.push_back(res->parameters); } + // Special handling for groupConcat with two arguments + if (name == "groupConcat" && arguments && arguments->children.size() == 2) + groupConcatArgumentOverride(res); + else + { + if (arguments) + { + res->arguments = arguments->clone(); + res->children.push_back(res->arguments); + } + if (parameters) + { + res->parameters = parameters->clone(); + res->children.push_back(res->parameters); + } + } if (window_definition) { diff --git a/src/Parsers/ASTFunction.h b/src/Parsers/ASTFunction.h index 1b4a5928d1c..b6aae46d21e 100644 --- a/src/Parsers/ASTFunction.h +++ b/src/Parsers/ASTFunction.h @@ -81,6 +81,8 @@ public: bool hasSecretParts() const override; + void groupConcatArgumentOverride(std::shared_ptr res) const; + protected: void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; diff --git a/tests/queries/0_stateless/03156_group_concat.sql b/tests/queries/0_stateless/03156_group_concat.sql index 8989bb20df9..c6fea68551a 100644 --- a/tests/queries/0_stateless/03156_group_concat.sql +++ b/tests/queries/0_stateless/03156_group_concat.sql @@ -44,9 +44,9 @@ SELECT length(groupConcat(number)) FROM numbers(100000); SELECT 'TESTING GroupConcat second argument overload'; -SELECT groupConcat(p_int, ',') FROM test_groupConcat; +SELECT groupConcat(',', p_int) FROM test_groupConcat; SELECT groupConcat('.')(p_string) FROM test_groupConcat; -SELECT groupConcat(p_array, '/') FROM test_groupConcat; +SELECT groupConcat('/', p_array) FROM test_groupConcat; DROP TABLE IF EXISTS test_groupConcat; From 802f98856a843e600a386fc56d4643daaf6d8b70 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:25:07 +0100 Subject: [PATCH 256/502] Add aliased version --- src/Parsers/ASTFunction.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 358b7b1e26b..bac512545d9 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -160,7 +160,7 @@ ASTPtr ASTFunction::clone() const res->children.clear(); // Special handling for groupConcat with two arguments - if (name == "groupConcat" && arguments && arguments->children.size() == 2) + if ((name == "groupConcat" || Poco::toLower(name) == "group_concat") && arguments && arguments->children.size() == 2) groupConcatArgumentOverride(res); else { From d0a11071d4736884acacefa03946d96efd5694ba Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 2 Dec 2024 13:25:59 +0100 Subject: [PATCH 257/502] Remove ostr from structure IAST::FormatSettings. --- programs/format/Format.cpp | 8 +- src/Client/ClientBase.cpp | 4 +- src/Functions/formatQuery.cpp | 4 +- src/Interpreters/InterpreterExplainQuery.cpp | 4 +- src/Interpreters/SystemLog.cpp | 2 +- src/Interpreters/executeQuery.cpp | 4 +- src/Parsers/ASTAlterNamedCollectionQuery.cpp | 28 +- src/Parsers/ASTAlterNamedCollectionQuery.h | 2 +- src/Parsers/ASTAlterQuery.cpp | 330 +++++++++--------- src/Parsers/ASTAlterQuery.h | 4 +- src/Parsers/ASTAssignment.h | 12 +- src/Parsers/ASTAsterisk.cpp | 10 +- src/Parsers/ASTAsterisk.h | 2 +- src/Parsers/ASTBackupQuery.cpp | 110 +++--- src/Parsers/ASTBackupQuery.h | 2 +- src/Parsers/ASTCheckQuery.h | 20 +- src/Parsers/ASTCollation.cpp | 7 +- src/Parsers/ASTCollation.h | 2 +- src/Parsers/ASTColumnDeclaration.cpp | 42 +-- src/Parsers/ASTColumnDeclaration.h | 2 +- src/Parsers/ASTColumnsMatcher.cpp | 64 ++-- src/Parsers/ASTColumnsMatcher.h | 8 +- src/Parsers/ASTColumnsTransformers.cpp | 56 +-- src/Parsers/ASTColumnsTransformers.h | 10 +- src/Parsers/ASTConstraintDeclaration.cpp | 8 +- src/Parsers/ASTConstraintDeclaration.h | 2 +- src/Parsers/ASTCreateFunctionQuery.cpp | 20 +- src/Parsers/ASTCreateFunctionQuery.h | 2 +- src/Parsers/ASTCreateIndexQuery.cpp | 24 +- src/Parsers/ASTCreateIndexQuery.h | 2 +- src/Parsers/ASTCreateNamedCollectionQuery.cpp | 22 +- src/Parsers/ASTCreateNamedCollectionQuery.h | 2 +- src/Parsers/ASTCreateQuery.cpp | 206 +++++------ src/Parsers/ASTCreateQuery.h | 6 +- src/Parsers/ASTCreateResourceQuery.cpp | 32 +- src/Parsers/ASTCreateResourceQuery.h | 2 +- src/Parsers/ASTCreateWorkloadQuery.cpp | 30 +- src/Parsers/ASTCreateWorkloadQuery.h | 2 +- src/Parsers/ASTDataType.cpp | 18 +- src/Parsers/ASTDataType.h | 2 +- src/Parsers/ASTDatabaseOrNone.cpp | 6 +- src/Parsers/ASTDatabaseOrNone.h | 2 +- src/Parsers/ASTDeleteQuery.cpp | 20 +- src/Parsers/ASTDeleteQuery.h | 2 +- src/Parsers/ASTDescribeCacheQuery.cpp | 4 +- src/Parsers/ASTDescribeCacheQuery.h | 2 +- src/Parsers/ASTDictionary.cpp | 68 ++-- src/Parsers/ASTDictionary.h | 10 +- .../ASTDictionaryAttributeDeclaration.cpp | 24 +- .../ASTDictionaryAttributeDeclaration.h | 2 +- src/Parsers/ASTDropFunctionQuery.cpp | 12 +- src/Parsers/ASTDropFunctionQuery.h | 2 +- src/Parsers/ASTDropIndexQuery.cpp | 20 +- src/Parsers/ASTDropIndexQuery.h | 2 +- src/Parsers/ASTDropNamedCollectionQuery.cpp | 10 +- src/Parsers/ASTDropNamedCollectionQuery.h | 2 +- src/Parsers/ASTDropQuery.cpp | 50 +-- src/Parsers/ASTDropQuery.h | 2 +- src/Parsers/ASTDropResourceQuery.cpp | 12 +- src/Parsers/ASTDropResourceQuery.h | 2 +- src/Parsers/ASTDropWorkloadQuery.cpp | 12 +- src/Parsers/ASTDropWorkloadQuery.h | 2 +- src/Parsers/ASTExplainQuery.h | 20 +- src/Parsers/ASTExpressionList.cpp | 28 +- src/Parsers/ASTExpressionList.h | 4 +- src/Parsers/ASTExternalDDLQuery.h | 8 +- src/Parsers/ASTFunction.cpp | 222 ++++++------ src/Parsers/ASTFunction.h | 4 +- .../ASTFunctionWithKeyValueArguments.cpp | 30 +- .../ASTFunctionWithKeyValueArguments.h | 4 +- src/Parsers/ASTIdentifier.cpp | 14 +- src/Parsers/ASTIdentifier.h | 2 +- src/Parsers/ASTIndexDeclaration.cpp | 24 +- src/Parsers/ASTIndexDeclaration.h | 2 +- src/Parsers/ASTInsertQuery.cpp | 42 +-- src/Parsers/ASTInsertQuery.h | 2 +- src/Parsers/ASTInterpolateElement.cpp | 6 +- src/Parsers/ASTInterpolateElement.h | 2 +- src/Parsers/ASTKillQueryQuery.cpp | 22 +- src/Parsers/ASTKillQueryQuery.h | 2 +- src/Parsers/ASTLiteral.cpp | 6 +- src/Parsers/ASTLiteral.h | 2 +- src/Parsers/ASTNameTypePair.cpp | 6 +- src/Parsers/ASTNameTypePair.h | 2 +- src/Parsers/ASTObjectTypeArgument.cpp | 14 +- src/Parsers/ASTObjectTypeArgument.h | 2 +- src/Parsers/ASTOptimizeQuery.cpp | 26 +- src/Parsers/ASTOptimizeQuery.h | 2 +- src/Parsers/ASTOrderByElement.cpp | 30 +- src/Parsers/ASTOrderByElement.h | 2 +- src/Parsers/ASTPartition.cpp | 10 +- src/Parsers/ASTPartition.h | 2 +- src/Parsers/ASTProjectionDeclaration.cpp | 10 +- src/Parsers/ASTProjectionDeclaration.h | 2 +- src/Parsers/ASTProjectionSelectQuery.cpp | 20 +- src/Parsers/ASTProjectionSelectQuery.h | 2 +- src/Parsers/ASTQualifiedAsterisk.cpp | 8 +- src/Parsers/ASTQualifiedAsterisk.h | 2 +- src/Parsers/ASTQueryParameter.cpp | 4 +- src/Parsers/ASTQueryParameter.h | 2 +- src/Parsers/ASTQueryWithOnCluster.cpp | 4 +- src/Parsers/ASTQueryWithOnCluster.h | 2 +- src/Parsers/ASTQueryWithOutput.cpp | 26 +- src/Parsers/ASTQueryWithOutput.h | 8 +- src/Parsers/ASTQueryWithTableAndOutput.h | 10 +- src/Parsers/ASTRefreshStrategy.cpp | 32 +- src/Parsers/ASTRefreshStrategy.h | 2 +- src/Parsers/ASTRenameQuery.h | 46 +-- src/Parsers/ASTSQLSecurity.cpp | 20 +- src/Parsers/ASTSQLSecurity.h | 2 +- src/Parsers/ASTSampleRatio.cpp | 4 +- src/Parsers/ASTSampleRatio.h | 2 +- src/Parsers/ASTSelectIntersectExceptQuery.cpp | 6 +- src/Parsers/ASTSelectIntersectExceptQuery.h | 2 +- src/Parsers/ASTSelectQuery.cpp | 118 +++---- src/Parsers/ASTSelectQuery.h | 2 +- src/Parsers/ASTSelectWithUnionQuery.cpp | 14 +- src/Parsers/ASTSelectWithUnionQuery.h | 2 +- src/Parsers/ASTSetQuery.cpp | 24 +- src/Parsers/ASTSetQuery.h | 2 +- src/Parsers/ASTShowColumnsQuery.cpp | 18 +- src/Parsers/ASTShowColumnsQuery.h | 2 +- src/Parsers/ASTShowFunctionsQuery.cpp | 6 +- src/Parsers/ASTShowFunctionsQuery.h | 2 +- src/Parsers/ASTShowIndexesQuery.cpp | 12 +- src/Parsers/ASTShowIndexesQuery.h | 2 +- src/Parsers/ASTShowSettingQuery.cpp | 4 +- src/Parsers/ASTShowSettingQuery.h | 2 +- src/Parsers/ASTShowTablesQuery.cpp | 58 +-- src/Parsers/ASTShowTablesQuery.h | 6 +- src/Parsers/ASTStatisticsDeclaration.cpp | 10 +- src/Parsers/ASTStatisticsDeclaration.h | 2 +- src/Parsers/ASTSubquery.cpp | 14 +- src/Parsers/ASTSubquery.h | 2 +- src/Parsers/ASTSystemQuery.cpp | 82 ++--- src/Parsers/ASTSystemQuery.h | 2 +- src/Parsers/ASTTTLElement.cpp | 32 +- src/Parsers/ASTTTLElement.h | 2 +- src/Parsers/ASTTableOverrides.cpp | 28 +- src/Parsers/ASTTableOverrides.h | 4 +- src/Parsers/ASTTablesInSelectQuery.cpp | 106 +++--- src/Parsers/ASTTablesInSelectQuery.h | 14 +- src/Parsers/ASTTimeInterval.cpp | 8 +- src/Parsers/ASTTimeInterval.h | 2 +- src/Parsers/ASTTransactionControl.cpp | 10 +- src/Parsers/ASTTransactionControl.h | 2 +- src/Parsers/ASTUndropQuery.cpp | 14 +- src/Parsers/ASTUndropQuery.h | 2 +- src/Parsers/ASTUseQuery.h | 6 +- src/Parsers/ASTViewTargets.cpp | 18 +- src/Parsers/ASTViewTargets.h | 6 +- src/Parsers/ASTWatchQuery.h | 16 +- src/Parsers/ASTWindowDefinition.cpp | 56 +-- src/Parsers/ASTWindowDefinition.h | 4 +- src/Parsers/ASTWithAlias.cpp | 24 +- src/Parsers/ASTWithAlias.h | 4 +- src/Parsers/ASTWithElement.cpp | 14 +- src/Parsers/ASTWithElement.h | 2 +- src/Parsers/Access/ASTAuthenticationData.cpp | 40 +-- src/Parsers/Access/ASTAuthenticationData.h | 2 +- src/Parsers/Access/ASTCheckGrantQuery.cpp | 8 +- src/Parsers/Access/ASTCheckGrantQuery.h | 2 +- src/Parsers/Access/ASTCreateQuotaQuery.cpp | 80 ++--- src/Parsers/Access/ASTCreateQuotaQuery.h | 2 +- src/Parsers/Access/ASTCreateRoleQuery.cpp | 40 +-- src/Parsers/Access/ASTCreateRoleQuery.h | 2 +- .../Access/ASTCreateRowPolicyQuery.cpp | 69 ++-- src/Parsers/Access/ASTCreateRowPolicyQuery.h | 2 +- .../Access/ASTCreateSettingsProfileQuery.cpp | 48 +-- .../Access/ASTCreateSettingsProfileQuery.h | 2 +- src/Parsers/Access/ASTCreateUserQuery.cpp | 140 ++++---- src/Parsers/Access/ASTCreateUserQuery.h | 2 +- .../Access/ASTDropAccessEntityQuery.cpp | 20 +- src/Parsers/Access/ASTDropAccessEntityQuery.h | 2 +- src/Parsers/Access/ASTGrantQuery.cpp | 38 +- src/Parsers/Access/ASTGrantQuery.h | 2 +- .../Access/ASTMoveAccessEntityQuery.cpp | 20 +- src/Parsers/Access/ASTMoveAccessEntityQuery.h | 2 +- src/Parsers/Access/ASTPublicSSHKey.cpp | 10 +- src/Parsers/Access/ASTPublicSSHKey.h | 2 +- src/Parsers/Access/ASTRolesOrUsersSet.cpp | 32 +- src/Parsers/Access/ASTRolesOrUsersSet.h | 2 +- src/Parsers/Access/ASTRowPolicyName.cpp | 36 +- src/Parsers/Access/ASTRowPolicyName.h | 4 +- src/Parsers/Access/ASTSetRoleQuery.cpp | 20 +- src/Parsers/Access/ASTSetRoleQuery.h | 2 +- .../Access/ASTSettingsProfileElement.cpp | 34 +- .../Access/ASTSettingsProfileElement.h | 4 +- .../Access/ASTShowAccessEntitiesQuery.cpp | 12 +- .../Access/ASTShowAccessEntitiesQuery.h | 2 +- .../Access/ASTShowCreateAccessEntityQuery.cpp | 24 +- .../Access/ASTShowCreateAccessEntityQuery.h | 2 +- src/Parsers/Access/ASTShowGrantsQuery.cpp | 12 +- src/Parsers/Access/ASTShowGrantsQuery.h | 2 +- src/Parsers/Access/ASTUserNameWithHost.cpp | 12 +- src/Parsers/Access/ASTUserNameWithHost.h | 4 +- src/Parsers/IAST.cpp | 6 +- src/Parsers/IAST.h | 18 +- src/Parsers/MySQL/ASTAlterCommand.h | 2 +- src/Parsers/MySQL/ASTAlterQuery.h | 2 +- src/Parsers/MySQL/ASTCreateDefines.h | 2 +- src/Parsers/MySQL/ASTCreateQuery.h | 2 +- src/Parsers/MySQL/ASTDeclareColumn.h | 2 +- src/Parsers/MySQL/ASTDeclareConstraint.h | 2 +- src/Parsers/MySQL/ASTDeclareIndex.h | 2 +- src/Parsers/MySQL/ASTDeclareOption.h | 2 +- src/Parsers/MySQL/ASTDeclarePartition.h | 2 +- .../MySQL/ASTDeclarePartitionOptions.h | 2 +- src/Parsers/MySQL/ASTDeclareReference.h | 2 +- src/Parsers/MySQL/ASTDeclareSubPartition.h | 2 +- src/Parsers/MySQL/ASTDropQuery.h | 2 +- src/Parsers/TablePropertiesQueriesASTs.h | 18 +- src/Parsers/formatAST.cpp | 4 +- src/Parsers/getInsertQuery.cpp | 3 +- src/Parsers/tests/gtest_format_hiliting.cpp | 4 +- src/Processors/QueryPlan/ReadFromRemote.cpp | 4 +- src/Storages/ColumnsDescription.cpp | 4 +- src/Storages/StorageDistributed.cpp | 8 +- src/Storages/StorageReplicatedMergeTree.cpp | 4 +- .../transformQueryForExternalDatabase.cpp | 3 +- 220 files changed, 1869 insertions(+), 1875 deletions(-) diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index 522b9a74cff..40d6673fb0d 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -277,10 +277,10 @@ int mainEntryClickHouseFormat(int argc, char ** argv) { WriteBufferFromOwnString str_buf; bool oneline_current_query = oneline || approx_query_length < max_line_length; - IAST::FormatSettings settings(str_buf, oneline_current_query, hilite); + IAST::FormatSettings settings(oneline_current_query, hilite); settings.show_secrets = true; settings.print_pretty_type_names = !oneline_current_query; - res->format(settings); + res->format(str_buf, settings); if (insert_query_payload) { @@ -324,10 +324,10 @@ int mainEntryClickHouseFormat(int argc, char ** argv) { WriteBufferFromOwnString str_buf; bool oneline_current_query = oneline || approx_query_length < max_line_length; - IAST::FormatSettings settings(str_buf, oneline_current_query, hilite); + IAST::FormatSettings settings(oneline_current_query, hilite); settings.show_secrets = true; settings.print_pretty_type_names = !oneline_current_query; - res->format(settings); + res->format(str_buf, settings); auto res_string = str_buf.str(); WriteBufferFromOStream res_cout(std::cout, 4096); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 93dea07a43f..ac41aaf07f0 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -348,11 +348,11 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Setting { output_stream << std::endl; WriteBufferFromOStream res_buf(output_stream, 4096); - IAST::FormatSettings format_settings(res_buf, /* one_line */ false); + IAST::FormatSettings format_settings(/* one_line */ false); format_settings.hilite = true; format_settings.show_secrets = true; format_settings.print_pretty_type_names = true; - res->format(format_settings); + res->format(res_buf, format_settings); res_buf.finalize(); output_stream << std::endl << std::endl; } diff --git a/src/Functions/formatQuery.cpp b/src/Functions/formatQuery.cpp index b850fd20035..50e720d897a 100644 --- a/src/Functions/formatQuery.cpp +++ b/src/Functions/formatQuery.cpp @@ -143,10 +143,10 @@ private: throw; } - IAST::FormatSettings settings(buf, output_formatting == OutputFormatting::SingleLine, /*hilite*/ false); + IAST::FormatSettings settings(output_formatting == OutputFormatting::SingleLine, /*hilite*/ false); settings.show_secrets = true; settings.print_pretty_type_names = print_pretty_type_names; - ast->format(settings); + ast->format(buf, settings); auto formatted = buf.stringView(); diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index 763d8201dff..85dd4bcc947 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -392,7 +392,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl() ExplainAnalyzedSyntaxVisitor::Data data(getContext()); ExplainAnalyzedSyntaxVisitor(data).visit(query); - ast.getExplainedQuery()->format(IAST::FormatSettings(buf, settings.oneline)); + ast.getExplainedQuery()->format(buf, IAST::FormatSettings(settings.oneline)); break; } case ASTExplainQuery::QueryTree: @@ -441,7 +441,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl() if (need_newline) buf << "\n\n"; - query_tree->toAST()->format(IAST::FormatSettings(buf, false)); + query_tree->toAST()->format(buf, IAST::FormatSettings(false)); } break; diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index cf0de1aa744..92d282e7e70 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -88,7 +88,7 @@ namespace throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method clone is not supported"); } - void formatImpl(const FormatSettings &, FormatState &, FormatStateStacked) const override + void formatImpl(WriteBuffer &, const FormatSettings &, FormatState &, FormatStateStacked) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported"); } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 2139353917e..f22db733dc5 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -1041,9 +1041,9 @@ static std::tuple executeQueryImpl( if (settings[Setting::enforce_strict_identifier_format]) { WriteBufferFromOwnString buf; - IAST::FormatSettings enforce_strict_identifier_format_settings(buf, true); + IAST::FormatSettings enforce_strict_identifier_format_settings(true); enforce_strict_identifier_format_settings.enforce_strict_identifier_format = true; - ast->format(enforce_strict_identifier_format_settings); + ast->format(buf, enforce_strict_identifier_format_settings); } if (auto * insert_query = ast->as()) diff --git a/src/Parsers/ASTAlterNamedCollectionQuery.cpp b/src/Parsers/ASTAlterNamedCollectionQuery.cpp index bb63a95182f..47e0e193798 100644 --- a/src/Parsers/ASTAlterNamedCollectionQuery.cpp +++ b/src/Parsers/ASTAlterNamedCollectionQuery.cpp @@ -12,46 +12,46 @@ ASTPtr ASTAlterNamedCollectionQuery::clone() const return std::make_shared(*this); } -void ASTAlterNamedCollectionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +void ASTAlterNamedCollectionQuery::formatImpl(WriteBuffer & ostr, const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "ALTER NAMED COLLECTION "; + ostr << (settings.hilite ? hilite_keyword : "") << "ALTER NAMED COLLECTION "; if (if_exists) - settings.ostr << "IF EXISTS "; - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(collection_name) << (settings.hilite ? hilite_none : ""); - formatOnCluster(settings); + ostr << "IF EXISTS "; + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(collection_name) << (settings.hilite ? hilite_none : ""); + formatOnCluster(ostr, settings); if (!changes.empty()) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " SET " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " SET " << (settings.hilite ? hilite_none : ""); bool first = true; for (const auto & change : changes) { if (!first) - settings.ostr << ", "; + ostr << ", "; else first = false; - formatSettingName(change.name, settings.ostr); + formatSettingName(change.name, ostr); if (settings.show_secrets) - settings.ostr << " = " << applyVisitor(FieldVisitorToString(), change.value); + ostr << " = " << applyVisitor(FieldVisitorToString(), change.value); else - settings.ostr << " = '[HIDDEN]'"; + ostr << " = '[HIDDEN]'"; auto override_value = overridability.find(change.name); if (override_value != overridability.end()) - settings.ostr << " " << (override_value->second ? "" : "NOT ") << "OVERRIDABLE"; + ostr << " " << (override_value->second ? "" : "NOT ") << "OVERRIDABLE"; } } if (!delete_keys.empty()) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " DELETE " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " DELETE " << (settings.hilite ? hilite_none : ""); bool first = true; for (const auto & key : delete_keys) { if (!first) - settings.ostr << ", "; + ostr << ", "; else first = false; - formatSettingName(key, settings.ostr); + formatSettingName(key, ostr); } } } diff --git a/src/Parsers/ASTAlterNamedCollectionQuery.h b/src/Parsers/ASTAlterNamedCollectionQuery.h index dc2670a8896..49944c56d57 100644 --- a/src/Parsers/ASTAlterNamedCollectionQuery.h +++ b/src/Parsers/ASTAlterNamedCollectionQuery.h @@ -21,7 +21,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index 8bcb6fdc2fc..9291d188578 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -68,270 +68,270 @@ ASTPtr ASTAlterCommand::clone() const return res; } -void ASTAlterCommand::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTAlterCommand::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { scope_guard closing_bracket_guard; if (format_alter_commands_with_parentheses) { - settings.ostr << "("; - closing_bracket_guard = make_scope_guard(std::function([&settings]() { settings.ostr << ")"; })); + ostr << "("; + closing_bracket_guard = make_scope_guard(std::function([&ostr]() { ostr << ")"; })); } if (type == ASTAlterCommand::ADD_COLUMN) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "ADD COLUMN " << (if_not_exists ? "IF NOT EXISTS " : "") + ostr << (settings.hilite ? hilite_keyword : "") << "ADD COLUMN " << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); - col_decl->formatImpl(settings, state, frame); + col_decl->formatImpl(ostr, settings, state, frame); if (first) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FIRST " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " FIRST " << (settings.hilite ? hilite_none : ""); else if (column) /// AFTER { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " AFTER " << (settings.hilite ? hilite_none : ""); - column->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " AFTER " << (settings.hilite ? hilite_none : ""); + column->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::DROP_COLUMN) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << (clear_column ? "CLEAR " : "DROP ") << "COLUMN " + ostr << (settings.hilite ? hilite_keyword : "") << (clear_column ? "CLEAR " : "DROP ") << "COLUMN " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); - column->formatImpl(settings, state, frame); + column->formatImpl(ostr, settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::MODIFY_COLUMN) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY COLUMN " << (if_exists ? "IF EXISTS " : "") + ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY COLUMN " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); - col_decl->formatImpl(settings, state, frame); + col_decl->formatImpl(ostr, settings, state, frame); if (!remove_property.empty()) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " REMOVE " << remove_property; + ostr << (settings.hilite ? hilite_keyword : "") << " REMOVE " << remove_property; } else if (settings_changes) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " MODIFY SETTING " << (settings.hilite ? hilite_none : ""); - settings_changes->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " MODIFY SETTING " << (settings.hilite ? hilite_none : ""); + settings_changes->formatImpl(ostr, settings, state, frame); } else if (settings_resets) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " RESET SETTING " << (settings.hilite ? hilite_none : ""); - settings_resets->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " RESET SETTING " << (settings.hilite ? hilite_none : ""); + settings_resets->formatImpl(ostr, settings, state, frame); } else { if (first) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FIRST " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " FIRST " << (settings.hilite ? hilite_none : ""); else if (column) /// AFTER { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " AFTER " << (settings.hilite ? hilite_none : ""); - column->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " AFTER " << (settings.hilite ? hilite_none : ""); + column->formatImpl(ostr, settings, state, frame); } } } else if (type == ASTAlterCommand::MATERIALIZE_COLUMN) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE COLUMN " << (settings.hilite ? hilite_none : ""); - column->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE COLUMN " << (settings.hilite ? hilite_none : ""); + column->formatImpl(ostr, settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::COMMENT_COLUMN) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "COMMENT COLUMN " << (if_exists ? "IF EXISTS " : "") + ostr << (settings.hilite ? hilite_keyword : "") << "COMMENT COLUMN " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); - column->formatImpl(settings, state, frame); - settings.ostr << " " << (settings.hilite ? hilite_none : ""); - comment->formatImpl(settings, state, frame); + column->formatImpl(ostr, settings, state, frame); + ostr << " " << (settings.hilite ? hilite_none : ""); + comment->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::MODIFY_COMMENT) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY COMMENT" << (settings.hilite ? hilite_none : ""); - settings.ostr << " " << (settings.hilite ? hilite_none : ""); - comment->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY COMMENT" << (settings.hilite ? hilite_none : ""); + ostr << " " << (settings.hilite ? hilite_none : ""); + comment->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::MODIFY_ORDER_BY) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY ORDER BY " << (settings.hilite ? hilite_none : ""); - order_by->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY ORDER BY " << (settings.hilite ? hilite_none : ""); + order_by->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::MODIFY_SAMPLE_BY) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY SAMPLE BY " << (settings.hilite ? hilite_none : ""); - sample_by->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY SAMPLE BY " << (settings.hilite ? hilite_none : ""); + sample_by->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::REMOVE_SAMPLE_BY) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "REMOVE SAMPLE BY" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "REMOVE SAMPLE BY" << (settings.hilite ? hilite_none : ""); } else if (type == ASTAlterCommand::ADD_INDEX) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "ADD INDEX " << (if_not_exists ? "IF NOT EXISTS " : "") + ostr << (settings.hilite ? hilite_keyword : "") << "ADD INDEX " << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); - index_decl->formatImpl(settings, state, frame); + index_decl->formatImpl(ostr, settings, state, frame); if (first) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FIRST " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " FIRST " << (settings.hilite ? hilite_none : ""); else if (index) /// AFTER { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " AFTER " << (settings.hilite ? hilite_none : ""); - index->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " AFTER " << (settings.hilite ? hilite_none : ""); + index->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::DROP_INDEX) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << (clear_index ? "CLEAR " : "DROP ") << "INDEX " + ostr << (settings.hilite ? hilite_keyword : "") << (clear_index ? "CLEAR " : "DROP ") << "INDEX " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); - index->formatImpl(settings, state, frame); + index->formatImpl(ostr, settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::MATERIALIZE_INDEX) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE INDEX " << (settings.hilite ? hilite_none : ""); - index->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE INDEX " << (settings.hilite ? hilite_none : ""); + index->formatImpl(ostr, settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::ADD_STATISTICS) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "ADD STATISTICS " << (if_not_exists ? "IF NOT EXISTS " : "") + ostr << (settings.hilite ? hilite_keyword : "") << "ADD STATISTICS " << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); - statistics_decl->formatImpl(settings, state, frame); + statistics_decl->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::MODIFY_STATISTICS) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY STATISTICS " + ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY STATISTICS " << (settings.hilite ? hilite_none : ""); - statistics_decl->formatImpl(settings, state, frame); + statistics_decl->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::DROP_STATISTICS) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << (clear_statistics ? "CLEAR " : "DROP ") << "STATISTICS " + ostr << (settings.hilite ? hilite_keyword : "") << (clear_statistics ? "CLEAR " : "DROP ") << "STATISTICS " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); - statistics_decl->formatImpl(settings, state, frame); + statistics_decl->formatImpl(ostr, settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::MATERIALIZE_STATISTICS) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE STATISTICS " << (settings.hilite ? hilite_none : ""); - statistics_decl->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE STATISTICS " << (settings.hilite ? hilite_none : ""); + statistics_decl->formatImpl(ostr, settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::ADD_CONSTRAINT) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "ADD CONSTRAINT " << (if_not_exists ? "IF NOT EXISTS " : "") + ostr << (settings.hilite ? hilite_keyword : "") << "ADD CONSTRAINT " << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); - constraint_decl->formatImpl(settings, state, frame); + constraint_decl->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::DROP_CONSTRAINT) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "DROP CONSTRAINT " << (if_exists ? "IF EXISTS " : "") + ostr << (settings.hilite ? hilite_keyword : "") << "DROP CONSTRAINT " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); - constraint->formatImpl(settings, state, frame); + constraint->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::ADD_PROJECTION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "ADD PROJECTION " << (if_not_exists ? "IF NOT EXISTS " : "") + ostr << (settings.hilite ? hilite_keyword : "") << "ADD PROJECTION " << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); - projection_decl->formatImpl(settings, state, frame); + projection_decl->formatImpl(ostr, settings, state, frame); if (first) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FIRST " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " FIRST " << (settings.hilite ? hilite_none : ""); else if (projection) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " AFTER " << (settings.hilite ? hilite_none : ""); - projection->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " AFTER " << (settings.hilite ? hilite_none : ""); + projection->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::DROP_PROJECTION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << (clear_projection ? "CLEAR " : "DROP ") << "PROJECTION " + ostr << (settings.hilite ? hilite_keyword : "") << (clear_projection ? "CLEAR " : "DROP ") << "PROJECTION " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); - projection->formatImpl(settings, state, frame); + projection->formatImpl(ostr, settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::MATERIALIZE_PROJECTION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE PROJECTION " << (settings.hilite ? hilite_none : ""); - projection->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE PROJECTION " << (settings.hilite ? hilite_none : ""); + projection->formatImpl(ostr, settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::DROP_PARTITION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << (detach ? "DETACH" : "DROP") << (part ? " PART " : " PARTITION ") + ostr << (settings.hilite ? hilite_keyword : "") << (detach ? "DETACH" : "DROP") << (part ? " PART " : " PARTITION ") << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + partition->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::DROP_DETACHED_PARTITION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "DROP DETACHED" << (part ? " PART " : " PARTITION ") + ostr << (settings.hilite ? hilite_keyword : "") << "DROP DETACHED" << (part ? " PART " : " PARTITION ") << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + partition->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::FORGET_PARTITION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "FORGET PARTITION " + ostr << (settings.hilite ? hilite_keyword : "") << "FORGET PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + partition->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::ATTACH_PARTITION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "ATTACH " << (part ? "PART " : "PARTITION ") + ostr << (settings.hilite ? hilite_keyword : "") << "ATTACH " << (part ? "PART " : "PARTITION ") << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + partition->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::MOVE_PARTITION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MOVE " << (part ? "PART " : "PARTITION ") + ostr << (settings.hilite ? hilite_keyword : "") << "MOVE " << (part ? "PART " : "PARTITION ") << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); - settings.ostr << " TO "; + partition->formatImpl(ostr, settings, state, frame); + ostr << " TO "; switch (move_destination_type) { case DataDestinationType::DISK: - settings.ostr << "DISK "; + ostr << "DISK "; break; case DataDestinationType::VOLUME: - settings.ostr << "VOLUME "; + ostr << "VOLUME "; break; case DataDestinationType::TABLE: - settings.ostr << "TABLE "; + ostr << "TABLE "; if (!to_database.empty()) { - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(to_database) + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(to_database) << (settings.hilite ? hilite_none : "") << "."; } - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(to_table) + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(to_table) << (settings.hilite ? hilite_none : ""); return; default: @@ -339,165 +339,165 @@ void ASTAlterCommand::formatImpl(const FormatSettings & settings, FormatState & } if (move_destination_type != DataDestinationType::TABLE) { - settings.ostr << quoteString(move_destination_name); + ostr << quoteString(move_destination_name); } } else if (type == ASTAlterCommand::REPLACE_PARTITION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << (replace ? "REPLACE" : "ATTACH") << " PARTITION " + ostr << (settings.hilite ? hilite_keyword : "") << (replace ? "REPLACE" : "ATTACH") << " PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : ""); if (!from_database.empty()) { - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(from_database) + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(from_database) << (settings.hilite ? hilite_none : "") << "."; } - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(from_table) << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(from_table) << (settings.hilite ? hilite_none : ""); } else if (type == ASTAlterCommand::FETCH_PARTITION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "FETCH " << (part ? "PART " : "PARTITION ") + ostr << (settings.hilite ? hilite_keyword : "") << "FETCH " << (part ? "PART " : "PARTITION ") << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << DB::quote << from; + partition->formatImpl(ostr, settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << DB::quote << from; } else if (type == ASTAlterCommand::FREEZE_PARTITION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "FREEZE PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "FREEZE PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); if (!with_name.empty()) { - settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "") << " " + ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "") << " " << DB::quote << with_name; } } else if (type == ASTAlterCommand::FREEZE_ALL) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "FREEZE" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "FREEZE" << (settings.hilite ? hilite_none : ""); if (!with_name.empty()) { - settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "") << " " + ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "") << " " << DB::quote << with_name; } } else if (type == ASTAlterCommand::UNFREEZE_PARTITION) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "UNFREEZE PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "UNFREEZE PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); if (!with_name.empty()) { - settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "") << " " + ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "") << " " << DB::quote << with_name; } } else if (type == ASTAlterCommand::UNFREEZE_ALL) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "UNFREEZE" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "UNFREEZE" << (settings.hilite ? hilite_none : ""); if (!with_name.empty()) { - settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "") << " " + ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "") << " " << DB::quote << with_name; } } else if (type == ASTAlterCommand::DELETE) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "DELETE" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "DELETE" << (settings.hilite ? hilite_none : ""); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); - predicate->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); + predicate->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::UPDATE) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "UPDATE " << (settings.hilite ? hilite_none : ""); - update_assignments->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "UPDATE " << (settings.hilite ? hilite_none : ""); + update_assignments->formatImpl(ostr, settings, state, frame); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); - predicate->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); + predicate->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::MODIFY_TTL) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY TTL " << (settings.hilite ? hilite_none : ""); - ttl->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY TTL " << (settings.hilite ? hilite_none : ""); + ttl->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::REMOVE_TTL) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "REMOVE TTL" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "REMOVE TTL" << (settings.hilite ? hilite_none : ""); } else if (type == ASTAlterCommand::MATERIALIZE_TTL) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE TTL" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE TTL" << (settings.hilite ? hilite_none : ""); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } } else if (type == ASTAlterCommand::MODIFY_SETTING) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY SETTING " << (settings.hilite ? hilite_none : ""); - settings_changes->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY SETTING " << (settings.hilite ? hilite_none : ""); + settings_changes->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::RESET_SETTING) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "RESET SETTING " << (settings.hilite ? hilite_none : ""); - settings_resets->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "RESET SETTING " << (settings.hilite ? hilite_none : ""); + settings_resets->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::MODIFY_DATABASE_SETTING) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY SETTING " << (settings.hilite ? hilite_none : ""); - settings_changes->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY SETTING " << (settings.hilite ? hilite_none : ""); + settings_changes->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::MODIFY_QUERY) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY QUERY" << settings.nl_or_ws + ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY QUERY" << settings.nl_or_ws << (settings.hilite ? hilite_none : ""); - select->formatImpl(settings, state, frame); + select->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::MODIFY_REFRESH) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY" << settings.nl_or_ws + ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY" << settings.nl_or_ws << (settings.hilite ? hilite_none : ""); - refresh->formatImpl(settings, state, frame); + refresh->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::RENAME_COLUMN) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "RENAME COLUMN " << (if_exists ? "IF EXISTS " : "") + ostr << (settings.hilite ? hilite_keyword : "") << "RENAME COLUMN " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); - column->formatImpl(settings, state, frame); + column->formatImpl(ostr, settings, state, frame); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " TO "; - rename_to->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " TO "; + rename_to->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::MODIFY_SQL_SECURITY) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY " << (settings.hilite ? hilite_none : ""); - sql_security->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY " << (settings.hilite ? hilite_none : ""); + sql_security->formatImpl(ostr, settings, state, frame); } else if (type == ASTAlterCommand::APPLY_DELETED_MASK) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "APPLY DELETED MASK" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "APPLY DELETED MASK" << (settings.hilite ? hilite_none : ""); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } } else @@ -614,58 +614,58 @@ ASTPtr ASTAlterQuery::clone() const return res; } -void ASTAlterQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTAlterQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.need_parens = false; std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str; + ostr << (settings.hilite ? hilite_keyword : "") << indent_str; switch (alter_object) { case AlterObjectType::TABLE: - settings.ostr << "ALTER TABLE "; + ostr << "ALTER TABLE "; break; case AlterObjectType::DATABASE: - settings.ostr << "ALTER DATABASE "; + ostr << "ALTER DATABASE "; break; default: break; } - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_none : ""); if (table) { - settings.ostr << indent_str; + ostr << indent_str; if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); } else if (alter_object == AlterObjectType::DATABASE && database) { - settings.ostr << indent_str; - database->formatImpl(settings, state, frame); + ostr << indent_str; + database->formatImpl(ostr, settings, state, frame); } - formatOnCluster(settings); + formatOnCluster(ostr, settings); FormatStateStacked frame_nested = frame; frame_nested.need_parens = false; if (settings.one_line) { frame_nested.expression_list_prepend_whitespace = true; - command_list->formatImpl(settings, state, frame_nested); + command_list->formatImpl(ostr, settings, state, frame_nested); } else { frame_nested.expression_list_always_start_on_new_line = true; - command_list->as().formatImplMultiline(settings, state, frame_nested); + command_list->as().formatImplMultiline(ostr, settings, state, frame_nested); } } diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index d7269bed2da..ab19ed9a55a 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -226,7 +226,7 @@ public: static void setFormatAlterCommandsWithParentheses(bool value) { format_alter_commands_with_parentheses = value; } protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void forEachPointerToChild(std::function f) override; @@ -273,7 +273,7 @@ public: QueryKind getQueryKind() const override { return QueryKind::Alter; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; bool isOneCommandTypeOnly(const ASTAlterCommand::Type & type) const; diff --git a/src/Parsers/ASTAssignment.h b/src/Parsers/ASTAssignment.h index 2b1391b5812..15832b1b955 100644 --- a/src/Parsers/ASTAssignment.h +++ b/src/Parsers/ASTAssignment.h @@ -26,16 +26,16 @@ public: } protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { - settings.ostr << (settings.hilite ? hilite_identifier : ""); - settings.writeIdentifier(column_name, /*ambiguous=*/false); - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_identifier : ""); + settings.writeIdentifier(ostr, column_name, /*ambiguous=*/false); + ostr << (settings.hilite ? hilite_none : ""); - settings.ostr << (settings.hilite ? hilite_operator : "") << " = " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_operator : "") << " = " << (settings.hilite ? hilite_none : ""); - expression()->formatImpl(settings, state, frame); + expression()->formatImpl(ostr, settings, state, frame); } }; diff --git a/src/Parsers/ASTAsterisk.cpp b/src/Parsers/ASTAsterisk.cpp index 7e872e944bd..f7d90119e03 100644 --- a/src/Parsers/ASTAsterisk.cpp +++ b/src/Parsers/ASTAsterisk.cpp @@ -27,19 +27,19 @@ void ASTAsterisk::appendColumnName(WriteBuffer & ostr) const ostr.write('*'); } -void ASTAsterisk::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTAsterisk::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (expression) { - expression->formatImpl(settings, state, frame); - settings.ostr << "."; + expression->formatImpl(ostr, settings, state, frame); + ostr << "."; } - settings.ostr << "*"; + ostr << "*"; if (transformers) { - transformers->formatImpl(settings, state, frame); + transformers->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTAsterisk.h b/src/Parsers/ASTAsterisk.h index 840b7996536..d6b6f094366 100644 --- a/src/Parsers/ASTAsterisk.h +++ b/src/Parsers/ASTAsterisk.h @@ -19,7 +19,7 @@ public: ASTPtr expression; ASTPtr transformers; protected: - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTBackupQuery.cpp b/src/Parsers/ASTBackupQuery.cpp index 9766190fe45..d3565631e34 100644 --- a/src/Parsers/ASTBackupQuery.cpp +++ b/src/Parsers/ASTBackupQuery.cpp @@ -15,162 +15,162 @@ namespace using Element = ASTBackupQuery::Element; using ElementType = ASTBackupQuery::ElementType; - void formatPartitions(const ASTs & partitions, const IAST::FormatSettings & format) + void formatPartitions(const ASTs & partitions, WriteBuffer & ostr, const IAST::FormatSettings & format) { - format.ostr << " " << (format.hilite ? IAST::hilite_keyword : "") << ((partitions.size() == 1) ? "PARTITION" : "PARTITIONS") << " " + ostr << " " << (format.hilite ? IAST::hilite_keyword : "") << ((partitions.size() == 1) ? "PARTITION" : "PARTITIONS") << " " << (format.hilite ? IAST::hilite_none : ""); bool need_comma = false; for (const auto & partition : partitions) { if (std::exchange(need_comma, true)) - format.ostr << ","; - format.ostr << " "; - partition->format(format); + ostr << ","; + ostr << " "; + partition->format(ostr, format); } } - void formatExceptDatabases(const std::set & except_databases, const IAST::FormatSettings & format) + void formatExceptDatabases(const std::set & except_databases, WriteBuffer & ostr, const IAST::FormatSettings & format) { if (except_databases.empty()) return; - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " EXCEPT " + ostr << (format.hilite ? IAST::hilite_keyword : "") << " EXCEPT " << (except_databases.size() == 1 ? "DATABASE" : "DATABASES") << " " << (format.hilite ? IAST::hilite_none : ""); bool need_comma = false; for (const auto & database_name : except_databases) { if (std::exchange(need_comma, true)) - format.ostr << ","; - format.ostr << backQuoteIfNeed(database_name); + ostr << ","; + ostr << backQuoteIfNeed(database_name); } } - void formatExceptTables(const std::set & except_tables, const IAST::FormatSettings & format, bool only_table_names=false) + void formatExceptTables(const std::set & except_tables, WriteBuffer & ostr, const IAST::FormatSettings & format, bool only_table_names=false) { if (except_tables.empty()) return; - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " EXCEPT " << (except_tables.size() == 1 ? "TABLE" : "TABLES") << " " + ostr << (format.hilite ? IAST::hilite_keyword : "") << " EXCEPT " << (except_tables.size() == 1 ? "TABLE" : "TABLES") << " " << (format.hilite ? IAST::hilite_none : ""); bool need_comma = false; for (const auto & table_name : except_tables) { if (std::exchange(need_comma, true)) - format.ostr << ", "; + ostr << ", "; if (!table_name.first.empty() && !only_table_names) - format.ostr << backQuoteIfNeed(table_name.first) << "."; - format.ostr << backQuoteIfNeed(table_name.second); + ostr << backQuoteIfNeed(table_name.first) << "."; + ostr << backQuoteIfNeed(table_name.second); } } - void formatElement(const Element & element, const IAST::FormatSettings & format) + void formatElement(const Element & element, WriteBuffer & ostr, const IAST::FormatSettings & format) { switch (element.type) { case ElementType::TABLE: { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << "TABLE " << (format.hilite ? IAST::hilite_none : ""); + ostr << (format.hilite ? IAST::hilite_keyword : "") << "TABLE " << (format.hilite ? IAST::hilite_none : ""); if (!element.database_name.empty()) - format.ostr << backQuoteIfNeed(element.database_name) << "."; - format.ostr << backQuoteIfNeed(element.table_name); + ostr << backQuoteIfNeed(element.database_name) << "."; + ostr << backQuoteIfNeed(element.table_name); if ((element.new_table_name != element.table_name) || (element.new_database_name != element.database_name)) { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " AS " << (format.hilite ? IAST::hilite_none : ""); + ostr << (format.hilite ? IAST::hilite_keyword : "") << " AS " << (format.hilite ? IAST::hilite_none : ""); if (!element.new_database_name.empty()) - format.ostr << backQuoteIfNeed(element.new_database_name) << "."; - format.ostr << backQuoteIfNeed(element.new_table_name); + ostr << backQuoteIfNeed(element.new_database_name) << "."; + ostr << backQuoteIfNeed(element.new_table_name); } if (element.partitions) - formatPartitions(*element.partitions, format); + formatPartitions(*element.partitions, ostr, format); break; } case ElementType::TEMPORARY_TABLE: { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << "TEMPORARY TABLE " << (format.hilite ? IAST::hilite_none : ""); - format.ostr << backQuoteIfNeed(element.table_name); + ostr << (format.hilite ? IAST::hilite_keyword : "") << "TEMPORARY TABLE " << (format.hilite ? IAST::hilite_none : ""); + ostr << backQuoteIfNeed(element.table_name); if (element.new_table_name != element.table_name) { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " AS " << (format.hilite ? IAST::hilite_none : ""); - format.ostr << backQuoteIfNeed(element.new_table_name); + ostr << (format.hilite ? IAST::hilite_keyword : "") << " AS " << (format.hilite ? IAST::hilite_none : ""); + ostr << backQuoteIfNeed(element.new_table_name); } break; } case ElementType::DATABASE: { - format.ostr << (format.hilite ? IAST::hilite_keyword : ""); - format.ostr << "DATABASE "; - format.ostr << (format.hilite ? IAST::hilite_none : ""); - format.ostr << backQuoteIfNeed(element.database_name); + ostr << (format.hilite ? IAST::hilite_keyword : ""); + ostr << "DATABASE "; + ostr << (format.hilite ? IAST::hilite_none : ""); + ostr << backQuoteIfNeed(element.database_name); if (element.new_database_name != element.database_name) { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " AS " << (format.hilite ? IAST::hilite_none : ""); - format.ostr << backQuoteIfNeed(element.new_database_name); + ostr << (format.hilite ? IAST::hilite_keyword : "") << " AS " << (format.hilite ? IAST::hilite_none : ""); + ostr << backQuoteIfNeed(element.new_database_name); } - formatExceptTables(element.except_tables, format, /*only_table_names*/true); + formatExceptTables(element.except_tables, ostr, format, /*only_table_names*/true); break; } case ElementType::ALL: { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << "ALL" << (format.hilite ? IAST::hilite_none : ""); - formatExceptDatabases(element.except_databases, format); - formatExceptTables(element.except_tables, format); + ostr << (format.hilite ? IAST::hilite_keyword : "") << "ALL" << (format.hilite ? IAST::hilite_none : ""); + formatExceptDatabases(element.except_databases, ostr, format); + formatExceptTables(element.except_tables, ostr, format); break; } } } - void formatElements(const std::vector & elements, const IAST::FormatSettings & format) + void formatElements(const std::vector & elements, WriteBuffer & ostr, const IAST::FormatSettings & format) { bool need_comma = false; for (const auto & element : elements) { if (std::exchange(need_comma, true)) - format.ostr << ", "; - formatElement(element, format); + ostr << ", "; + formatElement(element, ostr, format); } } - void formatSettings(const ASTPtr & settings, const ASTFunction * base_backup_name, const ASTPtr & cluster_host_ids, const IAST::FormatSettings & format) + void formatSettings(const ASTPtr & settings, const ASTFunction * base_backup_name, const ASTPtr & cluster_host_ids, WriteBuffer & ostr, const IAST::FormatSettings & format) { if (!settings && !base_backup_name && !cluster_host_ids) return; - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " SETTINGS " << (format.hilite ? IAST::hilite_none : ""); + ostr << (format.hilite ? IAST::hilite_keyword : "") << " SETTINGS " << (format.hilite ? IAST::hilite_none : ""); bool empty = true; if (base_backup_name) { - format.ostr << "base_backup = "; - base_backup_name->format(format); + ostr << "base_backup = "; + base_backup_name->format(ostr, format); empty = false; } if (settings) { if (!empty) - format.ostr << ", "; - settings->format(format); + ostr << ", "; + settings->format(ostr, format); empty = false; } if (cluster_host_ids) { if (!empty) - format.ostr << ", "; - format.ostr << "cluster_host_ids = "; - cluster_host_ids->format(format); + ostr << ", "; + ostr << "cluster_host_ids = "; + cluster_host_ids->format(ostr, format); } } @@ -267,18 +267,18 @@ ASTPtr ASTBackupQuery::clone() const } -void ASTBackupQuery::formatQueryImpl(const FormatSettings & fs, FormatState &, FormatStateStacked) const +void ASTBackupQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & fs, FormatState &, FormatStateStacked) const { - fs.ostr << (fs.hilite ? hilite_keyword : "") << ((kind == Kind::BACKUP) ? "BACKUP " : "RESTORE ") << (fs.hilite ? hilite_none : ""); + ostr << (fs.hilite ? hilite_keyword : "") << ((kind == Kind::BACKUP) ? "BACKUP " : "RESTORE ") << (fs.hilite ? hilite_none : ""); - formatElements(elements, fs); - formatOnCluster(fs); + formatElements(elements, ostr, fs); + formatOnCluster(ostr, fs); - fs.ostr << (fs.hilite ? hilite_keyword : "") << ((kind == Kind::BACKUP) ? " TO " : " FROM ") << (fs.hilite ? hilite_none : ""); - backup_name->format(fs); + ostr << (fs.hilite ? hilite_keyword : "") << ((kind == Kind::BACKUP) ? " TO " : " FROM ") << (fs.hilite ? hilite_none : ""); + backup_name->format(ostr, fs); if (settings || base_backup_name) - formatSettings(settings, base_backup_name, cluster_host_ids, fs); + formatSettings(settings, base_backup_name, cluster_host_ids, ostr, fs); } ASTPtr ASTBackupQuery::getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams & params) const diff --git a/src/Parsers/ASTBackupQuery.h b/src/Parsers/ASTBackupQuery.h index a56cdebc7b3..62b755203d5 100644 --- a/src/Parsers/ASTBackupQuery.h +++ b/src/Parsers/ASTBackupQuery.h @@ -91,7 +91,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatQueryImpl(const FormatSettings & fs, FormatState &, FormatStateStacked) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & fs, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override; QueryKind getQueryKind() const override; diff --git a/src/Parsers/ASTCheckQuery.h b/src/Parsers/ASTCheckQuery.h index 919555eb336..91b2a0817d4 100644 --- a/src/Parsers/ASTCheckQuery.h +++ b/src/Parsers/ASTCheckQuery.h @@ -36,32 +36,32 @@ struct ASTCheckTableQuery : public ASTQueryWithTableAndOutput } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "CHECK TABLE " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "CHECK TABLE " << (settings.hilite ? hilite_none : ""); if (table) { if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); } if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } if (!part_name.empty()) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " PART " << (settings.hilite ? hilite_none : "") + ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " PART " << (settings.hilite ? hilite_none : "") << quoteString(part_name); } } @@ -84,10 +84,10 @@ struct ASTCheckAllTablesQuery : public ASTQueryWithOutput QueryKind getQueryKind() const override { return QueryKind::Check; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & /* state */, FormatStateStacked frame) const override + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & /* state */, FormatStateStacked frame) const override { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "CHECK ALL TABLES" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "CHECK ALL TABLES" << (settings.hilite ? hilite_none : ""); } }; diff --git a/src/Parsers/ASTCollation.cpp b/src/Parsers/ASTCollation.cpp index 19716414581..6295849ec8a 100644 --- a/src/Parsers/ASTCollation.cpp +++ b/src/Parsers/ASTCollation.cpp @@ -9,13 +9,10 @@ namespace DB return res; } - void ASTCollation::formatImpl(const FormatSettings &s, FormatState &state, FormatStateStacked frame) const + void ASTCollation::formatImpl(WriteBuffer & ostr, const FormatSettings &s, FormatState &state, FormatStateStacked frame) const { if (collation) - { - collation->formatImpl(s, state, frame); - } - + collation->formatImpl(ostr, s, state, frame); } } diff --git a/src/Parsers/ASTCollation.h b/src/Parsers/ASTCollation.h index a735956a90e..3b5fb78fabc 100644 --- a/src/Parsers/ASTCollation.h +++ b/src/Parsers/ASTCollation.h @@ -14,7 +14,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index d96a52da61c..5b65c84f05c 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -62,69 +62,69 @@ ASTPtr ASTColumnDeclaration::clone() const return res; } -void ASTColumnDeclaration::formatImpl(const FormatSettings & format_settings, FormatState & state, FormatStateStacked frame) const +void ASTColumnDeclaration::formatImpl(WriteBuffer & ostr, const FormatSettings & format_settings, FormatState & state, FormatStateStacked frame) const { frame.need_parens = false; - format_settings.writeIdentifier(name, /*ambiguous=*/true); + format_settings.writeIdentifier(ostr, name, /*ambiguous=*/true); if (type) { - format_settings.ostr << ' '; - type->formatImpl(format_settings, state, frame); + ostr << ' '; + type->formatImpl(ostr, format_settings, state, frame); } if (null_modifier) { - format_settings.ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") + ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") << (*null_modifier ? "" : "NOT ") << "NULL" << (format_settings.hilite ? hilite_none : ""); } if (default_expression) { - format_settings.ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") << default_specifier << (format_settings.hilite ? hilite_none : ""); + ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") << default_specifier << (format_settings.hilite ? hilite_none : ""); if (!ephemeral_default) { - format_settings.ostr << ' '; - default_expression->formatImpl(format_settings, state, frame); + ostr << ' '; + default_expression->formatImpl(ostr, format_settings, state, frame); } } if (comment) { - format_settings.ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") << "COMMENT" << (format_settings.hilite ? hilite_none : "") << ' '; - comment->formatImpl(format_settings, state, frame); + ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") << "COMMENT" << (format_settings.hilite ? hilite_none : "") << ' '; + comment->formatImpl(ostr, format_settings, state, frame); } if (codec) { - format_settings.ostr << ' '; - codec->formatImpl(format_settings, state, frame); + ostr << ' '; + codec->formatImpl(ostr, format_settings, state, frame); } if (statistics_desc) { - format_settings.ostr << ' '; - statistics_desc->formatImpl(format_settings, state, frame); + ostr << ' '; + statistics_desc->formatImpl(ostr, format_settings, state, frame); } if (ttl) { - format_settings.ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") << "TTL" << (format_settings.hilite ? hilite_none : "") << ' '; - ttl->formatImpl(format_settings, state, frame); + ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") << "TTL" << (format_settings.hilite ? hilite_none : "") << ' '; + ttl->formatImpl(ostr, format_settings, state, frame); } if (collation) { - format_settings.ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") << "COLLATE" << (format_settings.hilite ? hilite_none : "") << ' '; - collation->formatImpl(format_settings, state, frame); + ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") << "COLLATE" << (format_settings.hilite ? hilite_none : "") << ' '; + collation->formatImpl(ostr, format_settings, state, frame); } if (settings) { - format_settings.ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") << "SETTINGS" << (format_settings.hilite ? hilite_none : "") << ' ' << '('; - settings->formatImpl(format_settings, state, frame); - format_settings.ostr << ')'; + ostr << ' ' << (format_settings.hilite ? hilite_keyword : "") << "SETTINGS" << (format_settings.hilite ? hilite_none : "") << ' ' << '('; + settings->formatImpl(ostr, format_settings, state, frame); + ostr << ')'; } } diff --git a/src/Parsers/ASTColumnDeclaration.h b/src/Parsers/ASTColumnDeclaration.h index 0c5076f0201..f2c462f0975 100644 --- a/src/Parsers/ASTColumnDeclaration.h +++ b/src/Parsers/ASTColumnDeclaration.h @@ -28,7 +28,7 @@ public: String getID(char delim) const override { return "ColumnDeclaration" + (delim + name); } ASTPtr clone() const override; - void formatImpl(const FormatSettings & format_settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & format_settings, FormatState & state, FormatStateStacked frame) const override; protected: void forEachPointerToChild(std::function f) override; diff --git a/src/Parsers/ASTColumnsMatcher.cpp b/src/Parsers/ASTColumnsMatcher.cpp index 518fd9b722d..f22694e8e62 100644 --- a/src/Parsers/ASTColumnsMatcher.cpp +++ b/src/Parsers/ASTColumnsMatcher.cpp @@ -39,23 +39,23 @@ void ASTColumnsRegexpMatcher::updateTreeHashImpl(SipHash & hash_state, bool igno IAST::updateTreeHashImpl(hash_state, ignore_aliases); } -void ASTColumnsRegexpMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTColumnsRegexpMatcher::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : ""); + ostr << (settings.hilite ? hilite_keyword : ""); if (expression) { - expression->formatImpl(settings, state, frame); - settings.ostr << "."; + expression->formatImpl(ostr, settings, state, frame); + ostr << "."; } - settings.ostr << "COLUMNS" << (settings.hilite ? hilite_none : "") << "("; - settings.ostr << quoteString(pattern); - settings.ostr << ")"; + ostr << "COLUMNS" << (settings.hilite ? hilite_none : "") << "("; + ostr << quoteString(pattern); + ostr << ")"; if (transformers) { - transformers->formatImpl(settings, state, frame); + transformers->formatImpl(ostr, settings, state, frame); } } @@ -101,31 +101,31 @@ void ASTColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const writeChar(')', ostr); } -void ASTColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTColumnsListMatcher::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : ""); + ostr << (settings.hilite ? hilite_keyword : ""); if (expression) { - expression->formatImpl(settings, state, frame); - settings.ostr << "."; + expression->formatImpl(ostr, settings, state, frame); + ostr << "."; } - settings.ostr << "COLUMNS" << (settings.hilite ? hilite_none : "") << "("; + ostr << "COLUMNS" << (settings.hilite ? hilite_none : "") << "("; for (ASTs::const_iterator it = column_list->children.begin(); it != column_list->children.end(); ++it) { if (it != column_list->children.begin()) { - settings.ostr << ", "; + ostr << ", "; } - (*it)->formatImpl(settings, state, frame); + (*it)->formatImpl(ostr, settings, state, frame); } - settings.ostr << ")"; + ostr << ")"; if (transformers) { - transformers->formatImpl(settings, state, frame); + transformers->formatImpl(ostr, settings, state, frame); } } @@ -167,19 +167,19 @@ void ASTQualifiedColumnsRegexpMatcher::updateTreeHashImpl(SipHash & hash_state, IAST::updateTreeHashImpl(hash_state, ignore_aliases); } -void ASTQualifiedColumnsRegexpMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTQualifiedColumnsRegexpMatcher::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : ""); + ostr << (settings.hilite ? hilite_keyword : ""); - qualifier->formatImpl(settings, state, frame); + qualifier->formatImpl(ostr, settings, state, frame); - settings.ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "("; - settings.ostr << quoteString(pattern); - settings.ostr << ")"; + ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "("; + ostr << quoteString(pattern); + ostr << ")"; if (transformers) { - transformers->formatImpl(settings, state, frame); + transformers->formatImpl(ostr, settings, state, frame); } } @@ -214,24 +214,24 @@ void ASTQualifiedColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const writeChar(')', ostr); } -void ASTQualifiedColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTQualifiedColumnsListMatcher::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : ""); - qualifier->formatImpl(settings, state, frame); - settings.ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "("; + ostr << (settings.hilite ? hilite_keyword : ""); + qualifier->formatImpl(ostr, settings, state, frame); + ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "("; for (ASTs::const_iterator it = column_list->children.begin(); it != column_list->children.end(); ++it) { if (it != column_list->children.begin()) - settings.ostr << ", "; + ostr << ", "; - (*it)->formatImpl(settings, state, frame); + (*it)->formatImpl(ostr, settings, state, frame); } - settings.ostr << ")"; + ostr << ")"; if (transformers) { - transformers->formatImpl(settings, state, frame); + transformers->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTColumnsMatcher.h b/src/Parsers/ASTColumnsMatcher.h index e5191b74b72..538e41bd450 100644 --- a/src/Parsers/ASTColumnsMatcher.h +++ b/src/Parsers/ASTColumnsMatcher.h @@ -25,7 +25,7 @@ public: ASTPtr expression; ASTPtr transformers; protected: - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; private: String pattern; @@ -43,7 +43,7 @@ public: ASTPtr column_list; ASTPtr transformers; protected: - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; /// Same as ASTColumnsRegexpMatcher. Qualified identifier is first child. @@ -61,7 +61,7 @@ public: ASTPtr qualifier; ASTPtr transformers; protected: - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; private: String pattern; @@ -79,7 +79,7 @@ public: ASTPtr column_list; ASTPtr transformers; protected: - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index 332ebca3bdb..cb9afdacc3f 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -20,12 +20,12 @@ namespace ErrorCodes extern const int CANNOT_COMPILE_REGEXP; } -void ASTColumnsTransformerList::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTColumnsTransformerList::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { for (const auto & child : children) { - settings.ostr << ' '; - child->formatImpl(settings, state, frame); + ostr << ' '; + child->formatImpl(ostr, settings, state, frame); } } @@ -45,33 +45,33 @@ void IASTColumnsTransformer::transform(const ASTPtr & transformer, ASTs & nodes) } } -void ASTColumnsApplyTransformer::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTColumnsApplyTransformer::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "APPLY" << (settings.hilite ? hilite_none : "") << " "; + ostr << (settings.hilite ? hilite_keyword : "") << "APPLY" << (settings.hilite ? hilite_none : "") << " "; if (!column_name_prefix.empty()) - settings.ostr << "("; + ostr << "("; if (lambda) { - lambda->formatImpl(settings, state, frame); + lambda->formatImpl(ostr, settings, state, frame); } else { - settings.ostr << func_name; + ostr << func_name; if (parameters) { auto nested_frame = frame; nested_frame.expression_list_prepend_whitespace = false; - settings.ostr << "("; - parameters->formatImpl(settings, state, nested_frame); - settings.ostr << ")"; + ostr << "("; + parameters->formatImpl(ostr, settings, state, nested_frame); + ostr << ")"; } } if (!column_name_prefix.empty()) - settings.ostr << ", '" << column_name_prefix << "')"; + ostr << ", '" << column_name_prefix << "')"; } void ASTColumnsApplyTransformer::transform(ASTs & nodes) const @@ -164,27 +164,27 @@ void ASTColumnsApplyTransformer::updateTreeHashImpl(SipHash & hash_state, bool i IAST::updateTreeHashImpl(hash_state, ignore_aliases); } -void ASTColumnsExceptTransformer::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTColumnsExceptTransformer::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "EXCEPT" << (is_strict ? " STRICT " : " ") << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "EXCEPT" << (is_strict ? " STRICT " : " ") << (settings.hilite ? hilite_none : ""); if (children.size() > 1) - settings.ostr << "("; + ostr << "("; for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) { if (it != children.begin()) { - settings.ostr << ", "; + ostr << ", "; } - (*it)->formatImpl(settings, state, frame); + (*it)->formatImpl(ostr, settings, state, frame); } if (pattern) - settings.ostr << quoteString(*pattern); + ostr << quoteString(*pattern); if (children.size() > 1) - settings.ostr << ")"; + ostr << ")"; } void ASTColumnsExceptTransformer::appendColumnName(WriteBuffer & ostr) const @@ -292,12 +292,12 @@ std::shared_ptr ASTColumnsExceptTransformer::getMatcher() const } void ASTColumnsReplaceTransformer::Replacement::formatImpl( - const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const + WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { assert(children.size() == 1); - children[0]->formatImpl(settings, state, frame); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(name); + children[0]->formatImpl(ostr, settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(name); } void ASTColumnsReplaceTransformer::Replacement::appendColumnName(WriteBuffer & ostr) const @@ -319,19 +319,19 @@ void ASTColumnsReplaceTransformer::Replacement::updateTreeHashImpl(SipHash & has IAST::updateTreeHashImpl(hash_state, ignore_aliases); } -void ASTColumnsReplaceTransformer::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTColumnsReplaceTransformer::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "REPLACE" << (is_strict ? " STRICT " : " ") << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "REPLACE" << (is_strict ? " STRICT " : " ") << (settings.hilite ? hilite_none : ""); - settings.ostr << "("; + ostr << "("; for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) { if (it != children.begin()) - settings.ostr << ", "; + ostr << ", "; - (*it)->formatImpl(settings, state, frame); + (*it)->formatImpl(ostr, settings, state, frame); } - settings.ostr << ")"; + ostr << ")"; } void ASTColumnsReplaceTransformer::appendColumnName(WriteBuffer & ostr) const diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h index cedf955bee2..d55d4f0b6da 100644 --- a/src/Parsers/ASTColumnsTransformers.h +++ b/src/Parsers/ASTColumnsTransformers.h @@ -24,7 +24,7 @@ public: } protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; class IASTColumnsTransformer : public IAST @@ -62,7 +62,7 @@ public: String column_name_prefix; protected: - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; class ASTColumnsExceptTransformer : public IASTColumnsTransformer @@ -83,7 +83,7 @@ public: void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; protected: - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; std::optional pattern; }; @@ -107,7 +107,7 @@ public: String name; protected: - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; bool is_strict = false; @@ -123,7 +123,7 @@ public: void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; protected: - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; private: static void replaceChildren(ASTPtr & node, const ASTPtr & replacement, const String & name); diff --git a/src/Parsers/ASTConstraintDeclaration.cpp b/src/Parsers/ASTConstraintDeclaration.cpp index 2b895b85996..07dae006114 100644 --- a/src/Parsers/ASTConstraintDeclaration.cpp +++ b/src/Parsers/ASTConstraintDeclaration.cpp @@ -19,11 +19,11 @@ ASTPtr ASTConstraintDeclaration::clone() const return res; } -void ASTConstraintDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +void ASTConstraintDeclaration::formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { - s.ostr << backQuoteIfNeed(name); - s.ostr << (s.hilite ? hilite_keyword : "") << (type == Type::CHECK ? " CHECK " : " ASSUME ") << (s.hilite ? hilite_none : ""); - expr->formatImpl(s, state, frame); + ostr << backQuoteIfNeed(name); + ostr << (s.hilite ? hilite_keyword : "") << (type == Type::CHECK ? " CHECK " : " ASSUME ") << (s.hilite ? hilite_none : ""); + expr->formatImpl(ostr, s, state, frame); } } diff --git a/src/Parsers/ASTConstraintDeclaration.h b/src/Parsers/ASTConstraintDeclaration.h index f48d7ef77fe..f4934fee7da 100644 --- a/src/Parsers/ASTConstraintDeclaration.h +++ b/src/Parsers/ASTConstraintDeclaration.h @@ -24,7 +24,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; void forEachPointerToChild(std::function f) override { diff --git a/src/Parsers/ASTCreateFunctionQuery.cpp b/src/Parsers/ASTCreateFunctionQuery.cpp index a12eca3d334..1f6357c04c6 100644 --- a/src/Parsers/ASTCreateFunctionQuery.cpp +++ b/src/Parsers/ASTCreateFunctionQuery.cpp @@ -21,26 +21,26 @@ ASTPtr ASTCreateFunctionQuery::clone() const return res; } -void ASTCreateFunctionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState & state, IAST::FormatStateStacked frame) const +void ASTCreateFunctionQuery::formatImpl(WriteBuffer & ostr, const IAST::FormatSettings & settings, IAST::FormatState & state, IAST::FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "CREATE "; + ostr << (settings.hilite ? hilite_keyword : "") << "CREATE "; if (or_replace) - settings.ostr << "OR REPLACE "; + ostr << "OR REPLACE "; - settings.ostr << "FUNCTION "; + ostr << "FUNCTION "; if (if_not_exists) - settings.ostr << "IF NOT EXISTS "; + ostr << "IF NOT EXISTS "; - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_none : ""); - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(getFunctionName()) << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(getFunctionName()) << (settings.hilite ? hilite_none : ""); - formatOnCluster(settings); + formatOnCluster(ostr, settings); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); - function_core->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); + function_core->formatImpl(ostr, settings, state, frame); } String ASTCreateFunctionQuery::getFunctionName() const diff --git a/src/Parsers/ASTCreateFunctionQuery.h b/src/Parsers/ASTCreateFunctionQuery.h index 8ce167ba7db..9520de21e43 100644 --- a/src/Parsers/ASTCreateFunctionQuery.h +++ b/src/Parsers/ASTCreateFunctionQuery.h @@ -20,7 +20,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } diff --git a/src/Parsers/ASTCreateIndexQuery.cpp b/src/Parsers/ASTCreateIndexQuery.cpp index f6def3ed85c..3d8ce47e7c0 100644 --- a/src/Parsers/ASTCreateIndexQuery.cpp +++ b/src/Parsers/ASTCreateIndexQuery.cpp @@ -30,37 +30,37 @@ ASTPtr ASTCreateIndexQuery::clone() const return res; } -void ASTCreateIndexQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTCreateIndexQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.need_parens = false; std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str; + ostr << (settings.hilite ? hilite_keyword : "") << indent_str; - settings.ostr << "CREATE " << (unique ? "UNIQUE " : "") << "INDEX " << (if_not_exists ? "IF NOT EXISTS " : ""); - index_name->formatImpl(settings, state, frame); - settings.ostr << " ON "; + ostr << "CREATE " << (unique ? "UNIQUE " : "") << "INDEX " << (if_not_exists ? "IF NOT EXISTS " : ""); + index_name->formatImpl(ostr, settings, state, frame); + ostr << " ON "; - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_none : ""); if (table) { if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); } - formatOnCluster(settings); + formatOnCluster(ostr, settings); - settings.ostr << " "; + ostr << " "; - index_decl->formatImpl(settings, state, frame); + index_decl->formatImpl(ostr, settings, state, frame); } ASTPtr ASTCreateIndexQuery::convertToASTAlterCommand() const diff --git a/src/Parsers/ASTCreateIndexQuery.h b/src/Parsers/ASTCreateIndexQuery.h index f5e35e270e9..c0e3f78c246 100644 --- a/src/Parsers/ASTCreateIndexQuery.h +++ b/src/Parsers/ASTCreateIndexQuery.h @@ -37,7 +37,7 @@ public: ASTPtr convertToASTAlterCommand() const; protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTCreateNamedCollectionQuery.cpp b/src/Parsers/ASTCreateNamedCollectionQuery.cpp index 54a1022102e..c2726b3f732 100644 --- a/src/Parsers/ASTCreateNamedCollectionQuery.cpp +++ b/src/Parsers/ASTCreateNamedCollectionQuery.cpp @@ -15,33 +15,33 @@ ASTPtr ASTCreateNamedCollectionQuery::clone() const return std::make_shared(*this); } -void ASTCreateNamedCollectionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +void ASTCreateNamedCollectionQuery::formatImpl(WriteBuffer & ostr, const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "CREATE NAMED COLLECTION "; + ostr << (settings.hilite ? hilite_keyword : "") << "CREATE NAMED COLLECTION "; if (if_not_exists) - settings.ostr << "IF NOT EXISTS "; - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(collection_name) << (settings.hilite ? hilite_none : ""); + ostr << "IF NOT EXISTS "; + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(collection_name) << (settings.hilite ? hilite_none : ""); - formatOnCluster(settings); + formatOnCluster(ostr, settings); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); bool first = true; for (const auto & change : changes) { if (!first) - settings.ostr << ", "; + ostr << ", "; else first = false; - formatSettingName(change.name, settings.ostr); + formatSettingName(change.name, ostr); if (settings.show_secrets) - settings.ostr << " = " << applyVisitor(FieldVisitorToString(), change.value); + ostr << " = " << applyVisitor(FieldVisitorToString(), change.value); else - settings.ostr << " = '[HIDDEN]'"; + ostr << " = '[HIDDEN]'"; auto override_value = overridability.find(change.name); if (override_value != overridability.end()) - settings.ostr << " " << (override_value->second ? "" : "NOT ") << "OVERRIDABLE"; + ostr << " " << (override_value->second ? "" : "NOT ") << "OVERRIDABLE"; } } diff --git a/src/Parsers/ASTCreateNamedCollectionQuery.h b/src/Parsers/ASTCreateNamedCollectionQuery.h index 47f2352b4cd..babcd8ad594 100644 --- a/src/Parsers/ASTCreateNamedCollectionQuery.h +++ b/src/Parsers/ASTCreateNamedCollectionQuery.h @@ -20,7 +20,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index 974b56e8ef6..854952952b9 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -14,33 +14,33 @@ namespace DB { -void ASTSQLSecurity::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTSQLSecurity::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (!type) return; if (definer || is_definer_current_user) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "DEFINER" << (settings.hilite ? hilite_none : ""); - settings.ostr << " = "; + ostr << (settings.hilite ? hilite_keyword : "") << "DEFINER" << (settings.hilite ? hilite_none : ""); + ostr << " = "; if (definer) - definer->formatImpl(settings, state, frame); + definer->formatImpl(ostr, settings, state, frame); else - settings.ostr << "CURRENT_USER"; - settings.ostr << " "; + ostr << "CURRENT_USER"; + ostr << " "; } - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SQL SECURITY" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "SQL SECURITY" << (settings.hilite ? hilite_none : ""); switch (*type) { case SQLSecurityType::INVOKER: - settings.ostr << " INVOKER"; + ostr << " INVOKER"; break; case SQLSecurityType::DEFINER: - settings.ostr << " DEFINER"; + ostr << " DEFINER"; break; case SQLSecurityType::NONE: - settings.ostr << " NONE"; + ostr << " NONE"; break; } } @@ -69,42 +69,42 @@ ASTPtr ASTStorage::clone() const return res; } -void ASTStorage::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +void ASTStorage::formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { if (engine) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "ENGINE" << (s.hilite ? hilite_none : "") << " = "; - engine->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "ENGINE" << (s.hilite ? hilite_none : "") << " = "; + engine->formatImpl(ostr, s, state, frame); } if (partition_by) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "PARTITION BY " << (s.hilite ? hilite_none : ""); - partition_by->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "PARTITION BY " << (s.hilite ? hilite_none : ""); + partition_by->formatImpl(ostr, s, state, frame); } if (primary_key) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "PRIMARY KEY " << (s.hilite ? hilite_none : ""); - primary_key->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "PRIMARY KEY " << (s.hilite ? hilite_none : ""); + primary_key->formatImpl(ostr, s, state, frame); } if (order_by) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "ORDER BY " << (s.hilite ? hilite_none : ""); - order_by->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "ORDER BY " << (s.hilite ? hilite_none : ""); + order_by->formatImpl(ostr, s, state, frame); } if (sample_by) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "SAMPLE BY " << (s.hilite ? hilite_none : ""); - sample_by->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "SAMPLE BY " << (s.hilite ? hilite_none : ""); + sample_by->formatImpl(ostr, s, state, frame); } if (ttl_table) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "TTL " << (s.hilite ? hilite_none : ""); - ttl_table->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "TTL " << (s.hilite ? hilite_none : ""); + ttl_table->formatImpl(ostr, s, state, frame); } if (settings) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "SETTINGS " << (s.hilite ? hilite_none : ""); - settings->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "SETTINGS " << (s.hilite ? hilite_none : ""); + settings->formatImpl(ostr, s, state, frame); } } @@ -124,7 +124,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; void forEachPointerToChild(std::function f) override { @@ -141,20 +141,20 @@ ASTPtr ASTColumnsElement::clone() const return res; } -void ASTColumnsElement::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +void ASTColumnsElement::formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { if (!elem) return; if (prefix.empty()) { - elem->formatImpl(s, state, frame); + elem->formatImpl(ostr, s, state, frame); return; } - s.ostr << (s.hilite ? hilite_keyword : "") << prefix << (s.hilite ? hilite_none : ""); - s.ostr << ' '; - elem->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << prefix << (s.hilite ? hilite_none : ""); + ostr << ' '; + elem->formatImpl(ostr, s, state, frame); } @@ -178,7 +178,7 @@ ASTPtr ASTColumns::clone() const return res; } -void ASTColumns::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +void ASTColumns::formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { ASTExpressionList list; @@ -226,9 +226,9 @@ void ASTColumns::formatImpl(const FormatSettings & s, FormatState & state, Forma if (!list.children.empty()) { if (s.one_line) - list.formatImpl(s, state, frame); + list.formatImpl(ostr, s, state, frame); else - list.formatImplMultiline(s, state, frame); + list.formatImplMultiline(ostr, s, state, frame); } } @@ -279,40 +279,40 @@ String ASTCreateQuery::getID(char delim) const return res; } -void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTCreateQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.need_parens = false; if (database && !table) { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << (attach ? "ATTACH DATABASE " : "CREATE DATABASE ") << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); - database->formatImpl(settings, state, frame); + database->formatImpl(ostr, settings, state, frame); if (uuid != UUIDHelpers::Nil) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " UUID " << (settings.hilite ? hilite_none : "") + ostr << (settings.hilite ? hilite_keyword : "") << " UUID " << (settings.hilite ? hilite_none : "") << quoteString(toString(uuid)); } - formatOnCluster(settings); + formatOnCluster(ostr, settings); if (storage) - storage->formatImpl(settings, state, frame); + storage->formatImpl(ostr, settings, state, frame); if (table_overrides) { - settings.ostr << settings.nl_or_ws; - table_overrides->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws; + table_overrides->formatImpl(ostr, settings, state, frame); } if (comment) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "COMMENT " << (settings.hilite ? hilite_none : ""); - comment->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "COMMENT " << (settings.hilite ? hilite_none : ""); + comment->formatImpl(ostr, settings, state, frame); } return; @@ -340,40 +340,40 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat else if (is_window_view) what = "WINDOW VIEW"; - settings.ostr << (settings.hilite ? hilite_keyword : "") << action << (settings.hilite ? hilite_none : ""); - settings.ostr << " "; - settings.ostr << (settings.hilite ? hilite_keyword : "") << (temporary ? "TEMPORARY " : "") + ostr << (settings.hilite ? hilite_keyword : "") << action << (settings.hilite ? hilite_none : ""); + ostr << " "; + ostr << (settings.hilite ? hilite_keyword : "") << (temporary ? "TEMPORARY " : "") << what << " " << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); if (uuid != UUIDHelpers::Nil) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " UUID " << (settings.hilite ? hilite_none : "") + ostr << (settings.hilite ? hilite_keyword : "") << " UUID " << (settings.hilite ? hilite_none : "") << quoteString(toString(uuid)); assert(attach || !attach_from_path); if (attach_from_path) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") + ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << quoteString(*attach_from_path); if (attach_as_replicated.has_value()) { if (attach_as_replicated.value()) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS REPLICATED" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " AS REPLICATED" << (settings.hilite ? hilite_none : ""); else - settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS NOT REPLICATED" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " AS NOT REPLICATED" << (settings.hilite ? hilite_none : ""); } - formatOnCluster(settings); + formatOnCluster(ostr, settings); } else { @@ -386,33 +386,33 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat action = "REPLACE"; /// Always DICTIONARY - settings.ostr << (settings.hilite ? hilite_keyword : "") << action << " DICTIONARY " + ostr << (settings.hilite ? hilite_keyword : "") << action << " DICTIONARY " << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); if (uuid != UUIDHelpers::Nil) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " UUID " << (settings.hilite ? hilite_none : "") + ostr << (settings.hilite ? hilite_keyword : "") << " UUID " << (settings.hilite ? hilite_none : "") << quoteString(toString(uuid)); - formatOnCluster(settings); + formatOnCluster(ostr, settings); } if (refresh_strategy) { - settings.ostr << settings.nl_or_ws; - refresh_strategy->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws; + refresh_strategy->formatImpl(ostr, settings, state, frame); } if (auto to_table_id = getTargetTableID(ViewTarget::To)) { - settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << toStringView(Keyword::TO) + ostr << " " << (settings.hilite ? hilite_keyword : "") << toStringView(Keyword::TO) << (settings.hilite ? hilite_none : "") << " " << (!to_table_id.database_name.empty() ? backQuoteIfNeed(to_table_id.database_name) + "." : "") << backQuoteIfNeed(to_table_id.table_name); @@ -420,7 +420,7 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat if (auto to_inner_uuid = getTargetInnerUUID(ViewTarget::To); to_inner_uuid != UUIDHelpers::Nil) { - settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << toStringView(Keyword::TO_INNER_UUID) + ostr << " " << (settings.hilite ? hilite_keyword : "") << toStringView(Keyword::TO_INNER_UUID) << (settings.hilite ? hilite_none : "") << " " << quoteString(toString(to_inner_uuid)); } @@ -430,7 +430,7 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat if (!should_add_empty) return; should_add_empty = false; - settings.ostr << (settings.hilite ? hilite_keyword : "") << " EMPTY" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " EMPTY" << (settings.hilite ? hilite_none : ""); }; bool should_add_clone = is_clone_as; @@ -439,14 +439,14 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat if (!should_add_clone) return; should_add_clone = false; - settings.ostr << (settings.hilite ? hilite_keyword : "") << " CLONE" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " CLONE" << (settings.hilite ? hilite_none : ""); }; if (!as_table.empty()) { add_empty_if_needed(); add_clone_if_needed(); - settings.ostr + ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "") << (!as_database.empty() ? backQuoteIfNeed(as_database) + "." : "") << backQuoteIfNeed(as_table); } @@ -456,108 +456,108 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat if (columns_list && !columns_list->empty()) { frame.expression_list_always_start_on_new_line = true; - settings.ostr << (settings.one_line ? " (" : "\n("); + ostr << (settings.one_line ? " (" : "\n("); FormatStateStacked frame_nested = frame; - columns_list->formatImpl(settings, state, frame_nested); - settings.ostr << (settings.one_line ? ")" : "\n)"); + columns_list->formatImpl(ostr, settings, state, frame_nested); + ostr << (settings.one_line ? ")" : "\n)"); frame.expression_list_always_start_on_new_line = false; } add_empty_if_needed(); add_clone_if_needed(); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); - as_table_function->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); + as_table_function->formatImpl(ostr, settings, state, frame); } frame.expression_list_always_start_on_new_line = true; if (columns_list && !columns_list->empty() && !as_table_function) { - settings.ostr << (settings.one_line ? " (" : "\n("); + ostr << (settings.one_line ? " (" : "\n("); FormatStateStacked frame_nested = frame; - columns_list->formatImpl(settings, state, frame_nested); - settings.ostr << (settings.one_line ? ")" : "\n)"); + columns_list->formatImpl(ostr, settings, state, frame_nested); + ostr << (settings.one_line ? ")" : "\n)"); } if (dictionary_attributes_list) { - settings.ostr << (settings.one_line ? " (" : "\n("); + ostr << (settings.one_line ? " (" : "\n("); FormatStateStacked frame_nested = frame; if (settings.one_line) - dictionary_attributes_list->formatImpl(settings, state, frame_nested); + dictionary_attributes_list->formatImpl(ostr, settings, state, frame_nested); else - dictionary_attributes_list->formatImplMultiline(settings, state, frame_nested); - settings.ostr << (settings.one_line ? ")" : "\n)"); + dictionary_attributes_list->formatImplMultiline(ostr, settings, state, frame_nested); + ostr << (settings.one_line ? ")" : "\n)"); } frame.expression_list_always_start_on_new_line = false; if (storage) - storage->formatImpl(settings, state, frame); + storage->formatImpl(ostr, settings, state, frame); if (auto inner_storage = getTargetInnerEngine(ViewTarget::Inner)) { - settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << toStringView(Keyword::INNER) << (settings.hilite ? hilite_none : ""); - inner_storage->formatImpl(settings, state, frame); + ostr << " " << (settings.hilite ? hilite_keyword : "") << toStringView(Keyword::INNER) << (settings.hilite ? hilite_none : ""); + inner_storage->formatImpl(ostr, settings, state, frame); } if (auto to_storage = getTargetInnerEngine(ViewTarget::To)) - to_storage->formatImpl(settings, state, frame); + to_storage->formatImpl(ostr, settings, state, frame); if (targets) { - targets->formatTarget(ViewTarget::Data, settings, state, frame); - targets->formatTarget(ViewTarget::Tags, settings, state, frame); - targets->formatTarget(ViewTarget::Metrics, settings, state, frame); + targets->formatTarget(ViewTarget::Data, ostr, settings, state, frame); + targets->formatTarget(ViewTarget::Tags, ostr, settings, state, frame); + targets->formatTarget(ViewTarget::Metrics, ostr, settings, state, frame); } if (dictionary) - dictionary->formatImpl(settings, state, frame); + dictionary->formatImpl(ostr, settings, state, frame); if (is_watermark_strictly_ascending) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WATERMARK STRICTLY_ASCENDING" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " WATERMARK STRICTLY_ASCENDING" << (settings.hilite ? hilite_none : ""); } else if (is_watermark_ascending) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WATERMARK ASCENDING" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " WATERMARK ASCENDING" << (settings.hilite ? hilite_none : ""); } else if (is_watermark_bounded) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WATERMARK " << (settings.hilite ? hilite_none : ""); - watermark_function->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " WATERMARK " << (settings.hilite ? hilite_none : ""); + watermark_function->formatImpl(ostr, settings, state, frame); } if (allowed_lateness) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " ALLOWED_LATENESS " << (settings.hilite ? hilite_none : ""); - lateness_function->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " ALLOWED_LATENESS " << (settings.hilite ? hilite_none : ""); + lateness_function->formatImpl(ostr, settings, state, frame); } if (is_populate) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " POPULATE" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " POPULATE" << (settings.hilite ? hilite_none : ""); add_empty_if_needed(); if (sql_security && supportSQLSecurity() && sql_security->as().type.has_value()) { - settings.ostr << settings.nl_or_ws; - sql_security->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws; + sql_security->formatImpl(ostr, settings, state, frame); } if (select) { - settings.ostr << settings.nl_or_ws; - settings.ostr << (settings.hilite ? hilite_keyword : "") << "AS " + ostr << settings.nl_or_ws; + ostr << (settings.hilite ? hilite_keyword : "") << "AS " << (comment ? "(" : "") << (settings.hilite ? hilite_none : ""); - select->formatImpl(settings, state, frame); - settings.ostr << (settings.hilite ? hilite_keyword : "") << (comment ? ")" : "") << (settings.hilite ? hilite_none : ""); + select->formatImpl(ostr, settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << (comment ? ")" : "") << (settings.hilite ? hilite_none : ""); } if (comment) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "COMMENT " << (settings.hilite ? hilite_none : ""); - comment->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "COMMENT " << (settings.hilite ? hilite_none : ""); + comment->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index 3621eae1f74..6a66b3fb67b 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -34,7 +34,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; bool isExtendedStorageDefinition() const; @@ -67,7 +67,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; bool empty() const { @@ -179,7 +179,7 @@ public: bool is_materialized_view_with_inner_table() const { return is_materialized_view && !hasTargetTableID(ViewTarget::To); } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void forEachPointerToChild(std::function f) override { diff --git a/src/Parsers/ASTCreateResourceQuery.cpp b/src/Parsers/ASTCreateResourceQuery.cpp index 3e40d76ba1b..7dd70034e6e 100644 --- a/src/Parsers/ASTCreateResourceQuery.cpp +++ b/src/Parsers/ASTCreateResourceQuery.cpp @@ -20,31 +20,31 @@ ASTPtr ASTCreateResourceQuery::clone() const return res; } -void ASTCreateResourceQuery::formatImpl(const IAST::FormatSettings & format, IAST::FormatState &, IAST::FormatStateStacked) const +void ASTCreateResourceQuery::formatImpl(WriteBuffer & ostr, const IAST::FormatSettings & format, IAST::FormatState &, IAST::FormatStateStacked) const { - format.ostr << (format.hilite ? hilite_keyword : "") << "CREATE "; + ostr << (format.hilite ? hilite_keyword : "") << "CREATE "; if (or_replace) - format.ostr << "OR REPLACE "; + ostr << "OR REPLACE "; - format.ostr << "RESOURCE "; + ostr << "RESOURCE "; if (if_not_exists) - format.ostr << "IF NOT EXISTS "; + ostr << "IF NOT EXISTS "; - format.ostr << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_none : ""); - format.ostr << (format.hilite ? hilite_identifier : "") << backQuoteIfNeed(getResourceName()) << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_identifier : "") << backQuoteIfNeed(getResourceName()) << (format.hilite ? hilite_none : ""); - formatOnCluster(format); + formatOnCluster(ostr, format); - format.ostr << " ("; + ostr << " ("; bool first = true; for (const auto & operation : operations) { if (!first) - format.ostr << ", "; + ostr << ", "; else first = false; @@ -52,25 +52,25 @@ void ASTCreateResourceQuery::formatImpl(const IAST::FormatSettings & format, IAS { case AccessMode::Read: { - format.ostr << (format.hilite ? hilite_keyword : "") << "READ "; + ostr << (format.hilite ? hilite_keyword : "") << "READ "; break; } case AccessMode::Write: { - format.ostr << (format.hilite ? hilite_keyword : "") << "WRITE "; + ostr << (format.hilite ? hilite_keyword : "") << "WRITE "; break; } } if (operation.disk) { - format.ostr << "DISK " << (format.hilite ? hilite_none : ""); - format.ostr << (format.hilite ? hilite_identifier : "") << backQuoteIfNeed(*operation.disk) << (format.hilite ? hilite_none : ""); + ostr << "DISK " << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_identifier : "") << backQuoteIfNeed(*operation.disk) << (format.hilite ? hilite_none : ""); } else - format.ostr << "ANY DISK" << (format.hilite ? hilite_none : ""); + ostr << "ANY DISK" << (format.hilite ? hilite_none : ""); } - format.ostr << ")"; + ostr << ")"; } String ASTCreateResourceQuery::getResourceName() const diff --git a/src/Parsers/ASTCreateResourceQuery.h b/src/Parsers/ASTCreateResourceQuery.h index 51933a375f8..fc4abb08fc3 100644 --- a/src/Parsers/ASTCreateResourceQuery.h +++ b/src/Parsers/ASTCreateResourceQuery.h @@ -36,7 +36,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & format, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & format, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } diff --git a/src/Parsers/ASTCreateWorkloadQuery.cpp b/src/Parsers/ASTCreateWorkloadQuery.cpp index 972ce733651..2ccba2c9351 100644 --- a/src/Parsers/ASTCreateWorkloadQuery.cpp +++ b/src/Parsers/ASTCreateWorkloadQuery.cpp @@ -27,47 +27,47 @@ ASTPtr ASTCreateWorkloadQuery::clone() const return res; } -void ASTCreateWorkloadQuery::formatImpl(const IAST::FormatSettings & format, IAST::FormatState &, IAST::FormatStateStacked) const +void ASTCreateWorkloadQuery::formatImpl(WriteBuffer & ostr, const IAST::FormatSettings & format, IAST::FormatState &, IAST::FormatStateStacked) const { - format.ostr << (format.hilite ? hilite_keyword : "") << "CREATE "; + ostr << (format.hilite ? hilite_keyword : "") << "CREATE "; if (or_replace) - format.ostr << "OR REPLACE "; + ostr << "OR REPLACE "; - format.ostr << "WORKLOAD "; + ostr << "WORKLOAD "; if (if_not_exists) - format.ostr << "IF NOT EXISTS "; + ostr << "IF NOT EXISTS "; - format.ostr << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_none : ""); - format.ostr << (format.hilite ? hilite_identifier : "") << backQuoteIfNeed(getWorkloadName()) << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_identifier : "") << backQuoteIfNeed(getWorkloadName()) << (format.hilite ? hilite_none : ""); - formatOnCluster(format); + formatOnCluster(ostr, format); if (hasParent()) { - format.ostr << (format.hilite ? hilite_keyword : "") << " IN " << (format.hilite ? hilite_none : ""); - format.ostr << (format.hilite ? hilite_identifier : "") << backQuoteIfNeed(getWorkloadParent()) << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << " IN " << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_identifier : "") << backQuoteIfNeed(getWorkloadParent()) << (format.hilite ? hilite_none : ""); } if (!changes.empty()) { - format.ostr << ' ' << (format.hilite ? hilite_keyword : "") << "SETTINGS" << (format.hilite ? hilite_none : "") << ' '; + ostr << ' ' << (format.hilite ? hilite_keyword : "") << "SETTINGS" << (format.hilite ? hilite_none : "") << ' '; bool first = true; for (const auto & change : changes) { if (!first) - format.ostr << ", "; + ostr << ", "; else first = false; - format.ostr << change.name << " = " << applyVisitor(FieldVisitorToString(), change.value); + ostr << change.name << " = " << applyVisitor(FieldVisitorToString(), change.value); if (!change.resource.empty()) { - format.ostr << ' ' << (format.hilite ? hilite_keyword : "") << "FOR" << (format.hilite ? hilite_none : "") << ' '; - format.ostr << (format.hilite ? hilite_identifier : "") << backQuoteIfNeed(change.resource) << (format.hilite ? hilite_none : ""); + ostr << ' ' << (format.hilite ? hilite_keyword : "") << "FOR" << (format.hilite ? hilite_none : "") << ' '; + ostr << (format.hilite ? hilite_identifier : "") << backQuoteIfNeed(change.resource) << (format.hilite ? hilite_none : ""); } } } diff --git a/src/Parsers/ASTCreateWorkloadQuery.h b/src/Parsers/ASTCreateWorkloadQuery.h index 8a4cecc001e..7ca218306d9 100644 --- a/src/Parsers/ASTCreateWorkloadQuery.h +++ b/src/Parsers/ASTCreateWorkloadQuery.h @@ -39,7 +39,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & format, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & format, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } diff --git a/src/Parsers/ASTDataType.cpp b/src/Parsers/ASTDataType.cpp index 4211347fb74..1d9635e1101 100644 --- a/src/Parsers/ASTDataType.cpp +++ b/src/Parsers/ASTDataType.cpp @@ -32,13 +32,13 @@ void ASTDataType::updateTreeHashImpl(SipHash & hash_state, bool) const /// Children are hashed automatically. } -void ASTDataType::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTDataType::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_function : "") << name; + ostr << (settings.hilite ? hilite_function : "") << name; if (arguments && !arguments->children.empty()) { - settings.ostr << '(' << (settings.hilite ? hilite_none : ""); + ostr << '(' << (settings.hilite ? hilite_none : ""); if (!settings.one_line && settings.print_pretty_type_names && name == "Tuple") { @@ -47,21 +47,21 @@ void ASTDataType::formatImpl(const FormatSettings & settings, FormatState & stat for (size_t i = 0, size = arguments->children.size(); i < size; ++i) { if (i != 0) - settings.ostr << ','; - settings.ostr << indent_str; - arguments->children[i]->formatImpl(settings, state, frame); + ostr << ','; + ostr << indent_str; + arguments->children[i]->formatImpl(ostr, settings, state, frame); } } else { frame.expression_list_prepend_whitespace = false; - arguments->formatImpl(settings, state, frame); + arguments->formatImpl(ostr, settings, state, frame); } - settings.ostr << (settings.hilite ? hilite_function : "") << ')'; + ostr << (settings.hilite ? hilite_function : "") << ')'; } - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/ASTDataType.h b/src/Parsers/ASTDataType.h index 71d3aeaa4eb..426efd10e16 100644 --- a/src/Parsers/ASTDataType.h +++ b/src/Parsers/ASTDataType.h @@ -16,7 +16,7 @@ public: String getID(char delim) const override; ASTPtr clone() const override; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; template diff --git a/src/Parsers/ASTDatabaseOrNone.cpp b/src/Parsers/ASTDatabaseOrNone.cpp index 0a0c800ff36..cab7ffe90f6 100644 --- a/src/Parsers/ASTDatabaseOrNone.cpp +++ b/src/Parsers/ASTDatabaseOrNone.cpp @@ -4,14 +4,14 @@ namespace DB { -void ASTDatabaseOrNone::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTDatabaseOrNone::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { if (none) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); return; } - settings.ostr << backQuoteIfNeed(database_name); + ostr << backQuoteIfNeed(database_name); } } diff --git a/src/Parsers/ASTDatabaseOrNone.h b/src/Parsers/ASTDatabaseOrNone.h index 65ee6a59126..80147b1c29c 100644 --- a/src/Parsers/ASTDatabaseOrNone.h +++ b/src/Parsers/ASTDatabaseOrNone.h @@ -14,7 +14,7 @@ public: bool isNone() const { return none; } String getID(char) const override { return "DatabaseOrNone"; } ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTDeleteQuery.cpp b/src/Parsers/ASTDeleteQuery.cpp index 434cc344a5a..60b03b22fb3 100644 --- a/src/Parsers/ASTDeleteQuery.cpp +++ b/src/Parsers/ASTDeleteQuery.cpp @@ -30,29 +30,29 @@ ASTPtr ASTDeleteQuery::clone() const return res; } -void ASTDeleteQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTDeleteQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "DELETE FROM " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "DELETE FROM " << (settings.hilite ? hilite_none : ""); if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); - formatOnCluster(settings); + formatOnCluster(ostr, settings); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " IN PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); - predicate->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); + predicate->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTDeleteQuery.h b/src/Parsers/ASTDeleteQuery.h index fcbeafac294..d3a3e83b084 100644 --- a/src/Parsers/ASTDeleteQuery.h +++ b/src/Parsers/ASTDeleteQuery.h @@ -27,7 +27,7 @@ public: ASTPtr predicate; protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTDescribeCacheQuery.cpp b/src/Parsers/ASTDescribeCacheQuery.cpp index 6daedfdb3a4..405822ddf3b 100644 --- a/src/Parsers/ASTDescribeCacheQuery.cpp +++ b/src/Parsers/ASTDescribeCacheQuery.cpp @@ -14,9 +14,9 @@ ASTPtr ASTDescribeCacheQuery::clone() const return res; } -void ASTDescribeCacheQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTDescribeCacheQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "DESCRIBE FILESYSTEM CACHE" << (settings.hilite ? hilite_none : "") + ostr << (settings.hilite ? hilite_keyword : "") << "DESCRIBE FILESYSTEM CACHE" << (settings.hilite ? hilite_none : "") << " " << quoteString(cache_name); } diff --git a/src/Parsers/ASTDescribeCacheQuery.h b/src/Parsers/ASTDescribeCacheQuery.h index 55b841e03f6..8d41d6c7114 100644 --- a/src/Parsers/ASTDescribeCacheQuery.h +++ b/src/Parsers/ASTDescribeCacheQuery.h @@ -15,7 +15,7 @@ public: ASTPtr clone() const override; protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTDictionary.cpp b/src/Parsers/ASTDictionary.cpp index e33a7de6836..68dfa92a68a 100644 --- a/src/Parsers/ASTDictionary.cpp +++ b/src/Parsers/ASTDictionary.cpp @@ -16,11 +16,12 @@ ASTPtr ASTDictionaryRange::clone() const } -void ASTDictionaryRange::formatImpl(const FormatSettings & settings, +void ASTDictionaryRange::formatImpl(WriteBuffer & ostr, + const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << "RANGE" << (settings.hilite ? hilite_none : "") << "(" @@ -44,11 +45,12 @@ ASTPtr ASTDictionaryLifetime::clone() const } -void ASTDictionaryLifetime::formatImpl(const FormatSettings & settings, +void ASTDictionaryLifetime::formatImpl(WriteBuffer & ostr, + const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << "LIFETIME" << (settings.hilite ? hilite_none : "") << "(" @@ -73,11 +75,12 @@ ASTPtr ASTDictionaryLayout::clone() const } -void ASTDictionaryLayout::formatImpl(const FormatSettings & settings, +void ASTDictionaryLayout::formatImpl(WriteBuffer & ostr, + const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << "LAYOUT" << (settings.hilite ? hilite_none : "") << "(" @@ -86,14 +89,14 @@ void ASTDictionaryLayout::formatImpl(const FormatSettings & settings, << (settings.hilite ? hilite_none : ""); if (has_brackets) - settings.ostr << "("; + ostr << "("; - if (parameters) parameters->formatImpl(settings, state, frame); + if (parameters) parameters->formatImpl(ostr, settings, state, frame); if (has_brackets) - settings.ostr << ")"; + ostr << ")"; - settings.ostr << ")"; + ostr << ")"; } ASTPtr ASTDictionarySettings::clone() const @@ -104,23 +107,24 @@ ASTPtr ASTDictionarySettings::clone() const return res; } -void ASTDictionarySettings::formatImpl(const FormatSettings & settings, - FormatState &, - FormatStateStacked) const +void ASTDictionarySettings::formatImpl(WriteBuffer & ostr, + const FormatSettings & settings, + FormatState &, + FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << "SETTINGS" << (settings.hilite ? hilite_none : "") << "("; for (auto it = changes.begin(); it != changes.end(); ++it) { if (it != changes.begin()) - settings.ostr << ", "; + ostr << ", "; - settings.ostr << it->name << " = " << applyVisitor(FieldVisitorToString(), it->value); + ostr << it->name << " = " << applyVisitor(FieldVisitorToString(), it->value); } - settings.ostr << (settings.hilite ? hilite_none : "") << ")"; + ostr << (settings.hilite ? hilite_none : "") << ")"; } @@ -150,46 +154,46 @@ ASTPtr ASTDictionary::clone() const } -void ASTDictionary::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTDictionary::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (primary_key) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "PRIMARY KEY " + ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "PRIMARY KEY " << (settings.hilite ? hilite_none : ""); - primary_key->formatImpl(settings, state, frame); + primary_key->formatImpl(ostr, settings, state, frame); } if (source) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "SOURCE" + ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "SOURCE" << (settings.hilite ? hilite_none : ""); - settings.ostr << "("; - source->formatImpl(settings, state, frame); - settings.ostr << ")"; + ostr << "("; + source->formatImpl(ostr, settings, state, frame); + ostr << ")"; } if (lifetime) { - settings.ostr << settings.nl_or_ws; - lifetime->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws; + lifetime->formatImpl(ostr, settings, state, frame); } if (layout) { - settings.ostr << settings.nl_or_ws; - layout->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws; + layout->formatImpl(ostr, settings, state, frame); } if (range) { - settings.ostr << settings.nl_or_ws; - range->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws; + range->formatImpl(ostr, settings, state, frame); } if (dict_settings) { - settings.ostr << settings.nl_or_ws; - dict_settings->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws; + dict_settings->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTDictionary.h b/src/Parsers/ASTDictionary.h index 8c332247d52..ecc93677d02 100644 --- a/src/Parsers/ASTDictionary.h +++ b/src/Parsers/ASTDictionary.h @@ -25,7 +25,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; /// AST for external dictionary layout. Has name and contain single parameter @@ -46,7 +46,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void forEachPointerToChild(std::function f) override { @@ -68,7 +68,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; class ASTDictionarySettings : public IAST @@ -80,7 +80,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; @@ -107,7 +107,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTDictionaryAttributeDeclaration.cpp b/src/Parsers/ASTDictionaryAttributeDeclaration.cpp index 43fef42be4f..5386c19559a 100644 --- a/src/Parsers/ASTDictionaryAttributeDeclaration.cpp +++ b/src/Parsers/ASTDictionaryAttributeDeclaration.cpp @@ -31,41 +31,41 @@ ASTPtr ASTDictionaryAttributeDeclaration::clone() const return res; } -void ASTDictionaryAttributeDeclaration::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTDictionaryAttributeDeclaration::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.need_parens = false; - settings.writeIdentifier(name, /*ambiguous=*/true); + settings.writeIdentifier(ostr, name, /*ambiguous=*/true); if (type) { - settings.ostr << ' '; - type->formatImpl(settings, state, frame); + ostr << ' '; + type->formatImpl(ostr, settings, state, frame); } if (default_value) { - settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "DEFAULT" << (settings.hilite ? hilite_none : "") << ' '; - default_value->formatImpl(settings, state, frame); + ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "DEFAULT" << (settings.hilite ? hilite_none : "") << ' '; + default_value->formatImpl(ostr, settings, state, frame); } if (expression) { - settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "EXPRESSION" << (settings.hilite ? hilite_none : "") << ' '; - expression->formatImpl(settings, state, frame); + ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "EXPRESSION" << (settings.hilite ? hilite_none : "") << ' '; + expression->formatImpl(ostr, settings, state, frame); } if (hierarchical) - settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "HIERARCHICAL" << (settings.hilite ? hilite_none : ""); + ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "HIERARCHICAL" << (settings.hilite ? hilite_none : ""); if (bidirectional) - settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "BIDIRECTIONAL" << (settings.hilite ? hilite_none : ""); + ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "BIDIRECTIONAL" << (settings.hilite ? hilite_none : ""); if (injective) - settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "INJECTIVE" << (settings.hilite ? hilite_none : ""); + ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "INJECTIVE" << (settings.hilite ? hilite_none : ""); if (is_object_id) - settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "IS_OBJECT_ID" << (settings.hilite ? hilite_none : ""); + ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "IS_OBJECT_ID" << (settings.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/ASTDictionaryAttributeDeclaration.h b/src/Parsers/ASTDictionaryAttributeDeclaration.h index 52103650684..1cff354933d 100644 --- a/src/Parsers/ASTDictionaryAttributeDeclaration.h +++ b/src/Parsers/ASTDictionaryAttributeDeclaration.h @@ -30,7 +30,7 @@ public: String getID(char delim) const override { return "DictionaryAttributeDeclaration" + (delim + name); } ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTDropFunctionQuery.cpp b/src/Parsers/ASTDropFunctionQuery.cpp index 3409d699042..2790ca6e36f 100644 --- a/src/Parsers/ASTDropFunctionQuery.cpp +++ b/src/Parsers/ASTDropFunctionQuery.cpp @@ -10,16 +10,16 @@ ASTPtr ASTDropFunctionQuery::clone() const return std::make_shared(*this); } -void ASTDropFunctionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +void ASTDropFunctionQuery::formatImpl(WriteBuffer & ostr, const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "DROP FUNCTION "; + ostr << (settings.hilite ? hilite_keyword : "") << "DROP FUNCTION "; if (if_exists) - settings.ostr << "IF EXISTS "; + ostr << "IF EXISTS "; - settings.ostr << (settings.hilite ? hilite_none : ""); - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(function_name) << (settings.hilite ? hilite_none : ""); - formatOnCluster(settings); + ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(function_name) << (settings.hilite ? hilite_none : ""); + formatOnCluster(ostr, settings); } } diff --git a/src/Parsers/ASTDropFunctionQuery.h b/src/Parsers/ASTDropFunctionQuery.h index da8fb1ba1c1..b7e309bb2b6 100644 --- a/src/Parsers/ASTDropFunctionQuery.h +++ b/src/Parsers/ASTDropFunctionQuery.h @@ -18,7 +18,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } diff --git a/src/Parsers/ASTDropIndexQuery.cpp b/src/Parsers/ASTDropIndexQuery.cpp index 1109f32f019..6d7c15da91e 100644 --- a/src/Parsers/ASTDropIndexQuery.cpp +++ b/src/Parsers/ASTDropIndexQuery.cpp @@ -25,33 +25,33 @@ ASTPtr ASTDropIndexQuery::clone() const return res; } -void ASTDropIndexQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTDropIndexQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.need_parens = false; std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str; + ostr << (settings.hilite ? hilite_keyword : "") << indent_str; - settings.ostr << "DROP INDEX " << (if_exists ? "IF EXISTS " : ""); - index_name->formatImpl(settings, state, frame); - settings.ostr << " ON "; + ostr << "DROP INDEX " << (if_exists ? "IF EXISTS " : ""); + index_name->formatImpl(ostr, settings, state, frame); + ostr << " ON "; - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_none : ""); if (table) { if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); } - formatOnCluster(settings); + formatOnCluster(ostr, settings); } ASTPtr ASTDropIndexQuery::convertToASTAlterCommand() const diff --git a/src/Parsers/ASTDropIndexQuery.h b/src/Parsers/ASTDropIndexQuery.h index 6c2aaeb5936..14119b14b01 100644 --- a/src/Parsers/ASTDropIndexQuery.h +++ b/src/Parsers/ASTDropIndexQuery.h @@ -37,7 +37,7 @@ public: ASTPtr convertToASTAlterCommand() const; protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTDropNamedCollectionQuery.cpp b/src/Parsers/ASTDropNamedCollectionQuery.cpp index e317681d33d..41d19050b8a 100644 --- a/src/Parsers/ASTDropNamedCollectionQuery.cpp +++ b/src/Parsers/ASTDropNamedCollectionQuery.cpp @@ -10,13 +10,13 @@ ASTPtr ASTDropNamedCollectionQuery::clone() const return std::make_shared(*this); } -void ASTDropNamedCollectionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +void ASTDropNamedCollectionQuery::formatImpl(WriteBuffer & ostr, const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "DROP NAMED COLLECTION "; + ostr << (settings.hilite ? hilite_keyword : "") << "DROP NAMED COLLECTION "; if (if_exists) - settings.ostr << "IF EXISTS "; - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(collection_name) << (settings.hilite ? hilite_none : ""); - formatOnCluster(settings); + ostr << "IF EXISTS "; + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(collection_name) << (settings.hilite ? hilite_none : ""); + formatOnCluster(ostr, settings); } } diff --git a/src/Parsers/ASTDropNamedCollectionQuery.h b/src/Parsers/ASTDropNamedCollectionQuery.h index 2ead6c72532..45a8c9e17a4 100644 --- a/src/Parsers/ASTDropNamedCollectionQuery.h +++ b/src/Parsers/ASTDropNamedCollectionQuery.h @@ -17,7 +17,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } diff --git a/src/Parsers/ASTDropQuery.cpp b/src/Parsers/ASTDropQuery.cpp index 593ca37aa2a..a4e0bd25648 100644 --- a/src/Parsers/ASTDropQuery.cpp +++ b/src/Parsers/ASTDropQuery.cpp @@ -33,43 +33,43 @@ ASTPtr ASTDropQuery::clone() const return res; } -void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTDropQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : ""); + ostr << (settings.hilite ? hilite_keyword : ""); if (kind == ASTDropQuery::Kind::Drop) - settings.ostr << "DROP "; + ostr << "DROP "; else if (kind == ASTDropQuery::Kind::Detach) - settings.ostr << "DETACH "; + ostr << "DETACH "; else if (kind == ASTDropQuery::Kind::Truncate) - settings.ostr << "TRUNCATE "; + ostr << "TRUNCATE "; else throw Exception(ErrorCodes::SYNTAX_ERROR, "Not supported kind of drop query."); if (temporary) - settings.ostr << "TEMPORARY "; + ostr << "TEMPORARY "; if (has_all_tables) - settings.ostr << "ALL TABLES FROM "; + ostr << "ALL TABLES FROM "; else if (!table && !database_and_tables && database) - settings.ostr << "DATABASE "; + ostr << "DATABASE "; else if (is_dictionary) - settings.ostr << "DICTIONARY "; + ostr << "DICTIONARY "; else if (is_view) - settings.ostr << "VIEW "; + ostr << "VIEW "; else - settings.ostr << "TABLE "; + ostr << "TABLE "; if (if_exists) - settings.ostr << "IF EXISTS "; + ostr << "IF EXISTS "; if (if_empty) - settings.ostr << "IF EMPTY "; + ostr << "IF EMPTY "; - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_none : ""); if (!table && !database_and_tables && database) { - database->formatImpl(settings, state, frame); + database->formatImpl(ostr, settings, state, frame); } else if (database_and_tables) { @@ -77,7 +77,7 @@ void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState for (auto * it = list.children.begin(); it != list.children.end(); ++it) { if (it != list.children.begin()) - settings.ostr << ", "; + ostr << ", "; auto identifier = dynamic_pointer_cast(*it); if (!identifier) @@ -85,34 +85,34 @@ void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState if (auto db = identifier->getDatabase()) { - db->formatImpl(settings, state, frame); - settings.ostr << '.'; + db->formatImpl(ostr, settings, state, frame); + ostr << '.'; } auto tb = identifier->getTable(); chassert(tb); - tb->formatImpl(settings, state, frame); + tb->formatImpl(ostr, settings, state, frame); } } else { if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); } - formatOnCluster(settings); + formatOnCluster(ostr, settings); if (permanently) - settings.ostr << " PERMANENTLY"; + ostr << " PERMANENTLY"; if (sync) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " SYNC" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " SYNC" << (settings.hilite ? hilite_none : ""); } ASTs ASTDropQuery::getRewrittenASTsOfSingleTable() diff --git a/src/Parsers/ASTDropQuery.h b/src/Parsers/ASTDropQuery.h index e18043b771b..341d8f6381b 100644 --- a/src/Parsers/ASTDropQuery.h +++ b/src/Parsers/ASTDropQuery.h @@ -58,7 +58,7 @@ public: QueryKind getQueryKind() const override { return QueryKind::Drop; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTDropResourceQuery.cpp b/src/Parsers/ASTDropResourceQuery.cpp index 753ac4e30e7..3ef260d279e 100644 --- a/src/Parsers/ASTDropResourceQuery.cpp +++ b/src/Parsers/ASTDropResourceQuery.cpp @@ -10,16 +10,16 @@ ASTPtr ASTDropResourceQuery::clone() const return std::make_shared(*this); } -void ASTDropResourceQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +void ASTDropResourceQuery::formatImpl(WriteBuffer & ostr, const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "DROP RESOURCE "; + ostr << (settings.hilite ? hilite_keyword : "") << "DROP RESOURCE "; if (if_exists) - settings.ostr << "IF EXISTS "; + ostr << "IF EXISTS "; - settings.ostr << (settings.hilite ? hilite_none : ""); - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(resource_name) << (settings.hilite ? hilite_none : ""); - formatOnCluster(settings); + ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(resource_name) << (settings.hilite ? hilite_none : ""); + formatOnCluster(ostr, settings); } } diff --git a/src/Parsers/ASTDropResourceQuery.h b/src/Parsers/ASTDropResourceQuery.h index e1534ea454a..0844a2986bc 100644 --- a/src/Parsers/ASTDropResourceQuery.h +++ b/src/Parsers/ASTDropResourceQuery.h @@ -18,7 +18,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } diff --git a/src/Parsers/ASTDropWorkloadQuery.cpp b/src/Parsers/ASTDropWorkloadQuery.cpp index 3192223c4b3..2e066a4cdd0 100644 --- a/src/Parsers/ASTDropWorkloadQuery.cpp +++ b/src/Parsers/ASTDropWorkloadQuery.cpp @@ -10,16 +10,16 @@ ASTPtr ASTDropWorkloadQuery::clone() const return std::make_shared(*this); } -void ASTDropWorkloadQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +void ASTDropWorkloadQuery::formatImpl(WriteBuffer & ostr, const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "DROP WORKLOAD "; + ostr << (settings.hilite ? hilite_keyword : "") << "DROP WORKLOAD "; if (if_exists) - settings.ostr << "IF EXISTS "; + ostr << "IF EXISTS "; - settings.ostr << (settings.hilite ? hilite_none : ""); - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(workload_name) << (settings.hilite ? hilite_none : ""); - formatOnCluster(settings); + ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(workload_name) << (settings.hilite ? hilite_none : ""); + formatOnCluster(ostr, settings); } } diff --git a/src/Parsers/ASTDropWorkloadQuery.h b/src/Parsers/ASTDropWorkloadQuery.h index 99c3a011447..7c42fc906fd 100644 --- a/src/Parsers/ASTDropWorkloadQuery.h +++ b/src/Parsers/ASTDropWorkloadQuery.h @@ -18,7 +18,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } diff --git a/src/Parsers/ASTExplainQuery.h b/src/Parsers/ASTExplainQuery.h index eb095b5dbbc..e3f34b818dc 100644 --- a/src/Parsers/ASTExplainQuery.h +++ b/src/Parsers/ASTExplainQuery.h @@ -112,30 +112,30 @@ public: QueryKind getQueryKind() const override { return QueryKind::Explain; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { - settings.ostr << (settings.hilite ? hilite_keyword : "") << toString(kind) << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << toString(kind) << (settings.hilite ? hilite_none : ""); if (ast_settings) { - settings.ostr << ' '; - ast_settings->formatImpl(settings, state, frame); + ostr << ' '; + ast_settings->formatImpl(ostr, settings, state, frame); } if (query) { - settings.ostr << settings.nl_or_ws; - query->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws; + query->formatImpl(ostr, settings, state, frame); } if (table_function) { - settings.ostr << settings.nl_or_ws; - table_function->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws; + table_function->formatImpl(ostr, settings, state, frame); } if (table_override) { - settings.ostr << settings.nl_or_ws; - table_override->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws; + table_override->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTExpressionList.cpp b/src/Parsers/ASTExpressionList.cpp index f345b0c6a6f..8f27eae09dd 100644 --- a/src/Parsers/ASTExpressionList.cpp +++ b/src/Parsers/ASTExpressionList.cpp @@ -12,18 +12,18 @@ ASTPtr ASTExpressionList::clone() const return clone; } -void ASTExpressionList::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTExpressionList::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (frame.expression_list_prepend_whitespace) - settings.ostr << ' '; + ostr << ' '; for (size_t i = 0, size = children.size(); i < size; ++i) { if (i) { if (separator) - settings.ostr << separator; - settings.ostr << ' '; + ostr << separator; + ostr << ' '; } FormatStateStacked frame_nested = frame; @@ -31,16 +31,16 @@ void ASTExpressionList::formatImpl(const FormatSettings & settings, FormatState frame_nested.list_element_index = i; if (frame.surround_each_list_element_with_parens) - settings.ostr << "("; + ostr << "("; - children[i]->formatImpl(settings, state, frame_nested); + children[i]->formatImpl(ostr, settings, state, frame_nested); if (frame.surround_each_list_element_with_parens) - settings.ostr << ")"; + ostr << ")"; } } -void ASTExpressionList::formatImplMultiline(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTExpressionList::formatImplMultiline(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { ++frame.indent; std::string indent_str = "\n" + std::string(4 * frame.indent, ' '); @@ -48,16 +48,16 @@ void ASTExpressionList::formatImplMultiline(const FormatSettings & settings, For if (frame.expression_list_prepend_whitespace) { if (!(children.size() > 1 || frame.expression_list_always_start_on_new_line)) - settings.ostr << ' '; + ostr << ' '; } for (size_t i = 0, size = children.size(); i < size; ++i) { if (i && separator) - settings.ostr << separator; + ostr << separator; if (size > 1 || frame.expression_list_always_start_on_new_line) - settings.ostr << indent_str; + ostr << indent_str; FormatStateStacked frame_nested = frame; frame_nested.expression_list_always_start_on_new_line = false; @@ -65,12 +65,12 @@ void ASTExpressionList::formatImplMultiline(const FormatSettings & settings, For frame_nested.list_element_index = i; if (frame.surround_each_list_element_with_parens) - settings.ostr << "("; + ostr << "("; - children[i]->formatImpl(settings, state, frame_nested); + children[i]->formatImpl(ostr, settings, state, frame_nested); if (frame.surround_each_list_element_with_parens) - settings.ostr << ")"; + ostr << ")"; } } diff --git a/src/Parsers/ASTExpressionList.h b/src/Parsers/ASTExpressionList.h index 0bafc17576b..9ec4326cc06 100644 --- a/src/Parsers/ASTExpressionList.h +++ b/src/Parsers/ASTExpressionList.h @@ -16,8 +16,8 @@ public: String getID(char) const override { return "ExpressionList"; } ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; - void formatImplMultiline(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImplMultiline(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; char separator; }; diff --git a/src/Parsers/ASTExternalDDLQuery.h b/src/Parsers/ASTExternalDDLQuery.h index 96600b07f29..f72403669c7 100644 --- a/src/Parsers/ASTExternalDDLQuery.h +++ b/src/Parsers/ASTExternalDDLQuery.h @@ -33,11 +33,11 @@ public: String getID(char) const override { return "external ddl query"; } - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked stacked) const override + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked stacked) const override { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "EXTERNAL DDL FROM " << (settings.hilite ? hilite_none : ""); - from->formatImpl(settings, state, stacked); - external_ddl->formatImpl(settings, state, stacked); + ostr << (settings.hilite ? hilite_keyword : "") << "EXTERNAL DDL FROM " << (settings.hilite ? hilite_none : ""); + from->formatImpl(ostr, settings, state, stacked); + external_ddl->formatImpl(ostr, settings, state, stacked); } QueryKind getQueryKind() const override { return QueryKind::ExternalDDL; } diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 11cfe2e584e..3bbe6dd3ad7 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -91,36 +91,36 @@ void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const } else { - FormatSettings format_settings{ostr, true /* one_line */}; + FormatSettings format_settings{true /* one_line */}; FormatState state; FormatStateStacked frame; writeCString("(", ostr); - window_definition->formatImpl(format_settings, state, frame); + window_definition->formatImpl(ostr, format_settings, state, frame); writeCString(")", ostr); } } } -void ASTFunction::finishFormatWithWindow(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTFunction::finishFormatWithWindow(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (nulls_action == NullsAction::RESPECT_NULLS) - settings.ostr << " RESPECT NULLS"; + ostr << " RESPECT NULLS"; else if (nulls_action == NullsAction::IGNORE_NULLS) - settings.ostr << " IGNORE NULLS"; + ostr << " IGNORE NULLS"; if (!is_window_function) return; - settings.ostr << " OVER "; + ostr << " OVER "; if (!window_name.empty()) { - settings.ostr << backQuoteIfNeed(window_name); + ostr << backQuoteIfNeed(window_name); } else { - settings.ostr << "("; - window_definition->formatImpl(settings, state, frame); - settings.ostr << ")"; + ostr << "("; + window_definition->formatImpl(ostr, settings, state, frame); + ostr << ")"; } } @@ -212,7 +212,7 @@ ASTPtr ASTFunction::toLiteral() const * * Another case is regexp match. Suppose the user types match(URL, 'www.clickhouse.com'). It often means that the user is unaware that . is a metacharacter. */ -static bool highlightStringLiteralWithMetacharacters(const ASTPtr & node, const IAST::FormatSettings & settings, const char * metacharacters) +static bool highlightStringLiteralWithMetacharacters(const ASTPtr & node, WriteBuffer & ostr, const char * metacharacters) { if (const auto * literal = node->as()) { @@ -225,7 +225,7 @@ static bool highlightStringLiteralWithMetacharacters(const ASTPtr & node, const { if (c == '\\') { - settings.ostr << c; + ostr << c; if (escaping == 2) escaping = 0; ++escaping; @@ -233,14 +233,14 @@ static bool highlightStringLiteralWithMetacharacters(const ASTPtr & node, const else if (nullptr != strchr(metacharacters, c)) { if (escaping == 2) /// Properly escaped metacharacter - settings.ostr << c; + ostr << c; else /// Unescaped metacharacter - settings.ostr << "\033[1;35m" << c << "\033[0m"; + ostr << "\033[1;35m" << c << "\033[0m"; escaping = 0; } else { - settings.ostr << c; + ostr << c; escaping = 0; } } @@ -263,7 +263,7 @@ ASTSelectWithUnionQuery * ASTFunction::tryGetQueryArgument() const } -static bool formatNamedArgWithHiddenValue(IAST * arg, const IAST::FormatSettings & settings, IAST::FormatState & state, IAST::FormatStateStacked frame) +static bool formatNamedArgWithHiddenValue(IAST * arg, WriteBuffer & ostr, const IAST::FormatSettings & settings, IAST::FormatState & state, IAST::FormatStateStacked frame) { const auto * equals_func = arg->as(); if (!equals_func || (equals_func->name != "equals")) @@ -275,14 +275,14 @@ static bool formatNamedArgWithHiddenValue(IAST * arg, const IAST::FormatSettings if (equal_args.size() != 2) return false; - equal_args[0]->formatImpl(settings, state, frame); - settings.ostr << (settings.hilite ? IAST::hilite_operator : "") << " = " << (settings.hilite ? IAST::hilite_none : ""); - settings.ostr << "'[HIDDEN]'"; + equal_args[0]->formatImpl(ostr, settings, state, frame); + ostr << (settings.hilite ? IAST::hilite_operator : "") << " = " << (settings.hilite ? IAST::hilite_none : ""); + ostr << "'[HIDDEN]'"; return true; } -void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTFunction::formatImplWithoutAlias(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.expression_list_prepend_whitespace = false; if (kind == Kind::CODEC || kind == Kind::STATISTICS || kind == Kind::BACKUP_NAME) @@ -296,15 +296,15 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format { std::string nl_or_nothing = settings.one_line ? "" : "\n"; std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); - settings.ostr << (settings.hilite ? hilite_function : "") << name << (settings.hilite ? hilite_none : ""); - settings.ostr << (settings.hilite ? hilite_function : "") << "(" << (settings.hilite ? hilite_none : ""); - settings.ostr << nl_or_nothing; + ostr << (settings.hilite ? hilite_function : "") << name << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_function : "") << "(" << (settings.hilite ? hilite_none : ""); + ostr << nl_or_nothing; FormatStateStacked frame_nested = frame; frame_nested.need_parens = false; ++frame_nested.indent; - query->formatImpl(settings, state, frame_nested); - settings.ostr << nl_or_nothing << indent_str; - settings.ostr << (settings.hilite ? hilite_function : "") << ")" << (settings.hilite ? hilite_none : ""); + query->formatImpl(ostr, settings, state, frame_nested); + ostr << nl_or_nothing << indent_str; + ostr << (settings.hilite ? hilite_function : "") << ")" << (settings.hilite ? hilite_none : ""); return; } @@ -357,21 +357,21 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format nested_need_parens.need_parens = false; if (outside_parens) - settings.ostr << '('; + ostr << '('; - settings.ostr << (settings.hilite ? hilite_operator : "") << func[1] << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_operator : "") << func[1] << (settings.hilite ? hilite_none : ""); if (inside_parens) - settings.ostr << '('; + ostr << '('; - arguments->formatImpl(settings, state, nested_need_parens); + arguments->formatImpl(ostr, settings, state, nested_need_parens); written = true; if (inside_parens) - settings.ostr << ')'; + ostr << ')'; if (outside_parens) - settings.ostr << ')'; + ostr << ')'; break; } @@ -395,11 +395,11 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format } if (frame.need_parens) - settings.ostr << '('; - arguments->formatImpl(settings, state, nested_need_parens); - settings.ostr << (settings.hilite ? hilite_operator : "") << func[1] << (settings.hilite ? hilite_none : ""); + ostr << '('; + arguments->formatImpl(ostr, settings, state, nested_need_parens); + ostr << (settings.hilite ? hilite_operator : "") << func[1] << (settings.hilite ? hilite_none : ""); if (frame.need_parens) - settings.ostr << ')'; + ostr << ')'; written = true; @@ -443,13 +443,13 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format if (name == std::string_view(func[0])) { if (frame.need_parens) - settings.ostr << '('; - arguments->children[0]->formatImpl(settings, state, nested_need_parens); - settings.ostr << (settings.hilite ? hilite_operator : "") << func[1] << (settings.hilite ? hilite_none : ""); + ostr << '('; + arguments->children[0]->formatImpl(ostr, settings, state, nested_need_parens); + ostr << (settings.hilite ? hilite_operator : "") << func[1] << (settings.hilite ? hilite_none : ""); bool special_hilite = settings.hilite && (name == "like" || name == "notLike" || name == "ilike" || name == "notILike") - && highlightStringLiteralWithMetacharacters(arguments->children[1], settings, "%_"); + && highlightStringLiteralWithMetacharacters(arguments->children[1], ostr, "%_"); /// Format x IN 1 as x IN (1): put parens around rhs even if there is a single element in set. const auto * second_arg_func = arguments->children[1]->as(); @@ -463,16 +463,16 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format if (extra_parents_around_in_rhs) { - settings.ostr << '('; - arguments->children[1]->formatImpl(settings, state, nested_dont_need_parens); - settings.ostr << ')'; + ostr << '('; + arguments->children[1]->formatImpl(ostr, settings, state, nested_dont_need_parens); + ostr << ')'; } if (!special_hilite && !extra_parents_around_in_rhs) - arguments->children[1]->formatImpl(settings, state, nested_need_parens); + arguments->children[1]->formatImpl(ostr, settings, state, nested_need_parens); if (frame.need_parens) - settings.ostr << ')'; + ostr << ')'; written = true; } } @@ -480,16 +480,16 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format if (!written && name == "arrayElement"sv) { if (frame.need_parens) - settings.ostr << '('; + ostr << '('; - arguments->children[0]->formatImpl(settings, state, nested_need_parens); - settings.ostr << (settings.hilite ? hilite_operator : "") << '[' << (settings.hilite ? hilite_none : ""); - arguments->children[1]->formatImpl(settings, state, nested_dont_need_parens); - settings.ostr << (settings.hilite ? hilite_operator : "") << ']' << (settings.hilite ? hilite_none : ""); + arguments->children[0]->formatImpl(ostr, settings, state, nested_need_parens); + ostr << (settings.hilite ? hilite_operator : "") << '[' << (settings.hilite ? hilite_none : ""); + arguments->children[1]->formatImpl(ostr, settings, state, nested_dont_need_parens); + ostr << (settings.hilite ? hilite_operator : "") << ']' << (settings.hilite ? hilite_none : ""); written = true; if (frame.need_parens) - settings.ostr << ')'; + ostr << ')'; } if (!written && name == "tupleElement"sv) @@ -530,15 +530,15 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format && lit_right->value.safeGet() >= 0) { if (frame.need_parens) - settings.ostr << '('; + ostr << '('; - arguments->children[0]->formatImpl(settings, state, nested_need_parens); - settings.ostr << (settings.hilite ? hilite_operator : "") << "." << (settings.hilite ? hilite_none : ""); - arguments->children[1]->formatImpl(settings, state, nested_dont_need_parens); + arguments->children[0]->formatImpl(ostr, settings, state, nested_need_parens); + ostr << (settings.hilite ? hilite_operator : "") << "." << (settings.hilite ? hilite_none : ""); + arguments->children[1]->formatImpl(ostr, settings, state, nested_dont_need_parens); written = true; if (frame.need_parens) - settings.ostr << ')'; + ostr << ')'; } } } @@ -559,24 +559,24 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format /// Example: f(x, (y -> z)) should not be printed as f((x, y) -> z). if (frame.need_parens || frame.list_element_index > 0) - settings.ostr << '('; + ostr << '('; if (first_argument_is_tuple && first_argument_function->arguments && (first_argument_function->arguments->children.size() == 1 || first_argument_function->arguments->children.empty())) { if (first_argument_function->arguments->children.size() == 1) - first_argument_function->arguments->children[0]->formatImpl(settings, state, nested_need_parens); + first_argument_function->arguments->children[0]->formatImpl(ostr, settings, state, nested_need_parens); else - settings.ostr << "()"; + ostr << "()"; } else - first_argument->formatImpl(settings, state, nested_need_parens); + first_argument->formatImpl(ostr, settings, state, nested_need_parens); - settings.ostr << (settings.hilite ? hilite_operator : "") << " -> " << (settings.hilite ? hilite_none : ""); - arguments->children[1]->formatImpl(settings, state, nested_need_parens); + ostr << (settings.hilite ? hilite_operator : "") << " -> " << (settings.hilite ? hilite_none : ""); + arguments->children[1]->formatImpl(ostr, settings, state, nested_need_parens); if (frame.need_parens || frame.list_element_index > 0) - settings.ostr << ')'; + ostr << ')'; written = true; } @@ -587,15 +587,15 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format auto indent0 = settings.one_line ? "" : String(4u * frame.indent, ' '); auto indent1 = settings.one_line ? "" : String(4u * (frame.indent + 1), ' '); auto indent2 = settings.one_line ? "" : String(4u * (frame.indent + 2), ' '); - settings.ostr << (settings.hilite ? hilite_function : "") << name << "(" << (settings.hilite ? hilite_none : "") << nl_or_nothing; + ostr << (settings.hilite ? hilite_function : "") << name << "(" << (settings.hilite ? hilite_none : "") << nl_or_nothing; FormatStateStacked frame_nested = frame; frame_nested.need_parens = false; frame_nested.indent += 2; - arguments->children[0]->formatImpl(settings, state, frame_nested); - settings.ostr << nl_or_nothing << indent1 << (settings.hilite ? hilite_keyword : "") << (settings.one_line ? " " : "") + arguments->children[0]->formatImpl(ostr, settings, state, frame_nested); + ostr << nl_or_nothing << indent1 << (settings.hilite ? hilite_keyword : "") << (settings.one_line ? " " : "") << "ELSE " << (settings.hilite ? hilite_none : "") << nl_or_nothing << indent2; - arguments->children[1]->formatImpl(settings, state, frame_nested); - settings.ostr << nl_or_nothing << indent0 << ")"; + arguments->children[1]->formatImpl(ostr, settings, state, frame_nested); + ostr << nl_or_nothing << indent0 << ")"; return; } } @@ -614,17 +614,17 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format if (name == std::string_view(func[0])) { if (frame.need_parens) - settings.ostr << '('; + ostr << '('; for (size_t i = 0; i < arguments->children.size(); ++i) { if (i != 0) - settings.ostr << (settings.hilite ? hilite_operator : "") << func[1] << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_operator : "") << func[1] << (settings.hilite ? hilite_none : ""); if (arguments->children[i]->as()) - settings.ostr << "SETTINGS "; - arguments->children[i]->formatImpl(settings, state, nested_need_parens); + ostr << "SETTINGS "; + arguments->children[i]->formatImpl(ostr, settings, state, nested_need_parens); } if (frame.need_parens) - settings.ostr << ')'; + ostr << ')'; written = true; } } @@ -632,72 +632,72 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format if (!written && name == "array"sv) { - settings.ostr << (settings.hilite ? hilite_operator : "") << '[' << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_operator : "") << '[' << (settings.hilite ? hilite_none : ""); for (size_t i = 0; i < arguments->children.size(); ++i) { if (i != 0) - settings.ostr << ", "; + ostr << ", "; if (arguments->children[i]->as()) - settings.ostr << "SETTINGS "; + ostr << "SETTINGS "; nested_dont_need_parens.list_element_index = i; - arguments->children[i]->formatImpl(settings, state, nested_dont_need_parens); + arguments->children[i]->formatImpl(ostr, settings, state, nested_dont_need_parens); } - settings.ostr << (settings.hilite ? hilite_operator : "") << ']' << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_operator : "") << ']' << (settings.hilite ? hilite_none : ""); written = true; } if (!written && arguments->children.size() >= 2 && name == "tuple"sv) { - settings.ostr << (settings.hilite ? hilite_operator : "") << ((frame.need_parens && !alias.empty()) ? "tuple" : "") << '(' + ostr << (settings.hilite ? hilite_operator : "") << ((frame.need_parens && !alias.empty()) ? "tuple" : "") << '(' << (settings.hilite ? hilite_none : ""); for (size_t i = 0; i < arguments->children.size(); ++i) { if (i != 0) - settings.ostr << ", "; + ostr << ", "; if (arguments->children[i]->as()) - settings.ostr << "SETTINGS "; + ostr << "SETTINGS "; nested_dont_need_parens.list_element_index = i; - arguments->children[i]->formatImpl(settings, state, nested_dont_need_parens); + arguments->children[i]->formatImpl(ostr, settings, state, nested_dont_need_parens); } - settings.ostr << (settings.hilite ? hilite_operator : "") << ')' << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_operator : "") << ')' << (settings.hilite ? hilite_none : ""); written = true; } if (!written && name == "map"sv) { - settings.ostr << (settings.hilite ? hilite_operator : "") << "map(" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_operator : "") << "map(" << (settings.hilite ? hilite_none : ""); for (size_t i = 0; i < arguments->children.size(); ++i) { if (i != 0) - settings.ostr << ", "; + ostr << ", "; if (arguments->children[i]->as()) - settings.ostr << "SETTINGS "; + ostr << "SETTINGS "; nested_dont_need_parens.list_element_index = i; - arguments->children[i]->formatImpl(settings, state, nested_dont_need_parens); + arguments->children[i]->formatImpl(ostr, settings, state, nested_dont_need_parens); } - settings.ostr << (settings.hilite ? hilite_operator : "") << ')' << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_operator : "") << ')' << (settings.hilite ? hilite_none : ""); written = true; } } if (written) { - finishFormatWithWindow(settings, state, frame); + finishFormatWithWindow(ostr, settings, state, frame); return; } - settings.ostr << (settings.hilite ? hilite_function : "") << name; + ostr << (settings.hilite ? hilite_function : "") << name; if (parameters) { - settings.ostr << '(' << (settings.hilite ? hilite_none : ""); - parameters->formatImpl(settings, state, nested_dont_need_parens); - settings.ostr << (settings.hilite ? hilite_function : "") << ')'; + ostr << '(' << (settings.hilite ? hilite_none : ""); + parameters->formatImpl(ostr, settings, state, nested_dont_need_parens); + ostr << (settings.hilite ? hilite_function : "") << ')'; } if ((arguments && !arguments->children.empty()) || !no_empty_args) - settings.ostr << '(' << (settings.hilite ? hilite_none : ""); + ostr << '(' << (settings.hilite ? hilite_none : ""); if (arguments) { @@ -712,11 +712,11 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format for (size_t i = 0, size = arguments->children.size(); i < size; ++i) { if (i != 0) - settings.ostr << ", "; + ostr << ", "; const auto & argument = arguments->children[i]; if (argument->as()) - settings.ostr << "SETTINGS "; + ostr << "SETTINGS "; if (!settings.show_secrets) { @@ -725,18 +725,18 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format if (secret_arguments.are_named) { if (const auto * func_ast = typeid_cast(argument.get())) - func_ast->arguments->children[0]->formatImpl(settings, state, nested_dont_need_parens); + func_ast->arguments->children[0]->formatImpl(ostr, settings, state, nested_dont_need_parens); else - argument->formatImpl(settings, state, nested_dont_need_parens); - settings.ostr << (settings.hilite ? hilite_operator : "") << " = " << (settings.hilite ? hilite_none : ""); + argument->formatImpl(ostr, settings, state, nested_dont_need_parens); + ostr << (settings.hilite ? hilite_operator : "") << " = " << (settings.hilite ? hilite_none : ""); } if (!secret_arguments.replacement.empty()) { - settings.ostr << "'" << secret_arguments.replacement << "'"; + ostr << "'" << secret_arguments.replacement << "'"; } else { - settings.ostr << "'[HIDDEN]'"; + ostr << "'[HIDDEN]'"; } if (size <= secret_arguments.start + secret_arguments.count && !secret_arguments.are_named) break; /// All other arguments should also be hidden. @@ -747,36 +747,36 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format if (function && function->arguments && std::count(secret_arguments.nested_maps.begin(), secret_arguments.nested_maps.end(), function->name) != 0) { /// headers('foo' = '[HIDDEN]', 'bar' = '[HIDDEN]') - settings.ostr << (settings.hilite ? hilite_function : "") << function->name << (settings.hilite ? hilite_none : "") << "("; + ostr << (settings.hilite ? hilite_function : "") << function->name << (settings.hilite ? hilite_none : "") << "("; for (size_t j = 0; j < function->arguments->children.size(); ++j) { if (j != 0) - settings.ostr << ", "; + ostr << ", "; auto inner_arg = function->arguments->children[j]; - if (!formatNamedArgWithHiddenValue(inner_arg.get(), settings, state, nested_dont_need_parens)) - inner_arg->formatImpl(settings, state, nested_dont_need_parens); + if (!formatNamedArgWithHiddenValue(inner_arg.get(), ostr, settings, state, nested_dont_need_parens)) + inner_arg->formatImpl(ostr, settings, state, nested_dont_need_parens); } - settings.ostr << ")"; + ostr << ")"; continue; } } if ((i == 1) && special_hilite_regexp - && highlightStringLiteralWithMetacharacters(argument, settings, "|()^$.[]?*+{:-")) + && highlightStringLiteralWithMetacharacters(argument, ostr, "|()^$.[]?*+{:-")) { continue; } nested_dont_need_parens.list_element_index = i; - argument->formatImpl(settings, state, nested_dont_need_parens); + argument->formatImpl(ostr, settings, state, nested_dont_need_parens); } } if ((arguments && !arguments->children.empty()) || !no_empty_args) - settings.ostr << (settings.hilite ? hilite_function : "") << ')'; + ostr << (settings.hilite ? hilite_function : "") << ')'; - settings.ostr << (settings.hilite ? hilite_none : ""); - finishFormatWithWindow(settings, state, frame); + ostr << (settings.hilite ? hilite_none : ""); + finishFormatWithWindow(ostr, settings, state, frame); } bool ASTFunction::hasSecretParts() const diff --git a/src/Parsers/ASTFunction.h b/src/Parsers/ASTFunction.h index 1b4a5928d1c..1d794aada72 100644 --- a/src/Parsers/ASTFunction.h +++ b/src/Parsers/ASTFunction.h @@ -82,10 +82,10 @@ public: bool hasSecretParts() const override; protected: - void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImplWithoutAlias(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; private: - void finishFormatWithWindow(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; + void finishFormatWithWindow(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; }; diff --git a/src/Parsers/ASTFunctionWithKeyValueArguments.cpp b/src/Parsers/ASTFunctionWithKeyValueArguments.cpp index abb2e42a5de..1ceca7ae430 100644 --- a/src/Parsers/ASTFunctionWithKeyValueArguments.cpp +++ b/src/Parsers/ASTFunctionWithKeyValueArguments.cpp @@ -23,39 +23,39 @@ ASTPtr ASTPair::clone() const } -void ASTPair::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTPair::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << Poco::toUpper(first) << " " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << Poco::toUpper(first) << " " << (settings.hilite ? hilite_none : ""); if (second_with_brackets) - settings.ostr << (settings.hilite ? hilite_keyword : "") << "("; + ostr << (settings.hilite ? hilite_keyword : "") << "("; if (!settings.show_secrets && (first == "password")) { /// Hide password in the definition of a dictionary: /// SOURCE(CLICKHOUSE(host 'example01-01-1' port 9000 user 'default' password '[HIDDEN]' db 'default' table 'ids')) - settings.ostr << "'[HIDDEN]'"; + ostr << "'[HIDDEN]'"; } else if (!settings.show_secrets && (first == "uri")) { // Hide password from URI in the defention of a dictionary WriteBufferFromOwnString temp_buf; - FormatSettings tmp_settings(temp_buf, settings.one_line); + FormatSettings tmp_settings(settings.one_line); FormatState tmp_state; - second->formatImpl(tmp_settings, tmp_state, frame); + second->formatImpl(temp_buf, tmp_settings, tmp_state, frame); maskURIPassword(&temp_buf.str()); - settings.ostr << temp_buf.str(); + ostr << temp_buf.str(); } else { - second->formatImpl(settings, state, frame); + second->formatImpl(ostr, settings, state, frame); } if (second_with_brackets) - settings.ostr << (settings.hilite ? hilite_keyword : "") << ")"; + ostr << (settings.hilite ? hilite_keyword : "") << ")"; - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_none : ""); } @@ -95,12 +95,12 @@ ASTPtr ASTFunctionWithKeyValueArguments::clone() const } -void ASTFunctionWithKeyValueArguments::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTFunctionWithKeyValueArguments::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << Poco::toUpper(name) << (settings.hilite ? hilite_none : "") << (has_brackets ? "(" : ""); - elements->formatImpl(settings, state, frame); - settings.ostr << (has_brackets ? ")" : ""); - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << Poco::toUpper(name) << (settings.hilite ? hilite_none : "") << (has_brackets ? "(" : ""); + elements->formatImpl(ostr, settings, state, frame); + ostr << (has_brackets ? ")" : ""); + ostr << (settings.hilite ? hilite_none : ""); } diff --git a/src/Parsers/ASTFunctionWithKeyValueArguments.h b/src/Parsers/ASTFunctionWithKeyValueArguments.h index ec2a793154f..8e2d790c36a 100644 --- a/src/Parsers/ASTFunctionWithKeyValueArguments.h +++ b/src/Parsers/ASTFunctionWithKeyValueArguments.h @@ -28,7 +28,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; bool hasSecretParts() const override; @@ -64,7 +64,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; }; diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index 902432f5ef3..ea4180c64e3 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -103,13 +103,13 @@ const String & ASTIdentifier::name() const return full_name; } -void ASTIdentifier::formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTIdentifier::formatImplWithoutAlias(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { auto format_element = [&](const String & elem_name) { - settings.ostr << (settings.hilite ? hilite_identifier : ""); - settings.writeIdentifier(elem_name, /*ambiguous=*/false); - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_identifier : ""); + settings.writeIdentifier(ostr, elem_name, /*ambiguous=*/false); + ostr << (settings.hilite ? hilite_none : ""); }; if (compound()) @@ -117,14 +117,14 @@ void ASTIdentifier::formatImplWithoutAlias(const FormatSettings & settings, Form for (size_t i = 0, j = 0, size = name_parts.size(); i < size; ++i) { if (i != 0) - settings.ostr << '.'; + ostr << '.'; /// Some AST rewriting code, like IdentifierSemantic::setColumnLongName, /// does not respect children of identifier. /// Here we also ignore children if they are empty. if (name_parts[i].empty() && j < children.size()) { - children[j]->formatImpl(settings, state, frame); + children[j]->formatImpl(ostr, settings, state, frame); ++j; } else @@ -135,7 +135,7 @@ void ASTIdentifier::formatImplWithoutAlias(const FormatSettings & settings, Form { const auto & name = shortName(); if (name.empty() && !children.empty()) - children.front()->formatImpl(settings, state, frame); + children.front()->formatImpl(ostr, settings, state, frame); else format_element(name); } diff --git a/src/Parsers/ASTIdentifier.h b/src/Parsers/ASTIdentifier.h index d986b9170f3..fdf674b45a9 100644 --- a/src/Parsers/ASTIdentifier.h +++ b/src/Parsers/ASTIdentifier.h @@ -58,7 +58,7 @@ public: protected: std::shared_ptr semantic; /// pimpl - void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImplWithoutAlias(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; private: diff --git a/src/Parsers/ASTIndexDeclaration.cpp b/src/Parsers/ASTIndexDeclaration.cpp index 80102479cb9..32b21ddd2eb 100644 --- a/src/Parsers/ASTIndexDeclaration.cpp +++ b/src/Parsers/ASTIndexDeclaration.cpp @@ -62,7 +62,7 @@ std::shared_ptr ASTIndexDeclaration::getType() const return func_ast; } -void ASTIndexDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +void ASTIndexDeclaration::formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { if (auto expr = getExpression()) { @@ -70,31 +70,31 @@ void ASTIndexDeclaration::formatImpl(const FormatSettings & s, FormatState & sta { if (expr->as()) { - s.ostr << "("; - expr->formatImpl(s, state, frame); - s.ostr << ")"; + ostr << "("; + expr->formatImpl(ostr, s, state, frame); + ostr << ")"; } else - expr->formatImpl(s, state, frame); + expr->formatImpl(ostr, s, state, frame); } else { - s.writeIdentifier(name, /*ambiguous=*/false); - s.ostr << " "; - expr->formatImpl(s, state, frame); + s.writeIdentifier(ostr, name, /*ambiguous=*/false); + ostr << " "; + expr->formatImpl(ostr, s, state, frame); } } if (auto type = getType()) { - s.ostr << (s.hilite ? hilite_keyword : "") << " TYPE " << (s.hilite ? hilite_none : ""); - type->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << " TYPE " << (s.hilite ? hilite_none : ""); + type->formatImpl(ostr, s, state, frame); } if (granularity) { - s.ostr << (s.hilite ? hilite_keyword : "") << " GRANULARITY " << (s.hilite ? hilite_none : ""); - s.ostr << granularity; + ostr << (s.hilite ? hilite_keyword : "") << " GRANULARITY " << (s.hilite ? hilite_none : ""); + ostr << granularity; } } diff --git a/src/Parsers/ASTIndexDeclaration.h b/src/Parsers/ASTIndexDeclaration.h index 72f3f017a99..0c123b62ae9 100644 --- a/src/Parsers/ASTIndexDeclaration.h +++ b/src/Parsers/ASTIndexDeclaration.h @@ -25,7 +25,7 @@ public: String getID(char) const override { return "Index"; } ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; ASTPtr getExpression() const; std::shared_ptr getType() const; diff --git a/src/Parsers/ASTInsertQuery.cpp b/src/Parsers/ASTInsertQuery.cpp index 8e3458539f3..2a073733968 100644 --- a/src/Parsers/ASTInsertQuery.cpp +++ b/src/Parsers/ASTInsertQuery.cpp @@ -46,54 +46,54 @@ void ASTInsertQuery::setTable(const String & name) table = std::make_shared(name); } -void ASTInsertQuery::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTInsertQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.need_parens = false; - settings.ostr << (settings.hilite ? hilite_keyword : "") << "INSERT INTO "; + ostr << (settings.hilite ? hilite_keyword : "") << "INSERT INTO "; if (table_function) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "FUNCTION "; - table_function->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "FUNCTION "; + table_function->formatImpl(ostr, settings, state, frame); if (partition_by) { - settings.ostr << " PARTITION BY "; - partition_by->formatImpl(settings, state, frame); + ostr << " PARTITION BY "; + partition_by->formatImpl(ostr, settings, state, frame); } } else if (table_id) { - settings.ostr << (settings.hilite ? hilite_none : "") + ostr << (settings.hilite ? hilite_none : "") << (!table_id.database_name.empty() ? backQuoteIfNeed(table_id.database_name) + "." : "") << backQuoteIfNeed(table_id.table_name); } else { if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); } if (columns) { - settings.ostr << " ("; - columns->formatImpl(settings, state, frame); - settings.ostr << ")"; + ostr << " ("; + columns->formatImpl(ostr, settings, state, frame); + ostr << ")"; } if (infile) { - settings.ostr + ostr << (settings.hilite ? hilite_keyword : "") << " FROM INFILE " << (settings.hilite ? hilite_none : "") << quoteString(infile->as().value.safeGet()); if (compression) - settings.ostr + ostr << (settings.hilite ? hilite_keyword : "") << " COMPRESSION " << (settings.hilite ? hilite_none : "") @@ -102,8 +102,8 @@ void ASTInsertQuery::formatImpl(const FormatSettings & settings, FormatState & s if (settings_ast) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "SETTINGS " << (settings.hilite ? hilite_none : ""); - settings_ast->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "SETTINGS " << (settings.hilite ? hilite_none : ""); + settings_ast->formatImpl(ostr, settings, state, frame); } /// Compatibility for INSERT without SETTINGS to format in oneline, i.e.: @@ -120,20 +120,20 @@ void ASTInsertQuery::formatImpl(const FormatSettings & settings, FormatState & s if (select) { - settings.ostr << delim; - select->formatImpl(settings, state, frame); + ostr << delim; + select->formatImpl(ostr, settings, state, frame); } if (!select) { if (!format.empty()) { - settings.ostr << delim + ostr << delim << (settings.hilite ? hilite_keyword : "") << "FORMAT " << (settings.hilite ? hilite_none : "") << format; } else if (!infile) { - settings.ostr << delim + ostr << delim << (settings.hilite ? hilite_keyword : "") << "VALUES" << (settings.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/ASTInsertQuery.h b/src/Parsers/ASTInsertQuery.h index aeab0f148be..8b3a4a0bde0 100644 --- a/src/Parsers/ASTInsertQuery.h +++ b/src/Parsers/ASTInsertQuery.h @@ -71,7 +71,7 @@ public: QueryKind getQueryKind() const override { return async_insert_flush ? QueryKind::AsyncInsertFlush : QueryKind::Insert; } protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; }; diff --git a/src/Parsers/ASTInterpolateElement.cpp b/src/Parsers/ASTInterpolateElement.cpp index 15531962eb9..bf4a964c7fb 100644 --- a/src/Parsers/ASTInterpolateElement.cpp +++ b/src/Parsers/ASTInterpolateElement.cpp @@ -7,10 +7,10 @@ namespace DB { -void ASTInterpolateElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTInterpolateElement::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << column << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); - expr->formatImpl(settings, state, frame); + ostr << column << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); + expr->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTInterpolateElement.h b/src/Parsers/ASTInterpolateElement.h index a278755de88..5e57fd78f43 100644 --- a/src/Parsers/ASTInterpolateElement.h +++ b/src/Parsers/ASTInterpolateElement.h @@ -25,7 +25,7 @@ public: protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTKillQueryQuery.cpp b/src/Parsers/ASTKillQueryQuery.cpp index a3c0f48f28a..ce848f3fdaa 100644 --- a/src/Parsers/ASTKillQueryQuery.cpp +++ b/src/Parsers/ASTKillQueryQuery.cpp @@ -9,37 +9,37 @@ String ASTKillQueryQuery::getID(char delim) const return String("KillQueryQuery") + delim + (where_expression ? where_expression->getID() : "") + delim + String(sync ? "SYNC" : "ASYNC"); } -void ASTKillQueryQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTKillQueryQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "KILL "; + ostr << (settings.hilite ? hilite_keyword : "") << "KILL "; switch (type) { case Type::Query: - settings.ostr << "QUERY"; + ostr << "QUERY"; break; case Type::Mutation: - settings.ostr << "MUTATION"; + ostr << "MUTATION"; break; case Type::PartMoveToShard: - settings.ostr << "PART_MOVE_TO_SHARD"; + ostr << "PART_MOVE_TO_SHARD"; break; case Type::Transaction: - settings.ostr << "TRANSACTION"; + ostr << "TRANSACTION"; break; } - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_none : ""); - formatOnCluster(settings); + formatOnCluster(ostr, settings); if (where_expression) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); - where_expression->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); + where_expression->formatImpl(ostr, settings, state, frame); } - settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << (test ? "TEST" : (sync ? "SYNC" : "ASYNC")) << (settings.hilite ? hilite_none : ""); + ostr << " " << (settings.hilite ? hilite_keyword : "") << (test ? "TEST" : (sync ? "SYNC" : "ASYNC")) << (settings.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/ASTKillQueryQuery.h b/src/Parsers/ASTKillQueryQuery.h index 89ba474e107..99a14c56d72 100644 --- a/src/Parsers/ASTKillQueryQuery.h +++ b/src/Parsers/ASTKillQueryQuery.h @@ -36,7 +36,7 @@ public: String getID(char) const override; - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { diff --git a/src/Parsers/ASTLiteral.cpp b/src/Parsers/ASTLiteral.cpp index 515f4f0cb9f..a3261a926ff 100644 --- a/src/Parsers/ASTLiteral.cpp +++ b/src/Parsers/ASTLiteral.cpp @@ -148,12 +148,12 @@ String FieldVisitorToStringPostgreSQL::operator() (const String & x) const return wb.str(); } -void ASTLiteral::formatImplWithoutAlias(const FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +void ASTLiteral::formatImplWithoutAlias(WriteBuffer & ostr, const FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const { if (settings.literal_escaping_style == LiteralEscapingStyle::Regular) - settings.ostr << applyVisitor(FieldVisitorToString(), value); + ostr << applyVisitor(FieldVisitorToString(), value); else - settings.ostr << applyVisitor(FieldVisitorToStringPostgreSQL(), value); + ostr << applyVisitor(FieldVisitorToStringPostgreSQL(), value); } } diff --git a/src/Parsers/ASTLiteral.h b/src/Parsers/ASTLiteral.h index b957e435e2d..f36a428352a 100644 --- a/src/Parsers/ASTLiteral.h +++ b/src/Parsers/ASTLiteral.h @@ -52,7 +52,7 @@ public: void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; protected: - void formatImplWithoutAlias(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImplWithoutAlias(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; diff --git a/src/Parsers/ASTNameTypePair.cpp b/src/Parsers/ASTNameTypePair.cpp index 1515700365f..aed14a9c606 100644 --- a/src/Parsers/ASTNameTypePair.cpp +++ b/src/Parsers/ASTNameTypePair.cpp @@ -21,10 +21,10 @@ ASTPtr ASTNameTypePair::clone() const } -void ASTNameTypePair::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTNameTypePair::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << backQuoteIfNeed(name) << ' '; - type->formatImpl(settings, state, frame); + ostr << backQuoteIfNeed(name) << ' '; + type->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTNameTypePair.h b/src/Parsers/ASTNameTypePair.h index 638e980cbdc..886cfea9405 100644 --- a/src/Parsers/ASTNameTypePair.h +++ b/src/Parsers/ASTNameTypePair.h @@ -21,7 +21,7 @@ public: ASTPtr clone() const override; protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; diff --git a/src/Parsers/ASTObjectTypeArgument.cpp b/src/Parsers/ASTObjectTypeArgument.cpp index 975f0389505..c0658506b6f 100644 --- a/src/Parsers/ASTObjectTypeArgument.cpp +++ b/src/Parsers/ASTObjectTypeArgument.cpp @@ -35,27 +35,27 @@ ASTPtr ASTObjectTypeArgument::clone() const return res; } -void ASTObjectTypeArgument::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTObjectTypeArgument::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (path_with_type) { - path_with_type->formatImpl(settings, state, frame); + path_with_type->formatImpl(ostr, settings, state, frame); } else if (parameter) { - parameter->formatImpl(settings, state, frame); + parameter->formatImpl(ostr, settings, state, frame); } else if (skip_path) { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); - settings.ostr << indent_str << "SKIP" << ' '; - skip_path->formatImpl(settings, state, frame); + ostr << indent_str << "SKIP" << ' '; + skip_path->formatImpl(ostr, settings, state, frame); } else if (skip_path_regexp) { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); - settings.ostr << indent_str << "SKIP REGEXP" << ' '; - skip_path_regexp->formatImpl(settings, state, frame); + ostr << indent_str << "SKIP REGEXP" << ' '; + skip_path_regexp->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTObjectTypeArgument.h b/src/Parsers/ASTObjectTypeArgument.h index ab18d00d770..19db28b0398 100644 --- a/src/Parsers/ASTObjectTypeArgument.h +++ b/src/Parsers/ASTObjectTypeArgument.h @@ -25,7 +25,7 @@ public: ASTPtr clone() const override; protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; diff --git a/src/Parsers/ASTOptimizeQuery.cpp b/src/Parsers/ASTOptimizeQuery.cpp index 397a37586fc..85682a92be4 100644 --- a/src/Parsers/ASTOptimizeQuery.cpp +++ b/src/Parsers/ASTOptimizeQuery.cpp @@ -5,40 +5,40 @@ namespace DB { -void ASTOptimizeQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTOptimizeQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "OPTIMIZE TABLE " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "OPTIMIZE TABLE " << (settings.hilite ? hilite_none : ""); if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); - formatOnCluster(settings); + formatOnCluster(ostr, settings); if (partition) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " PARTITION " << (settings.hilite ? hilite_none : ""); - partition->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " PARTITION " << (settings.hilite ? hilite_none : ""); + partition->formatImpl(ostr, settings, state, frame); } if (final) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FINAL" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " FINAL" << (settings.hilite ? hilite_none : ""); if (deduplicate) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " DEDUPLICATE" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " DEDUPLICATE" << (settings.hilite ? hilite_none : ""); if (cleanup) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " CLEANUP" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " CLEANUP" << (settings.hilite ? hilite_none : ""); if (deduplicate_by_columns) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " BY " << (settings.hilite ? hilite_none : ""); - deduplicate_by_columns->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " BY " << (settings.hilite ? hilite_none : ""); + deduplicate_by_columns->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTOptimizeQuery.h b/src/Parsers/ASTOptimizeQuery.h index 4c914c11912..738b4f3bd02 100644 --- a/src/Parsers/ASTOptimizeQuery.h +++ b/src/Parsers/ASTOptimizeQuery.h @@ -49,7 +49,7 @@ public: return res; } - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams & params) const override { diff --git a/src/Parsers/ASTOrderByElement.cpp b/src/Parsers/ASTOrderByElement.cpp index d87c296d398..b14ae547c56 100644 --- a/src/Parsers/ASTOrderByElement.cpp +++ b/src/Parsers/ASTOrderByElement.cpp @@ -15,16 +15,16 @@ void ASTOrderByElement::updateTreeHashImpl(SipHash & hash_state, bool ignore_ali IAST::updateTreeHashImpl(hash_state, ignore_aliases); } -void ASTOrderByElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTOrderByElement::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - children.front()->formatImpl(settings, state, frame); - settings.ostr << (settings.hilite ? hilite_keyword : "") + children.front()->formatImpl(ostr, settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << (direction == -1 ? " DESC" : " ASC") << (settings.hilite ? hilite_none : ""); if (nulls_direction_was_explicitly_specified) { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << " NULLS " << (nulls_direction == direction ? "LAST" : "FIRST") << (settings.hilite ? hilite_none : ""); @@ -32,32 +32,32 @@ void ASTOrderByElement::formatImpl(const FormatSettings & settings, FormatState if (auto collation = getCollation()) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " COLLATE " << (settings.hilite ? hilite_none : ""); - collation->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " COLLATE " << (settings.hilite ? hilite_none : ""); + collation->formatImpl(ostr, settings, state, frame); } if (with_fill) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WITH FILL" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " WITH FILL" << (settings.hilite ? hilite_none : ""); if (auto fill_from = getFillFrom()) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : ""); - fill_from->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : ""); + fill_from->formatImpl(ostr, settings, state, frame); } if (auto fill_to = getFillTo()) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : ""); - fill_to->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : ""); + fill_to->formatImpl(ostr, settings, state, frame); } if (auto fill_step = getFillStep()) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " STEP " << (settings.hilite ? hilite_none : ""); - fill_step->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " STEP " << (settings.hilite ? hilite_none : ""); + fill_step->formatImpl(ostr, settings, state, frame); } if (auto fill_staleness = getFillStaleness()) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " STALENESS " << (settings.hilite ? hilite_none : ""); - fill_staleness->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " STALENESS " << (settings.hilite ? hilite_none : ""); + fill_staleness->formatImpl(ostr, settings, state, frame); } } } diff --git a/src/Parsers/ASTOrderByElement.h b/src/Parsers/ASTOrderByElement.h index 4dc35dac217..3185968426b 100644 --- a/src/Parsers/ASTOrderByElement.h +++ b/src/Parsers/ASTOrderByElement.h @@ -54,7 +54,7 @@ public: void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; private: ASTPtr getChild(Child child) const diff --git a/src/Parsers/ASTPartition.cpp b/src/Parsers/ASTPartition.cpp index d08f9ae1ca5..f8972cd439b 100644 --- a/src/Parsers/ASTPartition.cpp +++ b/src/Parsers/ASTPartition.cpp @@ -61,20 +61,20 @@ ASTPtr ASTPartition::clone() const return res; } -void ASTPartition::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTPartition::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (value) { - value->formatImpl(settings, state, frame); + value->formatImpl(ostr, settings, state, frame); } else if (all) { - settings.ostr << "ALL"; + ostr << "ALL"; } else { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "ID " << (settings.hilite ? hilite_none : ""); - id->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "ID " << (settings.hilite ? hilite_none : ""); + id->formatImpl(ostr, settings, state, frame); } } } diff --git a/src/Parsers/ASTPartition.h b/src/Parsers/ASTPartition.h index b2c87bb446a..f5aa630687b 100644 --- a/src/Parsers/ASTPartition.h +++ b/src/Parsers/ASTPartition.h @@ -29,7 +29,7 @@ public: } protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTProjectionDeclaration.cpp b/src/Parsers/ASTProjectionDeclaration.cpp index 21a924c0aff..48dae06b28b 100644 --- a/src/Parsers/ASTProjectionDeclaration.cpp +++ b/src/Parsers/ASTProjectionDeclaration.cpp @@ -15,17 +15,17 @@ ASTPtr ASTProjectionDeclaration::clone() const } -void ASTProjectionDeclaration::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTProjectionDeclaration::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.writeIdentifier(name, /*ambiguous=*/false); + settings.writeIdentifier(ostr, name, /*ambiguous=*/false); std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); std::string nl_or_nothing = settings.one_line ? "" : "\n"; - settings.ostr << settings.nl_or_ws << indent_str << "(" << nl_or_nothing; + ostr << settings.nl_or_ws << indent_str << "(" << nl_or_nothing; FormatStateStacked frame_nested = frame; frame_nested.need_parens = false; ++frame_nested.indent; - query->formatImpl(settings, state, frame_nested); - settings.ostr << nl_or_nothing << indent_str << ")"; + query->formatImpl(ostr, settings, state, frame_nested); + ostr << nl_or_nothing << indent_str << ")"; } } diff --git a/src/Parsers/ASTProjectionDeclaration.h b/src/Parsers/ASTProjectionDeclaration.h index df7a7c832a6..894eeb768c9 100644 --- a/src/Parsers/ASTProjectionDeclaration.h +++ b/src/Parsers/ASTProjectionDeclaration.h @@ -17,7 +17,7 @@ public: String getID(char) const override { return "Projection"; } ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; void forEachPointerToChild(std::function f) override { diff --git a/src/Parsers/ASTProjectionSelectQuery.cpp b/src/Parsers/ASTProjectionSelectQuery.cpp index 4bb1d2eef30..d1001b03898 100644 --- a/src/Parsers/ASTProjectionSelectQuery.cpp +++ b/src/Parsers/ASTProjectionSelectQuery.cpp @@ -48,7 +48,7 @@ ASTPtr ASTProjectionSelectQuery::clone() const } -void ASTProjectionSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +void ASTProjectionSelectQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { frame.current_select = this; frame.need_parens = false; @@ -56,26 +56,26 @@ void ASTProjectionSelectQuery::formatImpl(const FormatSettings & s, FormatState if (with()) { - s.ostr << (s.hilite ? hilite_keyword : "") << indent_str << "WITH " << (s.hilite ? hilite_none : ""); - s.one_line ? with()->formatImpl(s, state, frame) : with()->as().formatImplMultiline(s, state, frame); - s.ostr << s.nl_or_ws; + ostr << (s.hilite ? hilite_keyword : "") << indent_str << "WITH " << (s.hilite ? hilite_none : ""); + s.one_line ? with()->formatImpl(ostr, s, state, frame) : with()->as().formatImplMultiline(ostr, s, state, frame); + ostr << s.nl_or_ws; } - s.ostr << (s.hilite ? hilite_keyword : "") << indent_str << "SELECT " << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << indent_str << "SELECT " << (s.hilite ? hilite_none : ""); - s.one_line ? select()->formatImpl(s, state, frame) : select()->as().formatImplMultiline(s, state, frame); + s.one_line ? select()->formatImpl(ostr, s, state, frame) : select()->as().formatImplMultiline(ostr, s, state, frame); if (groupBy()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "GROUP BY " << (s.hilite ? hilite_none : ""); - s.one_line ? groupBy()->formatImpl(s, state, frame) : groupBy()->as().formatImplMultiline(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "GROUP BY " << (s.hilite ? hilite_none : ""); + s.one_line ? groupBy()->formatImpl(ostr, s, state, frame) : groupBy()->as().formatImplMultiline(ostr, s, state, frame); } if (orderBy()) { /// Let's convert tuple ASTFunction into ASTExpressionList, which generates consistent format /// between GROUP BY and ORDER BY projection definition. - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "ORDER BY " << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "ORDER BY " << (s.hilite ? hilite_none : ""); ASTPtr order_by; if (auto * func = orderBy()->as(); func && func->name == "tuple") order_by = func->arguments; @@ -84,7 +84,7 @@ void ASTProjectionSelectQuery::formatImpl(const FormatSettings & s, FormatState order_by = std::make_shared(); order_by->children.push_back(orderBy()); } - s.one_line ? order_by->formatImpl(s, state, frame) : order_by->as().formatImplMultiline(s, state, frame); + s.one_line ? order_by->formatImpl(ostr, s, state, frame) : order_by->as().formatImplMultiline(ostr, s, state, frame); } } diff --git a/src/Parsers/ASTProjectionSelectQuery.h b/src/Parsers/ASTProjectionSelectQuery.h index d93c10b6e39..826d8b8ebe2 100644 --- a/src/Parsers/ASTProjectionSelectQuery.h +++ b/src/Parsers/ASTProjectionSelectQuery.h @@ -45,7 +45,7 @@ public: ASTPtr cloneToASTSelect() const; protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; private: std::unordered_map positions; diff --git a/src/Parsers/ASTQualifiedAsterisk.cpp b/src/Parsers/ASTQualifiedAsterisk.cpp index 2dcf481adb7..630b9dd3bea 100644 --- a/src/Parsers/ASTQualifiedAsterisk.cpp +++ b/src/Parsers/ASTQualifiedAsterisk.cpp @@ -11,14 +11,14 @@ void ASTQualifiedAsterisk::appendColumnName(WriteBuffer & ostr) const writeCString(".*", ostr); } -void ASTQualifiedAsterisk::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTQualifiedAsterisk::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - qualifier->formatImpl(settings, state, frame); - settings.ostr << ".*"; + qualifier->formatImpl(ostr, settings, state, frame); + ostr << ".*"; if (transformers) { - transformers->formatImpl(settings, state, frame); + transformers->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTQualifiedAsterisk.h b/src/Parsers/ASTQualifiedAsterisk.h index 079b83ae171..4e66b6ff911 100644 --- a/src/Parsers/ASTQualifiedAsterisk.h +++ b/src/Parsers/ASTQualifiedAsterisk.h @@ -35,7 +35,7 @@ public: ASTPtr qualifier; ASTPtr transformers; protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTQueryParameter.cpp b/src/Parsers/ASTQueryParameter.cpp index 9e98252e779..3aee01fc1de 100644 --- a/src/Parsers/ASTQueryParameter.cpp +++ b/src/Parsers/ASTQueryParameter.cpp @@ -7,9 +7,9 @@ namespace DB { -void ASTQueryParameter::formatImplWithoutAlias(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTQueryParameter::formatImplWithoutAlias(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr + ostr << (settings.hilite ? hilite_substitution : "") << '{' << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(name) << (settings.hilite ? hilite_substitution : "") << ':' diff --git a/src/Parsers/ASTQueryParameter.h b/src/Parsers/ASTQueryParameter.h index dd7f9bff863..fef509e8b31 100644 --- a/src/Parsers/ASTQueryParameter.h +++ b/src/Parsers/ASTQueryParameter.h @@ -24,7 +24,7 @@ public: void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; protected: - void formatImplWithoutAlias(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImplWithoutAlias(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; }; diff --git a/src/Parsers/ASTQueryWithOnCluster.cpp b/src/Parsers/ASTQueryWithOnCluster.cpp index 1281a97fa00..c0c34feca82 100644 --- a/src/Parsers/ASTQueryWithOnCluster.cpp +++ b/src/Parsers/ASTQueryWithOnCluster.cpp @@ -26,11 +26,11 @@ bool ASTQueryWithOnCluster::parse(Pos & pos, std::string & cluster_str, Expected } -void ASTQueryWithOnCluster::formatOnCluster(const IAST::FormatSettings & settings) const +void ASTQueryWithOnCluster::formatOnCluster(WriteBuffer & ostr, const IAST::FormatSettings & settings) const { if (!cluster.empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " ON CLUSTER " << (settings.hilite ? IAST::hilite_none : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " ON CLUSTER " << (settings.hilite ? IAST::hilite_none : "") << backQuoteIfNeed(cluster); } } diff --git a/src/Parsers/ASTQueryWithOnCluster.h b/src/Parsers/ASTQueryWithOnCluster.h index a9c59a4e2b2..92804b9d9cd 100644 --- a/src/Parsers/ASTQueryWithOnCluster.h +++ b/src/Parsers/ASTQueryWithOnCluster.h @@ -33,7 +33,7 @@ public: /// Returns a query prepared for execution on remote server std::string getRewrittenQueryWithoutOnCluster(const WithoutOnClusterASTRewriteParams & params = {}) const; - void formatOnCluster(const IAST::FormatSettings & settings) const; + void formatOnCluster(WriteBuffer & ostr, const IAST::FormatSettings & settings) const; /// Parses " CLUSTER [cluster|'cluster'] " clause static bool parse(Pos & pos, std::string & cluster_str, Expected & expected); diff --git a/src/Parsers/ASTQueryWithOutput.cpp b/src/Parsers/ASTQueryWithOutput.cpp index c57aa759969..4c04e91eec5 100644 --- a/src/Parsers/ASTQueryWithOutput.cpp +++ b/src/Parsers/ASTQueryWithOutput.cpp @@ -35,37 +35,37 @@ void ASTQueryWithOutput::cloneOutputOptions(ASTQueryWithOutput & cloned) const } } -void ASTQueryWithOutput::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +void ASTQueryWithOutput::formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { - formatQueryImpl(s, state, frame); + formatQueryImpl(ostr, s, state, frame); std::string indent_str = s.one_line ? "" : std::string(4u * frame.indent, ' '); if (out_file) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "INTO OUTFILE " << (s.hilite ? hilite_none : ""); - out_file->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "INTO OUTFILE " << (s.hilite ? hilite_none : ""); + out_file->formatImpl(ostr, s, state, frame); - s.ostr << (s.hilite ? hilite_keyword : ""); + ostr << (s.hilite ? hilite_keyword : ""); if (is_outfile_append) - s.ostr << " APPEND"; + ostr << " APPEND"; if (is_outfile_truncate) - s.ostr << " TRUNCATE"; + ostr << " TRUNCATE"; if (is_into_outfile_with_stdout) - s.ostr << " AND STDOUT"; - s.ostr << (s.hilite ? hilite_none : ""); + ostr << " AND STDOUT"; + ostr << (s.hilite ? hilite_none : ""); } if (format) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "FORMAT " << (s.hilite ? hilite_none : ""); - format->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "FORMAT " << (s.hilite ? hilite_none : ""); + format->formatImpl(ostr, s, state, frame); } if (settings_ast && assert_cast(settings_ast.get())->print_in_format) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "SETTINGS " << (s.hilite ? hilite_none : ""); - settings_ast->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "SETTINGS " << (s.hilite ? hilite_none : ""); + settings_ast->formatImpl(ostr, s, state, frame); } } diff --git a/src/Parsers/ASTQueryWithOutput.h b/src/Parsers/ASTQueryWithOutput.h index 6f9cafc89a9..3c8d6a1e612 100644 --- a/src/Parsers/ASTQueryWithOutput.h +++ b/src/Parsers/ASTQueryWithOutput.h @@ -23,7 +23,7 @@ public: ASTPtr compression; ASTPtr compression_level; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const final; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const final; /// Remove 'FORMAT and INTO OUTFILE ' if exists static bool resetOutputASTIfExist(IAST & ast); @@ -33,7 +33,7 @@ protected: void cloneOutputOptions(ASTQueryWithOutput & cloned) const; /// Format only the query part of the AST (without output options). - virtual void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const = 0; + virtual void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const = 0; }; @@ -54,9 +54,9 @@ public: } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << ASTIDAndQueryNames::Query << (settings.hilite ? hilite_none : ""); } }; diff --git a/src/Parsers/ASTQueryWithTableAndOutput.h b/src/Parsers/ASTQueryWithTableAndOutput.h index b0227d68672..e19647c7cb8 100644 --- a/src/Parsers/ASTQueryWithTableAndOutput.h +++ b/src/Parsers/ASTQueryWithTableAndOutput.h @@ -50,20 +50,20 @@ public: QueryKind getQueryKind() const override { return QueryKind::Show; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << (temporary ? AstIDAndQueryNames::QueryTemporary : AstIDAndQueryNames::Query) << " " << (settings.hilite ? hilite_none : ""); if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table != nullptr, "Table is empty for the ASTQueryWithTableAndOutputImpl."); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); } }; diff --git a/src/Parsers/ASTRefreshStrategy.cpp b/src/Parsers/ASTRefreshStrategy.cpp index d10c1b4e7f5..d1a229189c6 100644 --- a/src/Parsers/ASTRefreshStrategy.cpp +++ b/src/Parsers/ASTRefreshStrategy.cpp @@ -24,49 +24,49 @@ ASTPtr ASTRefreshStrategy::clone() const } void ASTRefreshStrategy::formatImpl( - const IAST::FormatSettings & f_settings, IAST::FormatState & state, IAST::FormatStateStacked frame) const + WriteBuffer & ostr, const IAST::FormatSettings & f_settings, IAST::FormatState & state, IAST::FormatStateStacked frame) const { frame.need_parens = false; - f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << "REFRESH " << (f_settings.hilite ? hilite_none : ""); + ostr << (f_settings.hilite ? hilite_keyword : "") << "REFRESH " << (f_settings.hilite ? hilite_none : ""); using enum RefreshScheduleKind; switch (schedule_kind) { case AFTER: - f_settings.ostr << "AFTER " << (f_settings.hilite ? hilite_none : ""); - period->formatImpl(f_settings, state, frame); + ostr << "AFTER " << (f_settings.hilite ? hilite_none : ""); + period->formatImpl(ostr, f_settings, state, frame); break; case EVERY: - f_settings.ostr << "EVERY " << (f_settings.hilite ? hilite_none : ""); - period->formatImpl(f_settings, state, frame); + ostr << "EVERY " << (f_settings.hilite ? hilite_none : ""); + period->formatImpl(ostr, f_settings, state, frame); if (offset) { - f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << " OFFSET " << (f_settings.hilite ? hilite_none : ""); - offset->formatImpl(f_settings, state, frame); + ostr << (f_settings.hilite ? hilite_keyword : "") << " OFFSET " << (f_settings.hilite ? hilite_none : ""); + offset->formatImpl(ostr, f_settings, state, frame); } break; default: - f_settings.ostr << (f_settings.hilite ? hilite_none : ""); + ostr << (f_settings.hilite ? hilite_none : ""); break; } if (spread) { - f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << " RANDOMIZE FOR " << (f_settings.hilite ? hilite_none : ""); - spread->formatImpl(f_settings, state, frame); + ostr << (f_settings.hilite ? hilite_keyword : "") << " RANDOMIZE FOR " << (f_settings.hilite ? hilite_none : ""); + spread->formatImpl(ostr, f_settings, state, frame); } if (dependencies) { - f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << " DEPENDS ON " << (f_settings.hilite ? hilite_none : ""); - dependencies->formatImpl(f_settings, state, frame); + ostr << (f_settings.hilite ? hilite_keyword : "") << " DEPENDS ON " << (f_settings.hilite ? hilite_none : ""); + dependencies->formatImpl(ostr, f_settings, state, frame); } if (settings) { - f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << " SETTINGS " << (f_settings.hilite ? hilite_none : ""); - settings->formatImpl(f_settings, state, frame); + ostr << (f_settings.hilite ? hilite_keyword : "") << " SETTINGS " << (f_settings.hilite ? hilite_none : ""); + settings->formatImpl(ostr, f_settings, state, frame); } if (append) - f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << " APPEND" << (f_settings.hilite ? hilite_none : ""); + ostr << (f_settings.hilite ? hilite_keyword : "") << " APPEND" << (f_settings.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/ASTRefreshStrategy.h b/src/Parsers/ASTRefreshStrategy.h index bb5ac97c054..b04fccb92af 100644 --- a/src/Parsers/ASTRefreshStrategy.h +++ b/src/Parsers/ASTRefreshStrategy.h @@ -30,7 +30,7 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTRenameQuery.h b/src/Parsers/ASTRenameQuery.h index 39fc4f787ec..50ca4300680 100644 --- a/src/Parsers/ASTRenameQuery.h +++ b/src/Parsers/ASTRenameQuery.h @@ -155,66 +155,66 @@ public: } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { if (database) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "RENAME DATABASE " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "RENAME DATABASE " << (settings.hilite ? hilite_none : ""); if (elements.at(0).if_exists) - settings.ostr << (settings.hilite ? hilite_keyword : "") << "IF EXISTS " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "IF EXISTS " << (settings.hilite ? hilite_none : ""); - elements.at(0).from.database->formatImpl(settings, state, frame); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : ""); - elements.at(0).to.database->formatImpl(settings, state, frame); - formatOnCluster(settings); + elements.at(0).from.database->formatImpl(ostr, settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : ""); + elements.at(0).to.database->formatImpl(ostr, settings, state, frame); + formatOnCluster(ostr, settings); return; } - settings.ostr << (settings.hilite ? hilite_keyword : ""); + ostr << (settings.hilite ? hilite_keyword : ""); if (exchange && dictionary) - settings.ostr << "EXCHANGE DICTIONARIES "; + ostr << "EXCHANGE DICTIONARIES "; else if (exchange) - settings.ostr << "EXCHANGE TABLES "; + ostr << "EXCHANGE TABLES "; else if (dictionary) - settings.ostr << "RENAME DICTIONARY "; + ostr << "RENAME DICTIONARY "; else - settings.ostr << "RENAME TABLE "; + ostr << "RENAME TABLE "; - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_none : ""); for (auto it = elements.cbegin(); it != elements.cend(); ++it) { if (it != elements.cbegin()) - settings.ostr << ", "; + ostr << ", "; if (it->if_exists) - settings.ostr << (settings.hilite ? hilite_keyword : "") << "IF EXISTS " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "IF EXISTS " << (settings.hilite ? hilite_none : ""); if (it->from.database) { - it->from.database->formatImpl(settings, state, frame); - settings.ostr << '.'; + it->from.database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(it->from.table); - it->from.table->formatImpl(settings, state, frame); + it->from.table->formatImpl(ostr, settings, state, frame); - settings.ostr << (settings.hilite ? hilite_keyword : "") << (exchange ? " AND " : " TO ") << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << (exchange ? " AND " : " TO ") << (settings.hilite ? hilite_none : ""); if (it->to.database) { - it->to.database->formatImpl(settings, state, frame); - settings.ostr << '.'; + it->to.database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(it->to.table); - it->to.table->formatImpl(settings, state, frame); + it->to.table->formatImpl(ostr, settings, state, frame); } - formatOnCluster(settings); + formatOnCluster(ostr, settings); } Elements elements; diff --git a/src/Parsers/ASTSQLSecurity.cpp b/src/Parsers/ASTSQLSecurity.cpp index 74408747290..5244848b5eb 100644 --- a/src/Parsers/ASTSQLSecurity.cpp +++ b/src/Parsers/ASTSQLSecurity.cpp @@ -5,33 +5,33 @@ namespace DB { -void ASTSQLSecurity::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTSQLSecurity::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (!type) return; if (definer || is_definer_current_user) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "DEFINER" << (settings.hilite ? hilite_none : ""); - settings.ostr << " = "; + ostr << (settings.hilite ? hilite_keyword : "") << "DEFINER" << (settings.hilite ? hilite_none : ""); + ostr << " = "; if (definer) - definer->formatImpl(settings, state, frame); + definer->formatImpl(ostr, settings, state, frame); else - settings.ostr << "CURRENT_USER"; - settings.ostr << " "; + ostr << "CURRENT_USER"; + ostr << " "; } - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SQL SECURITY" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "SQL SECURITY" << (settings.hilite ? hilite_none : ""); switch (*type) { case SQLSecurityType::INVOKER: - settings.ostr << " INVOKER"; + ostr << " INVOKER"; break; case SQLSecurityType::DEFINER: - settings.ostr << " DEFINER"; + ostr << " DEFINER"; break; case SQLSecurityType::NONE: - settings.ostr << " NONE"; + ostr << " NONE"; break; } } diff --git a/src/Parsers/ASTSQLSecurity.h b/src/Parsers/ASTSQLSecurity.h index 47fd8752a67..d94af44539f 100644 --- a/src/Parsers/ASTSQLSecurity.h +++ b/src/Parsers/ASTSQLSecurity.h @@ -20,7 +20,7 @@ public: String getID(char) const override { return "View SQL Security"; } ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTSampleRatio.cpp b/src/Parsers/ASTSampleRatio.cpp index 608bf18bb1e..2c80f2f8744 100644 --- a/src/Parsers/ASTSampleRatio.cpp +++ b/src/Parsers/ASTSampleRatio.cpp @@ -34,9 +34,9 @@ String ASTSampleRatio::toString(Rational ratio) return toString(ratio.numerator) + " / " + toString(ratio.denominator); } -void ASTSampleRatio::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +void ASTSampleRatio::formatImpl(WriteBuffer & ostr, const IAST::FormatSettings &, IAST::FormatState &, IAST::FormatStateStacked) const { - settings.ostr << toString(ratio); + ostr << toString(ratio); } } diff --git a/src/Parsers/ASTSampleRatio.h b/src/Parsers/ASTSampleRatio.h index 220f938335b..d29d31f3adb 100644 --- a/src/Parsers/ASTSampleRatio.h +++ b/src/Parsers/ASTSampleRatio.h @@ -31,7 +31,7 @@ public: static String toString(BigNum num); static String toString(Rational ratio); - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings &, FormatState &, FormatStateStacked) const override; }; inline bool operator==(const ASTSampleRatio::Rational & lhs, const ASTSampleRatio::Rational & rhs) diff --git a/src/Parsers/ASTSelectIntersectExceptQuery.cpp b/src/Parsers/ASTSelectIntersectExceptQuery.cpp index 75fbe2b5280..b4a12e3d5ad 100644 --- a/src/Parsers/ASTSelectIntersectExceptQuery.cpp +++ b/src/Parsers/ASTSelectIntersectExceptQuery.cpp @@ -18,7 +18,7 @@ ASTPtr ASTSelectIntersectExceptQuery::clone() const return res; } -void ASTSelectIntersectExceptQuery::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTSelectIntersectExceptQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); @@ -26,13 +26,13 @@ void ASTSelectIntersectExceptQuery::formatImpl(const FormatSettings & settings, { if (it != children.begin()) { - settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") + ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << fromOperator(final_operator) << (settings.hilite ? hilite_none : "") << settings.nl_or_ws; } - (*it)->formatImpl(settings, state, frame); + (*it)->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTSelectIntersectExceptQuery.h b/src/Parsers/ASTSelectIntersectExceptQuery.h index d2e3b1a7172..1c90c9ad03c 100644 --- a/src/Parsers/ASTSelectIntersectExceptQuery.h +++ b/src/Parsers/ASTSelectIntersectExceptQuery.h @@ -23,7 +23,7 @@ public: INTERSECT_DISTINCT, }; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; QueryKind getQueryKind() const override { return QueryKind::Select; } diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index 8b1f82e9ee8..ed35b9d2c4d 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -54,7 +54,7 @@ void ASTSelectQuery::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliase } -void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +void ASTSelectQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { frame.current_select = this; frame.need_parens = false; @@ -64,54 +64,54 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F if (with()) { - s.ostr << (s.hilite ? hilite_keyword : "") << indent_str << "WITH" << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << indent_str << "WITH" << (s.hilite ? hilite_none : ""); if (recursive_with) - s.ostr << (s.hilite ? hilite_keyword : "") << " RECURSIVE" << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << " RECURSIVE" << (s.hilite ? hilite_none : ""); s.one_line - ? with()->formatImpl(s, state, frame) - : with()->as().formatImplMultiline(s, state, frame); - s.ostr << s.nl_or_ws; + ? with()->formatImpl(ostr, s, state, frame) + : with()->as().formatImplMultiline(ostr, s, state, frame); + ostr << s.nl_or_ws; } - s.ostr << (s.hilite ? hilite_keyword : "") << indent_str << "SELECT" << (distinct ? " DISTINCT" : "") << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << indent_str << "SELECT" << (distinct ? " DISTINCT" : "") << (s.hilite ? hilite_none : ""); s.one_line - ? select()->formatImpl(s, state, frame) - : select()->as().formatImplMultiline(s, state, frame); + ? select()->formatImpl(ostr, s, state, frame) + : select()->as().formatImplMultiline(ostr, s, state, frame); if (tables()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "FROM" << (s.hilite ? hilite_none : ""); - tables()->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "FROM" << (s.hilite ? hilite_none : ""); + tables()->formatImpl(ostr, s, state, frame); } if (prewhere()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "PREWHERE " << (s.hilite ? hilite_none : ""); - prewhere()->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "PREWHERE " << (s.hilite ? hilite_none : ""); + prewhere()->formatImpl(ostr, s, state, frame); } if (where()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "WHERE " << (s.hilite ? hilite_none : ""); - where()->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "WHERE " << (s.hilite ? hilite_none : ""); + where()->formatImpl(ostr, s, state, frame); } if (!group_by_all && groupBy()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "GROUP BY" << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "GROUP BY" << (s.hilite ? hilite_none : ""); if (!group_by_with_grouping_sets) { s.one_line - ? groupBy()->formatImpl(s, state, frame) - : groupBy()->as().formatImplMultiline(s, state, frame); + ? groupBy()->formatImpl(ostr, s, state, frame) + : groupBy()->as().formatImplMultiline(ostr, s, state, frame); } } if (group_by_all) - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "GROUP BY ALL" << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "GROUP BY ALL" << (s.hilite ? hilite_none : ""); if (group_by_with_grouping_sets && groupBy()) { @@ -119,73 +119,73 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F nested_frame.surround_each_list_element_with_parens = true; nested_frame.expression_list_prepend_whitespace = false; nested_frame.indent++; - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "GROUPING SETS" << (s.hilite ? hilite_none : ""); - s.ostr << " ("; + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "GROUPING SETS" << (s.hilite ? hilite_none : ""); + ostr << " ("; s.one_line - ? groupBy()->formatImpl(s, state, nested_frame) - : groupBy()->as().formatImplMultiline(s, state, nested_frame); - s.ostr << ")"; + ? groupBy()->formatImpl(ostr, s, state, nested_frame) + : groupBy()->as().formatImplMultiline(ostr, s, state, nested_frame); + ostr << ")"; } if (group_by_with_rollup) - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH ROLLUP" << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH ROLLUP" << (s.hilite ? hilite_none : ""); if (group_by_with_cube) - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH CUBE" << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH CUBE" << (s.hilite ? hilite_none : ""); if (group_by_with_totals) - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH TOTALS" << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH TOTALS" << (s.hilite ? hilite_none : ""); if (having()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "HAVING " << (s.hilite ? hilite_none : ""); - having()->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "HAVING " << (s.hilite ? hilite_none : ""); + having()->formatImpl(ostr, s, state, frame); } if (window()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "WINDOW" << (s.hilite ? hilite_none : ""); - window()->as().formatImplMultiline(s, state, frame); + window()->as().formatImplMultiline(ostr, s, state, frame); } if (qualify()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "QUALIFY " << (s.hilite ? hilite_none : ""); - qualify()->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "QUALIFY " << (s.hilite ? hilite_none : ""); + qualify()->formatImpl(ostr, s, state, frame); } if (!order_by_all && orderBy()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "ORDER BY" << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "ORDER BY" << (s.hilite ? hilite_none : ""); s.one_line - ? orderBy()->formatImpl(s, state, frame) - : orderBy()->as().formatImplMultiline(s, state, frame); + ? orderBy()->formatImpl(ostr, s, state, frame) + : orderBy()->as().formatImplMultiline(ostr, s, state, frame); if (interpolate()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "INTERPOLATE" << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "INTERPOLATE" << (s.hilite ? hilite_none : ""); if (!interpolate()->children.empty()) { - s.ostr << " ("; - interpolate()->formatImpl(s, state, frame); - s.ostr << " )"; + ostr << " ("; + interpolate()->formatImpl(ostr, s, state, frame); + ostr << " )"; } } } if (order_by_all) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "ORDER BY ALL" << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "ORDER BY ALL" << (s.hilite ? hilite_none : ""); auto * elem = orderBy()->children[0]->as(); - s.ostr << (s.hilite ? hilite_keyword : "") + ostr << (s.hilite ? hilite_keyword : "") << (elem->direction == -1 ? " DESC" : " ASC") << (s.hilite ? hilite_none : ""); if (elem->nulls_direction_was_explicitly_specified) { - s.ostr << (s.hilite ? hilite_keyword : "") + ostr << (s.hilite ? hilite_keyword : "") << " NULLS " << (elem->nulls_direction == elem->direction ? "LAST" : "FIRST") << (s.hilite ? hilite_none : ""); @@ -194,41 +194,41 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F if (limitByLength()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "LIMIT " << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "LIMIT " << (s.hilite ? hilite_none : ""); if (limitByOffset()) { - limitByOffset()->formatImpl(s, state, frame); - s.ostr << ", "; + limitByOffset()->formatImpl(ostr, s, state, frame); + ostr << ", "; } - limitByLength()->formatImpl(s, state, frame); - s.ostr << (s.hilite ? hilite_keyword : "") << " BY" << (s.hilite ? hilite_none : ""); + limitByLength()->formatImpl(ostr, s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << " BY" << (s.hilite ? hilite_none : ""); s.one_line - ? limitBy()->formatImpl(s, state, frame) - : limitBy()->as().formatImplMultiline(s, state, frame); + ? limitBy()->formatImpl(ostr, s, state, frame) + : limitBy()->as().formatImplMultiline(ostr, s, state, frame); } if (limitLength()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "LIMIT " << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "LIMIT " << (s.hilite ? hilite_none : ""); if (limitOffset()) { - limitOffset()->formatImpl(s, state, frame); - s.ostr << ", "; + limitOffset()->formatImpl(ostr, s, state, frame); + ostr << ", "; } - limitLength()->formatImpl(s, state, frame); + limitLength()->formatImpl(ostr, s, state, frame); if (limit_with_ties) - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << " WITH TIES" << (s.hilite ? hilite_none : ""); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << " WITH TIES" << (s.hilite ? hilite_none : ""); } else if (limitOffset()) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "OFFSET " << (s.hilite ? hilite_none : ""); - limitOffset()->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "OFFSET " << (s.hilite ? hilite_none : ""); + limitOffset()->formatImpl(ostr, s, state, frame); } if (settings() && assert_cast(settings().get())->print_in_format) { - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "SETTINGS " << (s.hilite ? hilite_none : ""); - settings()->formatImpl(s, state, frame); + ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "SETTINGS " << (s.hilite ? hilite_none : ""); + settings()->formatImpl(ostr, s, state, frame); } } diff --git a/src/Parsers/ASTSelectQuery.h b/src/Parsers/ASTSelectQuery.h index a2a6791442a..a3964e50b7a 100644 --- a/src/Parsers/ASTSelectQuery.h +++ b/src/Parsers/ASTSelectQuery.h @@ -152,7 +152,7 @@ public: bool hasQueryParameters() const; protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; private: std::unordered_map positions; diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 811704171bd..1582b048ca2 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -27,7 +27,7 @@ ASTPtr ASTSelectWithUnionQuery::clone() const } -void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTSelectWithUnionQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); @@ -57,24 +57,24 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it) { if (it != list_of_selects->children.begin()) - settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") + ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << mode_to_str((is_normalized) ? union_mode : list_of_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : ""); if (auto * /*node*/ _ = (*it)->as()) { if (it != list_of_selects->children.begin()) - settings.ostr << settings.nl_or_ws; + ostr << settings.nl_or_ws; - settings.ostr << indent_str; + ostr << indent_str; auto sub_query = std::make_shared(*it); - sub_query->formatImpl(settings, state, frame); + sub_query->formatImpl(ostr, settings, state, frame); } else { if (it != list_of_selects->children.begin()) - settings.ostr << settings.nl_or_ws; - (*it)->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws; + (*it)->formatImpl(ostr, settings, state, frame); } } } diff --git a/src/Parsers/ASTSelectWithUnionQuery.h b/src/Parsers/ASTSelectWithUnionQuery.h index a775e217308..f9caea8df6d 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.h +++ b/src/Parsers/ASTSelectWithUnionQuery.h @@ -15,7 +15,7 @@ public: ASTPtr clone() const override; - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; QueryKind getQueryKind() const override { return QueryKind::Select; } diff --git a/src/Parsers/ASTSetQuery.cpp b/src/Parsers/ASTSetQuery.cpp index 94b44ed2fa7..278a35d94ee 100644 --- a/src/Parsers/ASTSetQuery.cpp +++ b/src/Parsers/ASTSetQuery.cpp @@ -66,48 +66,48 @@ void ASTSetQuery::updateTreeHashImpl(SipHash & hash_state, bool /*ignore_aliases } } -void ASTSetQuery::formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const +void ASTSetQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & format, FormatState &, FormatStateStacked) const { if (is_standalone) - format.ostr << (format.hilite ? hilite_keyword : "") << "SET " << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << "SET " << (format.hilite ? hilite_none : ""); bool first = true; for (const auto & change : changes) { if (!first) - format.ostr << ", "; + ostr << ", "; else first = false; - formatSettingName(change.name, format.ostr); + formatSettingName(change.name, ostr); CustomType custom; if (!format.show_secrets && change.value.tryGet(custom) && custom.isSecret()) - format.ostr << " = " << custom.toString(false); + ostr << " = " << custom.toString(false); else - format.ostr << " = " << applyVisitor(FieldVisitorToSetting(), change.value); + ostr << " = " << applyVisitor(FieldVisitorToSetting(), change.value); } for (const auto & setting_name : default_settings) { if (!first) - format.ostr << ", "; + ostr << ", "; else first = false; - formatSettingName(setting_name, format.ostr); - format.ostr << " = DEFAULT"; + formatSettingName(setting_name, ostr); + ostr << " = DEFAULT"; } for (const auto & [name, value] : query_parameters) { if (!first) - format.ostr << ", "; + ostr << ", "; else first = false; - formatSettingName(QUERY_PARAMETER_NAME_PREFIX + name, format.ostr); - format.ostr << " = " << quoteString(value); + formatSettingName(QUERY_PARAMETER_NAME_PREFIX + name, ostr); + ostr << " = " << quoteString(value); } } diff --git a/src/Parsers/ASTSetQuery.h b/src/Parsers/ASTSetQuery.h index b52662b246e..d2372e9ab62 100644 --- a/src/Parsers/ASTSetQuery.h +++ b/src/Parsers/ASTSetQuery.h @@ -32,7 +32,7 @@ public: ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & format, FormatState &, FormatStateStacked) const override; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; diff --git a/src/Parsers/ASTShowColumnsQuery.cpp b/src/Parsers/ASTShowColumnsQuery.cpp index a02dba9c288..4737a0fd7e5 100644 --- a/src/Parsers/ASTShowColumnsQuery.cpp +++ b/src/Parsers/ASTShowColumnsQuery.cpp @@ -15,22 +15,22 @@ ASTPtr ASTShowColumnsQuery::clone() const return res; } -void ASTShowColumnsQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTShowColumnsQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW " << (extended ? "EXTENDED " : "") << (full ? "FULL " : "") << "COLUMNS" << (settings.hilite ? hilite_none : ""); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(table); + ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(table); if (!database.empty()) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database); + ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database); if (!like.empty()) - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << (not_like ? " NOT" : "") << (case_insensitive_like ? " ILIKE " : " LIKE") << (settings.hilite ? hilite_none : "") @@ -38,14 +38,14 @@ void ASTShowColumnsQuery::formatQueryImpl(const FormatSettings & settings, Forma if (where_expression) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); - where_expression->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); + where_expression->formatImpl(ostr, settings, state, frame); } if (limit_length) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " LIMIT " << (settings.hilite ? hilite_none : ""); - limit_length->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " LIMIT " << (settings.hilite ? hilite_none : ""); + limit_length->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTShowColumnsQuery.h b/src/Parsers/ASTShowColumnsQuery.h index b49d688dcdb..417562befcd 100644 --- a/src/Parsers/ASTShowColumnsQuery.h +++ b/src/Parsers/ASTShowColumnsQuery.h @@ -28,7 +28,7 @@ public: QueryKind getQueryKind() const override { return QueryKind::Show; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTShowFunctionsQuery.cpp b/src/Parsers/ASTShowFunctionsQuery.cpp index 9253dcf5cb2..3016fc95e36 100644 --- a/src/Parsers/ASTShowFunctionsQuery.cpp +++ b/src/Parsers/ASTShowFunctionsQuery.cpp @@ -13,12 +13,12 @@ ASTPtr ASTShowFunctionsQuery::clone() const return res; } -void ASTShowFunctionsQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTShowFunctionsQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW FUNCTIONS" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW FUNCTIONS" << (settings.hilite ? hilite_none : ""); if (!like.empty()) - settings.ostr << (settings.hilite ? hilite_keyword : "") << (case_insensitive_like ? " ILIKE " : " LIKE ") + ostr << (settings.hilite ? hilite_keyword : "") << (case_insensitive_like ? " ILIKE " : " LIKE ") << (settings.hilite ? hilite_none : "") << DB::quote << like; } diff --git a/src/Parsers/ASTShowFunctionsQuery.h b/src/Parsers/ASTShowFunctionsQuery.h index 6993f939888..d81507152f1 100644 --- a/src/Parsers/ASTShowFunctionsQuery.h +++ b/src/Parsers/ASTShowFunctionsQuery.h @@ -17,7 +17,7 @@ public: QueryKind getQueryKind() const override { return QueryKind::Show; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTShowIndexesQuery.cpp b/src/Parsers/ASTShowIndexesQuery.cpp index df1e242a239..5757defd165 100644 --- a/src/Parsers/ASTShowIndexesQuery.cpp +++ b/src/Parsers/ASTShowIndexesQuery.cpp @@ -15,22 +15,22 @@ ASTPtr ASTShowIndexesQuery::clone() const return res; } -void ASTShowIndexesQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTShowIndexesQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW " << (extended ? "EXTENDED " : "") << "INDEXES" << (settings.hilite ? hilite_none : ""); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(table); + ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(table); if (!database.empty()) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database); + ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database); if (where_expression) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); - where_expression->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); + where_expression->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTShowIndexesQuery.h b/src/Parsers/ASTShowIndexesQuery.h index ab504bf71da..d94ac37a389 100644 --- a/src/Parsers/ASTShowIndexesQuery.h +++ b/src/Parsers/ASTShowIndexesQuery.h @@ -22,7 +22,7 @@ public: QueryKind getQueryKind() const override { return QueryKind::Show; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTShowSettingQuery.cpp b/src/Parsers/ASTShowSettingQuery.cpp index 267d462475d..279779f7f1d 100644 --- a/src/Parsers/ASTShowSettingQuery.cpp +++ b/src/Parsers/ASTShowSettingQuery.cpp @@ -16,9 +16,9 @@ ASTPtr ASTShowSettingQuery::clone() const return res; } -void ASTShowSettingQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTShowSettingQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW SETTING " << (settings.hilite ? hilite_none : "") + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW SETTING " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(setting_name); } diff --git a/src/Parsers/ASTShowSettingQuery.h b/src/Parsers/ASTShowSettingQuery.h index 2b4395f307d..0485efeb504 100644 --- a/src/Parsers/ASTShowSettingQuery.h +++ b/src/Parsers/ASTShowSettingQuery.h @@ -21,7 +21,7 @@ public: QueryKind getQueryKind() const override { return QueryKind::Show; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; private: String setting_name; diff --git a/src/Parsers/ASTShowTablesQuery.cpp b/src/Parsers/ASTShowTablesQuery.cpp index 5470bde10c8..3e00997cdcd 100644 --- a/src/Parsers/ASTShowTablesQuery.cpp +++ b/src/Parsers/ASTShowTablesQuery.cpp @@ -25,10 +25,10 @@ String ASTShowTablesQuery::getFrom() const return name; } -void ASTShowTablesQuery::formatLike(const FormatSettings & settings) const +void ASTShowTablesQuery::formatLike(WriteBuffer & ostr, const FormatSettings & settings) const { if (!like.empty()) - settings.ostr + ostr << (settings.hilite ? hilite_keyword : "") << (not_like ? " NOT" : "") << (case_insensitive_like ? " ILIKE " : " LIKE ") @@ -36,74 +36,74 @@ void ASTShowTablesQuery::formatLike(const FormatSettings & settings) const << DB::quote << like; } -void ASTShowTablesQuery::formatLimit(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTShowTablesQuery::formatLimit(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (limit_length) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " LIMIT " << (settings.hilite ? hilite_none : ""); - limit_length->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " LIMIT " << (settings.hilite ? hilite_none : ""); + limit_length->formatImpl(ostr, settings, state, frame); } } -void ASTShowTablesQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTShowTablesQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (databases) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW DATABASES" << (settings.hilite ? hilite_none : ""); - formatLike(settings); - formatLimit(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW DATABASES" << (settings.hilite ? hilite_none : ""); + formatLike(ostr, settings); + formatLimit(ostr, settings, state, frame); } else if (clusters) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CLUSTERS" << (settings.hilite ? hilite_none : ""); - formatLike(settings); - formatLimit(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CLUSTERS" << (settings.hilite ? hilite_none : ""); + formatLike(ostr, settings); + formatLimit(ostr, settings, state, frame); } else if (cluster) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CLUSTER" << (settings.hilite ? hilite_none : ""); - settings.ostr << " " << backQuoteIfNeed(cluster_str); + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CLUSTER" << (settings.hilite ? hilite_none : ""); + ostr << " " << backQuoteIfNeed(cluster_str); } else if (caches) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW FILESYSTEM CACHES" << (settings.hilite ? hilite_none : ""); - formatLike(settings); - formatLimit(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW FILESYSTEM CACHES" << (settings.hilite ? hilite_none : ""); + formatLike(ostr, settings); + formatLimit(ostr, settings, state, frame); } else if (m_settings) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW " << (changed ? "CHANGED " : "") << "SETTINGS" << + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW " << (changed ? "CHANGED " : "") << "SETTINGS" << (settings.hilite ? hilite_none : ""); - formatLike(settings); + formatLike(ostr, settings); } else if (merges) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW MERGES" << (settings.hilite ? hilite_none : ""); - formatLike(settings); - formatLimit(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW MERGES" << (settings.hilite ? hilite_none : ""); + formatLike(ostr, settings); + formatLimit(ostr, settings, state, frame); } else { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW " << (temporary ? "TEMPORARY " : "") << + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW " << (temporary ? "TEMPORARY " : "") << (dictionaries ? "DICTIONARIES" : "TABLES") << (settings.hilite ? hilite_none : ""); if (from) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : ""); - from->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : ""); + from->formatImpl(ostr, settings, state, frame); } - formatLike(settings); + formatLike(ostr, settings); if (where_expression) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); - where_expression->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); + where_expression->formatImpl(ostr, settings, state, frame); } - formatLimit(settings, state, frame); + formatLimit(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTShowTablesQuery.h b/src/Parsers/ASTShowTablesQuery.h index f6fec2d1230..d12b7ab9e64 100644 --- a/src/Parsers/ASTShowTablesQuery.h +++ b/src/Parsers/ASTShowTablesQuery.h @@ -43,9 +43,9 @@ public: String getFrom() const; protected: - void formatLike(const FormatSettings & settings) const; - void formatLimit(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatLike(WriteBuffer & ostr, const FormatSettings & settings) const; + void formatLimit(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTStatisticsDeclaration.cpp b/src/Parsers/ASTStatisticsDeclaration.cpp index f9b7a9e29db..5b731cfa5bb 100644 --- a/src/Parsers/ASTStatisticsDeclaration.cpp +++ b/src/Parsers/ASTStatisticsDeclaration.cpp @@ -45,14 +45,14 @@ std::vector ASTStatisticsDeclaration::getTypeNames() const } -void ASTStatisticsDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +void ASTStatisticsDeclaration::formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { - columns->formatImpl(s, state, frame); - s.ostr << (s.hilite ? hilite_keyword : ""); + columns->formatImpl(ostr, s, state, frame); + ostr << (s.hilite ? hilite_keyword : ""); if (types) { - s.ostr << " TYPE " << (s.hilite ? hilite_none : ""); - types->formatImpl(s, state, frame); + ostr << " TYPE " << (s.hilite ? hilite_none : ""); + types->formatImpl(ostr, s, state, frame); } } diff --git a/src/Parsers/ASTStatisticsDeclaration.h b/src/Parsers/ASTStatisticsDeclaration.h index f43567b3c70..4747031c676 100644 --- a/src/Parsers/ASTStatisticsDeclaration.h +++ b/src/Parsers/ASTStatisticsDeclaration.h @@ -22,7 +22,7 @@ public: std::vector getTypeNames() const; ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTSubquery.cpp b/src/Parsers/ASTSubquery.cpp index 7b4f8d141fd..f961af2d48c 100644 --- a/src/Parsers/ASTSubquery.cpp +++ b/src/Parsers/ASTSubquery.cpp @@ -25,7 +25,7 @@ void ASTSubquery::appendColumnNameImpl(WriteBuffer & ostr) const } } -void ASTSubquery::formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTSubquery::formatImplWithoutAlias(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { /// NOTE: due to trickery of filling cte_name (in interpreters) it is hard /// to print it without newline (for !oneline case), since if nl_or_ws @@ -34,21 +34,21 @@ void ASTSubquery::formatImplWithoutAlias(const FormatSettings & settings, Format /// (select 1 in ((select 1) as sub)) if (!cte_name.empty()) { - settings.ostr << (settings.hilite ? hilite_identifier : ""); - settings.writeIdentifier(cte_name, /*ambiguous=*/false); - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_identifier : ""); + settings.writeIdentifier(ostr, cte_name, /*ambiguous=*/false); + ostr << (settings.hilite ? hilite_none : ""); return; } std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); std::string nl_or_nothing = settings.one_line ? "" : "\n"; - settings.ostr << "(" << nl_or_nothing; + ostr << "(" << nl_or_nothing; FormatStateStacked frame_nested = frame; frame_nested.need_parens = false; ++frame_nested.indent; - children[0]->formatImpl(settings, state, frame_nested); - settings.ostr << nl_or_nothing << indent_str << ")"; + children[0]->formatImpl(ostr, settings, state, frame_nested); + ostr << nl_or_nothing << indent_str << ")"; } void ASTSubquery::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const diff --git a/src/Parsers/ASTSubquery.h b/src/Parsers/ASTSubquery.h index 88f26f84b4f..19ad89f2be1 100644 --- a/src/Parsers/ASTSubquery.h +++ b/src/Parsers/ASTSubquery.h @@ -38,7 +38,7 @@ public: String tryGetAlias() const override; protected: - void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImplWithoutAlias(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; }; diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index 7058c2cf564..6702770f918 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -91,37 +91,37 @@ void ASTSystemQuery::setTable(const String & name) } } -void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTSystemQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { auto print_identifier = [&](const String & identifier) -> WriteBuffer & { - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(identifier) + ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(identifier) << (settings.hilite ? hilite_none : ""); - return settings.ostr; + return ostr; }; auto print_keyword = [&](const auto & keyword) -> WriteBuffer & { - settings.ostr << (settings.hilite ? hilite_keyword : "") << keyword << (settings.hilite ? hilite_none : ""); - return settings.ostr; + ostr << (settings.hilite ? hilite_keyword : "") << keyword << (settings.hilite ? hilite_none : ""); + return ostr; }; auto print_database_table = [&]() -> WriteBuffer & { if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); - return settings.ostr; + table->formatImpl(ostr, settings, state, frame); + return ostr; }; auto print_drop_replica = [&] { - settings.ostr << " " << quoteString(replica); + ostr << " " << quoteString(replica); if (!shard.empty()) print_keyword(" FROM SHARD ") << quoteString(shard); @@ -151,7 +151,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s print_keyword("SYSTEM") << " "; print_keyword(typeToString(type)); if (!cluster.empty()) - formatOnCluster(settings); + formatOnCluster(ostr, settings); switch (type) { @@ -178,7 +178,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s { if (table) { - settings.ostr << ' '; + ostr << ' '; print_database_table(); } else if (!volume.empty()) @@ -197,37 +197,37 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s { if (table) { - settings.ostr << ' '; + ostr << ' '; print_database_table(); } if (sync_replica_mode != SyncReplicaMode::DEFAULT) { - settings.ostr << ' '; + ostr << ' '; print_keyword(magic_enum::enum_name(sync_replica_mode)); // If the mode is LIGHTWEIGHT and specific source replicas are specified if (sync_replica_mode == SyncReplicaMode::LIGHTWEIGHT && !src_replicas.empty()) { - settings.ostr << ' '; + ostr << ' '; print_keyword("FROM"); - settings.ostr << ' '; + ostr << ' '; bool first = true; for (const auto & src : src_replicas) { if (!first) - settings.ostr << ", "; + ostr << ", "; first = false; - settings.ostr << quoteString(src); + ostr << quoteString(src); } } } if (query_settings) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "SETTINGS " << (settings.hilite ? hilite_none : ""); - query_settings->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "SETTINGS " << (settings.hilite ? hilite_none : ""); + query_settings->formatImpl(ostr, settings, state, frame); } break; @@ -240,22 +240,22 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s { if (table) { - settings.ostr << ' '; + ostr << ' '; print_database_table(); } else if (!target_model.empty()) { - settings.ostr << ' '; + ostr << ' '; print_identifier(target_model); } else if (!target_function.empty()) { - settings.ostr << ' '; + ostr << ' '; print_identifier(target_function); } else if (!disk.empty()) { - settings.ostr << ' '; + ostr << ' '; print_identifier(disk); } @@ -263,7 +263,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s } case Type::SYNC_DATABASE_REPLICA: { - settings.ostr << ' '; + ostr << ' '; print_identifier(database->as()->name()); break; } @@ -292,7 +292,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s { if (!filesystem_cache_name.empty()) { - settings.ostr << ' ' << quoteString(filesystem_cache_name); + ostr << ' ' << quoteString(filesystem_cache_name); if (!key_to_drop.empty()) { print_keyword(" KEY "); @@ -300,7 +300,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s if (offset_to_drop.has_value()) { print_keyword(" OFFSET "); - settings.ostr << offset_to_drop.value(); + ostr << offset_to_drop.value(); } } } @@ -318,17 +318,17 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s case Type::UNFREEZE: { print_keyword(" WITH NAME "); - settings.ostr << quoteString(backup_name); + ostr << quoteString(backup_name); break; } case Type::START_LISTEN: case Type::STOP_LISTEN: { - settings.ostr << ' '; + ostr << ' '; print_keyword(ServerType::serverTypeToString(server_type.type)); if (server_type.type == ServerType::Type::CUSTOM) - settings.ostr << ' ' << quoteString(server_type.custom_name); + ostr << ' ' << quoteString(server_type.custom_name); bool comma = false; @@ -342,11 +342,11 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s continue; if (comma) - settings.ostr << ','; + ostr << ','; else comma = true; - settings.ostr << ' '; + ostr << ' '; print_keyword(ServerType::serverTypeToString(cur_type)); } @@ -355,13 +355,13 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s for (const auto & cur_name : server_type.exclude_custom_names) { if (comma) - settings.ostr << ','; + ostr << ','; else comma = true; - settings.ostr << ' '; + ostr << ' '; print_keyword(ServerType::serverTypeToString(ServerType::Type::CUSTOM)); - settings.ostr << " " << quoteString(cur_name); + ostr << " " << quoteString(cur_name); } } } @@ -371,7 +371,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s case Type::DISABLE_FAILPOINT: case Type::WAIT_FAILPOINT: { - settings.ostr << ' '; + ostr << ' '; print_identifier(fail_point_name); break; } @@ -381,25 +381,25 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s case Type::CANCEL_VIEW: case Type::WAIT_VIEW: { - settings.ostr << ' '; + ostr << ' '; print_database_table(); break; } case Type::TEST_VIEW: { - settings.ostr << ' '; + ostr << ' '; print_database_table(); if (!fake_time_for_view) { - settings.ostr << ' '; + ostr << ' '; print_keyword("UNSET FAKE TIME"); } else { - settings.ostr << ' '; + ostr << ' '; print_keyword("SET FAKE TIME"); - settings.ostr << " '" << LocalDateTime(*fake_time_for_view) << "'"; + ostr << " '" << LocalDateTime(*fake_time_for_view) << "'"; } break; } diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index d738ff5661e..bd3279d5faa 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -182,7 +182,7 @@ public: protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; diff --git a/src/Parsers/ASTTTLElement.cpp b/src/Parsers/ASTTTLElement.cpp index bb353194e8c..b8553f0370b 100644 --- a/src/Parsers/ASTTTLElement.cpp +++ b/src/Parsers/ASTTTLElement.cpp @@ -30,50 +30,50 @@ ASTPtr ASTTTLElement::clone() const return clone; } -void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTTTLElement::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - ttl()->formatImpl(settings, state, frame); + ttl()->formatImpl(ostr, settings, state, frame); if (mode == TTLMode::MOVE) { if (destination_type == DataDestinationType::DISK) - settings.ostr << " TO DISK "; + ostr << " TO DISK "; else if (destination_type == DataDestinationType::VOLUME) - settings.ostr << " TO VOLUME "; + ostr << " TO VOLUME "; else throw Exception(ErrorCodes::LOGICAL_ERROR, "Unsupported destination type {} for TTL MOVE", magic_enum::enum_name(destination_type)); if (if_exists) - settings.ostr << "IF EXISTS "; + ostr << "IF EXISTS "; - settings.ostr << quoteString(destination_name); + ostr << quoteString(destination_name); } else if (mode == TTLMode::GROUP_BY) { - settings.ostr << " GROUP BY "; + ostr << " GROUP BY "; for (const auto * it = group_by_key.begin(); it != group_by_key.end(); ++it) { if (it != group_by_key.begin()) - settings.ostr << ", "; - (*it)->formatImpl(settings, state, frame); + ostr << ", "; + (*it)->formatImpl(ostr, settings, state, frame); } if (!group_by_assignments.empty()) { - settings.ostr << " SET "; + ostr << " SET "; for (const auto * it = group_by_assignments.begin(); it != group_by_assignments.end(); ++it) { if (it != group_by_assignments.begin()) - settings.ostr << ", "; - (*it)->formatImpl(settings, state, frame); + ostr << ", "; + (*it)->formatImpl(ostr, settings, state, frame); } } } else if (mode == TTLMode::RECOMPRESS) { - settings.ostr << " RECOMPRESS "; - recompression_codec->formatImpl(settings, state, frame); + ostr << " RECOMPRESS "; + recompression_codec->formatImpl(ostr, settings, state, frame); } else if (mode == TTLMode::DELETE) { @@ -82,8 +82,8 @@ void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & st if (where()) { - settings.ostr << " WHERE "; - where()->formatImpl(settings, state, frame); + ostr << " WHERE "; + where()->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTTTLElement.h b/src/Parsers/ASTTTLElement.h index 9705cafbce3..8cef82ed293 100644 --- a/src/Parsers/ASTTTLElement.h +++ b/src/Parsers/ASTTTLElement.h @@ -44,7 +44,7 @@ public: void setWhere(ASTPtr && ast) { setExpression(where_expr_pos, std::forward(ast)); } protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; private: int ttl_expr_pos; diff --git a/src/Parsers/ASTTableOverrides.cpp b/src/Parsers/ASTTableOverrides.cpp index 8352e68b156..716b500d62f 100644 --- a/src/Parsers/ASTTableOverrides.cpp +++ b/src/Parsers/ASTTableOverrides.cpp @@ -22,7 +22,7 @@ ASTPtr ASTTableOverride::clone() const return res; } -void ASTTableOverride::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTTableOverride::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { String nl_or_nothing = settings.one_line ? "" : "\n"; String nl_or_ws = settings.one_line ? " " : "\n"; @@ -31,14 +31,14 @@ void ASTTableOverride::formatImpl(const FormatSettings & settings, FormatState & if (is_standalone) { - settings.ostr << hl_keyword << "TABLE OVERRIDE " << hl_none; - ASTIdentifier(table_name).formatImpl(settings, state, frame); + ostr << hl_keyword << "TABLE OVERRIDE " << hl_none; + ASTIdentifier(table_name).formatImpl(ostr, settings, state, frame); } auto override_frame = frame; if (is_standalone) { ++override_frame.indent; - settings.ostr << nl_or_ws << '(' << nl_or_nothing; + ostr << nl_or_ws << '(' << nl_or_nothing; } String indent_str = settings.one_line ? "" : String(4 * override_frame.indent, ' '); size_t override_elems = 0; @@ -46,9 +46,9 @@ void ASTTableOverride::formatImpl(const FormatSettings & settings, FormatState & { FormatStateStacked columns_frame = override_frame; columns_frame.expression_list_always_start_on_new_line = true; - settings.ostr << indent_str << hl_keyword << "COLUMNS" << hl_none << nl_or_ws << indent_str << "("; - columns->formatImpl(settings, state, columns_frame); - settings.ostr << nl_or_nothing << indent_str << ")"; + ostr << indent_str << hl_keyword << "COLUMNS" << hl_none << nl_or_ws << indent_str << "("; + columns->formatImpl(ostr, settings, state, columns_frame); + ostr << nl_or_nothing << indent_str << ")"; ++override_elems; } if (storage) @@ -57,10 +57,10 @@ void ASTTableOverride::formatImpl(const FormatSettings & settings, FormatState & { if (elem) { - settings.ostr << (override_elems++ ? nl_or_ws : "") + ostr << (override_elems++ ? nl_or_ws : "") << indent_str << hl_keyword << elem_name << hl_none << ' '; - elem->formatImpl(settings, state, override_frame); + elem->formatImpl(ostr, settings, state, override_frame); } }; format_storage_elem(storage->partition_by, "PARTITION BY"); @@ -71,7 +71,7 @@ void ASTTableOverride::formatImpl(const FormatSettings & settings, FormatState & } if (is_standalone) - settings.ostr << nl_or_nothing << ')'; + ostr << nl_or_nothing << ')'; } ASTPtr ASTTableOverrideList::clone() const @@ -121,19 +121,19 @@ bool ASTTableOverrideList::hasOverride(const String & name) const return positions.contains(name); } -void ASTTableOverrideList::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTTableOverrideList::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (frame.expression_list_prepend_whitespace) - settings.ostr << ' '; + ostr << ' '; for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) { if (it != children.begin()) { - settings.ostr << (settings.one_line ? ", " : ",\n"); + ostr << (settings.one_line ? ", " : ",\n"); } - (*it)->formatImpl(settings, state, frame); + (*it)->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTTableOverrides.h b/src/Parsers/ASTTableOverrides.h index 1df267acaa9..c1372fe6e17 100644 --- a/src/Parsers/ASTTableOverrides.h +++ b/src/Parsers/ASTTableOverrides.h @@ -26,7 +26,7 @@ public: bool is_standalone = true; String getID(char) const override { return "TableOverride " + table_name; } ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void forEachPointerToChild(std::function f) override { @@ -45,7 +45,7 @@ class ASTTableOverrideList : public IAST public: String getID(char) const override { return "TableOverrideList"; } ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void setTableOverride(const String & name, ASTPtr ast); void removeTableOverride(const String & name); ASTPtr tryGetTableOverride(const String & name) const; diff --git a/src/Parsers/ASTTablesInSelectQuery.cpp b/src/Parsers/ASTTablesInSelectQuery.cpp index 0046f603c7d..366fdbb410b 100644 --- a/src/Parsers/ASTTablesInSelectQuery.cpp +++ b/src/Parsers/ASTTablesInSelectQuery.cpp @@ -126,57 +126,57 @@ ASTPtr ASTTablesInSelectQuery::clone() const #undef CLONE -void ASTTableExpression::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTTableExpression::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.current_select = this; std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); if (database_and_table_name) { - settings.ostr << " "; - database_and_table_name->formatImpl(settings, state, frame); + ostr << " "; + database_and_table_name->formatImpl(ostr, settings, state, frame); } else if (table_function && !(table_function->as()->prefer_subquery_to_function_formatting && subquery)) { - settings.ostr << " "; - table_function->formatImpl(settings, state, frame); + ostr << " "; + table_function->formatImpl(ostr, settings, state, frame); } else if (subquery) { - settings.ostr << settings.nl_or_ws << indent_str; - subquery->formatImpl(settings, state, frame); + ostr << settings.nl_or_ws << indent_str; + subquery->formatImpl(ostr, settings, state, frame); } if (final) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << indent_str + ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << indent_str << "FINAL" << (settings.hilite ? hilite_none : ""); } if (sample_size) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << indent_str + ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << indent_str << "SAMPLE " << (settings.hilite ? hilite_none : ""); - sample_size->formatImpl(settings, state, frame); + sample_size->formatImpl(ostr, settings, state, frame); if (sample_offset) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << ' ' + ostr << (settings.hilite ? hilite_keyword : "") << ' ' << "OFFSET " << (settings.hilite ? hilite_none : ""); - sample_offset->formatImpl(settings, state, frame); + sample_offset->formatImpl(ostr, settings, state, frame); } } } -void ASTTableJoin::formatImplBeforeTable(const FormatSettings & settings, FormatState &, FormatStateStacked frame) const +void ASTTableJoin::formatImplBeforeTable(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : ""); + ostr << (settings.hilite ? hilite_keyword : ""); std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); if (kind != JoinKind::Comma) { - settings.ostr << settings.nl_or_ws << indent_str; + ostr << settings.nl_or_ws << indent_str; } switch (locality) @@ -185,7 +185,7 @@ void ASTTableJoin::formatImplBeforeTable(const FormatSettings & settings, Format case JoinLocality::Local: break; case JoinLocality::Global: - settings.ostr << "GLOBAL "; + ostr << "GLOBAL "; break; } @@ -197,19 +197,19 @@ void ASTTableJoin::formatImplBeforeTable(const FormatSettings & settings, Format break; case JoinStrictness::RightAny: case JoinStrictness::Any: - settings.ostr << "ANY "; + ostr << "ANY "; break; case JoinStrictness::All: - settings.ostr << "ALL "; + ostr << "ALL "; break; case JoinStrictness::Asof: - settings.ostr << "ASOF "; + ostr << "ASOF "; break; case JoinStrictness::Semi: - settings.ostr << "SEMI "; + ostr << "SEMI "; break; case JoinStrictness::Anti: - settings.ostr << "ANTI "; + ostr << "ANTI "; break; } } @@ -217,105 +217,105 @@ void ASTTableJoin::formatImplBeforeTable(const FormatSettings & settings, Format switch (kind) { case JoinKind::Inner: - settings.ostr << "INNER JOIN"; + ostr << "INNER JOIN"; break; case JoinKind::Left: - settings.ostr << "LEFT JOIN"; + ostr << "LEFT JOIN"; break; case JoinKind::Right: - settings.ostr << "RIGHT JOIN"; + ostr << "RIGHT JOIN"; break; case JoinKind::Full: - settings.ostr << "FULL OUTER JOIN"; + ostr << "FULL OUTER JOIN"; break; case JoinKind::Cross: - settings.ostr << "CROSS JOIN"; + ostr << "CROSS JOIN"; break; case JoinKind::Comma: - settings.ostr << ","; + ostr << ","; break; case JoinKind::Paste: - settings.ostr << "PASTE JOIN"; + ostr << "PASTE JOIN"; break; } - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_none : ""); } -void ASTTableJoin::formatImplAfterTable(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTTableJoin::formatImplAfterTable(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.need_parens = false; frame.expression_list_prepend_whitespace = false; if (using_expression_list) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " USING " << (settings.hilite ? hilite_none : ""); - settings.ostr << "("; - using_expression_list->formatImpl(settings, state, frame); - settings.ostr << ")"; + ostr << (settings.hilite ? hilite_keyword : "") << " USING " << (settings.hilite ? hilite_none : ""); + ostr << "("; + using_expression_list->formatImpl(ostr, settings, state, frame); + ostr << ")"; } else if (on_expression) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); /// If there is an alias for the whole expression parens should be added, otherwise it will be invalid syntax bool on_has_alias = !on_expression->tryGetAlias().empty(); if (on_has_alias) - settings.ostr << "("; - on_expression->formatImpl(settings, state, frame); + ostr << "("; + on_expression->formatImpl(ostr, settings, state, frame); if (on_has_alias) - settings.ostr << ")"; + ostr << ")"; } } -void ASTTableJoin::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTTableJoin::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - formatImplBeforeTable(settings, state, frame); - settings.ostr << " ..."; - formatImplAfterTable(settings, state, frame); + formatImplBeforeTable(ostr, settings, state, frame); + ostr << " ..."; + formatImplAfterTable(ostr, settings, state, frame); } -void ASTArrayJoin::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTArrayJoin::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); frame.expression_list_prepend_whitespace = true; - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << indent_str << (kind == Kind::Left ? "LEFT " : "") << "ARRAY JOIN" << (settings.hilite ? hilite_none : ""); settings.one_line - ? expression_list->formatImpl(settings, state, frame) - : expression_list->as().formatImplMultiline(settings, state, frame); + ? expression_list->formatImpl(ostr, settings, state, frame) + : expression_list->as().formatImplMultiline(ostr, settings, state, frame); } -void ASTTablesInSelectQueryElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTTablesInSelectQueryElement::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (table_expression) { if (table_join) - table_join->as().formatImplBeforeTable(settings, state, frame); + table_join->as().formatImplBeforeTable(ostr, settings, state, frame); - table_expression->formatImpl(settings, state, frame); + table_expression->formatImpl(ostr, settings, state, frame); if (table_join) - table_join->as().formatImplAfterTable(settings, state, frame); + table_join->as().formatImplAfterTable(ostr, settings, state, frame); } else if (array_join) { - array_join->formatImpl(settings, state, frame); + array_join->formatImpl(ostr, settings, state, frame); } } -void ASTTablesInSelectQuery::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTTablesInSelectQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { for (const auto & child : children) - child->formatImpl(settings, state, frame); + child->formatImpl(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTTablesInSelectQuery.h b/src/Parsers/ASTTablesInSelectQuery.h index ab32e51a1b4..8b13697164b 100644 --- a/src/Parsers/ASTTablesInSelectQuery.h +++ b/src/Parsers/ASTTablesInSelectQuery.h @@ -56,7 +56,7 @@ struct ASTTableExpression : public IAST using IAST::IAST; String getID(char) const override { return "TableExpression"; } ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; }; @@ -76,9 +76,9 @@ struct ASTTableJoin : public IAST String getID(char) const override { return "TableJoin"; } ASTPtr clone() const override; - void formatImplBeforeTable(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; - void formatImplAfterTable(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImplBeforeTable(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; + void formatImplAfterTable(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; protected: @@ -102,7 +102,7 @@ struct ASTArrayJoin : public IAST using IAST::IAST; String getID(char) const override { return "ArrayJoin"; } ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; }; @@ -120,7 +120,7 @@ struct ASTTablesInSelectQueryElement : public IAST using IAST::IAST; String getID(char) const override { return "TablesInSelectQueryElement"; } ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; @@ -130,7 +130,7 @@ struct ASTTablesInSelectQuery : public IAST using IAST::IAST; String getID(char) const override { return "TablesInSelectQuery"; } ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTTimeInterval.cpp b/src/Parsers/ASTTimeInterval.cpp index 44f6e577e94..fed8eb39720 100644 --- a/src/Parsers/ASTTimeInterval.cpp +++ b/src/Parsers/ASTTimeInterval.cpp @@ -12,16 +12,16 @@ ASTPtr ASTTimeInterval::clone() const return std::make_shared(*this); } -void ASTTimeInterval::formatImpl(const FormatSettings & f_settings, FormatState &, FormatStateStacked frame) const +void ASTTimeInterval::formatImpl(WriteBuffer & ostr, const FormatSettings & f_settings, FormatState &, FormatStateStacked frame) const { frame.need_parens = false; for (bool is_first = true; auto [kind, value] : interval.toIntervals()) { if (!std::exchange(is_first, false)) - f_settings.ostr << ' '; - f_settings.ostr << value << ' '; - f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << kind.toKeyword() << (f_settings.hilite ? hilite_none : ""); + ostr << ' '; + ostr << value << ' '; + ostr << (f_settings.hilite ? hilite_keyword : "") << kind.toKeyword() << (f_settings.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/ASTTimeInterval.h b/src/Parsers/ASTTimeInterval.h index a68acd0f8ea..c59f9cf2686 100644 --- a/src/Parsers/ASTTimeInterval.h +++ b/src/Parsers/ASTTimeInterval.h @@ -18,7 +18,7 @@ public: String getID(char) const override { return "TimeInterval"; } ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTTransactionControl.cpp b/src/Parsers/ASTTransactionControl.cpp index 6964441622d..5ed1c824fec 100644 --- a/src/Parsers/ASTTransactionControl.cpp +++ b/src/Parsers/ASTTransactionControl.cpp @@ -5,21 +5,21 @@ namespace DB { -void ASTTransactionControl::formatImpl(const FormatSettings & format /*state*/, FormatState &, FormatStateStacked /*frame*/) const +void ASTTransactionControl::formatImpl(WriteBuffer & ostr, const FormatSettings & format /*state*/, FormatState &, FormatStateStacked /*frame*/) const { switch (action) { case BEGIN: - format.ostr << (format.hilite ? hilite_keyword : "") << "BEGIN TRANSACTION" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << "BEGIN TRANSACTION" << (format.hilite ? hilite_none : ""); break; case COMMIT: - format.ostr << (format.hilite ? hilite_keyword : "") << "COMMIT" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << "COMMIT" << (format.hilite ? hilite_none : ""); break; case ROLLBACK: - format.ostr << (format.hilite ? hilite_keyword : "") << "ROLLBACK" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << "ROLLBACK" << (format.hilite ? hilite_none : ""); break; case SET_SNAPSHOT: - format.ostr << (format.hilite ? hilite_keyword : "") << "SET TRANSACTION SNAPSHOT " << (format.hilite ? hilite_none : "") << snapshot; + ostr << (format.hilite ? hilite_keyword : "") << "SET TRANSACTION SNAPSHOT " << (format.hilite ? hilite_none : "") << snapshot; break; } } diff --git a/src/Parsers/ASTTransactionControl.h b/src/Parsers/ASTTransactionControl.h index 84a1dcf0970..f4aac220a89 100644 --- a/src/Parsers/ASTTransactionControl.h +++ b/src/Parsers/ASTTransactionControl.h @@ -25,7 +25,7 @@ public: String getID(char /*delimiter*/) const override { return "ASTTransactionControl"; } ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & format, FormatState & /*state*/, FormatStateStacked /*frame*/) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & format, FormatState & /*state*/, FormatStateStacked /*frame*/) const override; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; QueryKind getQueryKind() const override; diff --git a/src/Parsers/ASTUndropQuery.cpp b/src/Parsers/ASTUndropQuery.cpp index 7212e264c0e..208149eb50d 100644 --- a/src/Parsers/ASTUndropQuery.cpp +++ b/src/Parsers/ASTUndropQuery.cpp @@ -19,9 +19,9 @@ ASTPtr ASTUndropQuery::clone() const return res; } -void ASTUndropQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTUndropQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << "UNDROP TABLE" << (settings.hilite ? hilite_none : "") << " "; @@ -32,19 +32,19 @@ void ASTUndropQuery::formatQueryImpl(const FormatSettings & settings, FormatStat { if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); } if (uuid != UUIDHelpers::Nil) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " UUID " << (settings.hilite ? hilite_none : "") + ostr << (settings.hilite ? hilite_keyword : "") << " UUID " << (settings.hilite ? hilite_none : "") << quoteString(toString(uuid)); - formatOnCluster(settings); + formatOnCluster(ostr, settings); } } diff --git a/src/Parsers/ASTUndropQuery.h b/src/Parsers/ASTUndropQuery.h index 7aac4c86c5b..f0d3f0fdabc 100644 --- a/src/Parsers/ASTUndropQuery.h +++ b/src/Parsers/ASTUndropQuery.h @@ -24,7 +24,7 @@ public: QueryKind getQueryKind() const override { return QueryKind::Undrop; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTUseQuery.h b/src/Parsers/ASTUseQuery.h index 873a316e653..10348f1e3bb 100644 --- a/src/Parsers/ASTUseQuery.h +++ b/src/Parsers/ASTUseQuery.h @@ -39,10 +39,10 @@ public: QueryKind getQueryKind() const override { return QueryKind::Use; } protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "USE " << (settings.hilite ? hilite_none : ""); - database->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << "USE " << (settings.hilite ? hilite_none : ""); + database->formatImpl(ostr, settings, state, frame); } }; diff --git a/src/Parsers/ASTViewTargets.cpp b/src/Parsers/ASTViewTargets.cpp index ffd746cc38a..eba118ba79a 100644 --- a/src/Parsers/ASTViewTargets.cpp +++ b/src/Parsers/ASTViewTargets.cpp @@ -204,29 +204,29 @@ ASTPtr ASTViewTargets::clone() const return res; } -void ASTViewTargets::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +void ASTViewTargets::formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { for (const auto & target : targets) - formatTarget(target, s, state, frame); + formatTarget(target, ostr, s, state, frame); } -void ASTViewTargets::formatTarget(ViewTarget::Kind kind, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +void ASTViewTargets::formatTarget(ViewTarget::Kind kind, WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { for (const auto & target : targets) { if (target.kind == kind) - formatTarget(target, s, state, frame); + formatTarget(target, ostr, s, state, frame); } } -void ASTViewTargets::formatTarget(const ViewTarget & target, const FormatSettings & s, FormatState & state, FormatStateStacked frame) +void ASTViewTargets::formatTarget(const ViewTarget & target, WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) { if (target.table_id) { auto keyword = getKeywordForTableID(target.kind); if (!keyword) throw Exception(ErrorCodes::LOGICAL_ERROR, "No keyword for table name of kind {}", toString(target.kind)); - s.ostr << " " << (s.hilite ? hilite_keyword : "") << toStringView(*keyword) + ostr << " " << (s.hilite ? hilite_keyword : "") << toStringView(*keyword) << (s.hilite ? hilite_none : "") << " " << (!target.table_id.database_name.empty() ? backQuoteIfNeed(target.table_id.database_name) + "." : "") << backQuoteIfNeed(target.table_id.table_name); @@ -237,7 +237,7 @@ void ASTViewTargets::formatTarget(const ViewTarget & target, const FormatSetting auto keyword = getKeywordForInnerUUID(target.kind); if (!keyword) throw Exception(ErrorCodes::LOGICAL_ERROR, "No prefix keyword for inner UUID of kind {}", toString(target.kind)); - s.ostr << " " << (s.hilite ? hilite_keyword : "") << toStringView(*keyword) + ostr << " " << (s.hilite ? hilite_keyword : "") << toStringView(*keyword) << (s.hilite ? hilite_none : "") << " " << quoteString(toString(target.inner_uuid)); } @@ -246,8 +246,8 @@ void ASTViewTargets::formatTarget(const ViewTarget & target, const FormatSetting auto keyword = getKeywordForInnerStorage(target.kind); if (!keyword) throw Exception(ErrorCodes::LOGICAL_ERROR, "No prefix keyword for table engine of kind {}", toString(target.kind)); - s.ostr << " " << (s.hilite ? hilite_keyword : "") << toStringView(*keyword) << (s.hilite ? hilite_none : ""); - target.inner_engine->formatImpl(s, state, frame); + ostr << " " << (s.hilite ? hilite_keyword : "") << toStringView(*keyword) << (s.hilite ? hilite_none : ""); + target.inner_engine->formatImpl(ostr, s, state, frame); } } diff --git a/src/Parsers/ASTViewTargets.h b/src/Parsers/ASTViewTargets.h index 7814dd5249c..234b41b38bd 100644 --- a/src/Parsers/ASTViewTargets.h +++ b/src/Parsers/ASTViewTargets.h @@ -106,11 +106,11 @@ public: ASTPtr clone() const override; - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; /// Formats information only about a specific target table. - void formatTarget(ViewTarget::Kind kind, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const; - static void formatTarget(const ViewTarget & target, const FormatSettings & s, FormatState & state, FormatStateStacked frame); + void formatTarget(ViewTarget::Kind kind, WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const; + static void formatTarget(const ViewTarget & target, WriteBuffer & ostr, const FormatSettings & s, FormatState & state, FormatStateStacked frame); /// Helper functions for class ParserViewTargets. Returns a prefix keyword matching a specified target kind. static std::optional getKeywordForTableID(ViewTarget::Kind kind); diff --git a/src/Parsers/ASTWatchQuery.h b/src/Parsers/ASTWatchQuery.h index a5b76c07605..7eb4c1a667c 100644 --- a/src/Parsers/ASTWatchQuery.h +++ b/src/Parsers/ASTWatchQuery.h @@ -40,30 +40,30 @@ public: QueryKind getQueryKind() const override { return QueryKind::Create; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); - settings.ostr << (settings.hilite ? hilite_keyword : "") << "WATCH " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "WATCH " << (settings.hilite ? hilite_none : ""); if (database) { - database->formatImpl(settings, state, frame); - settings.ostr << '.'; + database->formatImpl(ostr, settings, state, frame); + ostr << '.'; } chassert(table); - table->formatImpl(settings, state, frame); + table->formatImpl(ostr, settings, state, frame); if (is_watch_events) { - settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << "EVENTS" << (settings.hilite ? hilite_none : ""); + ostr << " " << (settings.hilite ? hilite_keyword : "") << "EVENTS" << (settings.hilite ? hilite_none : ""); } if (limit_length) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << indent_str << "LIMIT " << (settings.hilite ? hilite_none : ""); - limit_length->formatImpl(settings, state, frame); + ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << indent_str << "LIMIT " << (settings.hilite ? hilite_none : ""); + limit_length->formatImpl(ostr, settings, state, frame); } } }; diff --git a/src/Parsers/ASTWindowDefinition.cpp b/src/Parsers/ASTWindowDefinition.cpp index 21c44a166af..fcd7098128d 100644 --- a/src/Parsers/ASTWindowDefinition.cpp +++ b/src/Parsers/ASTWindowDefinition.cpp @@ -52,15 +52,15 @@ String ASTWindowDefinition::getID(char) const return "WindowDefinition"; } -void ASTWindowDefinition::formatImpl(const FormatSettings & settings, - FormatState & state, FormatStateStacked format_frame) const +void ASTWindowDefinition::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, + FormatState & state, FormatStateStacked format_frame) const { format_frame.expression_list_prepend_whitespace = false; bool need_space = false; if (!parent_window_name.empty()) { - settings.ostr << backQuoteIfNeed(parent_window_name); + ostr << backQuoteIfNeed(parent_window_name); need_space = true; } @@ -69,11 +69,11 @@ void ASTWindowDefinition::formatImpl(const FormatSettings & settings, { if (need_space) { - settings.ostr << " "; + ostr << " "; } - settings.ostr << "PARTITION BY "; - partition_by->formatImpl(settings, state, format_frame); + ostr << "PARTITION BY "; + partition_by->formatImpl(ostr, settings, state, format_frame); need_space = true; } @@ -82,11 +82,11 @@ void ASTWindowDefinition::formatImpl(const FormatSettings & settings, { if (need_space) { - settings.ostr << " "; + ostr << " "; } - settings.ostr << "ORDER BY "; - order_by->formatImpl(settings, state, format_frame); + ostr << "ORDER BY "; + order_by->formatImpl(ostr, settings, state, format_frame); need_space = true; } @@ -94,38 +94,38 @@ void ASTWindowDefinition::formatImpl(const FormatSettings & settings, if (!frame_is_default) { if (need_space) - settings.ostr << " "; + ostr << " "; format_frame.need_parens = true; - settings.ostr << frame_type << " BETWEEN "; + ostr << frame_type << " BETWEEN "; if (frame_begin_type == WindowFrame::BoundaryType::Current) { - settings.ostr << "CURRENT ROW"; + ostr << "CURRENT ROW"; } else if (frame_begin_type == WindowFrame::BoundaryType::Unbounded) { - settings.ostr << "UNBOUNDED PRECEDING"; + ostr << "UNBOUNDED PRECEDING"; } else { - frame_begin_offset->formatImpl(settings, state, format_frame); - settings.ostr << " " + frame_begin_offset->formatImpl(ostr, settings, state, format_frame); + ostr << " " << (!frame_begin_preceding ? "FOLLOWING" : "PRECEDING"); } - settings.ostr << " AND "; + ostr << " AND "; if (frame_end_type == WindowFrame::BoundaryType::Current) { - settings.ostr << "CURRENT ROW"; + ostr << "CURRENT ROW"; } else if (frame_end_type == WindowFrame::BoundaryType::Unbounded) { - settings.ostr << "UNBOUNDED FOLLOWING"; + ostr << "UNBOUNDED FOLLOWING"; } else { - frame_end_offset->formatImpl(settings, state, format_frame); - settings.ostr << " " + frame_end_offset->formatImpl(ostr, settings, state, format_frame); + ostr << " " << (!frame_end_preceding ? "FOLLOWING" : "PRECEDING"); } } @@ -134,10 +134,10 @@ void ASTWindowDefinition::formatImpl(const FormatSettings & settings, std::string ASTWindowDefinition::getDefaultWindowName() const { WriteBufferFromOwnString ostr; - FormatSettings settings{ostr, true /* one_line */}; + FormatSettings settings{true /* one_line */}; FormatState state; FormatStateStacked format_frame; - formatImpl(settings, state, format_frame); + formatImpl(ostr, settings, state, format_frame); return ostr.str(); } @@ -157,13 +157,13 @@ String ASTWindowListElement::getID(char) const return "WindowListElement"; } -void ASTWindowListElement::formatImpl(const FormatSettings & settings, - FormatState & state, FormatStateStacked frame) const +void ASTWindowListElement::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, + FormatState & state, FormatStateStacked frame) const { - settings.ostr << backQuoteIfNeed(name); - settings.ostr << " AS ("; - definition->formatImpl(settings, state, frame); - settings.ostr << ")"; + ostr << backQuoteIfNeed(name); + ostr << " AS ("; + definition->formatImpl(ostr, settings, state, frame); + ostr << ")"; } } diff --git a/src/Parsers/ASTWindowDefinition.h b/src/Parsers/ASTWindowDefinition.h index 72ad8f6abda..ecc6ae1a6c7 100644 --- a/src/Parsers/ASTWindowDefinition.h +++ b/src/Parsers/ASTWindowDefinition.h @@ -29,7 +29,7 @@ struct ASTWindowDefinition : public IAST String getID(char delimiter) const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; std::string getDefaultWindowName() const; }; @@ -45,7 +45,7 @@ struct ASTWindowListElement : public IAST String getID(char delimiter) const override; - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTWithAlias.cpp b/src/Parsers/ASTWithAlias.cpp index a5c285f315e..b7d4a68f120 100644 --- a/src/Parsers/ASTWithAlias.cpp +++ b/src/Parsers/ASTWithAlias.cpp @@ -7,38 +7,38 @@ namespace DB { -static void writeAlias(const String & name, const ASTWithAlias::FormatSettings & settings) +static void writeAlias(const String & name, WriteBuffer & ostr, const ASTWithAlias::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " AS " << (settings.hilite ? IAST::hilite_alias : ""); - settings.writeIdentifier(name, /*ambiguous=*/false); - settings.ostr << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " AS " << (settings.hilite ? IAST::hilite_alias : ""); + settings.writeIdentifier(ostr, name, /*ambiguous=*/false); + ostr << (settings.hilite ? IAST::hilite_none : ""); } -void ASTWithAlias::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTWithAlias::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { /// If we have previously output this node elsewhere in the query, now it is enough to output only the alias. /// This is needed because the query can become extraordinary large after substitution of aliases. if (!alias.empty() && !state.printed_asts_with_alias.emplace(frame.current_select, alias, getTreeHash(/*ignore_aliases=*/ true)).second) { - settings.ostr << (settings.hilite ? IAST::hilite_identifier : ""); - settings.writeIdentifier(alias, /*ambiguous=*/false); - settings.ostr << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_identifier : ""); + settings.writeIdentifier(ostr, alias, /*ambiguous=*/false); + ostr << (settings.hilite ? IAST::hilite_none : ""); } else { /// If there is an alias, then parentheses are required around the entire expression, including the alias. /// Because a record of the form `0 AS x + 0` is syntactically invalid. if (frame.need_parens && !alias.empty()) - settings.ostr << '('; + ostr << '('; - formatImplWithoutAlias(settings, state, frame); + formatImplWithoutAlias(ostr, settings, state, frame); if (!alias.empty()) { - writeAlias(alias, settings); + writeAlias(alias, ostr, settings); if (frame.need_parens) - settings.ostr << ')'; + ostr << ')'; } } } diff --git a/src/Parsers/ASTWithAlias.h b/src/Parsers/ASTWithAlias.h index 42adc221e2a..6d95ad12c69 100644 --- a/src/Parsers/ASTWithAlias.h +++ b/src/Parsers/ASTWithAlias.h @@ -31,11 +31,11 @@ public: void setAlias(const String & to) override { alias = to; } /// Calls formatImplWithoutAlias, and also outputs an alias. If necessary, encloses the entire expression in brackets. - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const final; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const final; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; - virtual void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const = 0; + virtual void formatImplWithoutAlias(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const = 0; protected: virtual void appendColumnNameImpl(WriteBuffer & ostr) const = 0; diff --git a/src/Parsers/ASTWithElement.cpp b/src/Parsers/ASTWithElement.cpp index a94b0a7062d..450453eb0e2 100644 --- a/src/Parsers/ASTWithElement.cpp +++ b/src/Parsers/ASTWithElement.cpp @@ -14,16 +14,16 @@ ASTPtr ASTWithElement::clone() const return res; } -void ASTWithElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTWithElement::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); - settings.ostr << (settings.hilite ? hilite_alias : ""); - settings.writeIdentifier(name, /*ambiguous=*/false); - settings.ostr << (settings.hilite ? hilite_none : ""); - settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS" << (settings.hilite ? hilite_none : ""); - settings.ostr << settings.nl_or_ws << indent_str; - dynamic_cast(*subquery).formatImplWithoutAlias(settings, state, frame); + ostr << (settings.hilite ? hilite_alias : ""); + settings.writeIdentifier(ostr, name, /*ambiguous=*/false); + ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " AS" << (settings.hilite ? hilite_none : ""); + ostr << settings.nl_or_ws << indent_str; + dynamic_cast(*subquery).formatImplWithoutAlias(ostr, settings, state, frame); } } diff --git a/src/Parsers/ASTWithElement.h b/src/Parsers/ASTWithElement.h index 97c68579fa1..a51d1897d94 100644 --- a/src/Parsers/ASTWithElement.h +++ b/src/Parsers/ASTWithElement.h @@ -19,7 +19,7 @@ public: ASTPtr clone() const override; protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index c7a6429f6aa..20655757499 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -16,10 +16,10 @@ namespace ErrorCodes namespace { - void formatValidUntil(const IAST & valid_until, const IAST::FormatSettings & settings) + void formatValidUntil(const IAST & valid_until, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " VALID UNTIL " << (settings.hilite ? IAST::hilite_none : ""); - valid_until.format(settings); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " VALID UNTIL " << (settings.hilite ? IAST::hilite_none : ""); + valid_until.format(ostr, settings); } } @@ -49,16 +49,16 @@ std::optional ASTAuthenticationData::getSalt() const return {}; } -void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTAuthenticationData::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { if (type && *type == AuthenticationType::NO_PASSWORD) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " no_password" + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " no_password" << (settings.hilite ? IAST::hilite_none : ""); if (valid_until) { - formatValidUntil(*valid_until, settings); + formatValidUntil(*valid_until, ostr, settings); } return; @@ -177,52 +177,52 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt if (!auth_type_name.empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << auth_type_name << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << auth_type_name << (settings.hilite ? IAST::hilite_none : ""); } if (!prefix.empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << prefix << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << prefix << (settings.hilite ? IAST::hilite_none : ""); } if (password) { - settings.ostr << " "; - children[0]->format(settings); + ostr << " "; + children[0]->format(ostr, settings); } if (salt) { - settings.ostr << " SALT "; - children[1]->format(settings); + ostr << " SALT "; + children[1]->format(ostr, settings); } if (parameter) { - settings.ostr << " "; - children[0]->format(settings); + ostr << " "; + children[0]->format(ostr, settings); } else if (parameters) { - settings.ostr << " "; + ostr << " "; bool need_comma = false; for (const auto & child : children) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - child->format(settings); + ostr << ", "; + child->format(ostr, settings); } } if (scheme) { - settings.ostr << " SCHEME "; - children[1]->format(settings); + ostr << " SCHEME "; + children[1]->format(ostr, settings); } if (valid_until) { - formatValidUntil(*valid_until, settings); + formatValidUntil(*valid_until, ostr, settings); } } diff --git a/src/Parsers/Access/ASTAuthenticationData.h b/src/Parsers/Access/ASTAuthenticationData.h index 24c4c015efd..2b0a391560f 100644 --- a/src/Parsers/Access/ASTAuthenticationData.h +++ b/src/Parsers/Access/ASTAuthenticationData.h @@ -44,7 +44,7 @@ public: ASTPtr valid_until; protected: - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/Access/ASTCheckGrantQuery.cpp b/src/Parsers/Access/ASTCheckGrantQuery.cpp index 301f930ee07..93474da66cc 100644 --- a/src/Parsers/Access/ASTCheckGrantQuery.cpp +++ b/src/Parsers/Access/ASTCheckGrantQuery.cpp @@ -20,13 +20,13 @@ ASTPtr ASTCheckGrantQuery::clone() const } -void ASTCheckGrantQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTCheckGrantQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "CHECK GRANT" + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "CHECK GRANT" << (settings.hilite ? IAST::hilite_none : ""); - settings.ostr << " "; - access_rights_elements.formatElementsWithoutOptions(settings.ostr, settings.hilite); + ostr << " "; + access_rights_elements.formatElementsWithoutOptions(ostr, settings.hilite); } diff --git a/src/Parsers/Access/ASTCheckGrantQuery.h b/src/Parsers/Access/ASTCheckGrantQuery.h index f7bb2441949..8c66eee9a1c 100644 --- a/src/Parsers/Access/ASTCheckGrantQuery.h +++ b/src/Parsers/Access/ASTCheckGrantQuery.h @@ -17,7 +17,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; void replaceEmptyDatabase(const String & current_database); QueryKind getQueryKind() const override { return QueryKind::Check; } }; diff --git a/src/Parsers/Access/ASTCreateQuotaQuery.cpp b/src/Parsers/Access/ASTCreateQuotaQuery.cpp index 091b62b0a9f..71135fb7861 100644 --- a/src/Parsers/Access/ASTCreateQuotaQuery.cpp +++ b/src/Parsers/Access/ASTCreateQuotaQuery.cpp @@ -10,16 +10,16 @@ namespace DB { namespace { - void formatKeyType(const QuotaKeyType & key_type, const IAST::FormatSettings & settings) + void formatKeyType(const QuotaKeyType & key_type, WriteBuffer & ostr, const IAST::FormatSettings & settings) { const auto & type_info = QuotaKeyTypeInfo::get(key_type); if (key_type == QuotaKeyType::NONE) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT KEYED" << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT KEYED" << (settings.hilite ? IAST::hilite_none : ""); return; } - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " KEYED BY " << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " KEYED BY " << (settings.hilite ? IAST::hilite_none : ""); if (!type_info.base_types.empty()) { @@ -27,49 +27,49 @@ namespace for (const auto & base_type : type_info.base_types) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << QuotaKeyTypeInfo::get(base_type).name; + ostr << ", "; + ostr << QuotaKeyTypeInfo::get(base_type).name; } return; } - settings.ostr << type_info.name; + ostr << type_info.name; } - void formatNames(const Strings & names, const IAST::FormatSettings & settings) + void formatNames(const Strings & names, WriteBuffer & ostr) { - settings.ostr << " "; + ostr << " "; bool need_comma = false; for (const String & name : names) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << backQuoteIfNeed(name); + ostr << ", "; + ostr << backQuoteIfNeed(name); } } - void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings) + void formatRenameTo(const String & new_name, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") << backQuote(new_name); } - void formatLimit(QuotaType quota_type, QuotaValue max_value, const IAST::FormatSettings & settings) + void formatLimit(QuotaType quota_type, QuotaValue max_value, WriteBuffer & ostr) { const auto & type_info = QuotaTypeInfo::get(quota_type); - settings.ostr << " " << type_info.name << " = " << type_info.valueToString(max_value); + ostr << " " << type_info.name << " = " << type_info.valueToString(max_value); } - void formatIntervalWithLimits(const ASTCreateQuotaQuery::Limits & limits, const IAST::FormatSettings & settings) + void formatIntervalWithLimits(const ASTCreateQuotaQuery::Limits & limits, WriteBuffer & ostr, const IAST::FormatSettings & settings) { auto interval_kind = IntervalKind::fromAvgSeconds(limits.duration.count()); Int64 num_intervals = limits.duration.count() / interval_kind.toAvgSeconds(); - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " FOR" << (limits.randomize_interval ? " RANDOMIZED" : "") << " INTERVAL" @@ -81,7 +81,7 @@ namespace if (limits.drop) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NO LIMITS" << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NO LIMITS" << (settings.hilite ? IAST::hilite_none : ""); } else { @@ -94,7 +94,7 @@ namespace } if (limit_found) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MAX" << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MAX" << (settings.hilite ? IAST::hilite_none : ""); bool need_comma = false; for (auto quota_type : collections::range(QuotaType::MAX)) { @@ -102,33 +102,33 @@ namespace if (limits.max[quota_type_i]) { if (std::exchange(need_comma, true)) - settings.ostr << ","; - formatLimit(quota_type, *limits.max[quota_type_i], settings); + ostr << ","; + formatLimit(quota_type, *limits.max[quota_type_i], ostr); } } } else - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TRACKING ONLY" << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TRACKING ONLY" << (settings.hilite ? IAST::hilite_none : ""); } } - void formatIntervalsWithLimits(const std::vector & all_limits, const IAST::FormatSettings & settings) + void formatIntervalsWithLimits(const std::vector & all_limits, WriteBuffer & ostr, const IAST::FormatSettings & settings) { bool need_comma = false; for (const auto & limits : all_limits) { if (need_comma) - settings.ostr << ","; + ostr << ","; need_comma = true; - formatIntervalWithLimits(limits, settings); + formatIntervalWithLimits(limits, ostr, settings); } } - void formatToRoles(const ASTRolesOrUsersSet & roles, const IAST::FormatSettings & settings) + void formatToRoles(const ASTRolesOrUsersSet & roles, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); - roles.format(settings); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); + roles.format(ostr, settings); } } @@ -150,44 +150,44 @@ ASTPtr ASTCreateQuotaQuery::clone() const } -void ASTCreateQuotaQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTCreateQuotaQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { if (attach) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "ATTACH QUOTA" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "ATTACH QUOTA" << (settings.hilite ? hilite_none : ""); } else { - settings.ostr << (settings.hilite ? hilite_keyword : "") << (alter ? "ALTER QUOTA" : "CREATE QUOTA") + ostr << (settings.hilite ? hilite_keyword : "") << (alter ? "ALTER QUOTA" : "CREATE QUOTA") << (settings.hilite ? hilite_none : ""); } if (if_exists) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IF EXISTS" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " IF EXISTS" << (settings.hilite ? hilite_none : ""); else if (if_not_exists) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IF NOT EXISTS" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " IF NOT EXISTS" << (settings.hilite ? hilite_none : ""); else if (or_replace) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " OR REPLACE" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " OR REPLACE" << (settings.hilite ? hilite_none : ""); - formatNames(names, settings); + formatNames(names, ostr); if (!storage_name.empty()) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IN " << (settings.hilite ? IAST::hilite_none : "") << backQuoteIfNeed(storage_name); - formatOnCluster(settings); + formatOnCluster(ostr, settings); if (!new_name.empty()) - formatRenameTo(new_name, settings); + formatRenameTo(new_name, ostr, settings); if (key_type) - formatKeyType(*key_type, settings); + formatKeyType(*key_type, ostr, settings); - formatIntervalsWithLimits(all_limits, settings); + formatIntervalsWithLimits(all_limits, ostr, settings); if (roles && (!roles->empty() || alter)) - formatToRoles(*roles, settings); + formatToRoles(*roles, ostr, settings); } diff --git a/src/Parsers/Access/ASTCreateQuotaQuery.h b/src/Parsers/Access/ASTCreateQuotaQuery.h index aecbbb00f9a..734e391ad57 100644 --- a/src/Parsers/Access/ASTCreateQuotaQuery.h +++ b/src/Parsers/Access/ASTCreateQuotaQuery.h @@ -53,7 +53,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; void replaceCurrentUserTag(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } diff --git a/src/Parsers/Access/ASTCreateRoleQuery.cpp b/src/Parsers/Access/ASTCreateRoleQuery.cpp index eeeb34c97e4..bfc7f68387d 100644 --- a/src/Parsers/Access/ASTCreateRoleQuery.cpp +++ b/src/Parsers/Access/ASTCreateRoleQuery.cpp @@ -8,28 +8,28 @@ namespace DB { namespace { - void formatNames(const Strings & names, const IAST::FormatSettings & settings) + void formatNames(const Strings & names, WriteBuffer & ostr) { - settings.ostr << " "; + ostr << " "; bool need_comma = false; for (const String & name : names) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << backQuoteIfNeed(name); + ostr << ", "; + ostr << backQuoteIfNeed(name); } } - void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings) + void formatRenameTo(const String & new_name, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") << quoteString(new_name); } - void formatSettings(const ASTSettingsProfileElements & settings, const IAST::FormatSettings & format) + void formatSettings(const ASTSettingsProfileElements & settings, WriteBuffer & ostr, const IAST::FormatSettings & format) { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " SETTINGS " << (format.hilite ? IAST::hilite_none : ""); - settings.format(format); + ostr << (format.hilite ? IAST::hilite_keyword : "") << " SETTINGS " << (format.hilite ? IAST::hilite_none : ""); + settings.format(ostr, format); } } @@ -51,39 +51,39 @@ ASTPtr ASTCreateRoleQuery::clone() const } -void ASTCreateRoleQuery::formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const +void ASTCreateRoleQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & format, FormatState &, FormatStateStacked) const { if (attach) { - format.ostr << (format.hilite ? hilite_keyword : "") << "ATTACH ROLE" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << "ATTACH ROLE" << (format.hilite ? hilite_none : ""); } else { - format.ostr << (format.hilite ? hilite_keyword : "") << (alter ? "ALTER ROLE" : "CREATE ROLE") + ostr << (format.hilite ? hilite_keyword : "") << (alter ? "ALTER ROLE" : "CREATE ROLE") << (format.hilite ? hilite_none : ""); } if (if_exists) - format.ostr << (format.hilite ? hilite_keyword : "") << " IF EXISTS" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << " IF EXISTS" << (format.hilite ? hilite_none : ""); else if (if_not_exists) - format.ostr << (format.hilite ? hilite_keyword : "") << " IF NOT EXISTS" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << " IF NOT EXISTS" << (format.hilite ? hilite_none : ""); else if (or_replace) - format.ostr << (format.hilite ? hilite_keyword : "") << " OR REPLACE" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << " OR REPLACE" << (format.hilite ? hilite_none : ""); - formatNames(names, format); + formatNames(names, ostr); if (!storage_name.empty()) - format.ostr << (format.hilite ? IAST::hilite_keyword : "") + ostr << (format.hilite ? IAST::hilite_keyword : "") << " IN " << (format.hilite ? IAST::hilite_none : "") << backQuoteIfNeed(storage_name); - formatOnCluster(format); + formatOnCluster(ostr, format); if (!new_name.empty()) - formatRenameTo(new_name, format); + formatRenameTo(new_name, ostr, format); if (settings && (!settings->empty() || alter)) - formatSettings(*settings, format); + formatSettings(*settings, ostr, format); } } diff --git a/src/Parsers/Access/ASTCreateRoleQuery.h b/src/Parsers/Access/ASTCreateRoleQuery.h index 4e465553164..a2a827d7f92 100644 --- a/src/Parsers/Access/ASTCreateRoleQuery.h +++ b/src/Parsers/Access/ASTCreateRoleQuery.h @@ -34,7 +34,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & format, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } QueryKind getQueryKind() const override { return QueryKind::Create; } diff --git a/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp b/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp index e95959703ee..1a646c57f80 100644 --- a/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp +++ b/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp @@ -13,57 +13,56 @@ namespace DB { namespace { - void formatRenameTo(const String & new_short_name, const IAST::FormatSettings & settings) + void formatRenameTo(const String & new_short_name, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") << backQuote(new_short_name); } - void formatAsRestrictiveOrPermissive(bool is_restrictive, const IAST::FormatSettings & settings) + void formatAsRestrictiveOrPermissive(bool is_restrictive, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " AS " << (settings.hilite ? IAST::hilite_none : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " AS " << (settings.hilite ? IAST::hilite_none : "") << (is_restrictive ? "restrictive" : "permissive"); } - void formatFilterExpression(const ASTPtr & expr, const IAST::FormatSettings & settings) + void formatFilterExpression(const ASTPtr & expr, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << " "; + ostr << " "; if (expr) - expr->format(settings); + expr->format(ostr, settings); else - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); } - void formatForClause(const boost::container::flat_set & commands, const String & filter, const String & check, bool alter, const IAST::FormatSettings & settings) + void formatForClause(const boost::container::flat_set & commands, const String & filter, const String & check, bool alter, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " FOR " << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " FOR " << (settings.hilite ? IAST::hilite_none : ""); bool need_comma = false; for (const auto & command : commands) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << command << (settings.hilite ? IAST::hilite_none : ""); + ostr << ", "; + ostr << (settings.hilite ? IAST::hilite_keyword : "") << command << (settings.hilite ? IAST::hilite_none : ""); } if (!filter.empty()) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " USING" << (settings.hilite ? IAST::hilite_none : "") << filter; + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " USING" << (settings.hilite ? IAST::hilite_none : "") << filter; if (!check.empty() && (alter || (check != filter))) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH CHECK" << (settings.hilite ? IAST::hilite_none : "") << check; + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH CHECK" << (settings.hilite ? IAST::hilite_none : "") << check; } - void formatForClauses(const std::vector> & filters, bool alter, const IAST::FormatSettings & settings) + void formatForClauses(const std::vector> & filters, bool alter, WriteBuffer & ostr, const IAST::FormatSettings & settings) { std::vector> filters_as_strings; WriteBufferFromOwnString temp_buf; - IAST::FormatSettings temp_settings(temp_buf, settings); for (const auto & [filter_type, filter] : filters) { - formatFilterExpression(filter, temp_settings); + formatFilterExpression(filter, temp_buf, settings); filters_as_strings.emplace_back(filter_type, temp_buf.str()); temp_buf.restart(); } @@ -102,16 +101,16 @@ namespace } if (!filter.empty() || !check.empty()) - formatForClause(commands, filter, check, alter, settings); + formatForClause(commands, filter, check, alter, ostr, settings); } while (!filter.empty() || !check.empty()); } - void formatToRoles(const ASTRolesOrUsersSet & roles, const IAST::FormatSettings & settings) + void formatToRoles(const ASTRolesOrUsersSet & roles, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); - roles.format(settings); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); + roles.format(ostr, settings); } } @@ -146,46 +145,46 @@ ASTPtr ASTCreateRowPolicyQuery::clone() const } -void ASTCreateRowPolicyQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTCreateRowPolicyQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { if (attach) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "ATTACH ROW POLICY"; + ostr << (settings.hilite ? hilite_keyword : "") << "ATTACH ROW POLICY"; } else { - settings.ostr << (settings.hilite ? hilite_keyword : "") << (alter ? "ALTER ROW POLICY" : "CREATE ROW POLICY") + ostr << (settings.hilite ? hilite_keyword : "") << (alter ? "ALTER ROW POLICY" : "CREATE ROW POLICY") << (settings.hilite ? hilite_none : ""); } if (if_exists) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IF EXISTS" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " IF EXISTS" << (settings.hilite ? hilite_none : ""); else if (if_not_exists) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " IF NOT EXISTS" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " IF NOT EXISTS" << (settings.hilite ? hilite_none : ""); else if (or_replace) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " OR REPLACE" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " OR REPLACE" << (settings.hilite ? hilite_none : ""); - settings.ostr << " "; - names->format(settings); + ostr << " "; + names->format(ostr, settings); if (!storage_name.empty()) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IN " << (settings.hilite ? IAST::hilite_none : "") << backQuoteIfNeed(storage_name); - formatOnCluster(settings); + formatOnCluster(ostr, settings); assert(names->cluster.empty()); if (!new_short_name.empty()) - formatRenameTo(new_short_name, settings); + formatRenameTo(new_short_name, ostr, settings); if (is_restrictive) - formatAsRestrictiveOrPermissive(*is_restrictive, settings); + formatAsRestrictiveOrPermissive(*is_restrictive, ostr, settings); - formatForClauses(filters, alter, settings); + formatForClauses(filters, alter, ostr, settings); if (roles && (!roles->empty() || alter)) - formatToRoles(*roles, settings); + formatToRoles(*roles, ostr, settings); } diff --git a/src/Parsers/Access/ASTCreateRowPolicyQuery.h b/src/Parsers/Access/ASTCreateRowPolicyQuery.h index 3f2418e7287..9ccb1c68465 100644 --- a/src/Parsers/Access/ASTCreateRowPolicyQuery.h +++ b/src/Parsers/Access/ASTCreateRowPolicyQuery.h @@ -47,7 +47,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } void replaceCurrentUserTag(const String & current_user_name) const; diff --git a/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp b/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp index e1b42bfb33d..e5b15b791cb 100644 --- a/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp +++ b/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp @@ -9,34 +9,34 @@ namespace DB { namespace { - void formatNames(const Strings & names, const IAST::FormatSettings & settings) + void formatNames(const Strings & names, WriteBuffer & ostr) { - settings.ostr << " "; + ostr << " "; bool need_comma = false; for (const String & name : names) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << backQuote(name); + ostr << ", "; + ostr << backQuote(name); } } - void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings) + void formatRenameTo(const String & new_name, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") << quoteString(new_name); } - void formatSettings(const ASTSettingsProfileElements & settings, const IAST::FormatSettings & format) + void formatSettings(const ASTSettingsProfileElements & settings, WriteBuffer & ostr, const IAST::FormatSettings & format) { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " SETTINGS " << (format.hilite ? IAST::hilite_none : ""); - settings.format(format); + ostr << (format.hilite ? IAST::hilite_keyword : "") << " SETTINGS " << (format.hilite ? IAST::hilite_none : ""); + settings.format(ostr, format); } - void formatToRoles(const ASTRolesOrUsersSet & roles, const IAST::FormatSettings & settings) + void formatToRoles(const ASTRolesOrUsersSet & roles, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); - roles.format(settings); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); + roles.format(ostr, settings); } } @@ -61,42 +61,42 @@ ASTPtr ASTCreateSettingsProfileQuery::clone() const } -void ASTCreateSettingsProfileQuery::formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const +void ASTCreateSettingsProfileQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & format, FormatState &, FormatStateStacked) const { if (attach) { - format.ostr << (format.hilite ? hilite_keyword : "") << "ATTACH SETTINGS PROFILE" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << "ATTACH SETTINGS PROFILE" << (format.hilite ? hilite_none : ""); } else { - format.ostr << (format.hilite ? hilite_keyword : "") << (alter ? "ALTER SETTINGS PROFILE" : "CREATE SETTINGS PROFILE") + ostr << (format.hilite ? hilite_keyword : "") << (alter ? "ALTER SETTINGS PROFILE" : "CREATE SETTINGS PROFILE") << (format.hilite ? hilite_none : ""); } if (if_exists) - format.ostr << (format.hilite ? hilite_keyword : "") << " IF EXISTS" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << " IF EXISTS" << (format.hilite ? hilite_none : ""); else if (if_not_exists) - format.ostr << (format.hilite ? hilite_keyword : "") << " IF NOT EXISTS" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << " IF NOT EXISTS" << (format.hilite ? hilite_none : ""); else if (or_replace) - format.ostr << (format.hilite ? hilite_keyword : "") << " OR REPLACE" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << " OR REPLACE" << (format.hilite ? hilite_none : ""); - formatNames(names, format); + formatNames(names, ostr); if (!storage_name.empty()) - format.ostr << (format.hilite ? IAST::hilite_keyword : "") + ostr << (format.hilite ? IAST::hilite_keyword : "") << " IN " << (format.hilite ? IAST::hilite_none : "") << backQuoteIfNeed(storage_name); - formatOnCluster(format); + formatOnCluster(ostr, format); if (!new_name.empty()) - formatRenameTo(new_name, format); + formatRenameTo(new_name, ostr, format); if (settings && (!settings->empty() || alter)) - formatSettings(*settings, format); + formatSettings(*settings, ostr, format); if (to_roles && (!to_roles->empty() || alter)) - formatToRoles(*to_roles, format); + formatToRoles(*to_roles, ostr, format); } diff --git a/src/Parsers/Access/ASTCreateSettingsProfileQuery.h b/src/Parsers/Access/ASTCreateSettingsProfileQuery.h index be01aae1e26..53618f2019d 100644 --- a/src/Parsers/Access/ASTCreateSettingsProfileQuery.h +++ b/src/Parsers/Access/ASTCreateSettingsProfileQuery.h @@ -39,7 +39,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & format, FormatState &, FormatStateStacked) const override; void replaceCurrentUserTag(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } QueryKind getQueryKind() const override { return QueryKind::Create; } diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index eb4503acf82..f0198252bfc 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -13,56 +13,56 @@ namespace DB namespace { - void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings) + void formatRenameTo(const String & new_name, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") << quoteString(new_name); } - void formatAuthenticationData(const std::vector> & authentication_methods, const IAST::FormatSettings & settings) + void formatAuthenticationData(const std::vector> & authentication_methods, WriteBuffer & ostr, const IAST::FormatSettings & settings) { // safe because this method is only called if authentication_methods.size > 1 // if the first type is present, include the `WITH` keyword if (authentication_methods[0]->type) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH" << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH" << (settings.hilite ? IAST::hilite_none : ""); } for (std::size_t i = 0; i < authentication_methods.size(); i++) { - authentication_methods[i]->format(settings); + authentication_methods[i]->format(ostr, settings); bool is_last = i < authentication_methods.size() - 1; if (is_last) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : ","); + ostr << (settings.hilite ? IAST::hilite_keyword : ","); } } } - void formatValidUntil(const IAST & valid_until, const IAST::FormatSettings & settings) + void formatValidUntil(const IAST & valid_until, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " VALID UNTIL " << (settings.hilite ? IAST::hilite_none : ""); - valid_until.format(settings); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " VALID UNTIL " << (settings.hilite ? IAST::hilite_none : ""); + valid_until.format(ostr, settings); } - void formatHosts(const char * prefix, const AllowedClientHosts & hosts, const IAST::FormatSettings & settings) + void formatHosts(const char * prefix, const AllowedClientHosts & hosts, WriteBuffer & ostr, const IAST::FormatSettings & settings) { if (prefix) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << prefix << " HOST " + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << prefix << " HOST " << (settings.hilite ? IAST::hilite_none : ""); else - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " HOST " << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " HOST " << (settings.hilite ? IAST::hilite_none : ""); if (hosts.empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); return; } if (hosts.containsAnyHost()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ANY" << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ANY" << (settings.hilite ? IAST::hilite_none : ""); return; } @@ -70,8 +70,8 @@ namespace if (hosts.containsLocalHost()) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "LOCAL" << (settings.hilite ? IAST::hilite_none : ""); + ostr << ", "; + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "LOCAL" << (settings.hilite ? IAST::hilite_none : ""); } const auto & addresses = hosts.getAddresses(); @@ -79,20 +79,20 @@ namespace if (!addresses.empty() || !subnets.empty()) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "IP " << (settings.hilite ? IAST::hilite_none : ""); + ostr << ", "; + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "IP " << (settings.hilite ? IAST::hilite_none : ""); bool need_comma2 = false; for (const auto & address : addresses) { if (std::exchange(need_comma2, true)) - settings.ostr << ", "; - settings.ostr << quoteString(address.toString()); + ostr << ", "; + ostr << quoteString(address.toString()); } for (const auto & subnet : subnets) { if (std::exchange(need_comma2, true)) - settings.ostr << ", "; - settings.ostr << quoteString(subnet.toString()); + ostr << ", "; + ostr << quoteString(subnet.toString()); } } @@ -100,14 +100,14 @@ namespace if (!names.empty()) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NAME " << (settings.hilite ? IAST::hilite_none : ""); + ostr << ", "; + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NAME " << (settings.hilite ? IAST::hilite_none : ""); bool need_comma2 = false; for (const auto & name : names) { if (std::exchange(need_comma2, true)) - settings.ostr << ", "; - settings.ostr << quoteString(name); + ostr << ", "; + ostr << quoteString(name); } } @@ -115,14 +115,14 @@ namespace if (!name_regexps.empty()) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "REGEXP " << (settings.hilite ? IAST::hilite_none : ""); + ostr << ", "; + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "REGEXP " << (settings.hilite ? IAST::hilite_none : ""); bool need_comma2 = false; for (const auto & host_regexp : name_regexps) { if (std::exchange(need_comma2, true)) - settings.ostr << ", "; - settings.ostr << quoteString(host_regexp); + ostr << ", "; + ostr << quoteString(host_regexp); } } @@ -130,43 +130,43 @@ namespace if (!like_patterns.empty()) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "LIKE " << (settings.hilite ? IAST::hilite_none : ""); + ostr << ", "; + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "LIKE " << (settings.hilite ? IAST::hilite_none : ""); bool need_comma2 = false; for (const auto & like_pattern : like_patterns) { if (std::exchange(need_comma2, true)) - settings.ostr << ", "; - settings.ostr << quoteString(like_pattern); + ostr << ", "; + ostr << quoteString(like_pattern); } } } - void formatDefaultRoles(const ASTRolesOrUsersSet & default_roles, const IAST::FormatSettings & settings) + void formatDefaultRoles(const ASTRolesOrUsersSet & default_roles, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " DEFAULT ROLE " << (settings.hilite ? IAST::hilite_none : ""); - default_roles.format(settings); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " DEFAULT ROLE " << (settings.hilite ? IAST::hilite_none : ""); + default_roles.format(ostr, settings); } - void formatSettings(const ASTSettingsProfileElements & settings, const IAST::FormatSettings & format) + void formatSettings(const ASTSettingsProfileElements & settings, WriteBuffer & ostr, const IAST::FormatSettings & format) { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " SETTINGS " << (format.hilite ? IAST::hilite_none : ""); - settings.format(format); + ostr << (format.hilite ? IAST::hilite_keyword : "") << " SETTINGS " << (format.hilite ? IAST::hilite_none : ""); + settings.format(ostr, format); } - void formatGrantees(const ASTRolesOrUsersSet & grantees, const IAST::FormatSettings & settings) + void formatGrantees(const ASTRolesOrUsersSet & grantees, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " GRANTEES " << (settings.hilite ? IAST::hilite_none : ""); - grantees.format(settings); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " GRANTEES " << (settings.hilite ? IAST::hilite_none : ""); + grantees.format(ostr, settings); } - void formatDefaultDatabase(const ASTDatabaseOrNone & default_database, const IAST::FormatSettings & settings) + void formatDefaultDatabase(const ASTDatabaseOrNone & default_database, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " DEFAULT DATABASE " << (settings.hilite ? IAST::hilite_none : ""); - default_database.format(settings); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " DEFAULT DATABASE " << (settings.hilite ? IAST::hilite_none : ""); + default_database.format(ostr, settings); } } @@ -209,83 +209,83 @@ ASTPtr ASTCreateUserQuery::clone() const } -void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const +void ASTCreateUserQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & format, FormatState &, FormatStateStacked) const { if (attach) { - format.ostr << (format.hilite ? hilite_keyword : "") << "ATTACH USER" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << "ATTACH USER" << (format.hilite ? hilite_none : ""); } else { - format.ostr << (format.hilite ? hilite_keyword : "") << (alter ? "ALTER USER" : "CREATE USER") + ostr << (format.hilite ? hilite_keyword : "") << (alter ? "ALTER USER" : "CREATE USER") << (format.hilite ? hilite_none : ""); } if (if_exists) - format.ostr << (format.hilite ? hilite_keyword : "") << " IF EXISTS" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << " IF EXISTS" << (format.hilite ? hilite_none : ""); else if (if_not_exists) - format.ostr << (format.hilite ? hilite_keyword : "") << " IF NOT EXISTS" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << " IF NOT EXISTS" << (format.hilite ? hilite_none : ""); else if (or_replace) - format.ostr << (format.hilite ? hilite_keyword : "") << " OR REPLACE" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << " OR REPLACE" << (format.hilite ? hilite_none : ""); - format.ostr << " "; - names->format(format); + ostr << " "; + names->format(ostr, format); if (!storage_name.empty()) - format.ostr << (format.hilite ? IAST::hilite_keyword : "") + ostr << (format.hilite ? IAST::hilite_keyword : "") << " IN " << (format.hilite ? IAST::hilite_none : "") << backQuoteIfNeed(storage_name); - formatOnCluster(format); + formatOnCluster(ostr, format); if (new_name) - formatRenameTo(*new_name, format); + formatRenameTo(*new_name, ostr, format); if (authentication_methods.empty()) { // If identification (auth method) is missing from query, we should serialize it in the form of `NO_PASSWORD` unless it is alter query if (!alter) { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH no_password" << (format.hilite ? IAST::hilite_none : ""); + ostr << (format.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH no_password" << (format.hilite ? IAST::hilite_none : ""); } } else { if (add_identified_with) { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " ADD" << (format.hilite ? IAST::hilite_none : ""); + ostr << (format.hilite ? IAST::hilite_keyword : "") << " ADD" << (format.hilite ? IAST::hilite_none : ""); } - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (format.hilite ? IAST::hilite_none : ""); - formatAuthenticationData(authentication_methods, format); + ostr << (format.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (format.hilite ? IAST::hilite_none : ""); + formatAuthenticationData(authentication_methods, ostr, format); } if (global_valid_until) { - formatValidUntil(*global_valid_until, format); + formatValidUntil(*global_valid_until, ostr, format); } if (hosts) - formatHosts(nullptr, *hosts, format); + formatHosts(nullptr, *hosts, ostr, format); if (add_hosts) - formatHosts("ADD", *add_hosts, format); + formatHosts("ADD", *add_hosts, ostr, format); if (remove_hosts) - formatHosts("DROP", *remove_hosts, format); + formatHosts("DROP", *remove_hosts, ostr, format); if (default_database) - formatDefaultDatabase(*default_database, format); + formatDefaultDatabase(*default_database, ostr, format); if (default_roles) - formatDefaultRoles(*default_roles, format); + formatDefaultRoles(*default_roles, ostr, format); if (settings && (!settings->empty() || alter)) - formatSettings(*settings, format); + formatSettings(*settings, ostr, format); if (grantees) - formatGrantees(*grantees, format); + formatGrantees(*grantees, ostr, format); if (reset_authentication_methods_to_new) - format.ostr << (format.hilite ? hilite_keyword : "") << " RESET AUTHENTICATION METHODS TO NEW" << (format.hilite ? hilite_none : ""); + ostr << (format.hilite ? hilite_keyword : "") << " RESET AUTHENTICATION METHODS TO NEW" << (format.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index 8926c7cad44..b43768bf325 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -66,7 +66,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & format, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } QueryKind getQueryKind() const override { return QueryKind::Create; } diff --git a/src/Parsers/Access/ASTDropAccessEntityQuery.cpp b/src/Parsers/Access/ASTDropAccessEntityQuery.cpp index bcd7105d0e9..c7d86c33037 100644 --- a/src/Parsers/Access/ASTDropAccessEntityQuery.cpp +++ b/src/Parsers/Access/ASTDropAccessEntityQuery.cpp @@ -8,14 +8,14 @@ namespace DB { namespace { - void formatNames(const Strings & names, const IAST::FormatSettings & settings) + void formatNames(const Strings & names, WriteBuffer & ostr) { bool need_comma = false; for (const auto & name : names) { if (std::exchange(need_comma, true)) - settings.ostr << ','; - settings.ostr << ' ' << backQuoteIfNeed(name); + ostr << ','; + ostr << ' ' << backQuoteIfNeed(name); } } } @@ -38,27 +38,27 @@ ASTPtr ASTDropAccessEntityQuery::clone() const } -void ASTDropAccessEntityQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTDropAccessEntityQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << "DROP " << AccessEntityTypeInfo::get(type).name << (if_exists ? " IF EXISTS" : "") << (settings.hilite ? hilite_none : ""); if (type == AccessEntityType::ROW_POLICY) { - settings.ostr << " "; - row_policy_names->format(settings); + ostr << " "; + row_policy_names->format(ostr, settings); } else - formatNames(names, settings); + formatNames(names, ostr); if (!storage_name.empty()) - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(storage_name); - formatOnCluster(settings); + formatOnCluster(ostr, settings); } diff --git a/src/Parsers/Access/ASTDropAccessEntityQuery.h b/src/Parsers/Access/ASTDropAccessEntityQuery.h index 32f4a8f8047..6cf17b20148 100644 --- a/src/Parsers/Access/ASTDropAccessEntityQuery.h +++ b/src/Parsers/Access/ASTDropAccessEntityQuery.h @@ -26,7 +26,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } void replaceEmptyDatabase(const String & current_database) const; diff --git a/src/Parsers/Access/ASTGrantQuery.cpp b/src/Parsers/Access/ASTGrantQuery.cpp index adc61e145a8..5ce5fb9a384 100644 --- a/src/Parsers/Access/ASTGrantQuery.cpp +++ b/src/Parsers/Access/ASTGrantQuery.cpp @@ -12,11 +12,11 @@ namespace ErrorCodes namespace { - void formatCurrentGrantsElements(const AccessRightsElements & elements, const IAST::FormatSettings & settings) + void formatCurrentGrantsElements(const AccessRightsElements & elements, WriteBuffer & ostr, const IAST::FormatSettings & settings) { - settings.ostr << "("; - elements.formatElementsWithoutOptions(settings.ostr, settings.hilite); - settings.ostr << ")"; + ostr << "("; + elements.formatElementsWithoutOptions(ostr, settings.hilite); + ostr << ")"; } } @@ -41,9 +41,9 @@ ASTPtr ASTGrantQuery::clone() const } -void ASTGrantQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTGrantQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << (attach_mode ? "ATTACH " : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << (attach_mode ? "ATTACH " : "") << (settings.hilite ? hilite_keyword : "") << (is_revoke ? "REVOKE" : "GRANT") << (settings.hilite ? IAST::hilite_none : ""); @@ -53,20 +53,20 @@ void ASTGrantQuery::formatImpl(const FormatSettings & settings, FormatState &, F throw Exception(ErrorCodes::LOGICAL_ERROR, "A partial revoke should be revoked, not granted"); bool grant_option = !access_rights_elements.empty() && access_rights_elements[0].grant_option; - formatOnCluster(settings); + formatOnCluster(ostr, settings); if (is_revoke) { if (grant_option) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " GRANT OPTION FOR" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " GRANT OPTION FOR" << (settings.hilite ? hilite_none : ""); else if (admin_option) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " ADMIN OPTION FOR" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " ADMIN OPTION FOR" << (settings.hilite ? hilite_none : ""); } - settings.ostr << " "; + ostr << " "; if (roles) { - roles->format(settings); + roles->format(ostr, settings); if (!access_rights_elements.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "ASTGrantQuery can contain either roles or access rights elements " @@ -74,27 +74,27 @@ void ASTGrantQuery::formatImpl(const FormatSettings & settings, FormatState &, F } else if (current_grants) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "CURRENT GRANTS" << (settings.hilite ? hilite_none : ""); - formatCurrentGrantsElements(access_rights_elements, settings); + ostr << (settings.hilite ? hilite_keyword : "") << "CURRENT GRANTS" << (settings.hilite ? hilite_none : ""); + formatCurrentGrantsElements(access_rights_elements, ostr, settings); } else { - access_rights_elements.formatElementsWithoutOptions(settings.ostr, settings.hilite); + access_rights_elements.formatElementsWithoutOptions(ostr, settings.hilite); } - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << (is_revoke ? " FROM " : " TO ") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << (is_revoke ? " FROM " : " TO ") << (settings.hilite ? IAST::hilite_none : ""); - grantees->format(settings); + grantees->format(ostr, settings); if (!is_revoke) { if (grant_option) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WITH GRANT OPTION" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " WITH GRANT OPTION" << (settings.hilite ? hilite_none : ""); else if (admin_option) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WITH ADMIN OPTION" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " WITH ADMIN OPTION" << (settings.hilite ? hilite_none : ""); if (replace_access || replace_granted_roles) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WITH REPLACE OPTION" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " WITH REPLACE OPTION" << (settings.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/Access/ASTGrantQuery.h b/src/Parsers/Access/ASTGrantQuery.h index 2ccbac3dac8..0fd9d649ee9 100644 --- a/src/Parsers/Access/ASTGrantQuery.h +++ b/src/Parsers/Access/ASTGrantQuery.h @@ -32,7 +32,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; void replaceEmptyDatabase(const String & current_database); void replaceCurrentUserTag(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } diff --git a/src/Parsers/Access/ASTMoveAccessEntityQuery.cpp b/src/Parsers/Access/ASTMoveAccessEntityQuery.cpp index 285f07854c0..21a83af6088 100644 --- a/src/Parsers/Access/ASTMoveAccessEntityQuery.cpp +++ b/src/Parsers/Access/ASTMoveAccessEntityQuery.cpp @@ -8,14 +8,14 @@ namespace DB { namespace { - void formatNames(const Strings & names, const IAST::FormatSettings & settings) + void formatNames(const Strings & names, WriteBuffer & ostr) { bool need_comma = false; for (const auto & name : names) { if (std::exchange(need_comma, true)) - settings.ostr << ','; - settings.ostr << ' ' << backQuoteIfNeed(name); + ostr << ','; + ostr << ' ' << backQuoteIfNeed(name); } } } @@ -35,25 +35,25 @@ ASTPtr ASTMoveAccessEntityQuery::clone() const return res; } -void ASTMoveAccessEntityQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTMoveAccessEntityQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << "MOVE " << AccessEntityTypeInfo::get(type).name << (settings.hilite ? hilite_none : ""); if (type == AccessEntityType::ROW_POLICY) { - settings.ostr << " "; - row_policy_names->format(settings); + ostr << " "; + row_policy_names->format(ostr, settings); } else - formatNames(names, settings); + formatNames(names, ostr); - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(storage_name); - formatOnCluster(settings); + formatOnCluster(ostr, settings); } void ASTMoveAccessEntityQuery::replaceEmptyDatabase(const String & current_database) const diff --git a/src/Parsers/Access/ASTMoveAccessEntityQuery.h b/src/Parsers/Access/ASTMoveAccessEntityQuery.h index aa2b3b0f98c..165f45e0910 100644 --- a/src/Parsers/Access/ASTMoveAccessEntityQuery.h +++ b/src/Parsers/Access/ASTMoveAccessEntityQuery.h @@ -22,7 +22,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } void replaceEmptyDatabase(const String & current_database) const; diff --git a/src/Parsers/Access/ASTPublicSSHKey.cpp b/src/Parsers/Access/ASTPublicSSHKey.cpp index 916202b96dc..bc7af7bf6d6 100644 --- a/src/Parsers/Access/ASTPublicSSHKey.cpp +++ b/src/Parsers/Access/ASTPublicSSHKey.cpp @@ -6,12 +6,12 @@ namespace DB { -void ASTPublicSSHKey::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTPublicSSHKey::formatImpl(WriteBuffer & ostr, const FormatSettings &, FormatState &, FormatStateStacked) const { - settings.ostr << "KEY "; - settings.ostr << backQuoteIfNeed(key_base64) << ' '; - settings.ostr << "TYPE "; - settings.ostr << backQuoteIfNeed(type); + ostr << "KEY "; + ostr << backQuoteIfNeed(key_base64) << ' '; + ostr << "TYPE "; + ostr << backQuoteIfNeed(type); } } diff --git a/src/Parsers/Access/ASTPublicSSHKey.h b/src/Parsers/Access/ASTPublicSSHKey.h index 549ab162a7e..0e666657455 100644 --- a/src/Parsers/Access/ASTPublicSSHKey.h +++ b/src/Parsers/Access/ASTPublicSSHKey.h @@ -19,7 +19,7 @@ public: {} String getID(char) const override { return "PublicSSHKey"; } ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings &, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/Access/ASTRolesOrUsersSet.cpp b/src/Parsers/Access/ASTRolesOrUsersSet.cpp index dc7626b90d6..e70ed8541c6 100644 --- a/src/Parsers/Access/ASTRolesOrUsersSet.cpp +++ b/src/Parsers/Access/ASTRolesOrUsersSet.cpp @@ -7,25 +7,25 @@ namespace DB { namespace { - void formatNameOrID(const String & str, bool is_id, const IAST::FormatSettings & settings) + void formatNameOrID(const String & str, bool is_id, WriteBuffer & ostr, const IAST::FormatSettings & settings) { if (is_id) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ID" << (settings.hilite ? IAST::hilite_none : "") << "(" + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ID" << (settings.hilite ? IAST::hilite_none : "") << "(" << quoteString(str) << ")"; } else { - settings.ostr << backQuoteIfNeed(str); + ostr << backQuoteIfNeed(str); } } } -void ASTRolesOrUsersSet::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTRolesOrUsersSet::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { if (empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); return; } @@ -34,8 +34,8 @@ void ASTRolesOrUsersSet::formatImpl(const FormatSettings & settings, FormatState if (all) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << (use_keyword_any ? "ANY" : "ALL") + ostr << ", "; + ostr << (settings.hilite ? IAST::hilite_keyword : "") << (use_keyword_any ? "ANY" : "ALL") << (settings.hilite ? IAST::hilite_none : ""); } else @@ -43,35 +43,35 @@ void ASTRolesOrUsersSet::formatImpl(const FormatSettings & settings, FormatState for (const auto & name : names) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - formatNameOrID(name, id_mode, settings); + ostr << ", "; + formatNameOrID(name, id_mode, ostr, settings); } if (current_user) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "CURRENT_USER" << (settings.hilite ? IAST::hilite_none : ""); + ostr << ", "; + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "CURRENT_USER" << (settings.hilite ? IAST::hilite_none : ""); } } if (except_current_user || !except_names.empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " EXCEPT " << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " EXCEPT " << (settings.hilite ? IAST::hilite_none : ""); need_comma = false; for (const auto & name : except_names) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - formatNameOrID(name, id_mode, settings); + ostr << ", "; + formatNameOrID(name, id_mode, ostr, settings); } if (except_current_user) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "CURRENT_USER" << (settings.hilite ? IAST::hilite_none : ""); + ostr << ", "; + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "CURRENT_USER" << (settings.hilite ? IAST::hilite_none : ""); } } } diff --git a/src/Parsers/Access/ASTRolesOrUsersSet.h b/src/Parsers/Access/ASTRolesOrUsersSet.h index 15d42ee39a0..17ffe8281f7 100644 --- a/src/Parsers/Access/ASTRolesOrUsersSet.h +++ b/src/Parsers/Access/ASTRolesOrUsersSet.h @@ -30,6 +30,6 @@ public: String getID(char) const override { return "RolesOrUsersSet"; } ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/Access/ASTRowPolicyName.cpp b/src/Parsers/Access/ASTRowPolicyName.cpp index 81a90de9d53..8325145388a 100644 --- a/src/Parsers/Access/ASTRowPolicyName.cpp +++ b/src/Parsers/Access/ASTRowPolicyName.cpp @@ -11,16 +11,16 @@ namespace ErrorCodes } -void ASTRowPolicyName::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTRowPolicyName::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { const String & database = full_name.database; const String & table_name = full_name.table_name; const String & short_name = full_name.short_name; - settings.ostr << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " + ostr << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : "") << (database.empty() ? String{} : backQuoteIfNeed(database) + ".") << backQuoteIfNeed(table_name); - formatOnCluster(settings); + formatOnCluster(ostr, settings); } @@ -36,7 +36,7 @@ String ASTRowPolicyNames::tableOrAsterisk(const String & table_name) const } -void ASTRowPolicyNames::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTRowPolicyNames::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { if (full_names.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "No names of row policies in AST"); @@ -66,19 +66,19 @@ void ASTRowPolicyNames::formatImpl(const FormatSettings & settings, FormatState if (same_short_name) { const String & short_name = full_names[0].short_name; - settings.ostr << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " + ostr << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); bool need_comma = false; for (const auto & full_name : full_names) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; + ostr << ", "; const String & database = full_name.database; const String & table_name = full_name.table_name; if (!database.empty()) - settings.ostr << backQuoteIfNeed(database) + "."; - settings.ostr << tableOrAsterisk(table_name); + ostr << backQuoteIfNeed(database) + "."; + ostr << tableOrAsterisk(table_name); } } else if (same_db_and_table_name) @@ -87,17 +87,17 @@ void ASTRowPolicyNames::formatImpl(const FormatSettings & settings, FormatState for (const auto & full_name : full_names) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; + ostr << ", "; const String & short_name = full_name.short_name; - settings.ostr << backQuoteIfNeed(short_name); + ostr << backQuoteIfNeed(short_name); } const String & database = full_names[0].database; const String & table_name = full_names[0].table_name; - settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); if (!database.empty()) - settings.ostr << backQuoteIfNeed(database) + "."; - settings.ostr << tableOrAsterisk(table_name); + ostr << backQuoteIfNeed(database) + "."; + ostr << tableOrAsterisk(table_name); } else { @@ -105,19 +105,19 @@ void ASTRowPolicyNames::formatImpl(const FormatSettings & settings, FormatState for (const auto & full_name : full_names) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; + ostr << ", "; const String & short_name = full_name.short_name; const String & database = full_name.database; const String & table_name = full_name.table_name; - settings.ostr << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " + ostr << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); if (!database.empty()) - settings.ostr << backQuoteIfNeed(database) + "."; - settings.ostr << tableOrAsterisk(table_name); + ostr << backQuoteIfNeed(database) + "."; + ostr << tableOrAsterisk(table_name); } } - formatOnCluster(settings); + formatOnCluster(ostr, settings); } diff --git a/src/Parsers/Access/ASTRowPolicyName.h b/src/Parsers/Access/ASTRowPolicyName.h index 86171475a0a..211f004669a 100644 --- a/src/Parsers/Access/ASTRowPolicyName.h +++ b/src/Parsers/Access/ASTRowPolicyName.h @@ -19,7 +19,7 @@ public: String getID(char) const override { return "RowPolicyName"; } ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } void replaceEmptyDatabase(const String & current_database); @@ -41,7 +41,7 @@ public: String getID(char) const override { return "RowPolicyNames"; } ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } void replaceEmptyDatabase(const String & current_database); diff --git a/src/Parsers/Access/ASTSetRoleQuery.cpp b/src/Parsers/Access/ASTSetRoleQuery.cpp index c26a7f18661..e33575138f5 100644 --- a/src/Parsers/Access/ASTSetRoleQuery.cpp +++ b/src/Parsers/Access/ASTSetRoleQuery.cpp @@ -26,27 +26,27 @@ ASTPtr ASTSetRoleQuery::clone() const } -void ASTSetRoleQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTSetRoleQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : ""); + ostr << (settings.hilite ? hilite_keyword : ""); switch (kind) { - case Kind::SET_ROLE: settings.ostr << "SET ROLE"; break; - case Kind::SET_ROLE_DEFAULT: settings.ostr << "SET ROLE DEFAULT"; break; - case Kind::SET_DEFAULT_ROLE: settings.ostr << "SET DEFAULT ROLE"; break; + case Kind::SET_ROLE: ostr << "SET ROLE"; break; + case Kind::SET_ROLE_DEFAULT: ostr << "SET ROLE DEFAULT"; break; + case Kind::SET_DEFAULT_ROLE: ostr << "SET DEFAULT ROLE"; break; } - settings.ostr << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_none : ""); if (kind == Kind::SET_ROLE_DEFAULT) return; - settings.ostr << " "; - roles->format(settings); + ostr << " "; + roles->format(ostr, settings); if (kind == Kind::SET_ROLE) return; - settings.ostr << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : ""); - to_users->format(settings); + ostr << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : ""); + to_users->format(ostr, settings); } } diff --git a/src/Parsers/Access/ASTSetRoleQuery.h b/src/Parsers/Access/ASTSetRoleQuery.h index 51cdedda29d..fd6c94841b8 100644 --- a/src/Parsers/Access/ASTSetRoleQuery.h +++ b/src/Parsers/Access/ASTSetRoleQuery.h @@ -26,7 +26,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; QueryKind getQueryKind() const override { return QueryKind::Set; } }; diff --git a/src/Parsers/Access/ASTSettingsProfileElement.cpp b/src/Parsers/Access/ASTSettingsProfileElement.cpp index 014b97132de..72a10ae99a5 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.cpp +++ b/src/Parsers/Access/ASTSettingsProfileElement.cpp @@ -9,46 +9,46 @@ namespace DB { namespace { - void formatProfileNameOrID(const String & str, bool is_id, const IAST::FormatSettings & settings) + void formatProfileNameOrID(const String & str, bool is_id, WriteBuffer & ostr, const IAST::FormatSettings & settings) { if (is_id) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ID" << (settings.hilite ? IAST::hilite_none : "") << "(" + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ID" << (settings.hilite ? IAST::hilite_none : "") << "(" << quoteString(str) << ")"; } else { - settings.ostr << backQuote(str); + ostr << backQuote(str); } } } -void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTSettingsProfileElement::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { if (!parent_profile.empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << (use_inherit_keyword ? "INHERIT" : "PROFILE") << " " + ostr << (settings.hilite ? IAST::hilite_keyword : "") << (use_inherit_keyword ? "INHERIT" : "PROFILE") << " " << (settings.hilite ? IAST::hilite_none : ""); - formatProfileNameOrID(parent_profile, id_mode, settings); + formatProfileNameOrID(parent_profile, id_mode, ostr, settings); return; } - formatSettingName(setting_name, settings.ostr); + formatSettingName(setting_name, ostr); if (value) { - settings.ostr << " = " << applyVisitor(FieldVisitorToString{}, *value); + ostr << " = " << applyVisitor(FieldVisitorToString{}, *value); } if (min_value) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MIN " << (settings.hilite ? IAST::hilite_none : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MIN " << (settings.hilite ? IAST::hilite_none : "") << applyVisitor(FieldVisitorToString{}, *min_value); } if (max_value) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MAX " << (settings.hilite ? IAST::hilite_none : "") + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MAX " << (settings.hilite ? IAST::hilite_none : "") << applyVisitor(FieldVisitorToString{}, *max_value); } @@ -57,15 +57,15 @@ void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, Form switch (*writability) { case SettingConstraintWritability::WRITABLE: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WRITABLE" + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WRITABLE" << (settings.hilite ? IAST::hilite_none : ""); break; case SettingConstraintWritability::CONST: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CONST" + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CONST" << (settings.hilite ? IAST::hilite_none : ""); break; case SettingConstraintWritability::CHANGEABLE_IN_READONLY: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CHANGEABLE_IN_READONLY" + ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CHANGEABLE_IN_READONLY" << (settings.hilite ? IAST::hilite_none : ""); break; case SettingConstraintWritability::MAX: break; @@ -83,11 +83,11 @@ bool ASTSettingsProfileElements::empty() const } -void ASTSettingsProfileElements::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTSettingsProfileElements::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { if (empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); + ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); return; } @@ -95,10 +95,10 @@ void ASTSettingsProfileElements::formatImpl(const FormatSettings & settings, For for (const auto & element : elements) { if (need_comma) - settings.ostr << ", "; + ostr << ", "; need_comma = true; - element->format(settings); + element->format(ostr, settings); } } diff --git a/src/Parsers/Access/ASTSettingsProfileElement.h b/src/Parsers/Access/ASTSettingsProfileElement.h index 13c1926d9b0..5877644473e 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.h +++ b/src/Parsers/Access/ASTSettingsProfileElement.h @@ -25,7 +25,7 @@ public: String getID(char) const override { return "SettingsProfileElement"; } ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; @@ -41,7 +41,7 @@ public: String getID(char) const override { return "SettingsProfileElements"; } ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; void setUseInheritKeyword(bool use_inherit_keyword_); }; diff --git a/src/Parsers/Access/ASTShowAccessEntitiesQuery.cpp b/src/Parsers/Access/ASTShowAccessEntitiesQuery.cpp index a682a5ef8e8..3fe70575319 100644 --- a/src/Parsers/Access/ASTShowAccessEntitiesQuery.cpp +++ b/src/Parsers/Access/ASTShowAccessEntitiesQuery.cpp @@ -24,20 +24,20 @@ String ASTShowAccessEntitiesQuery::getID(char) const return fmt::format("SHOW {} query", getKeyword()); } -void ASTShowAccessEntitiesQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTShowAccessEntitiesQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW " << getKeyword() << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW " << getKeyword() << (settings.hilite ? hilite_none : ""); if (!short_name.empty()) - settings.ostr << " " << backQuoteIfNeed(short_name); + ostr << " " << backQuoteIfNeed(short_name); if (database_and_table_name) { const String & database = database_and_table_name->first; const String & table_name = database_and_table_name->second; - settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); - settings.ostr << (database.empty() ? "" : backQuoteIfNeed(database) + "."); - settings.ostr << (table_name.empty() ? "*" : backQuoteIfNeed(table_name)); + ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); + ostr << (database.empty() ? "" : backQuoteIfNeed(database) + "."); + ostr << (table_name.empty() ? "*" : backQuoteIfNeed(table_name)); } } diff --git a/src/Parsers/Access/ASTShowAccessEntitiesQuery.h b/src/Parsers/Access/ASTShowAccessEntitiesQuery.h index 9957f8d5705..133d1ec10c9 100644 --- a/src/Parsers/Access/ASTShowAccessEntitiesQuery.h +++ b/src/Parsers/Access/ASTShowAccessEntitiesQuery.h @@ -34,7 +34,7 @@ public: QueryKind getQueryKind() const override { return QueryKind::Show; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; private: String getKeyword() const; diff --git a/src/Parsers/Access/ASTShowCreateAccessEntityQuery.cpp b/src/Parsers/Access/ASTShowCreateAccessEntityQuery.cpp index 12eda260712..c8a59686540 100644 --- a/src/Parsers/Access/ASTShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/Access/ASTShowCreateAccessEntityQuery.cpp @@ -8,14 +8,14 @@ namespace DB { namespace { - void formatNames(const Strings & names, const IAST::FormatSettings & settings) + void formatNames(const Strings & names, WriteBuffer & ostr) { bool need_comma = false; for (const auto & name : names) { if (std::exchange(need_comma, true)) - settings.ostr << ','; - settings.ostr << ' ' << backQuoteIfNeed(name); + ostr << ','; + ostr << ' ' << backQuoteIfNeed(name); } } } @@ -47,29 +47,29 @@ ASTPtr ASTShowCreateAccessEntityQuery::clone() const } -void ASTShowCreateAccessEntityQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTShowCreateAccessEntityQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CREATE " << getKeyword() << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CREATE " << getKeyword() << (settings.hilite ? hilite_none : ""); if (!names.empty()) - formatNames(names, settings); + formatNames(names, ostr); if (row_policy_names) { - settings.ostr << " "; - row_policy_names->format(settings); + ostr << " "; + row_policy_names->format(ostr, settings); } if (!short_name.empty()) - settings.ostr << " " << backQuoteIfNeed(short_name); + ostr << " " << backQuoteIfNeed(short_name); if (database_and_table_name) { const String & database = database_and_table_name->first; const String & table_name = database_and_table_name->second; - settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); - settings.ostr << (database.empty() ? "" : backQuoteIfNeed(database) + "."); - settings.ostr << (table_name.empty() ? "*" : backQuoteIfNeed(table_name)); + ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); + ostr << (database.empty() ? "" : backQuoteIfNeed(database) + "."); + ostr << (table_name.empty() ? "*" : backQuoteIfNeed(table_name)); } } diff --git a/src/Parsers/Access/ASTShowCreateAccessEntityQuery.h b/src/Parsers/Access/ASTShowCreateAccessEntityQuery.h index 657160a96dd..bd4fe4f59a2 100644 --- a/src/Parsers/Access/ASTShowCreateAccessEntityQuery.h +++ b/src/Parsers/Access/ASTShowCreateAccessEntityQuery.h @@ -44,7 +44,7 @@ public: protected: String getKeyword() const; - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/Access/ASTShowGrantsQuery.cpp b/src/Parsers/Access/ASTShowGrantsQuery.cpp index c037d8171af..6cbf34e5d07 100644 --- a/src/Parsers/Access/ASTShowGrantsQuery.cpp +++ b/src/Parsers/Access/ASTShowGrantsQuery.cpp @@ -23,9 +23,9 @@ ASTPtr ASTShowGrantsQuery::clone() const } -void ASTShowGrantsQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTShowGrantsQuery::formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW GRANTS" + ostr << (settings.hilite ? hilite_keyword : "") << "SHOW GRANTS" << (settings.hilite ? hilite_none : ""); if (for_roles->current_user && !for_roles->all && for_roles->names.empty() && for_roles->except_names.empty() @@ -34,20 +34,20 @@ void ASTShowGrantsQuery::formatQueryImpl(const FormatSettings & settings, Format } else { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FOR " + ostr << (settings.hilite ? hilite_keyword : "") << " FOR " << (settings.hilite ? hilite_none : ""); - for_roles->format(settings); + for_roles->format(ostr, settings); } if (with_implicit) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WITH IMPLICIT" + ostr << (settings.hilite ? hilite_keyword : "") << " WITH IMPLICIT" << (settings.hilite ? hilite_none : ""); } if (final) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FINAL" + ostr << (settings.hilite ? hilite_keyword : "") << " FINAL" << (settings.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/Access/ASTShowGrantsQuery.h b/src/Parsers/Access/ASTShowGrantsQuery.h index b205ce4b03e..086abb790d8 100644 --- a/src/Parsers/Access/ASTShowGrantsQuery.h +++ b/src/Parsers/Access/ASTShowGrantsQuery.h @@ -18,7 +18,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; QueryKind getQueryKind() const override { return QueryKind::Show; } }; diff --git a/src/Parsers/Access/ASTUserNameWithHost.cpp b/src/Parsers/Access/ASTUserNameWithHost.cpp index 667a8e37414..4f35e82392c 100644 --- a/src/Parsers/Access/ASTUserNameWithHost.cpp +++ b/src/Parsers/Access/ASTUserNameWithHost.cpp @@ -6,12 +6,12 @@ namespace DB { -void ASTUserNameWithHost::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTUserNameWithHost::formatImpl(WriteBuffer & ostr, const FormatSettings &, FormatState &, FormatStateStacked) const { - settings.ostr << backQuoteIfNeed(base_name); + ostr << backQuoteIfNeed(base_name); if (!host_pattern.empty()) - settings.ostr << "@" << backQuoteIfNeed(host_pattern); + ostr << "@" << backQuoteIfNeed(host_pattern); } String ASTUserNameWithHost::toString() const @@ -35,15 +35,15 @@ void ASTUserNameWithHost::replace(const String name_) } -void ASTUserNamesWithHost::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTUserNamesWithHost::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { assert(!names.empty()); bool need_comma = false; for (const auto & name : names) { if (std::exchange(need_comma, true)) - settings.ostr << ", "; - name->format(settings); + ostr << ", "; + name->format(ostr, settings); } } diff --git a/src/Parsers/Access/ASTUserNameWithHost.h b/src/Parsers/Access/ASTUserNameWithHost.h index 8e6a7e78987..cddbf16d929 100644 --- a/src/Parsers/Access/ASTUserNameWithHost.h +++ b/src/Parsers/Access/ASTUserNameWithHost.h @@ -26,7 +26,7 @@ public: explicit ASTUserNameWithHost(const String & name_) : base_name(name_) {} String getID(char) const override { return "UserNameWithHost"; } ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings &, FormatState &, FormatStateStacked) const override; void replace(String name_); }; @@ -48,7 +48,7 @@ public: String getID(char) const override { return "UserNamesWithHost"; } ASTPtr clone() const override { return std::make_shared(*this); } - void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index 0b1dff556f6..9ee421ff21f 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -179,12 +179,12 @@ String IAST::formatWithPossiblyHidingSensitiveData( IdentifierQuotingStyle identifier_quoting_style) const { WriteBufferFromOwnString buf; - FormatSettings settings(buf, one_line); + FormatSettings settings(one_line); settings.show_secrets = show_secrets; settings.print_pretty_type_names = print_pretty_type_names; settings.identifier_quoting_rule = identifier_quoting_rule; settings.identifier_quoting_style = identifier_quoting_style; - format(settings); + format(buf, settings); return wipeSensitiveDataAndCutToLength(buf.str(), max_length); } @@ -221,7 +221,7 @@ String IAST::getColumnNameWithoutAlias() const } -void IAST::FormatSettings::writeIdentifier(const String & name, bool ambiguous) const +void IAST::FormatSettings::writeIdentifier(WriteBuffer & ostr, const String & name, bool ambiguous) const { checkIdentifier(name); bool must_quote diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 07b32c7d8e2..851ae766419 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -193,7 +193,6 @@ public: /// Format settings. struct FormatSettings { - WriteBuffer & ostr; bool one_line; bool hilite; IdentifierQuotingRule identifier_quoting_rule; @@ -205,7 +204,6 @@ public: bool enforce_strict_identifier_format; explicit FormatSettings( - WriteBuffer & ostr_, bool one_line_, bool hilite_ = false, IdentifierQuotingRule identifier_quoting_rule_ = IdentifierQuotingRule::WhenNecessary, @@ -214,8 +212,7 @@ public: LiteralEscapingStyle literal_escaping_style_ = LiteralEscapingStyle::Regular, bool print_pretty_type_names_ = false, bool enforce_strict_identifier_format_ = false) - : ostr(ostr_) - , one_line(one_line_) + : one_line(one_line_) , hilite(hilite_) , identifier_quoting_rule(identifier_quoting_rule_) , identifier_quoting_style(identifier_quoting_style_) @@ -227,9 +224,8 @@ public: { } - FormatSettings(WriteBuffer & ostr_, const FormatSettings & other) - : ostr(ostr_) - , one_line(other.one_line) + FormatSettings(const FormatSettings & other) + : one_line(other.one_line) , hilite(other.hilite) , identifier_quoting_rule(other.identifier_quoting_rule) , identifier_quoting_style(other.identifier_quoting_style) @@ -241,7 +237,7 @@ public: { } - void writeIdentifier(const String & name, bool ambiguous) const; + void writeIdentifier(WriteBuffer & ostr, const String & name, bool ambiguous) const; void checkIdentifier(const String & name) const; }; @@ -270,13 +266,13 @@ public: const IAST * current_select = nullptr; }; - void format(const FormatSettings & settings) const + void format(WriteBuffer & ostr, const FormatSettings & settings) const { FormatState state; - formatImpl(settings, state, FormatStateStacked()); + formatImpl(ostr, settings, state, FormatStateStacked()); } - virtual void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const + virtual void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown element in AST: {}", getID()); } diff --git a/src/Parsers/MySQL/ASTAlterCommand.h b/src/Parsers/MySQL/ASTAlterCommand.h index 87b665ec6a5..d5ea6c5a6a5 100644 --- a/src/Parsers/MySQL/ASTAlterCommand.h +++ b/src/Parsers/MySQL/ASTAlterCommand.h @@ -76,7 +76,7 @@ public: String getID(char delim) const override { return "AlterCommand" + (delim + std::to_string(static_cast(type))); } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTAlterCommand."); } diff --git a/src/Parsers/MySQL/ASTAlterQuery.h b/src/Parsers/MySQL/ASTAlterQuery.h index 161e5e40086..5d254ac1ee3 100644 --- a/src/Parsers/MySQL/ASTAlterQuery.h +++ b/src/Parsers/MySQL/ASTAlterQuery.h @@ -28,7 +28,7 @@ public: String getID(char delim) const override { return "AlterQuery" + (delim + database) + delim + table; } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTAlterQuery."); } diff --git a/src/Parsers/MySQL/ASTCreateDefines.h b/src/Parsers/MySQL/ASTCreateDefines.h index 7c23d1cb87f..34e4622174c 100644 --- a/src/Parsers/MySQL/ASTCreateDefines.h +++ b/src/Parsers/MySQL/ASTCreateDefines.h @@ -27,7 +27,7 @@ public: String getID(char) const override { return "Create definitions"; } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTCreateDefines."); } diff --git a/src/Parsers/MySQL/ASTCreateQuery.h b/src/Parsers/MySQL/ASTCreateQuery.h index d01bed6b7d6..65e76ae8a4d 100644 --- a/src/Parsers/MySQL/ASTCreateQuery.h +++ b/src/Parsers/MySQL/ASTCreateQuery.h @@ -32,7 +32,7 @@ public: String getID(char) const override { return "create query"; } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTCreateQuery."); } diff --git a/src/Parsers/MySQL/ASTDeclareColumn.h b/src/Parsers/MySQL/ASTDeclareColumn.h index 30e16b7b84c..f1f33c93c84 100644 --- a/src/Parsers/MySQL/ASTDeclareColumn.h +++ b/src/Parsers/MySQL/ASTDeclareColumn.h @@ -26,7 +26,7 @@ public: String getID(char /*delimiter*/) const override { return "Column definition"; } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTDeclareColumn."); } diff --git a/src/Parsers/MySQL/ASTDeclareConstraint.h b/src/Parsers/MySQL/ASTDeclareConstraint.h index 82f2597bc4d..13eab7a869d 100644 --- a/src/Parsers/MySQL/ASTDeclareConstraint.h +++ b/src/Parsers/MySQL/ASTDeclareConstraint.h @@ -26,7 +26,7 @@ public: String getID(char /*delimiter*/) const override { return "constraint declaration"; } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTDeclareConstraint."); } diff --git a/src/Parsers/MySQL/ASTDeclareIndex.h b/src/Parsers/MySQL/ASTDeclareIndex.h index 5399c394137..4c4af82cbed 100644 --- a/src/Parsers/MySQL/ASTDeclareIndex.h +++ b/src/Parsers/MySQL/ASTDeclareIndex.h @@ -30,7 +30,7 @@ public: String getID(char /*delimiter*/) const override { return "index declaration"; } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTDeclareIndex."); } diff --git a/src/Parsers/MySQL/ASTDeclareOption.h b/src/Parsers/MySQL/ASTDeclareOption.h index 62fb42ded28..1bef9cefba9 100644 --- a/src/Parsers/MySQL/ASTDeclareOption.h +++ b/src/Parsers/MySQL/ASTDeclareOption.h @@ -38,7 +38,7 @@ public: String getID(char /*delimiter*/) const override { return "options declaration"; } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTDeclareOptions."); } diff --git a/src/Parsers/MySQL/ASTDeclarePartition.h b/src/Parsers/MySQL/ASTDeclarePartition.h index f2c4103a1ba..310c0a4152f 100644 --- a/src/Parsers/MySQL/ASTDeclarePartition.h +++ b/src/Parsers/MySQL/ASTDeclarePartition.h @@ -28,7 +28,7 @@ public: String getID(char /*delimiter*/) const override { return "partition declaration"; } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTDeclarePartition."); } diff --git a/src/Parsers/MySQL/ASTDeclarePartitionOptions.h b/src/Parsers/MySQL/ASTDeclarePartitionOptions.h index cee2d449291..cd3425452e8 100644 --- a/src/Parsers/MySQL/ASTDeclarePartitionOptions.h +++ b/src/Parsers/MySQL/ASTDeclarePartitionOptions.h @@ -30,7 +30,7 @@ public: String getID(char /*delimiter*/) const override { return "partition options declaration"; } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTDeclarePartitionOptions."); } diff --git a/src/Parsers/MySQL/ASTDeclareReference.h b/src/Parsers/MySQL/ASTDeclareReference.h index b04bf52e51c..05f1f931a04 100644 --- a/src/Parsers/MySQL/ASTDeclareReference.h +++ b/src/Parsers/MySQL/ASTDeclareReference.h @@ -44,7 +44,7 @@ public: String getID(char /*delimiter*/) const override { return "subpartition declaration"; } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTDeclareReference."); } diff --git a/src/Parsers/MySQL/ASTDeclareSubPartition.h b/src/Parsers/MySQL/ASTDeclareSubPartition.h index 82a1a5a3b0b..7f0b5efb0c6 100644 --- a/src/Parsers/MySQL/ASTDeclareSubPartition.h +++ b/src/Parsers/MySQL/ASTDeclareSubPartition.h @@ -25,7 +25,7 @@ public: String getID(char /*delimiter*/) const override { return "subpartition declaration"; } protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTDeclareSubPartition."); } diff --git a/src/Parsers/MySQL/ASTDropQuery.h b/src/Parsers/MySQL/ASTDropQuery.h index 742cf6ba421..7809fee4815 100644 --- a/src/Parsers/MySQL/ASTDropQuery.h +++ b/src/Parsers/MySQL/ASTDropQuery.h @@ -45,7 +45,7 @@ public: String getID(char /*delim*/) const override {return "ASTDropQuery" ;} protected: - void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + void formatImpl(WriteBuffer & /*ostr*/, const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported by MySQLParser::ASTDropQuery."); } diff --git a/src/Parsers/TablePropertiesQueriesASTs.h b/src/Parsers/TablePropertiesQueriesASTs.h index 81ad975aa37..e58f41657f7 100644 --- a/src/Parsers/TablePropertiesQueriesASTs.h +++ b/src/Parsers/TablePropertiesQueriesASTs.h @@ -95,11 +95,11 @@ public: } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { - settings.ostr << (settings.hilite ? hilite_keyword : "") << ASTExistsDatabaseQueryIDAndQueryNames::Query + ostr << (settings.hilite ? hilite_keyword : "") << ASTExistsDatabaseQueryIDAndQueryNames::Query << " " << (settings.hilite ? hilite_none : ""); - database->formatImpl(settings, state, frame); + database->formatImpl(ostr, settings, state, frame); } QueryKind getQueryKind() const override { return QueryKind::Exists; } @@ -117,11 +117,11 @@ public: } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { - settings.ostr << (settings.hilite ? hilite_keyword : "") << ASTShowCreateDatabaseQueryIDAndQueryNames::Query + ostr << (settings.hilite ? hilite_keyword : "") << ASTShowCreateDatabaseQueryIDAndQueryNames::Query << " " << (settings.hilite ? hilite_none : ""); - database->formatImpl(settings, state, frame); + database->formatImpl(ostr, settings, state, frame); } }; @@ -148,11 +148,11 @@ public: QueryKind getQueryKind() const override { return QueryKind::Describe; } protected: - void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override + void formatQueryImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { - settings.ostr << (settings.hilite ? hilite_keyword : "") + ostr << (settings.hilite ? hilite_keyword : "") << "DESCRIBE TABLE" << (settings.hilite ? hilite_none : ""); - table_expression->formatImpl(settings, state, frame); + table_expression->formatImpl(ostr, settings, state, frame); } }; diff --git a/src/Parsers/formatAST.cpp b/src/Parsers/formatAST.cpp index 9315279eae6..cad86fcadbb 100644 --- a/src/Parsers/formatAST.cpp +++ b/src/Parsers/formatAST.cpp @@ -6,9 +6,9 @@ namespace DB void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite, bool one_line, bool show_secrets) { - IAST::FormatSettings settings(buf, one_line, hilite); + IAST::FormatSettings settings(one_line, hilite); settings.show_secrets = show_secrets; - ast.format(settings); + ast.format(buf, settings); } String serializeAST(const IAST & ast) diff --git a/src/Parsers/getInsertQuery.cpp b/src/Parsers/getInsertQuery.cpp index 398d4fc6b5d..3303a673865 100644 --- a/src/Parsers/getInsertQuery.cpp +++ b/src/Parsers/getInsertQuery.cpp @@ -20,12 +20,11 @@ std::string getInsertQuery(const std::string & db_name, const std::string & tabl WriteBufferFromOwnString buf; IAST::FormatSettings settings( - /*ostr_=*/buf, /*one_line=*/true, /*hilite=*/false, /*identifier_quoting_rule=*/IdentifierQuotingRule::WhenNecessary, /*identifier_quoting_style=*/quoting); - query.IAST::format(settings); + query.IAST::format(buf, settings); return buf.str(); } } diff --git a/src/Parsers/tests/gtest_format_hiliting.cpp b/src/Parsers/tests/gtest_format_hiliting.cpp index 00e8197af1f..077440483e1 100644 --- a/src/Parsers/tests/gtest_format_hiliting.cpp +++ b/src/Parsers/tests/gtest_format_hiliting.cpp @@ -53,8 +53,8 @@ void compare(const String & expected, const String & query) ASTPtr ast = parseQuery(parser, query, 0, 0, 0); WriteBufferFromOwnString write_buffer; - IAST::FormatSettings settings(write_buffer, true, true); - ast->format(settings); + IAST::FormatSettings settings(true, true); + ast->format(write_buffer, settings); ASSERT_PRED2(HiliteComparator::are_equal_with_hilites_removed, expected, write_buffer.str()); ASSERT_PRED2(HiliteComparator::are_equal_with_hilites_and_end_without_hilite, expected, write_buffer.str()); diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 943c1e2bdfa..13cd9a73b13 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -102,8 +102,8 @@ static String formattedAST(const ASTPtr & ast) WriteBufferFromOwnString buf; IAST::FormatSettings ast_format_settings( - /*ostr_=*/buf, /*one_line=*/true, /*hilite=*/false, /*identifier_quoting_rule=*/IdentifierQuotingRule::Always); - ast->format(ast_format_settings); + /*one_line=*/true, /*hilite=*/false, /*identifier_quoting_rule=*/IdentifierQuotingRule::Always); + ast->format(buf, ast_format_settings); return buf.str(); } diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index 95d0f08fdf0..3213188aa2c 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -122,8 +122,8 @@ bool ColumnDescription::operator==(const ColumnDescription & other) const String formatASTStateAware(IAST & ast, IAST::FormatState & state) { WriteBufferFromOwnString buf; - IAST::FormatSettings settings(buf, true, false); - ast.formatImpl(settings, state, IAST::FormatStateStacked()); + IAST::FormatSettings settings(true, false); + ast.formatImpl(buf, settings, state, IAST::FormatStateStacked()); return buf.str(); } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 4f5a95ab508..4bc87050ac4 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1018,8 +1018,8 @@ std::optional StorageDistributed::distributedWriteBetweenDistribu { WriteBufferFromOwnString buf; IAST::FormatSettings ast_format_settings( - /*ostr_=*/buf, /*one_line*/ true, /*hilite*/ false, /*identifier_quoting_rule_=*/IdentifierQuotingRule::Always); - new_query->IAST::format(ast_format_settings); + /*one_line*/ true, /*hilite*/ false, /*identifier_quoting_rule_=*/IdentifierQuotingRule::Always); + new_query->IAST::format(buf, ast_format_settings); new_query_str = buf.str(); } @@ -1139,8 +1139,8 @@ std::optional StorageDistributed::distributedWriteFromClusterStor { WriteBufferFromOwnString buf; IAST::FormatSettings ast_format_settings( - /*ostr_=*/buf, /*one_line=*/true, /*hilite=*/false, /*identifier_quoting_rule=*/IdentifierQuotingRule::Always); - new_query->IAST::format(ast_format_settings); + /*one_line=*/true, /*hilite=*/false, /*identifier_quoting_rule=*/IdentifierQuotingRule::Always); + new_query->IAST::format(buf, ast_format_settings); new_query_str = buf.str(); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index bd476625081..5cf325761ee 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5789,8 +5789,8 @@ std::optional StorageReplicatedMergeTree::distributedWriteFromClu { WriteBufferFromOwnString buf; IAST::FormatSettings ast_format_settings( - /*ostr_=*/buf, /*one_line=*/true, /*hilite=*/false, /*identifier_quoting_rule=*/IdentifierQuotingRule::Always); - query.IAST::format(ast_format_settings); + /*one_line=*/true, /*hilite=*/false, /*identifier_quoting_rule=*/IdentifierQuotingRule::Always); + query.IAST::format(buf, ast_format_settings); query_str = buf.str(); } diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index a8e36cb7bbe..8a69b098e47 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -389,7 +389,6 @@ String transformQueryForExternalDatabaseImpl( IdentifierQuotingRule identifier_quoting_rule = IdentifierQuotingRule::Always; WriteBufferFromOwnString out; IAST::FormatSettings settings( - /*ostr_=*/out, /*one_line=*/true, /*hilite=*/false, /*identifier_quoting_rule=*/identifier_quoting_rule, @@ -397,7 +396,7 @@ String transformQueryForExternalDatabaseImpl( /*show_secrets_=*/true, /*literal_escaping_style=*/literal_escaping_style); - select->format(settings); + select->format(out, settings); return out.str(); } From 5b8955c2ae249bbe800abe6070e63ae8500b66d2 Mon Sep 17 00:00:00 2001 From: shuai-xu Date: Tue, 3 Dec 2024 09:39:05 +0800 Subject: [PATCH 258/502] add a test --- tests/queries/0_stateless/02353_translate.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02353_translate.sql b/tests/queries/0_stateless/02353_translate.sql index fd3f48f854d..62a4eda0115 100644 --- a/tests/queries/0_stateless/02353_translate.sql +++ b/tests/queries/0_stateless/02353_translate.sql @@ -17,4 +17,5 @@ SELECT translate('aAbBcC', 'abc', '12'); SELECT translate('aAbBcC', 'abc', '1235'); SELECT translate('aAbBcC', 'abc', ''); +SELECT translate('abc', 'abc', ''); SELECT translate('aAbBcC', '中文内码', '12'); -- { serverError BAD_ARGUMENTS } From d735d0f8cd66ca01d8898e81bddf317e43e97651 Mon Sep 17 00:00:00 2001 From: shuai-xu Date: Tue, 3 Dec 2024 10:34:52 +0800 Subject: [PATCH 259/502] add a test --- tests/queries/0_stateless/02353_translate.reference | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02353_translate.reference b/tests/queries/0_stateless/02353_translate.reference index 3c448d59ea2..16bf6bc6015 100644 --- a/tests/queries/0_stateless/02353_translate.reference +++ b/tests/queries/0_stateless/02353_translate.reference @@ -19,3 +19,4 @@ abc 1A2BC 1A2B3C ABC + From 99c8d1fd61895e15859a29bdcfcd51f26293ed82 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Tue, 3 Dec 2024 11:13:19 +0800 Subject: [PATCH 260/502] fix failure tests --- src/DataTypes/EnumValues.h | 4 ++-- src/Interpreters/convertFieldToType.cpp | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/DataTypes/EnumValues.h b/src/DataTypes/EnumValues.h index ec5991abff1..352f91e2851 100644 --- a/src/DataTypes/EnumValues.h +++ b/src/DataTypes/EnumValues.h @@ -10,7 +10,7 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; + extern const int UNKNOWN_ELEMENT_OF_ENUM; } template @@ -38,7 +38,7 @@ public: { auto it = value_to_name_map.find(value); if (it == value_to_name_map.end()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected value {} in enum", toString(value)); + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_OF_ENUM, "Unexpected value {} in enum", toString(value)); return it; } diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 8dd1eb56fbf..2d6f08eaffc 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -660,14 +660,18 @@ static bool decimalEqualsFloat(Field field, Float64 float_value) std::optional convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type, const FormatSettings & format_settings) { + //Field result_value = convertFieldToType(from_value, to_type, &from_type, format_settings); + Field result_value; try { result_value = convertFieldToType(from_value, to_type, &from_type, format_settings); } - catch (...) + catch (Exception & e) { - return {}; + if (isEnum(from_type) && e.code() == ErrorCodes::UNKNOWN_ELEMENT_OF_ENUM) + return {}; + throw; } if (Field::isDecimal(from_value.getType()) && Field::isDecimal(result_value.getType())) From 2df33734deeda01a70f04a3c8f7d4f33f6f37ece Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Tue, 3 Dec 2024 11:20:33 +0800 Subject: [PATCH 261/502] modify code as requested --- src/Interpreters/convertFieldToType.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 2d6f08eaffc..052b94eb1f8 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -669,7 +669,7 @@ std::optional convertFieldToTypeStrict(const Field & from_value, const ID } catch (Exception & e) { - if (isEnum(from_type) && e.code() == ErrorCodes::UNKNOWN_ELEMENT_OF_ENUM) + if (isEnum(to_type) && e.code() == ErrorCodes::UNKNOWN_ELEMENT_OF_ENUM) return {}; throw; } From fc4854e28455fba660092edec156840d7009a7bb Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Tue, 3 Dec 2024 11:30:36 +0800 Subject: [PATCH 262/502] fix style check --- src/Interpreters/convertFieldToType.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 052b94eb1f8..3fabedba873 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -38,6 +38,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int TYPE_MISMATCH; extern const int UNEXPECTED_DATA_AFTER_PARSED_VALUE; + extern const int UNKNOWN_ELEMENT_OF_ENUM; } From 94b4a5190f95ff552671345484ab1181e39ceee8 Mon Sep 17 00:00:00 2001 From: nauu Date: Tue, 3 Dec 2024 11:37:42 +0800 Subject: [PATCH 263/502] Change * to a specific column,because if the new version of system.tables adds a new column,it will cause an identifier error --- tests/integration/test_analyzer_compatibility/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_analyzer_compatibility/test.py b/tests/integration/test_analyzer_compatibility/test.py index 42bd140280b..78f3428e02e 100644 --- a/tests/integration/test_analyzer_compatibility/test.py +++ b/tests/integration/test_analyzer_compatibility/test.py @@ -43,7 +43,7 @@ def test_two_new_versions(start_cluster): query_id = str(uuid.uuid4()) current.query( - "SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables);", + "SELECT name FROM clusterAllReplicas('test_cluster_mixed', system.tables);", query_id=query_id, ) @@ -73,7 +73,7 @@ WHERE initial_query_id = '{query_id}';""" query_id = str(uuid.uuid4()) backward.query( - "SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables)", + "SELECT name FROM clusterAllReplicas('test_cluster_mixed', system.tables)", query_id=query_id, ) @@ -108,7 +108,7 @@ WHERE initial_query_id = '{query_id}';""" # to the remote server. query_id = str(uuid.uuid4()) current.query( - "SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables) SETTINGS enable_analyzer = 1;", + "SELECT name FROM clusterAllReplicas('test_cluster_mixed', system.tables) SETTINGS enable_analyzer = 1;", query_id=query_id, ) From e8ce561f63bb6e87f7da064d24a7ce5a064a994e Mon Sep 17 00:00:00 2001 From: nauu Date: Tue, 3 Dec 2024 12:12:56 +0800 Subject: [PATCH 264/502] Trigger CI From 7d0a62e351f5743c849ea73413df73356315ac20 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Tue, 3 Dec 2024 17:43:22 +0800 Subject: [PATCH 265/502] Support string search operator for Enum data type #72661 --- src/Functions/FunctionsStringSearch.h | 50 ++++++++++++++++- .../03278_enum_string_functions.reference | 36 ++++++++++++ .../03278_enum_string_functions.sql | 55 +++++++++++++++++++ 3 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03278_enum_string_functions.reference create mode 100644 tests/queries/0_stateless/03278_enum_string_functions.sql diff --git a/src/Functions/FunctionsStringSearch.h b/src/Functions/FunctionsStringSearch.h index 5a64091e600..e3f19119539 100644 --- a/src/Functions/FunctionsStringSearch.h +++ b/src/Functions/FunctionsStringSearch.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -140,7 +141,7 @@ public: const auto & haystack_type = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[0] : arguments[1]; const auto & needle_type = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[1] : arguments[0]; - if (!isStringOrFixedString(haystack_type)) + if (!(isStringOrFixedString(haystack_type) || isEnum(haystack_type))) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", @@ -173,11 +174,56 @@ public: return std::make_shared>(); } + template + static ColumnPtr genStringColumnFromEnumColumn(const ColumnWithTypeAndName & argument) + { + const auto * col = argument.column.get(); + const auto * type = argument.type.get(); + auto res = ColumnString::create(); + res->reserve(col->size()); + if constexpr (std::is_same_v) + { + const auto * enum_col = typeid_cast(col); + const auto * enum_type = typeid_cast(type); + const auto size = enum_col->size(); + for (size_t i = 0; i < size; ++i) + { + StringRef value = enum_type->getNameForValue(enum_col->getData()[i]); + res->insertData(value.data, value.size); + } + } + else if constexpr (std::is_same_v) + { + const auto * enum_col = typeid_cast(col); + const auto size = enum_col->size(); + const auto * enum_type = typeid_cast(type); + for (size_t i = 0; i < size; ++i) + { + StringRef value = enum_type->getNameForValue(enum_col->getData()[i]); + res->insertData(value.data, value.size); + } + } + return res; + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { - const ColumnPtr & column_haystack = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[0].column : arguments[1].column; + auto & haystack_argument = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[0] : arguments[1]; + ColumnPtr column_haystack = haystack_argument.column; const ColumnPtr & column_needle = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[1].column : arguments[0].column; + bool is_enum8 = isEnum8(haystack_argument.type); + bool is_enum16 = isEnum16(haystack_argument.type); + + if (is_enum8) + { + column_haystack = genStringColumnFromEnumColumn(haystack_argument); + } + if (is_enum16) + { + column_haystack = genStringColumnFromEnumColumn(haystack_argument); + } + ColumnPtr column_start_pos = nullptr; if (arguments.size() >= 3) column_start_pos = arguments[2].column; diff --git a/tests/queries/0_stateless/03278_enum_string_functions.reference b/tests/queries/0_stateless/03278_enum_string_functions.reference new file mode 100644 index 00000000000..1bc1be5cc77 --- /dev/null +++ b/tests/queries/0_stateless/03278_enum_string_functions.reference @@ -0,0 +1,36 @@ +a +a +1 +1 +1 +1 +1 +1 +1 +1 +0 +0 +1 +0 +1 +0 +0 +0 +0 +1 +0 +0 +1 +0 +0 +0 +0 +0 +0 +0 +3 +1 +3 +1 +3 +1 diff --git a/tests/queries/0_stateless/03278_enum_string_functions.sql b/tests/queries/0_stateless/03278_enum_string_functions.sql new file mode 100644 index 00000000000..e5d0f65b52b --- /dev/null +++ b/tests/queries/0_stateless/03278_enum_string_functions.sql @@ -0,0 +1,55 @@ +DROP TABLE IF EXISTS test_enum_string_functions; +CREATE TABLE test_enum_string_functions(e Enum('a'=1, 'b'=2)) ENGINE=TinyLog; +INSERT INTO test_enum_string_functions VALUES ('a'); +-- like -- +SELECT * from test_enum_string_functions WHERE e LIKE '%abc%'; +-- not like -- +SELECT * from test_enum_string_functions WHERE e NOT LIKE '%abc%'; +-- ilike -- +SELECT * from test_enum_string_functions WHERE e iLike '%a%'; +-- position -- +SELECT position(e, 'a') FROM test_enum_string_functions; +-- match -- +SELECT match(e, 'a') FROM test_enum_string_functions; +-- locate -- +SELECT locate('a', e) FROM test_enum_string_functions; +-- countsubstrings -- +SELECT countSubstrings(e, 'a') FROM test_enum_string_functions; +-- countSburstringsCaseInsensitive -- +SELECT countSubstringsCaseInsensitive(e, 'a') FROM test_enum_string_functions; +-- countSubstringsCaseInsensitiveUTF8 -- +SELECT countSubstringsCaseInsensitiveUTF8(e, 'a') FROM test_enum_string_functions; +-- hasToken -- +SELECT hasToken(e, 'a') FROM test_enum_string_functions; +-- hasTokenOrNull -- +SELECT hasTokenOrNull(e, 'a') FROM test_enum_string_functions; + +DROP TABLE jsons; +CREATE TABLE jsons +( + `json` Enum('a', '{"a":1}') +) +ENGINE = Memory; +INSERT INTO jsons VALUES ('{"a":1}'); +INSERT INTO jsons VALUES ('a'); +-- simpleJSONHas -- +SELECT simpleJSONHas(json, 'foo') FROM jsons; +SELECT simpleJSONHas(json, 'a') FROM jsons; +-- simpleJSONExtractString -- +SELECT simpleJSONExtractUInt(json, 'a') FROM jsons; +SELECT simpleJSONExtractUInt(json, 'not exsits') FROM jsons; +-- simpleJSONExtractInt -- +SELECT simpleJSONExtractInt(json, 'a') FROM jsons; +SELECT simpleJSONExtractInt(json, 'not exsits') FROM jsons; +-- simpleJSONExtractFloat -- +SELECT simpleJSONExtractFloat(json, 'a') FROM jsons; +SELECT simpleJSONExtractFloat(json, 'not exsits') FROM jsons; +-- simpleJSONExtractBool -- +SELECT simpleJSONExtractBool(json, 'a') FROM jsons; +SELECT simpleJSONExtractBool(json, 'not exsits') FROM jsons; +-- positionUTF8 -- +SELECT positionUTF8(json, 'a') FROM jsons; +-- positionCaseInsensitiveUTF8 -- +SELECT positionCaseInsensitiveUTF8(json, 'A') FROM jsons; +-- positionCaseInsensitive -- +SELECT positionCaseInsensitive(json, 'A') FROM jsons; From d314073a9da6e46bd839b0dc2411160e299460bc Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Tue, 3 Dec 2024 17:55:31 +0800 Subject: [PATCH 266/502] fix failed tests --- .../01263_type_conversion_nvartolomei.sql | 2 +- ...d_json_and_xml_on_http_exception.reference | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/01263_type_conversion_nvartolomei.sql b/tests/queries/0_stateless/01263_type_conversion_nvartolomei.sql index 343de3d0a12..10bf8be79f7 100644 --- a/tests/queries/0_stateless/01263_type_conversion_nvartolomei.sql +++ b/tests/queries/0_stateless/01263_type_conversion_nvartolomei.sql @@ -43,7 +43,7 @@ SELECT * FROM d; SELECT '---'; INSERT INTO m VALUES ('b'); -SELECT toString(v) FROM (SELECT v FROM d ORDER BY v) FORMAT Null; -- { serverError BAD_ARGUMENTS } +SELECT toString(v) FROM (SELECT v FROM d ORDER BY v) FORMAT Null; -- { serverError UNKNOWN_ELEMENT_OF_ENUM} DROP TABLE m; diff --git a/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.reference b/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.reference index 2cfba492361..a905323e683 100644 --- a/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.reference +++ b/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.reference @@ -324,13 +324,13 @@ JSON "rows": 3, - "exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONRowOutputFormat. (BAD_ARGUMENTS) " + "exception": "Code: 691. : Unexpected value 99 in enum: While executing JSONRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) " } JSONEachRow {"x":1,"s":"str1","y":"a"} {"x":2,"s":"str2","y":"a"} {"x":3,"s":"str3","y":"a"} -{"exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONEachRowRowOutputFormat. (BAD_ARGUMENTS) "} +{"exception": "Code: 691. : Unexpected value 99 in enum: While executing JSONEachRowRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) "} JSONCompact { "meta": @@ -358,19 +358,19 @@ JSONCompact "rows": 3, - "exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONCompactRowOutputFormat. (BAD_ARGUMENTS) " + "exception": "Code: 691. : Unexpected value 99 in enum: While executing JSONCompactRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) " } JSONCompactEachRow [1, "str1", "a"] [2, "str2", "a"] [3, "str3", "a"] -["Code: 36. : Unexpected value 99 in enum: While executing JSONCompactEachRowRowOutputFormat. (BAD_ARGUMENTS) "] +["Code: 691. : Unexpected value 99 in enum: While executing JSONCompactEachRowRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) "] JSONObjectEachRow { "row_1": {"x":1,"s":"str1","y":"a"}, "row_2": {"x":2,"s":"str2","y":"a"}, "row_3": {"x":3,"s":"str3","y":"a"}, - "exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONObjectEachRowRowOutputFormat. (BAD_ARGUMENTS) " + "exception": "Code: 691. : Unexpected value 99 in enum: While executing JSONObjectEachRowRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) " } XML @@ -409,7 +409,7 @@ XML 3 - Code: 36. : Unexpected value 99 in enum: While executing XMLRowOutputFormat. (BAD_ARGUMENTS) + Code: 691. : Unexpected value 99 in enum: While executing XMLRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) With parallel formatting JSON @@ -697,10 +697,10 @@ JSON "rows": 0, - "exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONRowOutputFormat. (BAD_ARGUMENTS) " + "exception": "Code: 691. : Unexpected value 99 in enum: While executing JSONRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) " } JSONEachRow -{"exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONEachRowRowOutputFormat. (BAD_ARGUMENTS) "} +{"exception": "Code: 691. : Unexpected value 99 in enum: While executing JSONEachRowRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) "} JSONCompact { "meta": @@ -726,13 +726,13 @@ JSONCompact "rows": 0, - "exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONCompactRowOutputFormat. (BAD_ARGUMENTS) " + "exception": "Code: 691. : Unexpected value 99 in enum: While executing JSONCompactRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) " } JSONCompactEachRow -["Code: 36. : Unexpected value 99 in enum: While executing JSONCompactEachRowRowOutputFormat. (BAD_ARGUMENTS) "] +["Code: 691. : Unexpected value 99 in enum: While executing JSONCompactEachRowRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) "] JSONObjectEachRow { - "exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONObjectEachRowRowOutputFormat. (BAD_ARGUMENTS) " + "exception": "Code: 691. : Unexpected value 99 in enum: While executing JSONObjectEachRowRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) " } XML @@ -756,7 +756,7 @@ XML 0 - Code: 36. : Unexpected value 99 in enum: While executing XMLRowOutputFormat. (BAD_ARGUMENTS) + Code: 691. : Unexpected value 99 in enum: While executing XMLRowOutputFormat. (UNKNOWN_ELEMENT_OF_ENUM) With parallel formatting JSON From bf6708069c1c1f89e89c3a8ad96cb4b14f0637f0 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Tue, 3 Dec 2024 10:40:01 +0000 Subject: [PATCH 267/502] add a check to debug LOGICAL ERROR of dictionary update --- src/Dictionaries/DictionaryHelpers.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Dictionaries/DictionaryHelpers.h b/src/Dictionaries/DictionaryHelpers.h index 04828d4d76f..32de8681fdb 100644 --- a/src/Dictionaries/DictionaryHelpers.h +++ b/src/Dictionaries/DictionaryHelpers.h @@ -600,6 +600,7 @@ void mergeBlockWithPipe( while (executor.pull(block)) { convertToFullIfSparse(block); + block.checkNumberOfRows(); Columns block_key_columns; block_key_columns.reserve(key_columns_size); From 6ceebe6ec7a9275b45cc83990144916353124947 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Sat, 5 Oct 2024 17:39:18 +0200 Subject: [PATCH 268/502] Initial commit --- utils/chcache/.gitignore | 1 + utils/chcache/Cargo.lock | 1561 +++++++++++++++++++++++++++++++++++++ utils/chcache/Cargo.toml | 14 + utils/chcache/README.md | 23 + utils/chcache/src/main.rs | 378 +++++++++ 5 files changed, 1977 insertions(+) create mode 100644 utils/chcache/.gitignore create mode 100644 utils/chcache/Cargo.lock create mode 100644 utils/chcache/Cargo.toml create mode 100644 utils/chcache/README.md create mode 100644 utils/chcache/src/main.rs diff --git a/utils/chcache/.gitignore b/utils/chcache/.gitignore new file mode 100644 index 00000000000..ea8c4bf7f35 --- /dev/null +++ b/utils/chcache/.gitignore @@ -0,0 +1 @@ +/target diff --git a/utils/chcache/Cargo.lock b/utils/chcache/Cargo.lock new file mode 100644 index 00000000000..c1872446a23 --- /dev/null +++ b/utils/chcache/Cargo.lock @@ -0,0 +1,1561 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" + +[[package]] +name = "anstyle-parse" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + +[[package]] +name = "arrayref" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "aws-lc-rs" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070" +dependencies = [ + "aws-lc-sys", + "mirai-annotations", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5055edc4a9a1b2a917a818258cdfb86a535947feebd9981adc99667a062c6f85" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", +] + +[[package]] +name = "backtrace" +version = "0.3.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "blake3" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + +[[package]] +name = "bstr" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +dependencies = [ + "memchr", +] + +[[package]] +name = "bytes" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" + +[[package]] +name = "cc" +version = "1.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chcache-rust" +version = "0.1.0" +dependencies = [ + "blake3", + "clickhouse", + "env_logger", + "log", + "serde", + "tokio", + "toml", + "xdg", +] + +[[package]] +name = "cityhash-rs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93a719913643003b84bd13022b4b7e703c09342cd03b679c4641c7d2e50dc34d" + +[[package]] +name = "clang-sys" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clickhouse" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3093f817c4f81c8bd174ed8dd30eac785821a8a7eef27a7dcb7f8cd0d0f6548" +dependencies = [ + "bstr", + "bytes", + "cityhash-rs", + "clickhouse-derive", + "futures", + "futures-channel", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "lz4_flex", + "replace_with", + "sealed", + "serde", + "static_assertions", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "clickhouse-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d70f3e2893f7d3e017eeacdc9a708fbc29a10488e3ebca21f9df6a5d2b616dbb" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "dunce" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "env_filter" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http", + "hyper", + "hyper-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" + +[[package]] +name = "libloading" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +dependencies = [ + "cfg-if", + "windows-targets", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +dependencies = [ + "hermit-abi", + "libc", + "wasi", + "windows-sys 0.52.0", +] + +[[package]] +name = "mirai-annotations" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "object" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "prettyplease" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + +[[package]] +name = "replace_with" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a8614ee435691de62bcffcf4a66d91b3594bf1428a5722e79103249a095690" + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +dependencies = [ + "base64", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" + +[[package]] +name = "rustls-webpki" +version = "0.102.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "schannel" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sealed" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a8caec23b7800fb97971a1c6ae365b6239aaeddfb934d6265f8505e795699d" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +dependencies = [ + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "webpki-roots" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +dependencies = [ + "memchr", +] + +[[package]] +name = "xdg" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546" + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/utils/chcache/Cargo.toml b/utils/chcache/Cargo.toml new file mode 100644 index 00000000000..df54fa193e9 --- /dev/null +++ b/utils/chcache/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "chcache-rust" +version = "0.1.0" +edition = "2021" + +[dependencies] +blake3 = "1.5.4" +clickhouse = { version = "0.12.2", features = ["rustls-tls"] } +env_logger = "0.11.5" +log = "0.4.22" +serde = { version = "1.0.210", features = ["serde_derive"] } +tokio = { version = "1.40.0", features = ["full"] } +toml = "0.8.19" +xdg = "2.5.2" diff --git a/utils/chcache/README.md b/utils/chcache/README.md new file mode 100644 index 00000000000..a6140e12ed8 --- /dev/null +++ b/utils/chcache/README.md @@ -0,0 +1,23 @@ +## chcache + +#### Usage instructions + +First, build the binary. If you don't have Rust toolchain, get one by installing `rustup` with your package manager, then enable the default toolchain: +```bash +rustup default stable +``` + +Then, build the `chcache`: +```bash +cargo build --release +``` + +Place this in your `~/.config/chcache/config.toml` +```toml +TBD +``` + +Go to your `~/src/clickhouse/cmake-build-debug` (only `cmake-build-debug` part is important) and run CMake like this, adjusting the path to `chcache` binary: +```bash +cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_BUILD_PATH_MAPPING=1 -DCMAKE_CXX_COMPILER_LAUNCHER=/home/thevar1able/src/chcache/chcache-rust/target/release/chcache-rust -DCMAKE_C_COMPILER_LAUNCHER=/home/thevar1able/src/chcache/chcache-rust/target/release/chcache-rust -DCOMPILER_CACHE=disabled .. +``` diff --git a/utils/chcache/src/main.rs b/utils/chcache/src/main.rs new file mode 100644 index 00000000000..5474fc0e426 --- /dev/null +++ b/utils/chcache/src/main.rs @@ -0,0 +1,378 @@ +use blake3::Hasher; +use std::fs; +use log::{info, trace, warn}; + + +#[derive(Debug, serde::Deserialize)] +struct Config { + hostname: String, + user: String, + password: String, +} + +#[tokio::main] +async fn main() { + let config_path = xdg::BaseDirectories::with_prefix("chcache") + .unwrap() + .place_config_file("config.toml") + .unwrap(); + + if !config_path.exists() { + panic!("Config file not found at {}", config_path.display()); + } + + let config = fs::read_to_string(config_path).expect("Missing config file?"); + let config: Config = toml::from_str(&config).expect("Unable to load config, is it a valid toml?"); + + env_logger::init(); + + compiler_cache_entrypoint(&config).await; +} + +fn assume_base_path(args: &Vec) -> String { + let cwd: String = match std::env::current_dir() { + Ok(pathbuf) => pathbuf.into_os_string().into_string().unwrap(), + _ => { + panic!("Couldn't get current directory"); + } + }; + + let mut maybe_basepath: Vec = vec![cwd]; + + for (i, arg) in args.iter().enumerate() { + if arg.starts_with("-I") { + maybe_basepath.push(arg[2..].to_string()); + continue; + } + if arg == "-isystem" || arg == "-c" { + maybe_basepath.push(args[i + 1].to_string()); + continue; + } + if arg.starts_with("--gcc-toolchain") { + maybe_basepath.push(arg[16..].to_string()); + continue; + } + if arg.starts_with("--sysroot") { + maybe_basepath.push(arg[10..].to_string()); + continue; + } + } + + let maybe_basepaths_sep_by_slash: Vec> = maybe_basepath + .into_iter() + .map(|x| x.split("/").map(|x| x.to_string()).collect()) + .collect(); + let mut basepath = "".to_string(); + + for i in 0..maybe_basepaths_sep_by_slash[0].len() { + for j in 1..maybe_basepaths_sep_by_slash.len() { + if maybe_basepaths_sep_by_slash[0][i] != maybe_basepaths_sep_by_slash[j][i] { + return basepath.trim_end_matches('/').to_string(); + } + } + basepath.push_str(&maybe_basepaths_sep_by_slash[0][i]); + basepath.push_str("/"); + } + + basepath.trim_end_matches('/').to_string() +} + +fn compiler_version() -> String { + let find_compiler_vars = vec![ + "CMAKE_CXX_COMPILER", + "CXX", + "CMAKE_C_COMPILER", + "CC", + ]; + + let compiler_from_env = find_compiler_vars + .iter() + .map(|x| std::env::var(x)) + .find(|x| x.is_ok()) + .unwrap_or_else(|| Ok(String::from("clang"))) + .unwrap(); + + trace!("Using compiler: {}", compiler_from_env); + + let compiler_version = std::process::Command::new(compiler_from_env) + .arg("-dM") + .arg("-E") + .arg("-x") + .arg("c") + .arg("/dev/null") + .output() + .expect("Failed to execute command"); + + let compiler_version = String::from_utf8_lossy(&compiler_version.stdout); + + let compiler_version = compiler_version + .lines() + .find(|x| x.starts_with("#define __VERSION__")) + .unwrap() + .split_whitespace() + .skip(2) + .collect::>() + .join(" "); + + compiler_version.trim_matches('"').to_string() +} + +fn get_output_from_args(args: &Vec) -> String { + let mut target = String::new(); + for (i, arg) in args.iter().enumerate() { + if arg == "-o" { + target = args[i + 1].to_string(); + break; + } + } + target +} + +fn get_input_from_args(args: &Vec) -> String { + let mut target = String::new(); + for (i, arg) in args.iter().enumerate() { + if arg == "-c" { + target = args[i + 1].to_string(); + break; + } + } + target +} + +fn hash_preprocessed_compiler_output(compiler: String, args: &Vec) -> String { + let mut preprocess_args = vec![ + "-E".to_string(), + "-P".to_string(), + "-fminimize-whitespace".to_string(), + ]; + preprocess_args.extend(args.clone()); + + let output_flag_index = preprocess_args.iter().position(|x| x == "-o").unwrap(); + preprocess_args.remove(output_flag_index); + preprocess_args.remove(output_flag_index); + + let output = std::process::Command::new(compiler) + .args(preprocess_args) + .output() + .expect("Failed to execute command"); + let output = String::from_utf8_lossy(&output.stdout); + + assert!(output.len() > 0); + + let mut hasher = Hasher::new(); + hasher.update(output.as_bytes()); + + let hash = hasher.finalize(); + hash.to_hex().to_string() +} + +fn hash_compiler_target(args: &Vec, stripped_args: &Vec) -> String { + let target: String = get_input_from_args(args); + let mut hasher = Hasher::new(); + + let data = fs::read(&target).expect("Unable to read file"); + hasher.update(&data); + + let target = get_output_from_args(stripped_args); + hasher.update(&target.as_bytes()); + + let hash = hasher.finalize(); + hash.to_hex().to_string() +} + +fn path_from_hash(hash: &String) -> String { + return format!( + "{}/{}/{}", + hash.get(0..2).unwrap(), + hash.get(2..4).unwrap(), + hash + ); +} + +fn get_from_fscache(hash: &String) -> Option> { + let cache_file = xdg::BaseDirectories::with_prefix("chcache") + .unwrap() + .get_cache_file(path_from_hash(hash)); + + trace!("Cache file: {:?}", cache_file); + + if cache_file.exists() { + let data = fs::read(cache_file).expect("Unable to read file"); + return Some(data); + } + + None +} + +fn write_to_fscache(hash: &String, data: &Vec) { + let cache_file = xdg::BaseDirectories::with_prefix("chcache") + .unwrap() + .place_cache_file(path_from_hash(hash)) + .unwrap(); + + if cache_file.exists() { + return; + } + + let cache_file_dir = cache_file.parent().unwrap(); + + if !cache_file_dir.exists() { + fs::create_dir_all(cache_file_dir).expect("Unable to create directory"); + } + + fs::write(cache_file, data).expect("Unable to write file"); +} + +async fn load_from_clickhouse( + client: &clickhouse::Client, + hash: &String, + compiler_version: &String, +) -> Result>, clickhouse::error::Error> { + let mut cursor = client + .query("SELECT ?fields FROM default.build_cache WHERE hash = ? and compiler_version = ? LIMIT 1") + .bind(hash) + .bind(compiler_version) + .fetch::() + .unwrap(); + + while let Some(row) = cursor.next().await? { + return Ok(Some(row.blob.into())); + } + + Ok(None) +} + +#[derive(Debug, clickhouse::Row, serde::Serialize, serde::Deserialize)] +struct MyRow { + blob: Vec, + hash: String, + compiler_version: String, +} + +async fn load_to_clickhouse( + client: &clickhouse::Client, + hash: &String, + compiler_version: &String, + data: &Vec, +) -> Result<(), clickhouse::error::Error> { + let mut insert = client.insert("default.build_cache").unwrap(); + + let row = MyRow { + blob: data.clone(), + hash: hash.clone(), + compiler_version: compiler_version.clone(), + }; + + insert.write(&row).await.unwrap(); + insert.end().await +} + +async fn compiler_cache_entrypoint(config: &Config) { + let compiler: String = std::env::args().nth(1).unwrap(); + let rest_of_args: Vec = std::env::args().skip(2).collect(); + + trace!("Compiler: {}", compiler); + // assert!(compiler.contains("clang") || compiler.contains("clang++")); + + trace!("Args: {:?}", rest_of_args); + + let assumed_base_path = assume_base_path(&rest_of_args); + trace!("Assumed base path: {}", assumed_base_path); + + let stripped_args = rest_of_args + .iter() + .map(|x| x.replace(&assumed_base_path, "/")) + .collect::>(); + + let mut hasher = Hasher::new(); + stripped_args.iter().map(|x| x.as_bytes()).for_each(|x| { + hasher.update(&x); + }); + let args_hash = hasher.finalize().to_string(); + + let compiler_version = compiler_version(); + + trace!("Compiler version: {}", compiler_version); + + let compiler_target_hash = hash_compiler_target(&rest_of_args, &stripped_args); + let preprocessed_output_hash = + hash_preprocessed_compiler_output(compiler.clone(), &rest_of_args); + + trace!("Compiler target hash: {}", compiler_target_hash); + trace!("Preprocessed output hash: {}", preprocessed_output_hash); + trace!("Args hash: {}", args_hash); + + hasher.reset(); + + hasher.update(compiler_version.as_bytes()); + hasher.update(compiler_target_hash.as_bytes()); + hasher.update(preprocessed_output_hash.as_bytes()); + hasher.update(args_hash.as_bytes()); + + let total_hash = hasher.finalize().to_string(); + trace!("Total hash: {}", total_hash); + + let client = clickhouse::Client::default() + .with_url(&config.hostname) + .with_user(&config.user) + .with_password(&config.password) + .with_compression(clickhouse::Compression::Lz4) + .with_option("async_insert", "1") + .with_option("wait_for_async_insert", "0"); + + let mut did_load_from_cache = false; + + let compiled_bytes: Vec = match get_from_fscache(&total_hash) { + Some(bytes) => { + info!("Local cache hit"); + did_load_from_cache = true; + + fs::write(get_output_from_args(&rest_of_args), &bytes).expect("Unable to write file"); + + bytes + } + None => { + trace!("Cache miss"); + + let compiled_bytes = + match load_from_clickhouse(&client, &total_hash, &compiler_version).await { + Ok(Some(bytes)) => { + did_load_from_cache = true; + info!("Loaded from ClickHouse"); + + fs::write(get_output_from_args(&rest_of_args), &bytes) + .expect("Unable to write file"); + + bytes + } + Ok(None) | Err(_) => { + let output = std::process::Command::new(compiler) + .args(&rest_of_args) + .output() + .unwrap(); + if output.status.code().unwrap() != 0 { + println!("{}", String::from_utf8_lossy(&output.stdout)); + eprintln!("{}", String::from_utf8_lossy(&output.stderr)); + return; + } + fs::read(get_output_from_args(&rest_of_args)).expect("Unable to read file") + } + }; + + compiled_bytes + } + }; + + write_to_fscache(&total_hash, &compiled_bytes); + if !did_load_from_cache { + loop { + let upload_result = + load_to_clickhouse(&client, &total_hash, &compiler_version, &compiled_bytes).await; + if upload_result.is_ok() { + info!("Uploaded to ClickHouse"); + break; + } + warn!("Failed to upload to ClickHouse, retrying..."); + } + } +} From ddde64300f9a1e10cc04d540189e9523bd403bc0 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Sat, 19 Oct 2024 03:26:44 +0200 Subject: [PATCH 269/502] temp-commit --- CMakeLists.txt | 13 + cmake/ccache.cmake | 19 +- docker/packager/binary-builder/Dockerfile | 2 +- docker/packager/packager | 15 +- docker/test/fasttest/Dockerfile | 50 +- tests/ci/build_check.py | 1 - tests/ci/fast_test_check.py | 11 +- tests/docker_scripts/fasttest_runner.sh | 1 + utils/chcache/Cargo.lock | 816 ++++++++++------------ utils/chcache/Cargo.toml | 10 +- utils/chcache/src/main.rs | 57 +- 11 files changed, 490 insertions(+), 505 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a165be799c0..a835a8b15bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -591,6 +591,19 @@ endif() include (cmake/sanitize_targets.cmake) +if (COMPILER_CACHE STREQUAL "chcache") + set (ENABLE_BUILD_PATH_MAPPING_DEFAULT ON) + + get_all_targets(all_targets) + set (chcache_targets _cargo-build_chcache cargo-build_chcache cargo-prebuild_chcache) + foreach(target ${all_targets}) + if (target IN_LIST chcache_targets) + continue() + endif() + add_dependencies(${target} chcache) + endforeach() +endif() + # Build native targets if necessary get_property(NATIVE_BUILD_TARGETS GLOBAL PROPERTY NATIVE_BUILD_TARGETS) if (NATIVE_BUILD_TARGETS diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake index 0df70d82d2c..2ef21affd87 100644 --- a/cmake/ccache.cmake +++ b/cmake/ccache.cmake @@ -9,7 +9,7 @@ if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MA return() endif() -set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (sccache, then ccache), 'ccache', 'sccache', or 'disabled'") +set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (sccache, then ccache), 'ccache', 'sccache', 'chcache', or 'disabled'") if(COMPILER_CACHE STREQUAL "auto") find_program (CCACHE_EXECUTABLE NAMES sccache ccache) @@ -17,11 +17,23 @@ elseif (COMPILER_CACHE STREQUAL "ccache") find_program (CCACHE_EXECUTABLE ccache) elseif(COMPILER_CACHE STREQUAL "sccache") find_program (CCACHE_EXECUTABLE sccache) +elseif(COMPILER_CACHE STREQUAL "chcache") + list (APPEND CMAKE_MODULE_PATH "${ClickHouse_SOURCE_DIR}/contrib/corrosion/cmake") + find_package(Rust REQUIRED) + + include ("${ClickHouse_SOURCE_DIR}/contrib/corrosion/cmake/Corrosion.cmake") + corrosion_import_crate( + MANIFEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/utils/chcache/Cargo.toml + LOCKED + ) + set_target_properties(chcache PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/programs/) + + set(CCACHE_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/programs/chcache) elseif(COMPILER_CACHE STREQUAL "disabled") message(STATUS "Using *ccache: no (disabled via configuration)") return() else() - message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|sccache|ccache|disabled), value: '${COMPILER_CACHE}'") + message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|sccache|ccache|chcache|disabled), value: '${COMPILER_CACHE}'") endif() @@ -60,6 +72,9 @@ if (CCACHE_EXECUTABLE MATCHES "/ccache$") elseif(CCACHE_EXECUTABLE MATCHES "/sccache$") message(STATUS "Using sccache: ${CCACHE_EXECUTABLE}") set(LAUNCHER ${CCACHE_EXECUTABLE}) +elseif(CCACHE_EXECUTABLE MATCHES "/chcache$") + message(STATUS "Using chcache: ${CCACHE_EXECUTABLE}") + set(LAUNCHER ${CCACHE_EXECUTABLE}) endif() set (CMAKE_CXX_COMPILER_LAUNCHER ${LAUNCHER} ${CMAKE_CXX_COMPILER_LAUNCHER}) diff --git a/docker/packager/binary-builder/Dockerfile b/docker/packager/binary-builder/Dockerfile index 7d6acdcd856..cbe4b51b2bf 100644 --- a/docker/packager/binary-builder/Dockerfile +++ b/docker/packager/binary-builder/Dockerfile @@ -14,7 +14,7 @@ ENV PATH="/rust/cargo/bin:${PATH}" RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \ chmod 777 -R /rust && \ rustup toolchain install nightly-2024-04-01 && \ - rustup default nightly-2024-04-01 && \ + rustup default nightly-2024-11-01 && \ rustup toolchain remove stable && \ rustup component add rust-src && \ rustup target add x86_64-unknown-linux-gnu && \ diff --git a/docker/packager/packager b/docker/packager/packager index e7c0f4b3a00..725bd426e6c 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -130,6 +130,7 @@ def parse_env_variables( sanitizer: str, package_type: str, cache: str, + chcache_password: str, s3_bucket: str, s3_directory: str, s3_rw_access: bool, @@ -322,6 +323,12 @@ def parse_env_variables( if not s3_rw_access: result.append("SCCACHE_S3_NO_CREDENTIALS=true") + if cache == "chcache": + cmake_flags.append("-DCOMPILER_CACHE=chcache") + result.append(f"CH_PASSWORD={chcache_password}") + result.append("CH_USER=ci_builder") + result.append("CH_HOSTNAME=https://lr5v5i0nr3.eu-west-1.aws.clickhouse-staging.com") + if clang_tidy: # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above. # It's there to have the clang-tidy cache embedded into our standard `CCACHE_DIR` @@ -437,10 +444,15 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--clang-tidy", action="store_true") parser.add_argument( "--cache", - choices=("ccache", "sccache", ""), + choices=("ccache", "sccache", "chcache", ""), default="", help="ccache or sccache for objects caching; sccache uses only S3 buckets", ) + parser.add_argument( + "--chcache-password", + default="", + help="", + ) parser.add_argument( "--ccache-dir", default=Path.home() / ".ccache", @@ -530,6 +542,7 @@ def main() -> None: args.sanitizer, args.package_type, args.cache, + args.chcache_password, args.s3_bucket, args.s3_directory, args.s3_rw_access, diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 318f437a108..c3bb89834cc 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -40,42 +40,22 @@ RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld # https://salsa.debian.org/pkg-llvm-team/llvm-toolchain/-/commit/992e52c0b156a5ba9c6a8a54f8c4857ddd3d371d RUN sed -i '/_IMPORT_CHECK_FILES_FOR_\(mlir-\|llvm-bolt\|merge-fdata\|MLIR\)/ {s|^|#|}' /usr/lib/llvm-${LLVM_VERSION}/lib/cmake/llvm/LLVMExports-*.cmake -ARG CCACHE_VERSION=4.6.1 -RUN mkdir /tmp/ccache \ - && cd /tmp/ccache \ - && curl -L \ - -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz \ - -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz.asc \ - && gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 5A939A71A46792CF57866A51996DDA075594ADB8 \ - && gpg --verify ccache-4.6.1.tar.xz.asc \ - && tar xf ccache-$CCACHE_VERSION.tar.xz \ - && cd /tmp/ccache/ccache-$CCACHE_VERSION \ - && cmake -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_BUILD_TYPE=None \ - -DZSTD_FROM_INTERNET=ON \ - -DREDIS_STORAGE_BACKEND=OFF \ - -Wno-dev \ - -B build \ - -S . \ - && make VERBOSE=1 -C build \ - && make install -C build \ - && cd / \ - && rm -rf /tmp/ccache +# LLVM changes paths for compiler-rt libraries. For some reason clang-18.1.8 cannot catch up libraries from default install path. +# It's very dirty workaround, better to build compiler and LLVM ourself and use it. Details: https://github.com/llvm/llvm-project/issues/95792 +RUN test ! -d /usr/lib/llvm-18/lib/clang/18/lib/x86_64-pc-linux-gnu || ln -s /usr/lib/llvm-18/lib/clang/18/lib/x86_64-pc-linux-gnu /usr/lib/llvm-18/lib/clang/18/lib/x86_64-unknown-linux-gnu -ARG TARGETARCH -ARG SCCACHE_VERSION=v0.7.7 -ENV SCCACHE_IGNORE_SERVER_IO_ERROR=1 -# sccache requires a value for the region. So by default we use The Default Region -ENV SCCACHE_REGION=us-east-1 -RUN arch=${TARGETARCH:-amd64} \ - && case $arch in \ - amd64) rarch=x86_64 ;; \ - arm64) rarch=aarch64 ;; \ - esac \ - && curl -Ls "https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl.tar.gz" | \ - tar xz -C /tmp \ - && mv "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl/sccache" /usr/bin \ - && rm "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl" -r +# Rust toolchain and libraries +ENV RUSTUP_HOME=/rust/rustup +ENV CARGO_HOME=/rust/cargo +ENV PATH="/rust/cargo/bin:${PATH}" +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \ + chmod 777 -R /rust && \ + rustup toolchain install nightly-2024-11-01 && \ + rustup default nightly-2024-11-01 && \ + rustup toolchain remove stable && \ + rustup component add rust-src && \ + rustup target add x86_64-unknown-linux-gnu && \ + rustup target add aarch64-unknown-linux-gnu # Give suid to gdb to grant it attach permissions # chmod 777 to make the container user independent diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 77d91c8400b..0b2e6853cad 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -61,7 +61,6 @@ def get_packager_cmd( if build_config.tidy: cmd += " --clang-tidy" - cmd += " --cache=sccache" cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 55eefcf9714..4acd45b16af 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -8,11 +8,12 @@ from pathlib import Path from typing import Tuple from docker_images_helper import DockerImage, get_docker_image, pull_image -from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH +from env_helper import REPO_COPY, TEMP_PATH from pr_info import PRInfo from report import ERROR, FAILURE, SUCCESS, JobReport, TestResults, read_test_results from stopwatch import Stopwatch from tee_popen import TeePopen +from get_robot_token import get_parameter_from_ssm # Will help to avoid errors like _csv.Error: field larger than field limit (131072) csv.field_size_limit(sys.maxsize) @@ -26,16 +27,17 @@ def get_fasttest_cmd( commit_sha: str, image: DockerImage, ) -> str: + chcache_password = get_parameter_from_ssm("chcache_password") return ( f"docker run --cap-add=SYS_PTRACE --user={os.geteuid()}:{os.getegid()} " "--security-opt seccomp=unconfined " # required to issue io_uring sys-calls "--network=host " # required to get access to IAM credentials f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " f"-e FASTTEST_SOURCE=/repo " - f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " + f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=chcache' " f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " - f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " + f"-e CH_HOSTNAME='https://lr5v5i0nr3.eu-west-1.aws.clickhouse-staging.com' -e CH_USER=ci_builder -e CH_PASSWORD='{chcache_password}' " "-e stage=clone_submodules " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/repo " f"--volume={output_path}:/test_output {image} /repo/tests/docker_scripts/fasttest_runner.sh" @@ -97,7 +99,8 @@ def main(): pr_info.sha, docker_image, ) - logging.info("Going to run fasttest with cmd %s", run_cmd) + sanitized_run_cmd = run_cmd.replace(get_parameter_from_ssm("chcache_password"), "****") + logging.info("Going to run fasttest with cmd %s", sanitized_run_cmd) logs_path = temp_path / "fasttest-logs" logs_path.mkdir(parents=True, exist_ok=True) diff --git a/tests/docker_scripts/fasttest_runner.sh b/tests/docker_scripts/fasttest_runner.sh index 2b0e5428e85..898bc1cf3fb 100755 --- a/tests/docker_scripts/fasttest_runner.sh +++ b/tests/docker_scripts/fasttest_runner.sh @@ -158,6 +158,7 @@ function clone_submodules contrib/libfiu contrib/incbin contrib/yaml-cpp + contrib/corrosion ) git submodule sync diff --git a/utils/chcache/Cargo.lock b/utils/chcache/Cargo.lock index c1872446a23..b46433676cc 100644 --- a/utils/chcache/Cargo.lock +++ b/utils/chcache/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aho-corasick" @@ -26,60 +26,11 @@ dependencies = [ "memchr", ] -[[package]] -name = "anstream" -version = "0.6.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" - -[[package]] -name = "anstyle-parse" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" -dependencies = [ - "anstyle", - "windows-sys 0.52.0", -] - [[package]] name = "arrayref" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" @@ -89,15 +40,15 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-rs" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070" +checksum = "cdd82dba44d209fddb11c190e0a94b78651f95299598e472215667417a03ff1d" dependencies = [ "aws-lc-sys", "mirai-annotations", @@ -107,9 +58,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5055edc4a9a1b2a917a818258cdfb86a535947feebd9981adc99667a062c6f85" +checksum = "df7a4168111d7eb622a31b214057b8509c0a7e1794f44c546d742330dc793972" dependencies = [ "bindgen", "cc", @@ -122,30 +73,24 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags", "cexpr", @@ -194,15 +139,15 @@ dependencies = [ [[package]] name = "bytes" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "cc" -version = "1.1.18" +version = "1.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" +checksum = "baee610e9452a8f6f0a1b6194ec09ff9e2d85dea54432acdae41aa0761c95d70" dependencies = [ "jobserver", "libc", @@ -246,9 +191,9 @@ checksum = "93a719913643003b84bd13022b4b7e703c09342cd03b679c4641c7d2e50dc34d" [[package]] name = "clang-sys" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -257,9 +202,9 @@ dependencies = [ [[package]] name = "clickhouse" -version = "0.12.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3093f817c4f81c8bd174ed8dd30eac785821a8a7eef27a7dcb7f8cd0d0f6548" +checksum = "2135bb9638e8c8c1e3d794f242099e57987059ba52e7e3de597e1d99b2c4a5a3" dependencies = [ "bstr", "bytes", @@ -273,6 +218,7 @@ dependencies = [ "hyper-util", "lz4_flex", "replace_with", + "rustls", "sealed", "serde", "static_assertions", @@ -295,19 +241,13 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.50" +version = "0.1.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" dependencies = [ "cc", ] -[[package]] -name = "colorchoice" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" - [[package]] name = "constant_time_eq" version = "0.3.1" @@ -315,26 +255,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] -name = "core-foundation" -version = "0.9.4" +name = "displaydoc" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "core-foundation-sys", - "libc", + "proc-macro2", + "quote", + "syn", ] -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "dunce" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "either" @@ -349,7 +284,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" dependencies = [ "log", - "regex", ] [[package]] @@ -358,10 +292,7 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" dependencies = [ - "anstream", - "anstyle", "env_filter", - "humantime", "log", ] @@ -378,7 +309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -404,9 +335,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -419,9 +350,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -429,15 +360,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -446,15 +377,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -463,21 +394,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -504,9 +435,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -516,15 +447,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "hermit-abi" @@ -538,7 +463,7 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -577,21 +502,15 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -608,17 +527,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http", "hyper", "hyper-util", - "log", "rustls", - "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -628,9 +545,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", @@ -641,37 +558,159 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] [[package]] -name = "idna" -version = "0.5.0" +name = "icu_collections" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", "hashbrown", ] -[[package]] -name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - [[package]] name = "itertools" version = "0.12.1" @@ -710,15 +749,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.162" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" [[package]] name = "libloading" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", "windows-targets", @@ -731,14 +770,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] -name = "lock_api" -version = "0.4.12" +name = "litemap" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" [[package]] name = "log" @@ -766,11 +801,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] @@ -782,7 +817,7 @@ dependencies = [ "hermit-abi", "libc", "wasi", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -803,47 +838,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "parking_lot" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "paste" @@ -857,31 +863,11 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "pin-project" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -891,9 +877,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "prettyplease" -version = "0.2.15" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", "syn", @@ -901,9 +887,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -917,20 +903,11 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "redox_syscall" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" -dependencies = [ - "bitflags", -] - [[package]] name = "regex" -version = "1.10.6" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -940,9 +917,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", @@ -951,9 +928,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "replace_with" @@ -973,7 +950,7 @@ dependencies = [ "libc", "spin", "untrusted", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -990,25 +967,24 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.36" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" dependencies = [ "aws-lc-rs", - "log", "once_cell", "rustls-pki-types", "rustls-webpki", @@ -1016,40 +992,17 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "rustls-pki-types", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" -dependencies = [ - "base64", - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "aws-lc-rs", "ring", @@ -1057,70 +1010,31 @@ dependencies = [ "untrusted", ] -[[package]] -name = "schannel" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "sealed" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a8caec23b7800fb97971a1c6ae365b6239aaeddfb934d6265f8505e795699d" +checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" dependencies = [ - "heck", "proc-macro2", "quote", "syn", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "serde" -version = "1.0.210" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", @@ -1140,9 +1054,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -1153,15 +1067,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "signal-hook-registry" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" -dependencies = [ - "libc", -] - [[package]] name = "slab" version = "0.4.9" @@ -1184,7 +1089,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -1193,6 +1098,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -1207,9 +1118,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.77" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -1217,19 +1128,10 @@ dependencies = [ ] [[package]] -name = "thiserror" -version = "1.0.63" +name = "synstructure" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.63" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", @@ -1237,36 +1139,48 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.8.0" +name = "thiserror" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ - "tinyvec_macros", + "thiserror-impl", ] [[package]] -name = "tinyvec_macros" -version = "0.1.1" +name = "thiserror-impl" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", - "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", - "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -1314,9 +1228,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap", "serde", @@ -1325,27 +1239,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - [[package]] name = "tower-service" version = "0.3.3" @@ -1377,26 +1270,11 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "unicode-bidi" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" - [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-normalization" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" -dependencies = [ - "tinyvec", -] +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "untrusted" @@ -1406,9 +1284,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", @@ -1416,10 +1294,16 @@ dependencies = [ ] [[package]] -name = "utf8parse" -version = "0.2.2" +name = "utf16_iter" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "want" @@ -1438,9 +1322,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "webpki-roots" -version = "0.26.3" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] @@ -1466,15 +1350,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -1541,21 +1416,100 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "xdg" version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546" +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/utils/chcache/Cargo.toml b/utils/chcache/Cargo.toml index df54fa193e9..b173ab71601 100644 --- a/utils/chcache/Cargo.toml +++ b/utils/chcache/Cargo.toml @@ -5,10 +5,14 @@ edition = "2021" [dependencies] blake3 = "1.5.4" -clickhouse = { version = "0.12.2", features = ["rustls-tls"] } -env_logger = "0.11.5" +clickhouse = { version = "0.13.1", features = ["rustls-tls"] } +env_logger = { version = "0.11.5", default-features = false } log = "0.4.22" serde = { version = "1.0.210", features = ["serde_derive"] } -tokio = { version = "1.40.0", features = ["full"] } +tokio = { version = "1.40.0", features = ["rt-multi-thread"] } toml = "0.8.19" xdg = "2.5.2" + +[[bin]] +name = "chcache" +path = "src/main.rs" # FIXME diff --git a/utils/chcache/src/main.rs b/utils/chcache/src/main.rs index 5474fc0e426..cfaa0975098 100644 --- a/utils/chcache/src/main.rs +++ b/utils/chcache/src/main.rs @@ -12,17 +12,22 @@ struct Config { #[tokio::main] async fn main() { - let config_path = xdg::BaseDirectories::with_prefix("chcache") - .unwrap() - .place_config_file("config.toml") - .unwrap(); + // let config_path = xdg::BaseDirectories::with_prefix("chcache") + // .unwrap() + // .place_config_file("config.toml") + // .unwrap(); + // + // if !config_path.exists() { + // panic!("Config file not found at {}", config_path.display()); + // } - if !config_path.exists() { - panic!("Config file not found at {}", config_path.display()); - } - - let config = fs::read_to_string(config_path).expect("Missing config file?"); - let config: Config = toml::from_str(&config).expect("Unable to load config, is it a valid toml?"); + // let config = fs::read_to_string(config_path).expect("Missing config file?"); + // let config: Config = toml::from_str(&config).expect("Unable to load config, is it a valid toml?"); + let config: Config = Config { + hostname: std::env::var("CH_HOSTNAME").unwrap(), + user: std::env::var("CH_USER").unwrap(), + password: std::env::var("CH_PASSWORD").unwrap(), + }; env_logger::init(); @@ -77,24 +82,22 @@ fn assume_base_path(args: &Vec) -> String { basepath.trim_end_matches('/').to_string() } -fn compiler_version() -> String { - let find_compiler_vars = vec![ - "CMAKE_CXX_COMPILER", - "CXX", - "CMAKE_C_COMPILER", - "CC", - ]; +fn compiler_version(compiler: String) -> String { + // let find_compiler_vars = vec![ + // "CXX", + // "CC", + // ]; + // + // let compiler_from_env = find_compiler_vars + // .iter() + // .map(|x| std::env::var(x)) + // .find(|x| x.is_ok()) + // .unwrap_or_else(|| Ok(String::from("clang"))) + // .unwrap(); - let compiler_from_env = find_compiler_vars - .iter() - .map(|x| std::env::var(x)) - .find(|x| x.is_ok()) - .unwrap_or_else(|| Ok(String::from("clang"))) - .unwrap(); + trace!("Using compiler: {}", compiler); - trace!("Using compiler: {}", compiler_from_env); - - let compiler_version = std::process::Command::new(compiler_from_env) + let compiler_version = std::process::Command::new(compiler) .arg("-dM") .arg("-E") .arg("-x") @@ -290,7 +293,7 @@ async fn compiler_cache_entrypoint(config: &Config) { }); let args_hash = hasher.finalize().to_string(); - let compiler_version = compiler_version(); + let compiler_version = compiler_version(compiler.clone()); trace!("Compiler version: {}", compiler_version); From c0f66a3d0e67f70230563463ffd179df536c37f8 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Fri, 22 Nov 2024 19:52:06 +0100 Subject: [PATCH 270/502] temp-commit --- CMakeLists.txt | 3 +- cmake/ccache.cmake | 1 + utils/chcache/Cargo.lock | 14 +++++++-- utils/chcache/Cargo.toml | 5 +-- utils/chcache/src/main.rs | 64 ++++++++++++++++++++++----------------- 5 files changed, 54 insertions(+), 33 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a835a8b15bf..1d6a0ae1c49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -600,7 +600,8 @@ if (COMPILER_CACHE STREQUAL "chcache") if (target IN_LIST chcache_targets) continue() endif() - add_dependencies(${target} chcache) + + add_dependencies(${target} cargo-build_chcache) endforeach() endif() diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake index 2ef21affd87..7457733a9fb 100644 --- a/cmake/ccache.cmake +++ b/cmake/ccache.cmake @@ -24,6 +24,7 @@ elseif(COMPILER_CACHE STREQUAL "chcache") include ("${ClickHouse_SOURCE_DIR}/contrib/corrosion/cmake/Corrosion.cmake") corrosion_import_crate( MANIFEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/utils/chcache/Cargo.toml + PROFILE release LOCKED ) set_target_properties(chcache PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/programs/) diff --git a/utils/chcache/Cargo.lock b/utils/chcache/Cargo.lock index b46433676cc..ea2e8cb8340 100644 --- a/utils/chcache/Cargo.lock +++ b/utils/chcache/Cargo.lock @@ -170,14 +170,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "chcache-rust" -version = "0.1.0" +name = "chcache" +version = "1.0.0" dependencies = [ "blake3", "clickhouse", "env_logger", "log", "serde", + "serde_bytes", "tokio", "toml", "xdg", @@ -1030,6 +1031,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.214" diff --git a/utils/chcache/Cargo.toml b/utils/chcache/Cargo.toml index b173ab71601..33ac0e4f79d 100644 --- a/utils/chcache/Cargo.toml +++ b/utils/chcache/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "chcache-rust" -version = "0.1.0" +name = "chcache" +version = "1.0.0" edition = "2021" [dependencies] @@ -9,6 +9,7 @@ clickhouse = { version = "0.13.1", features = ["rustls-tls"] } env_logger = { version = "0.11.5", default-features = false } log = "0.4.22" serde = { version = "1.0.210", features = ["serde_derive"] } +serde_bytes = "0.11.15" tokio = { version = "1.40.0", features = ["rt-multi-thread"] } toml = "0.8.19" xdg = "2.5.2" diff --git a/utils/chcache/src/main.rs b/utils/chcache/src/main.rs index cfaa0975098..fc44a36fdbf 100644 --- a/utils/chcache/src/main.rs +++ b/utils/chcache/src/main.rs @@ -11,27 +11,46 @@ struct Config { } #[tokio::main] -async fn main() { - // let config_path = xdg::BaseDirectories::with_prefix("chcache") - // .unwrap() - // .place_config_file("config.toml") - // .unwrap(); - // - // if !config_path.exists() { - // panic!("Config file not found at {}", config_path.display()); - // } +async fn main() -> Result<(), Box> { + let config_path = xdg::BaseDirectories::with_prefix("chcache") + .unwrap() + .place_config_file("config.toml") + .unwrap(); - // let config = fs::read_to_string(config_path).expect("Missing config file?"); - // let config: Config = toml::from_str(&config).expect("Unable to load config, is it a valid toml?"); - let config: Config = Config { - hostname: std::env::var("CH_HOSTNAME").unwrap(), - user: std::env::var("CH_USER").unwrap(), - password: std::env::var("CH_PASSWORD").unwrap(), + let config: Config = match config_path.exists() { + true => { + trace!("Loading config file contents from {}", config_path.display()); + + let config_text = fs::read_to_string(config_path).expect("Missing config file?"); + toml::from_str(&config_text).expect("Unable to load config, is it a valid toml?") + } + false => { + trace!("Config file not found at {}, trying env vars", config_path.display()); + + let required_env_vars = vec!["CH_HOSTNAME", "CH_USER", "CH_PASSWORD"]; + for var in required_env_vars { + if std::env::var(var).is_err() { + return Err(format!( + "Please set a missing environment variable {} or place a config file at {}", + var, + config_path.display(), + ).into()); + } + } + + Config { + hostname: std::env::var("CH_HOSTNAME").unwrap(), + user: std::env::var("CH_USER").unwrap(), + password: std::env::var("CH_PASSWORD").unwrap(), + } + } }; env_logger::init(); compiler_cache_entrypoint(&config).await; + + Ok(()) } fn assume_base_path(args: &Vec) -> String { @@ -83,18 +102,6 @@ fn assume_base_path(args: &Vec) -> String { } fn compiler_version(compiler: String) -> String { - // let find_compiler_vars = vec![ - // "CXX", - // "CC", - // ]; - // - // let compiler_from_env = find_compiler_vars - // .iter() - // .map(|x| std::env::var(x)) - // .find(|x| x.is_ok()) - // .unwrap_or_else(|| Ok(String::from("clang"))) - // .unwrap(); - trace!("Using compiler: {}", compiler); let compiler_version = std::process::Command::new(compiler) @@ -247,6 +254,7 @@ async fn load_from_clickhouse( #[derive(Debug, clickhouse::Row, serde::Serialize, serde::Deserialize)] struct MyRow { + #[serde(with = "serde_bytes")] blob: Vec, hash: String, compiler_version: String, @@ -353,7 +361,7 @@ async fn compiler_cache_entrypoint(config: &Config) { .args(&rest_of_args) .output() .unwrap(); - if output.status.code().unwrap() != 0 { + if !output.status.success() { println!("{}", String::from_utf8_lossy(&output.stdout)); eprintln!("{}", String::from_utf8_lossy(&output.stderr)); return; From ebb321b980822def2ec4613a003986bd4bc4705a Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Fri, 22 Nov 2024 20:13:20 +0100 Subject: [PATCH 271/502] Add reasonable defaults --- utils/chcache/src/main.rs | 60 ++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/utils/chcache/src/main.rs b/utils/chcache/src/main.rs index fc44a36fdbf..8a4875332e2 100644 --- a/utils/chcache/src/main.rs +++ b/utils/chcache/src/main.rs @@ -1,7 +1,6 @@ use blake3::Hasher; -use std::fs; use log::{info, trace, warn}; - +use std::fs; #[derive(Debug, serde::Deserialize)] struct Config { @@ -10,6 +9,16 @@ struct Config { password: String, } +impl Default for Config { + fn default() -> Self { + Config { + hostname: "https://build-cache.clickhouse-staging.com".to_string(), + user: "reader".to_string(), + password: "reader".to_string(), + } + } +} + #[tokio::main] async fn main() -> Result<(), Box> { let config_path = xdg::BaseDirectories::with_prefix("chcache") @@ -17,26 +26,32 @@ async fn main() -> Result<(), Box> { .place_config_file("config.toml") .unwrap(); - let config: Config = match config_path.exists() { - true => { - trace!("Loading config file contents from {}", config_path.display()); + let mut env_vars_available = true; + let required_env_vars = vec!["CH_HOSTNAME", "CH_USER", "CH_PASSWORD"]; + for var in required_env_vars { + if std::env::var(var).is_ok() { + continue; + } + + env_vars_available = false; + break; + } + + let config: Config = match (config_path.exists(), env_vars_available) { + (true, _) => { + trace!( + "Loading config file contents from {}", + config_path.display() + ); let config_text = fs::read_to_string(config_path).expect("Missing config file?"); toml::from_str(&config_text).expect("Unable to load config, is it a valid toml?") } - false => { - trace!("Config file not found at {}, trying env vars", config_path.display()); - - let required_env_vars = vec!["CH_HOSTNAME", "CH_USER", "CH_PASSWORD"]; - for var in required_env_vars { - if std::env::var(var).is_err() { - return Err(format!( - "Please set a missing environment variable {} or place a config file at {}", - var, - config_path.display(), - ).into()); - } - } + (_, true) => { + trace!( + "Config file not found at {}, trying env vars", + config_path.display() + ); Config { hostname: std::env::var("CH_HOSTNAME").unwrap(), @@ -44,6 +59,13 @@ async fn main() -> Result<(), Box> { password: std::env::var("CH_PASSWORD").unwrap(), } } + (false, false) => { + trace!( + "Config file not found at {}, and env vars are missing, using defaults", + config_path.display() + ); + Config::default() + } }; env_logger::init(); @@ -375,7 +397,7 @@ async fn compiler_cache_entrypoint(config: &Config) { }; write_to_fscache(&total_hash, &compiled_bytes); - if !did_load_from_cache { + if !did_load_from_cache && config.user != "reader" { loop { let upload_result = load_to_clickhouse(&client, &total_hash, &compiler_version, &compiled_bytes).await; From b2841f643d71e2ad46277e88c5505c9ab696f524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 3 Dec 2024 12:18:57 +0100 Subject: [PATCH 272/502] Make the test work with sanitizers --- tests/queries/0_stateless/02995_new_settings_history.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02995_new_settings_history.sh b/tests/queries/0_stateless/02995_new_settings_history.sh index 6405f7f16ab..5f279dbd695 100755 --- a/tests/queries/0_stateless/02995_new_settings_history.sh +++ b/tests/queries/0_stateless/02995_new_settings_history.sh @@ -62,7 +62,12 @@ $CLICKHOUSE_LOCAL --query " SELECT arrayJoin(tupleElement(changes, 'name')) FROM system.settings_changes WHERE type = 'Core' AND splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 11 - )) + )) AND ( + -- Different values for sanitizers + ( SELECT count() FROM system.build_options WHERE name = 'CXX_FLAGS' AND position('sanitize' IN value) = 1 ) + AND + name NOT IN ('query_profiler_cpu_time_period_ns', 'query_profiler_real_time_period_ns') + ) ) UNION ALL ( From eb9d5705a9055c505cc5aad169118927e4d321d4 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 3 Dec 2024 12:22:22 +0100 Subject: [PATCH 273/502] Correct merge --- src/Databases/Iceberg/RestCatalog.cpp | 11 ++++++++++- src/Interpreters/Context.h | 3 +-- .../ObjectStorage/DataLakes/IcebergMetadata.cpp | 2 +- .../ObjectStorage/DataLakes/IcebergMetadata.h | 1 + 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index a3d8a2a2d5a..262ac53e99f 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,11 @@ namespace DB::ErrorCodes extern const int NOT_IMPLEMENTED; } +namespace DB::Setting +{ + extern const SettingsBool iceberg_engine_ignore_schema_evolution; +} + namespace Iceberg { @@ -571,8 +577,11 @@ bool RestCatalog::getTableMetadataImpl( if (result.requiresSchema()) { + const auto & settings = getContext()->getSettingsRef(); int format_version = metadata_object->getValue("format-version"); - result.setSchema(DB::IcebergMetadata::parseTableSchema(metadata_object, format_version, true).first); + auto schema = DB::IcebergMetadata::parseTableSchema( + metadata_object, log, format_version, settings[DB::Setting::iceberg_engine_ignore_schema_evolution]).first; + result.setSchema(schema); } if (result.requiresCredentials() && object->has("config")) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 6ddb3c1c0f5..9286de76427 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1077,8 +1077,6 @@ public: void clearMarkCache() const; ThreadPool & getLoadMarksThreadpool() const; - ThreadPool & getIcebergCatalogThreadpool() const; - void setPrimaryIndexCache(const String & cache_policy, size_t max_cache_size_in_bytes, double size_ratio); void updatePrimaryIndexCacheConfiguration(const Poco::Util::AbstractConfiguration & config); std::shared_ptr getPrimaryIndexCache() const; @@ -1125,6 +1123,7 @@ public: size_t getPrefetchThreadpoolSize() const; ThreadPool & getBuildVectorSimilarityIndexThreadPool() const; + ThreadPool & getIcebergCatalogThreadpool() const; /// Settings for MergeTree background tasks stored in config.xml BackgroundTaskSchedulingSettings getBackgroundProcessingTaskSchedulingSettings() const; diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index a93af4ee780..7b2ad165972 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -321,7 +321,7 @@ parseTableSchemaV1Method(const Poco::JSON::Object::Ptr & metadata_object, bool i return {schema, current_schema_id}; } -std::pair parseTableSchema( +std::pair IcebergMetadata::parseTableSchema( const Poco::JSON::Object::Ptr & metadata_object, LoggerPtr metadata_logger, int format_version, bool ignore_schema_evolution) { Poco::JSON::Object::Ptr schema; diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index b28de471b40..5f62dbfb2ef 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -98,6 +98,7 @@ public: static std::pair parseTableSchema( const Poco::JSON::Object::Ptr & metadata_object, + LoggerPtr metadata_logger, int format_version, bool ignore_schema_evolution); From f26c63eb4e23f4aea1d40f71a950ed1a5e0cf3af Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Dec 2024 12:32:19 +0100 Subject: [PATCH 274/502] Update Rust vendored dependencies --- cmake/ccache.cmake | 1 + contrib/rust_vendor | 2 +- docker/packager/binary-builder/Dockerfile | 4 +- docker/test/fasttest/Dockerfile | 4 +- rust/VENDOR.md | 8 +- rust/workspace/Cargo.lock | 482 +++++++++++++--------- utils/chcache/Cargo.lock | 179 ++++---- 7 files changed, 380 insertions(+), 300 deletions(-) diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake index 7457733a9fb..06537b2b572 100644 --- a/cmake/ccache.cmake +++ b/cmake/ccache.cmake @@ -26,6 +26,7 @@ elseif(COMPILER_CACHE STREQUAL "chcache") MANIFEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/utils/chcache/Cargo.toml PROFILE release LOCKED + FLAGS --offline ) set_target_properties(chcache PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/programs/) diff --git a/contrib/rust_vendor b/contrib/rust_vendor index 08e82ca6543..8046b94f716 160000 --- a/contrib/rust_vendor +++ b/contrib/rust_vendor @@ -1 +1 @@ -Subproject commit 08e82ca6543683abe4770305ad811a942186a520 +Subproject commit 8046b94f716108920f35e8936c8750a419f22274 diff --git a/docker/packager/binary-builder/Dockerfile b/docker/packager/binary-builder/Dockerfile index cbe4b51b2bf..dea5e6a9843 100644 --- a/docker/packager/binary-builder/Dockerfile +++ b/docker/packager/binary-builder/Dockerfile @@ -13,8 +13,8 @@ ENV CARGO_HOME=/rust/cargo ENV PATH="/rust/cargo/bin:${PATH}" RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \ chmod 777 -R /rust && \ - rustup toolchain install nightly-2024-04-01 && \ - rustup default nightly-2024-11-01 && \ + rustup toolchain install nightly-2024-12-01 && \ + rustup default nightly-2024-12-01 && \ rustup toolchain remove stable && \ rustup component add rust-src && \ rustup target add x86_64-unknown-linux-gnu && \ diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index c3bb89834cc..32416993caa 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -50,8 +50,8 @@ ENV CARGO_HOME=/rust/cargo ENV PATH="/rust/cargo/bin:${PATH}" RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \ chmod 777 -R /rust && \ - rustup toolchain install nightly-2024-11-01 && \ - rustup default nightly-2024-11-01 && \ + rustup toolchain install nightly-2024-12-01 && \ + rustup default nightly-2024-12-01 && \ rustup toolchain remove stable && \ rustup component add rust-src && \ rustup target add x86_64-unknown-linux-gnu && \ diff --git a/rust/VENDOR.md b/rust/VENDOR.md index 76aa9b9e1ab..f09196e1980 100644 --- a/rust/VENDOR.md +++ b/rust/VENDOR.md @@ -1,7 +1,7 @@ As we have multiple projects we use a workspace to manage them (it's way simpler and leads to less issues). In order to vendor all the dependencies we need to store both the registry and the packages themselves. -Note that this includes the exact `std` dependencies for the rustc version used in CI (currently nightly-2024-04-01), +Note that this includes the exact `std` dependencies for the rustc version used in CI (currently nightly-2024-12-01), so you need to install `rustup component add rust-src` for the specific version. * First step: (Re)-generate the Cargo.lock file (run under `workspace/`). @@ -16,7 +16,7 @@ Note that we use both commands to vendor both registry and crates. No idea why b * First we need to install the tool if you don't already have it: ```bash -cargo install --version 0.2.6 cargo-local-registry +cargo install --version 0.2.7 cargo-local-registry ``` * Now add the local packages: @@ -28,9 +28,9 @@ export RUSTC_ROOT=$(rustc --print=sysroot) cd "$CH_TOP_DIR"/rust/workspace cargo local-registry --git --sync Cargo.lock "$CH_TOP_DIR"/contrib/rust_vendor -cp "$RUSTC_ROOT"/lib/rustlib/src/rust/Cargo.lock "$RUSTC_ROOT"/lib/rustlib/src/rust/library/std/ +cp "$RUSTC_ROOT"/lib/rustlib/src/rust/library/Cargo.lock "$RUSTC_ROOT"/lib/rustlib/src/rust/library/std/ cargo local-registry --no-delete --git --sync "$RUSTC_ROOT"/lib/rustlib/src/rust/library/std/Cargo.lock "$CH_TOP_DIR"/contrib/rust_vendor -cp "$RUSTC_ROOT"/lib/rustlib/src/rust/Cargo.lock "$RUSTC_ROOT"/lib/rustlib/src/rust/library/test/ +cp "$RUSTC_ROOT"/lib/rustlib/src/rust/library/Cargo.lock "$RUSTC_ROOT"/lib/rustlib/src/rust/library/test/ cargo local-registry --no-delete --git --sync "$RUSTC_ROOT"/lib/rustlib/src/rust/library/test/Cargo.lock "$CH_TOP_DIR"/contrib/rust_vendor cargo vendor --no-delete --locked "$CH_TOP_DIR"/contrib/rust_vendor diff --git a/rust/workspace/Cargo.lock b/rust/workspace/Cargo.lock index 38e29be254f..d309b26f88d 100644 --- a/rust/workspace/Cargo.lock +++ b/rust/workspace/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "_ch_rust_prql" @@ -23,18 +23,18 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.21.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "ahash" @@ -59,9 +59,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android-tzdata" @@ -80,47 +80,48 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.13" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.6" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ "windows-sys", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", "windows-sys", @@ -128,18 +129,18 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.81" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" dependencies = [ "backtrace", ] [[package]] name = "ariadne" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd002a6223f12c7a95cdd4b1cb3a0149d22d37f7a9ecdb2cb691a071fe236c29" +checksum = "44055e597c674aef7cb903b2b9f6e4cba1277ed0d2d61dae7cd52d7ffa81f8e2" dependencies = [ "unicode-width", "yansi", @@ -147,29 +148,29 @@ dependencies = [ [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "autocfg" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -186,9 +187,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bumpalo" @@ -198,9 +199,12 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "cc" -version = "1.0.92" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2678b2e3449475e95b0aa6f9b506a28e61b3dc8996592b983695e8ebb58a8b41" +checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" @@ -210,9 +214,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", @@ -228,10 +232,36 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eebd66744a15ded14960ab4ccdbfb51ad3b81f51f3f04a80adac98c985396c9" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", "stacker", ] +[[package]] +name = "clap" +version = "4.5.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +dependencies = [ + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_lex" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -244,15 +274,15 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "crossbeam" @@ -269,9 +299,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ "crossbeam-utils", ] @@ -306,15 +336,15 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "csv" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" dependencies = [ "csv-core", "itoa", @@ -333,46 +363,61 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.121" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21db378d04296a84d8b7d047c36bb3954f0b46529db725d7e62fb02f9ba53ccc" +checksum = "05e1ec88093d2abd9cf1b09ffd979136b8e922bf31cad966a8fe0d73233112ef" dependencies = [ "cc", + "cxxbridge-cmd", "cxxbridge-flags", "cxxbridge-macro", + "foldhash", "link-cplusplus", ] [[package]] name = "cxx-build" -version = "1.0.121" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5262a7fa3f0bae2a55b767c223ba98032d7c328f5c13fa5cdc980b77fc0658" +checksum = "9afa390d956ee7ccb41aeed7ed7856ab3ffb4fc587e7216be7e0f83e949b4e6c" dependencies = [ "cc", "codespan-reporting", - "once_cell", "proc-macro2", "quote", "scratch", - "syn 2.0.58", + "syn 2.0.90", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c23bfff654d6227cbc83de8e059d2f8678ede5fc3a6c5a35d5c379983cc61e6" +dependencies = [ + "clap", + "codespan-reporting", + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] name = "cxxbridge-flags" -version = "1.0.121" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8dcadd2e2fb4a501e1d9e93d6e88e6ea494306d8272069c92d5a9edf8855c0" +checksum = "f7c01b36e22051bc6928a78583f1621abaaf7621561c2ada1b00f7878fbe2caa" [[package]] name = "cxxbridge-macro" -version = "1.0.121" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad08a837629ad949b73d032c637653d069e909cffe4ee7870b02301939ce39cc" +checksum = "f6e14013136fac689345d17b9a6df55977251f11d333c0a571e8d963b55e1f95" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "rustversion", + "syn 2.0.90", ] [[package]] @@ -395,7 +440,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 1.0.109", ] @@ -483,20 +528,20 @@ dependencies = [ [[package]] name = "either" -version = "1.10.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "enum-as-inner" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.90", ] [[package]] @@ -511,6 +556,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "fuzzy-matcher" version = "0.3.7" @@ -522,9 +573,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -533,31 +584,37 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", ] [[package]] -name = "heck" -version = "0.4.1" +name = "hashbrown" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -584,14 +641,20 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "indexmap" -version = "2.2.6" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.12.1" @@ -603,30 +666,31 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" dependencies = [ + "once_cell", "wasm-bindgen", ] [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" [[package]] name = "libredox" @@ -634,7 +698,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "libc", ] @@ -649,15 +713,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -676,11 +740,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] @@ -726,27 +790,27 @@ checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] name = "object" -version = "0.32.2" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "pin-utils" @@ -762,9 +826,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "proc-macro2" -version = "1.0.79" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -825,18 +889,18 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.21" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" dependencies = [ "cc", ] [[package]] name = "quote" -version = "1.0.35" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -863,9 +927,9 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", @@ -874,9 +938,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.4" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -886,9 +950,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -897,27 +961,27 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustversion" -version = "1.0.15" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "scratch" @@ -927,40 +991,41 @@ checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" [[package]] name = "semver" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" dependencies = [ "serde", ] [[package]] name = "serde" -version = "1.0.197" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.90", ] [[package]] name = "serde_json" -version = "1.0.115" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -978,6 +1043,12 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "skim" version = "0.10.4" @@ -1005,11 +1076,10 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.3" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790" dependencies = [ - "itertools", "nom", "unicode_categories", ] @@ -1026,15 +1096,15 @@ dependencies = [ [[package]] name = "stacker" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" dependencies = [ "cc", "cfg-if", "libc", "psm", - "winapi", + "windows-sys", ] [[package]] @@ -1044,25 +1114,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] -name = "strum" -version = "0.26.2" +name = "strsim" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 2.0.58", + "syn 2.0.90", ] [[package]] @@ -1078,9 +1154,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.58" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -1109,22 +1185,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.90", ] [[package]] @@ -1139,9 +1215,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "num-conv", @@ -1180,15 +1256,15 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode_categories" @@ -1204,15 +1280,15 @@ checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vte" @@ -1227,9 +1303,9 @@ dependencies = [ [[package]] name = "vte_generate_state_changes" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" dependencies = [ "proc-macro2", "quote", @@ -1243,34 +1319,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.90", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1278,22 +1355,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" [[package]] name = "winapi" @@ -1313,11 +1390,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi", + "windows-sys", ] [[package]] @@ -1337,22 +1414,23 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.52.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", + "windows_i686_gnullvm", "windows_i686_msvc", "windows_x86_64_gnu", "windows_x86_64_gnullvm", @@ -1361,68 +1439,74 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "yansi" -version = "0.5.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.90", ] diff --git a/utils/chcache/Cargo.lock b/utils/chcache/Cargo.lock index ea2e8cb8340..b5005631276 100644 --- a/utils/chcache/Cargo.lock +++ b/utils/chcache/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -46,21 +46,20 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-rs" -version = "1.10.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdd82dba44d209fddb11c190e0a94b78651f95299598e472215667417a03ff1d" +checksum = "f47bb8cc16b669d267eeccf585aea077d0882f4777b1c1f740217885d6e6e5a3" dependencies = [ "aws-lc-sys", - "mirai-annotations", "paste", "zeroize", ] [[package]] name = "aws-lc-sys" -version = "0.22.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df7a4168111d7eb622a31b214057b8509c0a7e1794f44c546d742330dc793972" +checksum = "a2101df3813227bbaaaa0b04cd61c534c7954b22bd68d399b440be937dc63ff7" dependencies = [ "bindgen", "cc", @@ -117,9 +116,9 @@ checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "blake3" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" dependencies = [ "arrayref", "arrayvec", @@ -130,24 +129,24 @@ dependencies = [ [[package]] name = "bstr" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +checksum = "1a68f1f47cdf0ec8ee4b941b2eee2a80cb796db73118c0dd09ac63fbe405be22" dependencies = [ "memchr", ] [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cc" -version = "1.1.36" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baee610e9452a8f6f0a1b6194ec09ff9e2d85dea54432acdae41aa0761c95d70" +checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" dependencies = [ "jobserver", "libc", @@ -242,9 +241,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e" dependencies = [ "cc", ] @@ -305,12 +304,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -448,15 +447,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "hashbrown" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" - -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "home" @@ -464,7 +457,7 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -509,9 +502,9 @@ checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "hyper" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" dependencies = [ "bytes", "futures-channel", @@ -704,9 +697,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", "hashbrown", @@ -723,9 +716,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jobserver" @@ -750,15 +743,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.162" +version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" +checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" [[package]] name = "libloading" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", "windows-targets", @@ -772,9 +765,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "litemap" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] name = "log" @@ -811,22 +804,15 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi", "libc", "wasi", - "windows-sys", + "windows-sys 0.52.0", ] -[[package]] -name = "mirai-annotations" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" - [[package]] name = "nom" version = "7.1.3" @@ -888,9 +874,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -918,9 +904,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -951,7 +937,7 @@ dependencies = [ "libc", "spin", "untrusted", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -968,22 +954,22 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.39" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.16" +version = "0.23.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" dependencies = [ "aws-lc-rs", "once_cell", @@ -1024,9 +1010,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] @@ -1042,9 +1028,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -1094,12 +1080,12 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1128,9 +1114,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.87" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -1150,18 +1136,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", @@ -1190,7 +1176,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1257,9 +1243,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "pin-project-lite", "tracing-core", @@ -1267,9 +1253,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", ] @@ -1282,9 +1268,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "untrusted" @@ -1294,9 +1280,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.3" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -1332,9 +1318,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "webpki-roots" -version = "0.26.6" +version = "0.26.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" dependencies = [ "rustls-pki-types", ] @@ -1360,6 +1346,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -1453,9 +1448,9 @@ checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546" [[package]] name = "yoke" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", @@ -1465,9 +1460,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", @@ -1477,18 +1472,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", From f771b6f2280a4c636c669f66ef3371c389f6f641 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Dec 2024 12:36:42 +0100 Subject: [PATCH 275/502] Undo integrating into CI --- docker/packager/packager | 15 +-------------- tests/ci/build_check.py | 1 + tests/ci/fast_test_check.py | 11 ++++------- utils/chcache/README.md | 23 ----------------------- 4 files changed, 6 insertions(+), 44 deletions(-) delete mode 100644 utils/chcache/README.md diff --git a/docker/packager/packager b/docker/packager/packager index 725bd426e6c..e7c0f4b3a00 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -130,7 +130,6 @@ def parse_env_variables( sanitizer: str, package_type: str, cache: str, - chcache_password: str, s3_bucket: str, s3_directory: str, s3_rw_access: bool, @@ -323,12 +322,6 @@ def parse_env_variables( if not s3_rw_access: result.append("SCCACHE_S3_NO_CREDENTIALS=true") - if cache == "chcache": - cmake_flags.append("-DCOMPILER_CACHE=chcache") - result.append(f"CH_PASSWORD={chcache_password}") - result.append("CH_USER=ci_builder") - result.append("CH_HOSTNAME=https://lr5v5i0nr3.eu-west-1.aws.clickhouse-staging.com") - if clang_tidy: # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above. # It's there to have the clang-tidy cache embedded into our standard `CCACHE_DIR` @@ -444,15 +437,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--clang-tidy", action="store_true") parser.add_argument( "--cache", - choices=("ccache", "sccache", "chcache", ""), + choices=("ccache", "sccache", ""), default="", help="ccache or sccache for objects caching; sccache uses only S3 buckets", ) - parser.add_argument( - "--chcache-password", - default="", - help="", - ) parser.add_argument( "--ccache-dir", default=Path.home() / ".ccache", @@ -542,7 +530,6 @@ def main() -> None: args.sanitizer, args.package_type, args.cache, - args.chcache_password, args.s3_bucket, args.s3_directory, args.s3_rw_access, diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 0b2e6853cad..77d91c8400b 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -61,6 +61,7 @@ def get_packager_cmd( if build_config.tidy: cmd += " --clang-tidy" + cmd += " --cache=sccache" cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 4acd45b16af..55eefcf9714 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -8,12 +8,11 @@ from pathlib import Path from typing import Tuple from docker_images_helper import DockerImage, get_docker_image, pull_image -from env_helper import REPO_COPY, TEMP_PATH +from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH from pr_info import PRInfo from report import ERROR, FAILURE, SUCCESS, JobReport, TestResults, read_test_results from stopwatch import Stopwatch from tee_popen import TeePopen -from get_robot_token import get_parameter_from_ssm # Will help to avoid errors like _csv.Error: field larger than field limit (131072) csv.field_size_limit(sys.maxsize) @@ -27,17 +26,16 @@ def get_fasttest_cmd( commit_sha: str, image: DockerImage, ) -> str: - chcache_password = get_parameter_from_ssm("chcache_password") return ( f"docker run --cap-add=SYS_PTRACE --user={os.geteuid()}:{os.getegid()} " "--security-opt seccomp=unconfined " # required to issue io_uring sys-calls "--network=host " # required to get access to IAM credentials f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " f"-e FASTTEST_SOURCE=/repo " - f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=chcache' " + f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " - f"-e CH_HOSTNAME='https://lr5v5i0nr3.eu-west-1.aws.clickhouse-staging.com' -e CH_USER=ci_builder -e CH_PASSWORD='{chcache_password}' " + f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " "-e stage=clone_submodules " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/repo " f"--volume={output_path}:/test_output {image} /repo/tests/docker_scripts/fasttest_runner.sh" @@ -99,8 +97,7 @@ def main(): pr_info.sha, docker_image, ) - sanitized_run_cmd = run_cmd.replace(get_parameter_from_ssm("chcache_password"), "****") - logging.info("Going to run fasttest with cmd %s", sanitized_run_cmd) + logging.info("Going to run fasttest with cmd %s", run_cmd) logs_path = temp_path / "fasttest-logs" logs_path.mkdir(parents=True, exist_ok=True) diff --git a/utils/chcache/README.md b/utils/chcache/README.md deleted file mode 100644 index a6140e12ed8..00000000000 --- a/utils/chcache/README.md +++ /dev/null @@ -1,23 +0,0 @@ -## chcache - -#### Usage instructions - -First, build the binary. If you don't have Rust toolchain, get one by installing `rustup` with your package manager, then enable the default toolchain: -```bash -rustup default stable -``` - -Then, build the `chcache`: -```bash -cargo build --release -``` - -Place this in your `~/.config/chcache/config.toml` -```toml -TBD -``` - -Go to your `~/src/clickhouse/cmake-build-debug` (only `cmake-build-debug` part is important) and run CMake like this, adjusting the path to `chcache` binary: -```bash -cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_BUILD_PATH_MAPPING=1 -DCMAKE_CXX_COMPILER_LAUNCHER=/home/thevar1able/src/chcache/chcache-rust/target/release/chcache-rust -DCMAKE_C_COMPILER_LAUNCHER=/home/thevar1able/src/chcache/chcache-rust/target/release/chcache-rust -DCOMPILER_CACHE=disabled .. -``` From 21131689bd8f6fa74ebc8890603dbf5aacdea24c Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Dec 2024 12:44:08 +0100 Subject: [PATCH 276/502] Update docs --- docker/test/fasttest/Dockerfile | 50 +++++++++++++++++++++++---------- docs/en/development/build.md | 4 +-- docs/ja/development/build.md | 4 +-- 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 32416993caa..318f437a108 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -40,22 +40,42 @@ RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld # https://salsa.debian.org/pkg-llvm-team/llvm-toolchain/-/commit/992e52c0b156a5ba9c6a8a54f8c4857ddd3d371d RUN sed -i '/_IMPORT_CHECK_FILES_FOR_\(mlir-\|llvm-bolt\|merge-fdata\|MLIR\)/ {s|^|#|}' /usr/lib/llvm-${LLVM_VERSION}/lib/cmake/llvm/LLVMExports-*.cmake -# LLVM changes paths for compiler-rt libraries. For some reason clang-18.1.8 cannot catch up libraries from default install path. -# It's very dirty workaround, better to build compiler and LLVM ourself and use it. Details: https://github.com/llvm/llvm-project/issues/95792 -RUN test ! -d /usr/lib/llvm-18/lib/clang/18/lib/x86_64-pc-linux-gnu || ln -s /usr/lib/llvm-18/lib/clang/18/lib/x86_64-pc-linux-gnu /usr/lib/llvm-18/lib/clang/18/lib/x86_64-unknown-linux-gnu +ARG CCACHE_VERSION=4.6.1 +RUN mkdir /tmp/ccache \ + && cd /tmp/ccache \ + && curl -L \ + -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz \ + -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz.asc \ + && gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 5A939A71A46792CF57866A51996DDA075594ADB8 \ + && gpg --verify ccache-4.6.1.tar.xz.asc \ + && tar xf ccache-$CCACHE_VERSION.tar.xz \ + && cd /tmp/ccache/ccache-$CCACHE_VERSION \ + && cmake -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_BUILD_TYPE=None \ + -DZSTD_FROM_INTERNET=ON \ + -DREDIS_STORAGE_BACKEND=OFF \ + -Wno-dev \ + -B build \ + -S . \ + && make VERBOSE=1 -C build \ + && make install -C build \ + && cd / \ + && rm -rf /tmp/ccache -# Rust toolchain and libraries -ENV RUSTUP_HOME=/rust/rustup -ENV CARGO_HOME=/rust/cargo -ENV PATH="/rust/cargo/bin:${PATH}" -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \ - chmod 777 -R /rust && \ - rustup toolchain install nightly-2024-12-01 && \ - rustup default nightly-2024-12-01 && \ - rustup toolchain remove stable && \ - rustup component add rust-src && \ - rustup target add x86_64-unknown-linux-gnu && \ - rustup target add aarch64-unknown-linux-gnu +ARG TARGETARCH +ARG SCCACHE_VERSION=v0.7.7 +ENV SCCACHE_IGNORE_SERVER_IO_ERROR=1 +# sccache requires a value for the region. So by default we use The Default Region +ENV SCCACHE_REGION=us-east-1 +RUN arch=${TARGETARCH:-amd64} \ + && case $arch in \ + amd64) rarch=x86_64 ;; \ + arm64) rarch=aarch64 ;; \ + esac \ + && curl -Ls "https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl.tar.gz" | \ + tar xz -C /tmp \ + && mv "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl/sccache" /usr/bin \ + && rm "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl" -r # Give suid to gdb to grant it attach permissions # chmod 777 to make the container user independent diff --git a/docs/en/development/build.md b/docs/en/development/build.md index e130e57aeb2..f8e02c81245 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -70,8 +70,8 @@ enable sanitizers you must use a version that matches the exact same `std` as th the crates): ```bash -rustup toolchain install nightly-2024-04-01 -rustup default nightly-2024-04-01 +rustup toolchain install nightly-2024-12-01 +rustup default nightly-2024-12-01 rustup component add rust-src ``` diff --git a/docs/ja/development/build.md b/docs/ja/development/build.md index 48d2649fb79..1c135936b0f 100644 --- a/docs/ja/development/build.md +++ b/docs/ja/development/build.md @@ -67,8 +67,8 @@ C++ 依存関係と同様に、ClickHouse はベンダリングを使用して サニタイザを有効にする予定がある場合は、CI で使用されるものと同じ `std` と一致するバージョンを使用する必要があります（crates をベンダリングしています）: ```bash -rustup toolchain install nightly-2024-04-01 -rustup default nightly-2024-04-01 +rustup toolchain install nightly-2024-12-01 +rustup default nightly-2024-12-01 rustup component add rust-src ``` From 0d91a9b60cf8ce6bdbdcd650d1f1e72fa837b881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 3 Dec 2024 13:13:05 +0100 Subject: [PATCH 277/502] Use an Enum for type in system.settings_changes --- .../system-tables/settings_changes.md | 2 +- .../System/StorageSystemSettingsChanges.cpp | 32 ++++++++++++++----- ...26_settings_changes_system_table.reference | 2 +- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/docs/en/operations/system-tables/settings_changes.md b/docs/en/operations/system-tables/settings_changes.md index f380023c4fa..34a52d25528 100644 --- a/docs/en/operations/system-tables/settings_changes.md +++ b/docs/en/operations/system-tables/settings_changes.md @@ -7,7 +7,7 @@ Contains information about setting changes in previous ClickHouse versions. Columns: -- `type` ([String](../../sql-reference/data-types/string.md)) - The group of settings (Core, MergeTree...) +- `type` ([Enum](../../sql-reference/data-types/enum.md)) - The settings type: `Core` (general / query settings), `MergeTree`. - `version` ([String](../../sql-reference/data-types/string.md)) — The ClickHouse version in which settings were changed - `changes` ([Array](../../sql-reference/data-types/array.md) of [Tuple](../../sql-reference/data-types/tuple.md)) — A description of the setting changes: (setting name, previous value, new value, reason for the change) diff --git a/src/Storages/System/StorageSystemSettingsChanges.cpp b/src/Storages/System/StorageSystemSettingsChanges.cpp index 34455316630..286b0ab2e5e 100644 --- a/src/Storages/System/StorageSystemSettingsChanges.cpp +++ b/src/Storages/System/StorageSystemSettingsChanges.cpp @@ -1,18 +1,34 @@ -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include namespace DB { + +namespace +{ +DataTypePtr getSettingsTypeEnum() +{ + return std::make_shared( + DataTypeEnum8::Values + { + {"Core", 0}, + {"MergeTree", 1}, + }); +} +} + + ColumnsDescription StorageSystemSettingsChanges::getColumnsDescription() { /// TODO: Fill in all the comments return ColumnsDescription { - {"type", std::make_shared(), "The group of settings (Core, MergeTree...)"}, + {"type", getSettingsTypeEnum(), "The group of settings (Core, MergeTree...)"}, {"version", std::make_shared(), "The ClickHouse server version."}, {"changes", std::make_shared(std::make_shared( @@ -30,7 +46,7 @@ void StorageSystemSettingsChanges::fillData(MutableColumns & res_columns, Contex const auto & settings_changes_history = getSettingsChangesHistory(); for (auto it = settings_changes_history.rbegin(); it != settings_changes_history.rend(); ++it) { - res_columns[0]->insert("Core"); + res_columns[0]->insert(0); res_columns[1]->insert(it->first.toString()); Array changes; for (const auto & change : it->second) @@ -41,7 +57,7 @@ void StorageSystemSettingsChanges::fillData(MutableColumns & res_columns, Contex const auto & mergetree_settings_changes_history = getMergeTreeSettingsChangesHistory(); for (auto it = mergetree_settings_changes_history.rbegin(); it != mergetree_settings_changes_history.rend(); ++it) { - res_columns[0]->insert("MergeTree"); + res_columns[0]->insert(1); res_columns[1]->insert(it->first.toString()); Array changes; for (const auto & change : it->second) diff --git a/tests/queries/0_stateless/02326_settings_changes_system_table.reference b/tests/queries/0_stateless/02326_settings_changes_system_table.reference index 026b96cd16c..465e01a0ade 100644 --- a/tests/queries/0_stateless/02326_settings_changes_system_table.reference +++ b/tests/queries/0_stateless/02326_settings_changes_system_table.reference @@ -1,4 +1,4 @@ -type String The group of settings (Core, MergeTree...) +type Enum8(\'Core\' = 0, \'MergeTree\' = 1) The group of settings (Core, MergeTree...) version String The ClickHouse server version. changes Array(Tuple(\n name String,\n previous_value String,\n new_value String,\n reason String)) The list of changes in settings which changed the behaviour of ClickHouse. Core 22.5 [('memory_overcommit_ratio_denominator','0','1073741824','Enable memory overcommit feature by default'),('memory_overcommit_ratio_denominator_for_user','0','1073741824','Enable memory overcommit feature by default')] From c89e6259c474f9555eedce37734dd1349f813140 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Dec 2024 13:37:24 +0100 Subject: [PATCH 278/502] Update list-licenses.sh --- utils/list-licenses/list-licenses.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/list-licenses/list-licenses.sh b/utils/list-licenses/list-licenses.sh index 101b7c9b18a..675d3c9bcd8 100755 --- a/utils/list-licenses/list-licenses.sh +++ b/utils/list-licenses/list-licenses.sh @@ -97,4 +97,4 @@ do done # Special care for Rust -find "${LIBS_PATH}/rust_vendor/" -name 'Cargo.toml' | xargs ${GREP_CMD} 'license = ' | (${GREP_CMD} -v -P 'MIT|Apache|MPL' && echo "Fatal error: unrecognized licenses in the Rust code" >&2 && exit 1 || true) +find "${LIBS_PATH}/rust_vendor/" -name 'Cargo.toml' | xargs ${GREP_CMD} 'license = ' | (${GREP_CMD} -v -P 'MIT|Apache|MPL|ISC|BSD|Unicode|Zlib' && echo "Fatal error: unrecognized licenses in the Rust code" >&2 && exit 1 || true) From 599218849cca4d31273ce0f445f253ec796f2cbb Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 3 Dec 2024 13:23:08 +0100 Subject: [PATCH 279/502] Fix no-such-key in s3queue --- .../ObjectStorageQueueIFileMetadata.cpp | 24 ++++++++++++ .../ObjectStorageQueueIFileMetadata.h | 3 ++ .../ObjectStorageQueueSource.cpp | 38 ++++++++++++++++++- .../ObjectStorageQueueSource.h | 4 ++ ...bjectStorageQueueUnorderedFileMetadata.cpp | 31 +++++++++++---- .../ObjectStorageQueueUnorderedFileMetadata.h | 3 ++ .../StorageObjectStorageQueue.cpp | 14 +++++-- 7 files changed, 106 insertions(+), 11 deletions(-) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp index 6fac519849d..f8c4ceb91f4 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp @@ -62,6 +62,15 @@ void ObjectStorageQueueIFileMetadata::FileStatus::onFailed(const std::string & e last_exception = exception; } +void ObjectStorageQueueIFileMetadata::FileStatus::reset() +{ + state = FileStatus::State::None; + processing_start_time = {}; + processing_end_time = {}; + processed_rows = 0; + retries = 0; +} + void ObjectStorageQueueIFileMetadata::FileStatus::updateState(State state_) { state = state_; @@ -245,6 +254,21 @@ bool ObjectStorageQueueIFileMetadata::setProcessing() return success; } +void ObjectStorageQueueIFileMetadata::resetProcessing() +{ + auto state = file_status->state.load(); + if (state != FileStatus::State::Processing) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot reset non-processing state: {}", state); + + resetProcessingImpl(); + file_status->reset(); +} + +void ObjectStorageQueueIFileMetadata::resetProcessingImpl() +{ + throw Exception(ErrorCodes::LOGICAL_ERROR, "resetProcessingImpl is not implemented"); +} + void ObjectStorageQueueIFileMetadata::setProcessed() { LOG_TRACE(log, "Setting file {} as processed (path: {})", path, processed_node_path); diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.h index 9bab9e7f075..59741d9f052 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.h @@ -22,6 +22,7 @@ public: void setProcessingEndTime(); void onProcessing(); void onProcessed(); + void reset(); void onFailed(const std::string & exception); void updateState(State state_); @@ -54,6 +55,7 @@ public: bool setProcessing(); void setProcessed(); + void resetProcessing(); void setFailed(const std::string & exception_message, bool reduce_retry_count, bool overwrite_status); virtual void setProcessedAtStartRequests( @@ -78,6 +80,7 @@ public: protected: virtual std::pair setProcessingImpl() = 0; virtual void setProcessedImpl() = 0; + virtual void resetProcessingImpl(); void setFailedNonRetriable(); void setFailedRetriable(); diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp index eaedfdd4dfd..8034c9bf8ae 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp @@ -49,11 +49,15 @@ ObjectStorageQueueSource::ObjectStorageQueueObjectInfo::ObjectStorageQueueObject ObjectStorageQueueSource::FileIterator::FileIterator( std::shared_ptr metadata_, std::unique_ptr glob_iterator_, + ObjectStoragePtr object_storage_, + bool file_deletion_on_processed_enabled_, std::atomic & shutdown_called_, LoggerPtr logger_) : StorageObjectStorageSource::IIterator("ObjectStorageQueueIterator") , metadata(metadata_) + , object_storage(object_storage_) , glob_iterator(std::move(glob_iterator_)) + , file_deletion_on_processed_enabled(file_deletion_on_processed_enabled_) , shutdown_called(shutdown_called_) , log(logger_) { @@ -114,7 +118,39 @@ ObjectStorageQueueSource::Source::ObjectInfoPtr ObjectStorageQueueSource::FileIt auto file_metadata = metadata->getFileMetadata(object_info->relative_path, bucket_info); if (file_metadata->setProcessing()) + { + if (file_deletion_on_processed_enabled + && !object_storage->exists(StoredObject(object_info->getPath()))) + { + /// Imagine the following case: + /// Replica A processed fileA and deletes it afterwards. + /// Replica B has a list request batch (by default list batch is 1000 elements) + /// and this batch was collected from object storage before replica A processed fileA. + /// fileA could be somewhere in the middle of this batch of replica B + /// and replica A processed it before replica B reached fileA in this batch. + /// All would be alright, unless user has tracked_files_size_limit or tracked_files_ttl_limit + /// which could expire before replica B reached fileA in this list batch. + /// It would mean that replica B listed this file while it no longer + /// exists in object storage at the moment it wants to process it, but + /// because of tracked_files_size(ttl)_limit expiration - we no longer + /// have information in keeper that the file was actually processed before, + /// so replica B would successfully set itself as processor of this file in keeper + /// and face "The specified key does not exist" after that. + /// + /// This existance check here is enough, + /// only because we do applyActionAfterProcessing BEFORE setting file as processed + /// and because at this exact place we already successfully set file as processing, + /// e.g. file deletion and marking file as processed in keeper already took place. + /// + /// Note: this all applies only for Unordered mode. + LOG_TRACE(log, "Ignoring {} because of the race with list & delete", object_info->getPath()); + + file_metadata->resetProcessing(); + continue; + } + return std::make_shared(*object_info, file_metadata); + } } return {}; } @@ -628,8 +664,8 @@ void ObjectStorageQueueSource::commit(bool success, const std::string & exceptio { if (success) { - file_metadata->setProcessed(); applyActionAfterProcessing(file_metadata->getPath()); + file_metadata->setProcessed(); } else { diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h index 8b7d5a7e65b..b9d85539d8f 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h @@ -40,6 +40,8 @@ public: FileIterator( std::shared_ptr metadata_, std::unique_ptr glob_iterator_, + ObjectStoragePtr object_storage_, + bool file_deletion_on_processed_enabled_, std::atomic & shutdown_called_, LoggerPtr logger_); @@ -67,7 +69,9 @@ public: using Processor = ObjectStorageQueueMetadata::Processor; const std::shared_ptr metadata; + const ObjectStoragePtr object_storage; const std::unique_ptr glob_iterator; + const bool file_deletion_on_processed_enabled; std::atomic & shutdown_called; std::mutex mutex; diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.cpp index 2050797a2ea..b5ec887162d 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.cpp @@ -99,6 +99,16 @@ void ObjectStorageQueueUnorderedFileMetadata::setProcessedAtStartRequests( } void ObjectStorageQueueUnorderedFileMetadata::setProcessedImpl() +{ + setProcessedImpl(false); +} + +void ObjectStorageQueueUnorderedFileMetadata::resetProcessingImpl() +{ + setProcessedImpl(true); +} + +void ObjectStorageQueueUnorderedFileMetadata::setProcessedImpl(bool remove_processing_nodes_only) { /// In one zookeeper transaction do the following: enum RequestType @@ -124,16 +134,19 @@ void ObjectStorageQueueUnorderedFileMetadata::setProcessedImpl() request_index[CHECK_PROCESSING_ID_PATH] = 0; request_index[REMOVE_PROCESSING_ID_PATH] = 1; request_index[REMOVE_PROCESSING_PATH] = 2; - request_index[SET_PROCESSED_PATH] = 3; + + if (!remove_processing_nodes_only) + request_index[SET_PROCESSED_PATH] = 3; } - else + else if (!remove_processing_nodes_only) { request_index[SET_PROCESSED_PATH] = 0; } - requests.push_back( - zkutil::makeCreateRequest( - processed_node_path, node_metadata.toString(), zkutil::CreateMode::Persistent)); + if (!remove_processing_nodes_only) + requests.push_back( + zkutil::makeCreateRequest( + processed_node_path, node_metadata.toString(), zkutil::CreateMode::Persistent)); Coordination::Responses responses; auto is_request_failed = [&](RequestType type) @@ -147,6 +160,9 @@ void ObjectStorageQueueUnorderedFileMetadata::setProcessedImpl() const auto code = zk_client->tryMulti(requests, responses); if (code == Coordination::Error::ZOK) { + if (remove_processing_nodes_only) + return; + if (max_loading_retries && zk_client->tryRemove(failed_node_path + ".retriable", -1) == Coordination::Error::ZOK) { @@ -181,7 +197,7 @@ void ObjectStorageQueueUnorderedFileMetadata::setProcessedImpl() /// This is normal in case of expired session with keeper as this node is ephemeral. failure_reason = "Failed to remove processing path"; } - else if (is_request_failed(SET_PROCESSED_PATH)) + else if (!remove_processing_nodes_only && is_request_failed(SET_PROCESSED_PATH)) { unexpected_error = true; failure_reason = "Cannot create a persistent node in /processed since it already exists"; @@ -192,7 +208,8 @@ void ObjectStorageQueueUnorderedFileMetadata::setProcessedImpl() if (unexpected_error) throw Exception(ErrorCodes::LOGICAL_ERROR, "{}", failure_reason); - LOG_WARNING(log, "Cannot set file {} as processed: {}. Reason: {}", path, code, failure_reason); + if (!remove_processing_nodes_only) + LOG_WARNING(log, "Cannot set file {} as processed: {}. Reason: {}", path, code, failure_reason); } } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.h index cc5d8a09ec9..8df838d07bb 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.h @@ -27,6 +27,9 @@ public: private: std::pair setProcessingImpl() override; void setProcessedImpl() override; + void resetProcessingImpl() override; + + void setProcessedImpl(bool remove_processing_nodes_only); }; } diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 6630d5a6c8f..4ac96b3367a 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -731,13 +731,21 @@ zkutil::ZooKeeperPtr StorageObjectStorageQueue::getZooKeeper() const return getContext()->getZooKeeper(); } -std::shared_ptr StorageObjectStorageQueue::createFileIterator(ContextPtr local_context, const ActionsDAG::Node * predicate) +std::shared_ptr +StorageObjectStorageQueue::createFileIterator(ContextPtr local_context, const ActionsDAG::Node * predicate) { auto settings = configuration->getQuerySettings(local_context); auto glob_iterator = std::make_unique( - object_storage, configuration, predicate, getVirtualsList(), local_context, nullptr, settings.list_object_keys_size, settings.throw_on_zero_files_match); + object_storage, configuration, predicate, getVirtualsList(), local_context, + nullptr, settings.list_object_keys_size, settings.throw_on_zero_files_match); - return std::make_shared(files_metadata, std::move(glob_iterator), shutdown_called, log); + const auto & table_metadata = getTableMetadata(); + bool file_deletion_enabled = table_metadata.getMode() == ObjectStorageQueueMode::UNORDERED + && table_metadata.tracked_files_ttl_sec + && table_metadata.tracked_files_limit; + + return std::make_shared( + files_metadata, std::move(glob_iterator), object_storage, file_deletion_enabled, shutdown_called, log); } ObjectStorageQueueSettings StorageObjectStorageQueue::getSettings() const From ca5a1463e2fdb0f0503cd301ad3021837724e258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 3 Dec 2024 14:05:03 +0100 Subject: [PATCH 280/502] Update setting changes with master --- src/Core/SettingsChangesHistory.cpp | 1 + src/Storages/MergeTree/MergeTreeSettings.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index c1ec68a34f3..f9451afa483 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -620,6 +620,7 @@ static std::initializer_list Date: Tue, 3 Dec 2024 13:15:14 +0000 Subject: [PATCH 281/502] Support JSON in notEmpty function --- src/Functions/empty.cpp | 30 ++++++++++++------- src/Functions/notEmpty.cpp | 25 ---------------- .../03279_not_empty_json.reference | 12 ++++++++ .../0_stateless/03279_not_empty_json.sql | 23 ++++++++++++++ 4 files changed, 54 insertions(+), 36 deletions(-) delete mode 100644 src/Functions/notEmpty.cpp create mode 100644 tests/queries/0_stateless/03279_not_empty_json.reference create mode 100644 tests/queries/0_stateless/03279_not_empty_json.sql diff --git a/src/Functions/empty.cpp b/src/Functions/empty.cpp index 0864cb9e2fb..f258cd8a369 100644 --- a/src/Functions/empty.cpp +++ b/src/Functions/empty.cpp @@ -23,13 +23,17 @@ struct NameEmpty static constexpr auto name = "empty"; }; -using FunctionEmpty = FunctionStringOrArrayToT, NameEmpty, UInt8, false>; +struct NameNotEmpty +{ + static constexpr auto name = "notEmpty"; +}; /// Implements the empty function for JSON type. +template class ExecutableFunctionJSONEmpty : public IExecutableFunction { public: - std::string getName() const override { return NameEmpty::name; } + std::string getName() const override { return negative ? NameNotEmpty::name : NameEmpty::name; } private: bool useDefaultImplementationForConstants() const override { return true; } @@ -48,7 +52,7 @@ private: /// If object column has at least 1 typed path, it will never be empty, because these paths always have values. if (!typed_paths.empty()) { - data.resize_fill(size, 0); + data.resize_fill(size, negative); return res; } @@ -76,19 +80,20 @@ private: } } - data.push_back(empty); + data.push_back(negative ^ empty); } return res; } }; +template class FunctionEmptyJSON final : public IFunctionBase { public: FunctionEmptyJSON(const DataTypes & argument_types_, const DataTypePtr & return_type_) : argument_types(argument_types_), return_type(return_type_) {} - String getName() const override { return NameEmpty::name; } + String getName() const override { return negative ? NameNotEmpty::name : NameEmpty::name; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } @@ -97,7 +102,7 @@ public: ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName &) const override { - return std::make_unique(); + return std::make_unique>(); } private: @@ -105,17 +110,18 @@ private: DataTypePtr return_type; }; +template class FunctionEmptyOverloadResolver final : public IFunctionOverloadResolver { public: - static constexpr auto name = NameEmpty::name; + static constexpr auto name = negative ? NameNotEmpty::name : NameEmpty::name; static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique(); } - String getName() const override { return NameEmpty::name; } + String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override @@ -126,9 +132,9 @@ public: argument_types.push_back(arg.type); if (argument_types.size() == 1 && isObject(argument_types[0])) - return std::make_shared(argument_types, return_type); + return std::make_shared>(argument_types, return_type); - return std::make_shared(std::make_shared(), argument_types, return_type); + return std::make_shared(std::make_shared, NameEmpty, UInt8, false>>(), argument_types, return_type); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -155,7 +161,9 @@ public: REGISTER_FUNCTION(Empty) { - factory.registerFunction(); + factory.registerFunction>(); + factory.registerFunction>(); + } } diff --git a/src/Functions/notEmpty.cpp b/src/Functions/notEmpty.cpp deleted file mode 100644 index f7d58e15143..00000000000 --- a/src/Functions/notEmpty.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#include -#include -#include -#include - - -namespace DB -{ -namespace -{ - -struct NameNotEmpty -{ - static constexpr auto name = "notEmpty"; -}; -using FunctionNotEmpty = FunctionStringOrArrayToT, NameNotEmpty, UInt8, false>; - -} - -REGISTER_FUNCTION(NotEmpty) -{ - factory.registerFunction(); -} - -} diff --git a/tests/queries/0_stateless/03279_not_empty_json.reference b/tests/queries/0_stateless/03279_not_empty_json.reference new file mode 100644 index 00000000000..6da41258c26 --- /dev/null +++ b/tests/queries/0_stateless/03279_not_empty_json.reference @@ -0,0 +1,12 @@ +{} 0 +{"a":"42"} 1 +{"b":{"c":"42"}} 1 +{"a":0} 1 +{"a":42} 1 +{"a":0,"b":{"c":"42"}} 1 +{} 0 +{"a":"42"} 1 +{"b":{"c":"42"}} 1 +{} 0 +{"a":"42"} 1 +{"b":{"c":"42"}} 1 diff --git a/tests/queries/0_stateless/03279_not_empty_json.sql b/tests/queries/0_stateless/03279_not_empty_json.sql new file mode 100644 index 00000000000..d4080b725b8 --- /dev/null +++ b/tests/queries/0_stateless/03279_not_empty_json.sql @@ -0,0 +1,23 @@ +set enable_json_type=1; + +create table test (json JSON) engine=Memory; +insert into test values ('{}'), ('{"a" : 42}'), ('{"b" : {"c" : 42}}'); +select json, notEmpty(json) from test; +drop table test; + +create table test (json JSON(a UInt32)) engine=Memory; +insert into test values ('{}'), ('{"a" : 42}'), ('{"b" : {"c" : 42}}'); +select json, notEmpty(json) from test; +drop table test; + +create table test (json JSON(max_dynamic_paths=1)) engine=Memory; +insert into test values ('{}'), ('{"a" : 42}'), ('{"b" : {"c" : 42}}'); +select json, notEmpty(json) from test; +drop table test; + +create table test (json JSON(max_dynamic_paths=0)) engine=Memory; +insert into test values ('{}'), ('{"a" : 42}'), ('{"b" : {"c" : 42}}'); +select json, notEmpty(json) from test; +drop table test; + + From d9b1665327c3c82e705bc1bbfba5fcb8d9cb6f5a Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Dec 2024 14:58:56 +0100 Subject: [PATCH 282/502] Update rust_vendor submodule --- contrib/rust_vendor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/rust_vendor b/contrib/rust_vendor index 8046b94f716..a73510df047 160000 --- a/contrib/rust_vendor +++ b/contrib/rust_vendor @@ -1 +1 @@ -Subproject commit 8046b94f716108920f35e8936c8750a419f22274 +Subproject commit a73510df0475fad6897f9c422c15d82f10abc6f0 From abbff40cd0ba13dbe802dd7fc893ad1b8da07f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Tue, 3 Dec 2024 14:23:59 +0000 Subject: [PATCH 283/502] Make JOIN query also deterministic --- ...2_common_expression_optimization.reference | 40 +++++++++---------- .../03262_common_expression_optimization.sql | 11 ++++- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.reference b/tests/queries/0_stateless/03262_common_expression_optimization.reference index 035769cce67..70da99ae413 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.reference +++ b/tests/queries/0_stateless/03262_common_expression_optimization.reference @@ -830,26 +830,26 @@ QUERY id: 0 SELECT count() AS `count()` FROM default.x AS __table1 WHERE __table1.A OR (__table1.B AND ((__table1.C AND __table1.D) OR (__table1.C AND __table1.E))) --9222995085389227671 0 0 0 0 0 1 -9180023040793172172 0 0 1 0 1 1 --9222995085389227671 0 0 0 0 0 1 -9172372182410586193 0 1 1 0 1 1 --9222995085389227671 0 0 0 0 0 1 -9140643988322994670 0 1 1 1 0 1 --9222995085389227671 0 0 0 0 0 1 -9131908548041292850 0 0 1 0 0 0 --9222995085389227671 0 0 0 0 0 1 -9125905281269664989 0 1 1 0 1 0 --9222995085389227671 0 0 0 0 0 1 -9122596639456104649 0 0 1 0 1 1 --9222995085389227671 0 0 0 0 0 1 -9072373031481714042 0 0 1 0 0 1 --9222995085389227671 0 0 0 0 0 1 -9026061943293167962 0 1 1 0 1 1 --9222995085389227671 0 0 0 0 0 1 -9004164398646425447 0 1 1 0 0 0 --9222995085389227671 0 0 0 0 0 1 -8974795392131287614 0 1 1 0 1 0 --9222995085389227671 0 0 0 0 0 1 -9180023040793172172 0 0 1 0 1 1 --9222995085389227671 0 0 0 0 0 1 -9172372182410586193 0 1 1 0 1 1 --9222995085389227671 0 0 0 0 0 1 -9140643988322994670 0 1 1 1 0 1 --9222995085389227671 0 0 0 0 0 1 -9131908548041292850 0 0 1 0 0 0 --9222995085389227671 0 0 0 0 0 1 -9125905281269664989 0 1 1 0 1 0 --9222995085389227671 0 0 0 0 0 1 -9122596639456104649 0 0 1 0 1 1 --9222995085389227671 0 0 0 0 0 1 -9072373031481714042 0 0 1 0 0 1 --9222995085389227671 0 0 0 0 0 1 -9026061943293167962 0 1 1 0 1 1 --9222995085389227671 0 0 0 0 0 1 -9004164398646425447 0 1 1 0 0 0 --9222995085389227671 0 0 0 0 0 1 -8974795392131287614 0 1 1 0 1 0 +-9222995085389227671 0 0 0 0 0 1 -9212191261079457524 0 0 1 0 1 0 +-9222995085389227671 0 0 0 0 0 1 -9210944775923669427 0 1 1 1 0 0 +-9222995085389227671 0 0 0 0 0 1 -9202601338732071287 0 1 1 1 1 1 +-9222995085389227671 0 0 0 0 0 1 -9185234005543064629 0 1 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9181371158682533758 0 1 1 1 0 0 +-9222995085389227671 0 0 0 0 0 1 -9173688614647112626 0 0 1 0 0 1 +-9222995085389227671 0 0 0 0 0 1 -9130964659455442087 0 1 1 1 1 1 +-9222995085389227671 0 0 0 0 0 1 -9109186410997511387 0 1 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9071619989214493680 0 1 1 0 1 0 +-9222995085389227671 0 0 0 0 0 1 -9071253360041153133 0 1 1 1 0 1 +-9222995085389227671 0 0 0 0 0 1 -9212191261079457524 0 0 1 0 1 0 +-9222995085389227671 0 0 0 0 0 1 -9210944775923669427 0 1 1 1 0 0 +-9222995085389227671 0 0 0 0 0 1 -9202601338732071287 0 1 1 1 1 1 +-9222995085389227671 0 0 0 0 0 1 -9185234005543064629 0 1 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9181371158682533758 0 1 1 1 0 0 +-9222995085389227671 0 0 0 0 0 1 -9173688614647112626 0 0 1 0 0 1 +-9222995085389227671 0 0 0 0 0 1 -9130964659455442087 0 1 1 1 1 1 +-9222995085389227671 0 0 0 0 0 1 -9109186410997511387 0 1 1 0 1 1 +-9222995085389227671 0 0 0 0 0 1 -9071619989214493680 0 1 1 0 1 0 +-9222995085389227671 0 0 0 0 0 1 -9071253360041153133 0 1 1 1 0 1 QUERY id: 0 PROJECTION COLUMNS count() UInt64 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index af8a4dd8e73..80d8d1718b2 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -96,7 +96,16 @@ EXPLAIN QUERY TREE dump_ast = 1 SELECT count() FROM x WHERE A OR (B AND ((C AND DROP TABLE IF EXISTS y; CREATE TABLE y (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) ENGINE = MergeTree ORDER BY x; -INSERT INTO y SELECT x, A%2 AS A, B%2 AS B, C%2 AS C, D%2 AS D, E%2 AS E, F%2 AS F FROM generateRandom('x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8', 43) LIMIT 2000; +INSERT INTO y + SELECT + murmurHash3_64(number) AS x, + murmurHash3_64(number + 1) % 2 AS A, + murmurHash3_64(number + 2) % 2 AS B, + murmurHash3_64(number + 3) % 2 AS C, + murmurHash3_64(number + 4) % 2 AS D, + murmurHash3_64(number + 5) % 2 AS E, + murmurHash3_64(number + 6) % 2 AS F + FROM numbers(2000); -- JOIN expressions -- As the optimization code is shared between ON and WHERE, it is enough to test that the optimization is done also in ON From 8c6c7f5f7da3b2631769b6867d310997fd061a38 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 3 Dec 2024 15:46:29 +0100 Subject: [PATCH 284/502] Remove the trivial copy constructor in IAST::FormatSettings. --- src/Parsers/IAST.h | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 851ae766419..0d921e317a8 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -224,19 +224,6 @@ public: { } - FormatSettings(const FormatSettings & other) - : one_line(other.one_line) - , hilite(other.hilite) - , identifier_quoting_rule(other.identifier_quoting_rule) - , identifier_quoting_style(other.identifier_quoting_style) - , show_secrets(other.show_secrets) - , nl_or_ws(other.nl_or_ws) - , literal_escaping_style(other.literal_escaping_style) - , print_pretty_type_names(other.print_pretty_type_names) - , enforce_strict_identifier_format(other.enforce_strict_identifier_format) - { - } - void writeIdentifier(WriteBuffer & ostr, const String & name, bool ambiguous) const; void checkIdentifier(const String & name) const; }; From 6cc668bd88cb12685c30705ac623cbd1d85465f5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 3 Dec 2024 15:00:05 +0000 Subject: [PATCH 285/502] Add serializetion to some query plan steps. --- programs/server/Server.cpp | 3 + src/Analyzer/Resolve/IdentifierResolver.cpp | 2 +- src/Analyzer/Resolve/IdentifierResolver.h | 3 +- src/Analyzer/Resolve/QueryAnalyzer.cpp | 7 +- src/Analyzer/SetUtils.cpp | 13 +- src/Analyzer/SetUtils.h | 2 +- src/Columns/ColumnFunction.h | 3 + src/Core/BaseSettings.h | 43 ++ src/Core/Field.h | 4 + src/Core/SortDescription.cpp | 59 +++ src/Core/SortDescription.h | 7 + src/Functions/FunctionsMiscellaneous.h | 150 +++--- src/Interpreters/ActionsDAG.cpp | 434 ++++++++++++++++++ src/Interpreters/ActionsDAG.h | 6 + src/Interpreters/ActionsVisitor.cpp | 48 +- src/Interpreters/AggregateDescription.cpp | 81 ++++ src/Interpreters/AggregateDescription.h | 4 + src/Interpreters/Aggregator.cpp | 64 +-- src/Interpreters/Aggregator.h | 18 +- src/Interpreters/InterpreterSelectQuery.cpp | 34 +- src/Interpreters/PreparedSets.cpp | 84 ++-- src/Interpreters/PreparedSets.h | 30 +- src/Interpreters/SetSerialization.h | 32 ++ src/Planner/Planner.cpp | 29 +- src/Planner/PlannerActionsVisitor.cpp | 8 +- src/Processors/QueryPlan/AggregatingStep.cpp | 219 ++++++++- src/Processors/QueryPlan/AggregatingStep.h | 11 +- src/Processors/QueryPlan/ArrayJoinStep.cpp | 49 ++ src/Processors/QueryPlan/ArrayJoinStep.h | 5 + .../QueryPlan/BuildQueryPipelineSettings.cpp | 5 + .../QueryPlan/BuildQueryPipelineSettings.h | 3 + src/Processors/QueryPlan/CreatingSetsStep.h | 2 + src/Processors/QueryPlan/DistinctStep.cpp | 68 +++ src/Processors/QueryPlan/DistinctStep.h | 10 + src/Processors/QueryPlan/ExpressionStep.cpp | 26 ++ src/Processors/QueryPlan/ExpressionStep.h | 3 + src/Processors/QueryPlan/ExtremesStep.cpp | 17 + src/Processors/QueryPlan/ExtremesStep.h | 3 + src/Processors/QueryPlan/FilterStep.cpp | 43 ++ src/Processors/QueryPlan/FilterStep.h | 4 + src/Processors/QueryPlan/IQueryPlanStep.cpp | 8 + src/Processors/QueryPlan/IQueryPlanStep.h | 8 + src/Processors/QueryPlan/LimitByStep.cpp | 39 +- src/Processors/QueryPlan/LimitByStep.h | 6 +- src/Processors/QueryPlan/LimitStep.cpp | 45 ++ src/Processors/QueryPlan/LimitStep.h | 4 + src/Processors/QueryPlan/OffsetStep.cpp | 20 + src/Processors/QueryPlan/OffsetStep.h | 4 + .../QueryPlan/Optimizations/limitPushDown.cpp | 7 + .../QueryPlanSerializationSettings.cpp | 8 + .../QueryPlanSerializationSettings.h | 59 +++ .../QueryPlan/QueryPlanStepRegistry.cpp | 70 +++ .../QueryPlan/QueryPlanStepRegistry.h | 32 ++ .../QueryPlan/ReadFromMergeTree.cpp | 3 +- src/Processors/QueryPlan/Serialization.h | 30 ++ src/Processors/QueryPlan/SortingStep.cpp | 80 ++++ src/Processors/QueryPlan/SortingStep.h | 10 +- .../QueryPlan/SourceStepWithFilter.h | 3 + src/Processors/QueryPlan/TotalsHavingStep.cpp | 80 +++- src/Processors/QueryPlan/TotalsHavingStep.h | 9 +- src/Processors/QueryPlan/UnionStep.cpp | 23 +- src/Processors/QueryPlan/UnionStep.h | 5 +- .../TTL/TTLAggregationAlgorithm.cpp | 18 +- src/QueryPipeline/SizeLimits.cpp | 29 ++ src/QueryPipeline/SizeLimits.h | 6 + 65 files changed, 2030 insertions(+), 212 deletions(-) create mode 100644 src/Interpreters/SetSerialization.h create mode 100644 src/Processors/QueryPlan/QueryPlanSerializationSettings.cpp create mode 100644 src/Processors/QueryPlan/QueryPlanSerializationSettings.h create mode 100644 src/Processors/QueryPlan/QueryPlanStepRegistry.cpp create mode 100644 src/Processors/QueryPlan/QueryPlanStepRegistry.h create mode 100644 src/Processors/QueryPlan/Serialization.h diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 7d98178e096..6b5aa1ac0b2 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -924,6 +925,8 @@ try registerRemoteFileMetadatas(); registerSchedulerNodes(); + QueryPlanStepRegistry::registerPlanSteps(); + CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision()); CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index 317a02a60f2..a59867091bd 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -393,7 +393,7 @@ QueryTreeNodePtr IdentifierResolver::wrapExpressionNodeInTupleElement(QueryTreeN /// Resolve identifier functions implementation /// Try resolve table identifier from database catalog -QueryTreeNodePtr IdentifierResolver::tryResolveTableIdentifierFromDatabaseCatalog(const Identifier & table_identifier, ContextPtr context) +std::shared_ptr IdentifierResolver::tryResolveTableIdentifierFromDatabaseCatalog(const Identifier & table_identifier, ContextPtr context) { size_t parts_size = table_identifier.getPartsSize(); if (parts_size < 1 || parts_size > 2) diff --git a/src/Analyzer/Resolve/IdentifierResolver.h b/src/Analyzer/Resolve/IdentifierResolver.h index cdbd7610b5e..7402c69db72 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.h +++ b/src/Analyzer/Resolve/IdentifierResolver.h @@ -21,6 +21,7 @@ class QueryExpressionsAliasVisitor ; class QueryNode; class JoinNode; class ColumnNode; +class TableNode; using ProjectionName = String; using ProjectionNames = std::vector; @@ -86,7 +87,7 @@ public: /// Resolve identifier functions - static QueryTreeNodePtr tryResolveTableIdentifierFromDatabaseCatalog(const Identifier & table_identifier, ContextPtr context); + static std::shared_ptr tryResolveTableIdentifierFromDatabaseCatalog(const Identifier & table_identifier, ContextPtr context); QueryTreeNodePtr tryResolveIdentifierFromCompoundExpression(const Identifier & expression_identifier, size_t identifier_bind_size, diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index d118cb281ae..f9b62618e9a 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -3470,11 +3470,8 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi auto set = std::make_shared(size_limits_for_set, 0, settings[Setting::transform_null_in]); - set->setHeader(result_block.cloneEmpty().getColumnsWithTypeAndName()); - set->insertFromBlock(result_block.getColumnsWithTypeAndName()); - set->finishInsert(); - - auto future_set = std::make_shared(std::move(set)); + auto hash = function_arguments[1]->getTreeHash(); + auto future_set = std::make_shared(hash, std::move(result_block), settings[Setting::transform_null_in], size_limits_for_set); /// Create constant set column for constant folding diff --git a/src/Analyzer/SetUtils.cpp b/src/Analyzer/SetUtils.cpp index 59a243b27f3..5ffa4f58307 100644 --- a/src/Analyzer/SetUtils.cpp +++ b/src/Analyzer/SetUtils.cpp @@ -62,7 +62,7 @@ size_t getCompoundTypeDepth(const IDataType & type) } template -Block createBlockFromCollection(const Collection & collection, const DataTypes& value_types, const DataTypes & block_types, bool transform_null_in) +ColumnsWithTypeAndName createBlockFromCollection(const Collection & collection, const DataTypes& value_types, const DataTypes & block_types, bool transform_null_in) { assert(collection.size() == value_types.size()); size_t columns_size = block_types.size(); @@ -132,16 +132,19 @@ Block createBlockFromCollection(const Collection & collection, const DataTypes& columns[i]->insert(tuple_values[i]); } - Block res; + ColumnsWithTypeAndName res(columns_size); for (size_t i = 0; i < columns_size; ++i) - res.insert(ColumnWithTypeAndName{std::move(columns[i]), block_types[i], "argument_" + toString(i)}); + { + res[i].type = block_types[i]; + res[i].column = std::move(columns[i]); + } return res; } } -Block getSetElementsForConstantValue(const DataTypePtr & expression_type, const Field & value, const DataTypePtr & value_type, bool transform_null_in) +ColumnsWithTypeAndName getSetElementsForConstantValue(const DataTypePtr & expression_type, const Field & value, const DataTypePtr & value_type, bool transform_null_in) { DataTypes set_element_types = {expression_type}; const auto * lhs_tuple_type = typeid_cast(expression_type.get()); @@ -158,7 +161,7 @@ Block getSetElementsForConstantValue(const DataTypePtr & expression_type, const size_t lhs_type_depth = getCompoundTypeDepth(*expression_type); size_t rhs_type_depth = getCompoundTypeDepth(*value_type); - Block result_block; + ColumnsWithTypeAndName result_block; if (lhs_type_depth == rhs_type_depth) { diff --git a/src/Analyzer/SetUtils.h b/src/Analyzer/SetUtils.h index aef906a6576..17f9721b7d9 100644 --- a/src/Analyzer/SetUtils.h +++ b/src/Analyzer/SetUtils.h @@ -19,6 +19,6 @@ using SetPtr = std::shared_ptr; * Example: SELECT id FROM test_table WHERE id IN (1, 2, 3, 4); * Example: SELECT id FROM test_table WHERE id IN ((1, 2), (3, 4)); */ -Block getSetElementsForConstantValue(const DataTypePtr & expression_type, const Field & value, const DataTypePtr & value_type, bool transform_null_in); +ColumnsWithTypeAndName getSetElementsForConstantValue(const DataTypePtr & expression_type, const Field & value, const DataTypePtr & value_type, bool transform_null_in); } diff --git a/src/Columns/ColumnFunction.h b/src/Columns/ColumnFunction.h index 8df9e23c0e8..a0551a9847a 100644 --- a/src/Columns/ColumnFunction.h +++ b/src/Columns/ColumnFunction.h @@ -198,6 +198,9 @@ public: /// Create copy of this column, but with recursively_convert_result_to_full_column_if_low_cardinality = true ColumnPtr recursivelyConvertResultToFullColumnIfLowCardinality() const; + const FunctionBasePtr & getFunction() const { return function; } + const ColumnsWithTypeAndName & getCapturedColumns() const { return captured_columns; } + private: size_t elements_size; FunctionBasePtr function; diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index 201b586f067..04f80210443 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -145,6 +145,9 @@ public: void write(WriteBuffer & out, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT) const; void read(ReadBuffer & in, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT); + void writeChangedBinary(WriteBuffer & out) const; + void readBinary(ReadBuffer & in); + // A debugging aid. std::string toString() const; @@ -479,6 +482,46 @@ void BaseSettings::write(WriteBuffer & out, SettingsWriteFormat format) BaseSettingsHelpers::writeString(std::string_view{}, out); } +template +void BaseSettings::writeChangedBinary(WriteBuffer & out) const +{ + const auto & accessor = Traits::Accessor::instance(); + + size_t num_settings = 0; + for (auto it = this->begin(); it != this->end(); ++it) + ++num_settings; + + writeVarUInt(num_settings, out); + + for (const auto & field : *this) + { + BaseSettingsHelpers::writeString(field.getName(), out); + using Flags = BaseSettingsHelpers::Flags; + Flags flags{0}; + BaseSettingsHelpers::writeFlags(flags, out); + accessor.writeBinary(*this, field.index, out); + } +} + +template +void BaseSettings::readBinary(ReadBuffer & in) +{ + const auto & accessor = Traits::Accessor::instance(); + + size_t num_settings = 0; + readVarUInt(num_settings, in); + + for (size_t i = 0; i < num_settings; ++i) + { + String read_name = BaseSettingsHelpers::readString(in); + std::string_view name = TTraits::resolveName(read_name); + size_t index = accessor.find(name); + + std::ignore = BaseSettingsHelpers::readFlags(in); + accessor.readBinary(*this, index, in); + } +} + template void BaseSettings::read(ReadBuffer & in, SettingsWriteFormat format) { diff --git a/src/Core/Field.h b/src/Core/Field.h index 5a6ee9cdf29..b7f7225dae1 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -1004,6 +1004,10 @@ void readQuoted(DecimalField & x, ReadBuffer & buf); void writeFieldText(const Field & x, WriteBuffer & buf); + +void writeFieldBinary(const Field & x, WriteBuffer & buf); +Field readFieldBinary(ReadBuffer & buf); + String toString(const Field & x); } diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index 1b3f81f8547..01f22b9b9e5 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -15,6 +15,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + void dumpSortDescription(const SortDescription & description, WriteBuffer & out) { bool first = true; @@ -209,4 +214,58 @@ JSONBuilder::ItemPtr explainSortDescription(const SortDescription & description) return json_array; } +void serializeSortDescription(const SortDescription & sort_description, WriteBuffer & out) +{ + writeVarUInt(sort_description.size(), out); + for (const auto & desc : sort_description) + { + writeStringBinary(desc.column_name, out); + + UInt8 flags = 0; + if (desc.direction > 0) + flags |= 1; + if (desc.nulls_direction > 0) + flags |= 2; + if (desc.collator) + flags |= 4; + if (desc.with_fill) + flags |= 8; + + writeIntBinary(flags, out); + + if (desc.collator) + writeStringBinary(desc.collator->getLocale(), out); + + if (desc.with_fill) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "WITH FILL is not supported in serialized sort description"); + } +} + +void deserializeSortDescription(SortDescription & sort_description, ReadBuffer & in) +{ + size_t size = 0; + readVarUInt(size, in); + sort_description.resize(size); + for (auto & desc : sort_description) + { + readStringBinary(desc.column_name, in); + UInt8 flags = 0; + readIntBinary(flags, in); + + desc.direction = (flags & 1) ? 1 : -1; + desc.nulls_direction = (flags & 2) ? 1 : -1; + + if (flags & 4) + { + String collator_locale; + readStringBinary(collator_locale, in); + if (!collator_locale.empty()) + desc.collator = std::make_shared(collator_locale); + } + + if (flags & 8) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "WITH FILL is not supported in deserialized sort description"); + } +} + } diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index 522732dad90..48a87e9e2e9 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -143,4 +143,11 @@ void dumpSortDescription(const SortDescription & description, WriteBuffer & out) std::string dumpSortDescription(const SortDescription & description); JSONBuilder::ItemPtr explainSortDescription(const SortDescription & description); + +class WriteBuffer; +class ReadBuffer; + +void serializeSortDescription(const SortDescription & sort_description, WriteBuffer & out); +void deserializeSortDescription(SortDescription & sort_description, ReadBuffer & in); + } diff --git a/src/Functions/FunctionsMiscellaneous.h b/src/Functions/FunctionsMiscellaneous.h index cea11cfe677..93ce7660f45 100644 --- a/src/Functions/FunctionsMiscellaneous.h +++ b/src/Functions/FunctionsMiscellaneous.h @@ -1,13 +1,13 @@ #pragma once -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include namespace DB @@ -18,6 +18,18 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +struct LambdaCapture +{ + Names captured_names; + DataTypes captured_types; + NamesAndTypesList lambda_arguments; + String return_name; + DataTypePtr return_type; + bool allow_constant_folding; +}; + +using LambdaCapturePtr = std::shared_ptr; + class ExecutableFunctionExpression : public IExecutableFunction { public: @@ -30,14 +42,20 @@ public: using SignaturePtr = std::shared_ptr; ExecutableFunctionExpression(ExpressionActionsPtr expression_actions_, SignaturePtr signature_) - : expression_actions(std::move(expression_actions_)) - , signature(std::move(signature_)) - {} + : expression_actions(std::move(expression_actions_)), signature(std::move(signature_)) + { + } String getName() const override { return "FunctionExpression"; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { + if (input_rows_count == 0) + return result_type->createColumn(); + + if (!expression_actions) + throw Exception(ErrorCodes::LOGICAL_ERROR, "No actions were passed to FunctionExpression"); + DB::Block expr_columns; for (size_t i = 0; i < arguments.size(); ++i) { @@ -48,7 +66,7 @@ public: expression_actions->execute(expr_columns); - return expr_columns.getByName(signature->return_name).column; + return expr_columns.getByName(signature->return_name).column; } bool useDefaultImplementationForNulls() const override { return false; } @@ -71,13 +89,27 @@ public: using Signature = ExecutableFunctionExpression::Signature; using SignaturePtr = ExecutableFunctionExpression::SignaturePtr; - FunctionExpression(ExpressionActionsPtr expression_actions_, - DataTypes argument_types_, const Names & argument_names_, - DataTypePtr return_type_, const std::string & return_name_) - : expression_actions(std::move(expression_actions_)) - , signature(std::make_shared(Signature{argument_names_, return_name_})) - , argument_types(std::move(argument_types_)), return_type(std::move(return_type_)) + FunctionExpression(LambdaCapturePtr capture_, ExpressionActionsPtr expression_actions_) + : expression_actions(std::move(expression_actions_)) + , capture(std::move(capture_)) { + Names names; + DataTypes types; + + names.reserve(capture->captured_names.size() + capture->lambda_arguments.size()); + names.insert(names.end(), capture->captured_names.begin(), capture->captured_names.end()); + + types.reserve(capture->captured_types.size() + capture->lambda_arguments.size()); + types.insert(types.end(), capture->captured_types.begin(), capture->captured_types.end()); + + for (const auto & lambda_argument : capture->lambda_arguments) + { + names.push_back(lambda_argument.name); + types.push_back(lambda_argument.type); + } + + argument_types = std::move(types); + signature = std::make_shared(Signature{names, capture->return_name}); } String getName() const override { return "FunctionExpression"; } @@ -85,7 +117,10 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } const DataTypes & getArgumentTypes() const override { return argument_types; } - const DataTypePtr & getResultType() const override { return return_type; } + const DataTypePtr & getResultType() const override { return capture->return_type; } + + const LambdaCapture & getCapture() const { return *capture; } + const ActionsDAG & getAcionsDAG() const { return expression_actions->getActionsDAG(); } ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName &) const override { @@ -94,9 +129,11 @@ public: private: ExpressionActionsPtr expression_actions; + LambdaCapturePtr capture; + + /// This is redundant and is built from capture. SignaturePtr signature; DataTypes argument_types; - DataTypePtr return_type; }; /// Captures columns which are used by lambda function but not in argument list. @@ -106,20 +143,10 @@ private: class ExecutableFunctionCapture : public IExecutableFunction { public: - struct Capture + ExecutableFunctionCapture(ExpressionActionsPtr expression_actions_, LambdaCapturePtr capture_) + : expression_actions(std::move(expression_actions_)), capture(std::move(capture_)) { - Names captured_names; - DataTypes captured_types; - NamesAndTypesList lambda_arguments; - String return_name; - DataTypePtr return_type; - bool allow_constant_folding; - }; - - using CapturePtr = std::shared_ptr; - - ExecutableFunctionCapture(ExpressionActionsPtr expression_actions_, CapturePtr capture_) - : expression_actions(std::move(expression_actions_)), capture(std::move(capture_)) {} + } String getName() const override { return "FunctionCapture"; } @@ -148,8 +175,7 @@ public: types.push_back(lambda_argument.type); } - auto function = std::make_unique(expression_actions, types, names, - capture->return_type, capture->return_name); + auto function = std::make_unique(capture, expression_actions); /// If all the captured arguments are constant, let's also return ColumnConst (with ColumnFunction inside it). /// Consequently, it allows to treat higher order functions with constant arrays and constant captured columns @@ -175,17 +201,15 @@ public: private: ExpressionActionsPtr expression_actions; - CapturePtr capture; + LambdaCapturePtr capture; }; class FunctionCapture : public IFunctionBase { public: - using CapturePtr = ExecutableFunctionCapture::CapturePtr; - FunctionCapture( ExpressionActionsPtr expression_actions_, - CapturePtr capture_, + LambdaCapturePtr capture_, DataTypePtr return_type_, String name_) : expression_actions(std::move(expression_actions_)) @@ -207,9 +231,12 @@ public: return std::make_unique(expression_actions, capture); } + const LambdaCapture & getCapture() const { return *capture; } + const ActionsDAG & getAcionsDAG() const { return expression_actions->getActionsDAG(); } + private: ExpressionActionsPtr expression_actions; - CapturePtr capture; + LambdaCapturePtr capture; DataTypePtr return_type; String name; }; @@ -217,28 +244,23 @@ private: class FunctionCaptureOverloadResolver : public IFunctionOverloadResolver { public: - using Capture = ExecutableFunctionCapture::Capture; - using CapturePtr = ExecutableFunctionCapture::CapturePtr; - FunctionCaptureOverloadResolver( - ExpressionActionsPtr expression_actions_, - const Names & captured_names, - const NamesAndTypesList & lambda_arguments, - const DataTypePtr & function_return_type, - const String & expression_return_name, - bool allow_constant_folding) - : expression_actions(std::move(expression_actions_)) + ActionsDAG actions_dag, + const ExpressionActionsSettings & actions_settings, + const Names & captured_names, + const NamesAndTypesList & lambda_arguments, + const DataTypePtr & function_return_type, + const String & expression_return_name, + bool allow_constant_folding) { /// Check that expression does not contain unusual actions that will break columns structure. - for (const auto & action : expression_actions->getActions()) - if (action.node->type == ActionsDAG::ActionType::ARRAY_JOIN) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expression with arrayJoin or other unusual action cannot be captured"); + if (actions_dag.hasArrayJoin()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expression with arrayJoin or other unusual action cannot be captured"); std::unordered_map arguments_map; - const auto & all_arguments = expression_actions->getRequiredColumnsWithTypes(); - for (const auto & arg : all_arguments) - arguments_map[arg.name] = arg.type; + for (const auto * input : actions_dag.getInputs()) + arguments_map[input->result_name] = input->result_type; DataTypes captured_types; captured_types.reserve(captured_names.size()); @@ -263,14 +285,16 @@ public: name = "Capture[" + toString(captured_types) + "](" + toString(argument_types) + ") -> " + function_return_type->getName(); - capture = std::make_shared(Capture{ - .captured_names = captured_names, - .captured_types = std::move(captured_types), - .lambda_arguments = lambda_arguments, - .return_name = expression_return_name, - .return_type = function_return_type, - .allow_constant_folding = allow_constant_folding, + capture = std::make_shared(LambdaCapture{ + .captured_names = captured_names, + .captured_types = std::move(captured_types), + .lambda_arguments = lambda_arguments, + .return_name = expression_return_name, + .return_type = function_return_type, + .allow_constant_folding = allow_constant_folding, }); + + expression_actions = std::make_shared(std::move(actions_dag), actions_settings); } String getName() const override { return name; } @@ -288,7 +312,7 @@ public: private: ExpressionActionsPtr expression_actions; - CapturePtr capture; + LambdaCapturePtr capture; DataTypePtr return_type; String name; diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 8d95b6520c1..2ef874fb84a 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -3,15 +3,21 @@ #include #include #include +#include +#include +#include #include +#include #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -20,6 +26,7 @@ #include #include #include +#include #include #include @@ -38,6 +45,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int NOT_FOUND_COLUMN_IN_BLOCK; extern const int BAD_ARGUMENTS; + extern const int INCORRECT_DATA; } namespace @@ -3205,4 +3213,430 @@ const ActionsDAG::Node * FindOriginalNodeForOutputName::find(const String & outp return it->second; } + +static void serializeCapture(const LambdaCapture & capture, WriteBuffer & out) +{ + writeStringBinary(capture.return_name, out); + encodeDataType(capture.return_type, out); + + writeVarUInt(capture.captured_names.size(), out); + for (const auto & name : capture.captured_names) + writeStringBinary(name, out); + + writeVarUInt(capture.captured_types.size(), out); + for (const auto & type : capture.captured_types) + encodeDataType(type, out); + + writeVarUInt(capture.lambda_arguments.size(), out); + for (const auto & item : capture.lambda_arguments) + { + writeStringBinary(item.name, out); + encodeDataType(item.type, out); + } +} + +static void deserializeCapture(LambdaCapture & capture, ReadBuffer & in) +{ + readStringBinary(capture.return_name, in); + capture.return_type = decodeDataType(in); + + UInt64 num_names; + readVarUInt(num_names, in); + capture.captured_names.resize(num_names); + for (auto & name : capture.captured_names) + readStringBinary(name, in); + + UInt64 num_types; + readVarUInt(num_types, in); + capture.captured_types.resize(num_types); + for (auto & type : capture.captured_types) + type = decodeDataType(in); + + UInt64 num_args; + readVarUInt(num_args, in); + capture.lambda_arguments.clear(); + for (size_t i = 0; i < num_args; ++i) + { + NameAndTypePair name_and_type; + readStringBinary(name_and_type.name, in); + name_and_type.type = decodeDataType(in); + capture.lambda_arguments.push_back(std::move(name_and_type)); + } +} + +static void serialzieConstant( + const IDataType & type, + const IColumn & value, + WriteBuffer & out, + SerializedSetsRegistry & registry) +{ + if (WhichDataType(type).isSet()) + { + const IColumn * maybe_set = &value; + if (const auto * column_const = typeid_cast(maybe_set)) + maybe_set = &column_const->getDataColumn(); + + const auto * column_set = typeid_cast(maybe_set); + if (!column_set) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "ColumnSet is expected for DataTypeSet. Got {}", value.getName()); + + auto hash = column_set->getData()->getHash(); + writeBinary(hash, out); + registry.sets.emplace(hash, column_set->getData()); + return; + } + + if (WhichDataType(type).isFunction()) + { + const IColumn * maybe_function = &value; + if (const auto * column_const = typeid_cast(maybe_function)) + maybe_function = &column_const->getDataColumn(); + + const auto * column_function = typeid_cast(maybe_function); + if (!column_function) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "ColumnSet is expected for DataTypeSet. Got {}", value.getName()); + + auto function = column_function->getFunction(); + const auto * function_expression = typeid_cast(function.get()); + if (!function_expression) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected FunctionExpression for ColumnFunction. Got {}", function->getName()); + + serializeCapture(function_expression->getCapture(), out); + function_expression->getAcionsDAG().serialize(out, registry); + + const auto & captured_columns = column_function->getCapturedColumns(); + writeVarUInt(captured_columns.size(), out); + for (const auto & captured_column : captured_columns) + { + encodeDataType(captured_column.type, out); + serialzieConstant(*captured_column.type, *captured_column.column, out, registry); + } + + return; + } + + const auto * const_column = typeid_cast(&value); + if (!const_column) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cannot serialize non-constant column {}", value.getName()); + + const auto & data = const_column->getDataColumn(); + type.getDefaultSerialization()->serializeBinary(data, 0, out, FormatSettings{}); +} + +static MutableColumnPtr deserializeConstant( + const IDataType & type, + ReadBuffer & in, + DeserializedSetsRegistry & registry, + const ContextPtr & context) +{ + if (WhichDataType(type).isSet()) + { + FutureSet::Hash hash; + readBinary(hash, in); + + auto column_set = ColumnSet::create(0, nullptr); + registry.sets[hash].push_back(column_set.get()); + + return column_set; + } + + if (WhichDataType(type).isFunction()) + { + LambdaCapture capture; + deserializeCapture(capture, in); + auto capture_dag = ActionsDAG::deserialize(in, registry, context); + + UInt64 num_captured_columns; + readVarUInt(num_captured_columns, in); + ColumnsWithTypeAndName captured_columns(num_captured_columns); + + for (auto & captured_column : captured_columns) + { + captured_column.type = decodeDataType(in); + captured_column.column = deserializeConstant(*captured_column.type, in, registry, context); + } + + auto function_expression = std::make_shared( + std::make_shared(std::move(capture)), + std::make_shared( + std::move(capture_dag), + ExpressionActionsSettings::fromContext(context, CompileExpressions::yes))); + + return ColumnFunction::create(1, std::move(function_expression), std::move(captured_columns)); + } + + auto column = type.createColumn(); + type.getDefaultSerialization()->deserializeBinary(*column, in, FormatSettings{}); + return ColumnConst::create(std::move(column), 0); +} + +void ActionsDAG::serialize(WriteBuffer & out, SerializedSetsRegistry & registry) const +{ + size_t nodes_size = nodes.size(); + writeVarUInt(nodes_size, out); + + std::unordered_map node_to_id; + for (const auto & node : nodes) + node_to_id.emplace(&node, node_to_id.size()); + + if (nodes_size != node_to_id.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Duplicate nodes in ActionsDAG"); + + for (const auto & node : nodes) + { + writeIntBinary(static_cast(node.type), out); + writeStringBinary(node.result_name, out); + encodeDataType(node.result_type, out); + + writeVarUInt(node.children.size(), out); + for (const auto * child : node.children) + writeVarUInt(node_to_id.at(child), out); + + /// Serialize column if it is present + const bool has_column = node.column != nullptr; + UInt8 column_flags = 0; + if (has_column) + { + column_flags |= 1; + if (node.is_deterministic_constant) + column_flags |= 2; + } + + const auto * function_capture = typeid_cast(node.function_base.get()); + if (function_capture) + column_flags |= 4; + + writeIntBinary(column_flags, out); + + if (has_column) + serialzieConstant(*node.result_type, *node.column, out, registry); + + if (node.type == ActionType::INPUT) + { + /// nothing to serialize + } + else if (node.type == ActionType::COLUMN) + { + /// nothing to serialize, column is already serialized + } + else if (node.type == ActionType::ALIAS) + { + /// nothing to serialize + } + else if (node.type == ActionType::FUNCTION) + { + writeStringBinary(node.function_base->getName(), out); + if (function_capture) + { + serializeCapture(function_capture->getCapture(), out); + function_capture->getAcionsDAG().serialize(out, registry); + } + } + else if (node.type == ActionType::ARRAY_JOIN) + { + /// nothing to serialize + } + else + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown node type {}", static_cast(node.type)); + } + } + + writeVarUInt(inputs.size(), out); + for (const auto * input : inputs) + writeVarUInt(node_to_id.at(input), out); + + writeVarUInt(outputs.size(), out); + for (const auto * output : outputs) + writeVarUInt(node_to_id.at(output), out); +} + +ActionsDAG ActionsDAG::deserialize(ReadBuffer & in, DeserializedSetsRegistry & registry, const ContextPtr & context) +{ + size_t nodes_size; + readVarUInt(nodes_size, in); + + std::list nodes; + std::unordered_map id_to_node; + for (size_t i = 0; i < nodes_size; ++i) + id_to_node.emplace(i, &nodes.emplace_back(Node{})); + + for (size_t i = 0; i < nodes_size; ++i) + { + Node & node = *id_to_node.at(i); + + UInt8 action_type{0}; + readIntBinary(action_type, in); + if (action_type > static_cast(ActionType::FUNCTION)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown action type {}", size_t(action_type)); + node.type = static_cast(action_type); + + readStringBinary(node.result_name, in); + node.result_type = decodeDataType(in); + + size_t children_size; + readVarUInt(children_size, in); + for (size_t j = 0; j < children_size; ++j) + { + size_t child_id; + readVarUInt(child_id, in); + node.children.push_back(id_to_node.at(child_id)); + } + + UInt8 column_flags = 0; + readIntBinary(column_flags, in); + + /// Deserialize column if it is present + if (column_flags & 1) + { + if ((column_flags & 2) == 0) + node.is_deterministic_constant = false; + + node.column = deserializeConstant(*node.result_type, in, registry, context); + } + + if (node.type == ActionType::INPUT) + { + /// nothing to deserialize + if (!node.children.empty()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Deserialized input can't have children"); + } + else if (node.type == ActionType::COLUMN) + { + /// Column is already deserialized + if (!node.children.empty()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Deserialized column can't have children"); + } + else if (node.type == ActionType::ALIAS) + { + /// nothing to deserialize + if (node.children.size() != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "Deserialized alias must have one children"); + } + else if (node.type == ActionType::FUNCTION) + { + String function_name; + readStringBinary(function_name, in); + + ColumnsWithTypeAndName arguments; + arguments.reserve(node.children.size()); + for (const auto * child : node.children) + { + ColumnWithTypeAndName argument; + argument.column = child->column; + argument.type = child->result_type; + argument.name = child->result_name; + + arguments.emplace_back(std::move(argument)); + } + + if (column_flags & 4) + { + LambdaCapture capture; + deserializeCapture(capture, in); + auto capture_dag = ActionsDAG::deserialize(in, registry, context); + + node.function_base = std::make_shared( + std::make_shared( + std::move(capture_dag), + ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)), + std::make_shared(std::move(capture)), + node.result_type, + function_name); + + node.function = node.function_base->prepare(arguments); + node.is_function_compiled = false; + } + else + { + auto function = FunctionFactory::instance().get(function_name, context); + + node.function_base = function->build(arguments); + node.function = node.function_base->prepare(arguments); + node.is_function_compiled = false; + + auto lhs_type = node.result_type; + if (const auto * lhs_tuple = typeid_cast(lhs_type.get())) + lhs_type = std::make_shared(lhs_tuple->getElements()); + + auto rhs_type = node.function_base->getResultType(); + if (const auto * rhs_tuple = typeid_cast(rhs_type.get())) + rhs_type = std::make_shared(rhs_tuple->getElements()); + + if (!lhs_type->equals(*lhs_type)) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Deserialized function {} has invalid type. Expected {}, deserialized {}.", + function_name, + rhs_type->getName(), + lhs_type->getName()); + } + } + else if (node.type == ActionType::ARRAY_JOIN) + { + /// nothing to deserialize + if (node.children.size() != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "Deserialized array join must have one children"); + } + else + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown node type {}", static_cast(node.type)); + } + } + + size_t inputs_size; + readVarUInt(inputs_size, in); + std::vector inputs; + std::unordered_set inputs_set; + inputs.reserve(inputs_size); + for (size_t i = 0; i < inputs_size; ++i) + { + size_t input_id; + readVarUInt(input_id, in); + const auto * input = id_to_node.at(input_id); + + if (input->type != ActionType::INPUT) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Deserialized input {} has type {}", + input->result_name, magic_enum::enum_name(input->type)); + + if (!inputs_set.emplace(input).second) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Duplicate input {}", input->result_name); + + inputs.push_back(input); + } + + size_t outputs_size; + readVarUInt(outputs_size, in); + std::vector outputs; + outputs.reserve(outputs_size); + for (size_t i = 0; i < outputs_size; ++i) + { + size_t output_id; + readVarUInt(output_id, in); + outputs.push_back(id_to_node.at(output_id)); + } + + for (const auto & node : nodes) + if (node.type == ActionType::INPUT && !inputs_set.contains(&node)) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Deserialized input {} is not in the inputs list", + node.result_name); + + ActionsDAG dag; + dag.nodes = std::move(nodes); + dag.inputs = std::move(inputs); + dag.outputs = std::move(outputs); + + return dag; +} + } diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 7e88f77a835..bc9b1c50db0 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -35,6 +35,9 @@ namespace JSONBuilder class SortDescription; +struct SerializedSetsRegistry; +struct DeserializedSetsRegistry; + /// Directed acyclic graph of expressions. /// This is an intermediate representation of actions which is usually built from expression list AST. /// Node of DAG describe calculation of a single column with known type, name, and constant value (if applicable). @@ -130,6 +133,9 @@ public: std::string dumpNames() const; std::string dumpDAG() const; + void serialize(WriteBuffer & out, SerializedSetsRegistry & registry) const; + static ActionsDAG deserialize(ReadBuffer & in, DeserializedSetsRegistry & registry, const ContextPtr & context); + const Node & addInput(std::string name, DataTypePtr type); const Node & addInput(ColumnWithTypeAndName column); const Node & addColumn(ColumnWithTypeAndName column); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 696021b418c..f1cbcef83c1 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -111,7 +111,7 @@ static size_t getTypeDepth(const DataTypePtr & type) /// 33.33 in the set is converted to 33.3, but it is not equal to 33.3 in the column, so the result should still be empty. /// We can not include values that don't represent any possible value from the type of filtered column to the set. template -static Block createBlockFromCollection(const Collection & collection, const DataTypes & value_types, const DataTypes & types, bool transform_null_in) +static ColumnsWithTypeAndName createBlockFromCollection(const Collection & collection, const DataTypes & value_types, const DataTypes & types, bool transform_null_in) { size_t columns_num = types.size(); MutableColumns columns(columns_num); @@ -169,9 +169,13 @@ static Block createBlockFromCollection(const Collection & collection, const Data } } - Block res; + ColumnsWithTypeAndName res(columns_num); for (size_t i = 0; i < columns_num; ++i) - res.insert(ColumnWithTypeAndName{std::move(columns[i]), types[i], "_" + toString(i)}); + { + res[i].type = types[i]; + res[i].column = std::move(columns[i]); + } + return res; } @@ -189,16 +193,14 @@ static Field extractValueFromNode(const ASTPtr & node, const IDataType & type, C throw Exception(ErrorCodes::INCORRECT_ELEMENT_OF_SET, "Incorrect element of set. Must be literal or constant expression."); } -static Block createBlockFromAST(const ASTPtr & node, const DataTypes & types, ContextPtr context) +static ColumnsWithTypeAndName createBlockFromAST(const ASTPtr & node, const DataTypes & types, ContextPtr context) { /// Will form a block with values from the set. - Block header; size_t num_columns = types.size(); + MutableColumns columns(num_columns); for (size_t i = 0; i < num_columns; ++i) - header.insert(ColumnWithTypeAndName(types[i]->createColumn(), types[i], "_" + toString(i))); - - MutableColumns columns = header.cloneEmptyColumns(); + columns[i] = types[i]->createColumn(); DataTypePtr tuple_type; Row tuple_values; @@ -290,7 +292,14 @@ static Block createBlockFromAST(const ASTPtr & node, const DataTypes & types, Co throw Exception(ErrorCodes::INCORRECT_ELEMENT_OF_SET, "Incorrect element of set"); } - return header.cloneWithColumns(std::move(columns)); + ColumnsWithTypeAndName res(num_columns); + for (size_t i = 0; i < num_columns; ++i) + { + res[i].type = types[i]; + res[i].column = std::move(columns[i]); + } + + return res; } @@ -304,7 +313,7 @@ namespace * We need special implementation for ASTFunction, because in case, when we interpret * large tuple or array as function, `evaluateConstantExpression` works extremely slow. */ -Block createBlockForSet( +ColumnsWithTypeAndName createBlockForSet( const DataTypePtr & left_arg_type, const ASTPtr & right_arg, const DataTypes & set_element_types, @@ -321,7 +330,7 @@ Block createBlockForSet( type->getName()); }; - Block block; + ColumnsWithTypeAndName block; bool tranform_null_in = context->getSettingsRef()[Setting::transform_null_in]; /// 1 in 1; (1, 2) in (1, 2); identity(tuple(tuple(tuple(1)))) in tuple(tuple(tuple(1))); etc. @@ -360,7 +369,7 @@ Block createBlockForSet( * 'set_element_types' - types of what are on the left hand side of IN. * 'right_arg' - Literal - Tuple or Array. */ -Block createBlockForSet( +ColumnsWithTypeAndName createBlockForSet( const DataTypePtr & left_arg_type, const std::shared_ptr & right_arg, const DataTypes & set_element_types, @@ -442,14 +451,14 @@ FutureSetPtr makeExplicitSet( if (const auto * low_cardinality_type = typeid_cast(element_type.get())) element_type = low_cardinality_type->getDictionaryType(); - Block block; + ColumnsWithTypeAndName block; const auto & right_arg_func = std::dynamic_pointer_cast(right_arg); if (right_arg_func && (right_arg_func->name == "tuple" || right_arg_func->name == "array")) block = createBlockForSet(left_arg_type, right_arg_func, set_element_types, context); else block = createBlockForSet(left_arg_type, right_arg, set_element_types, context); - return prepared_sets.addFromTuple(set_key, block, context->getSettingsRef()); + return prepared_sets.addFromTuple(set_key, std::move(block), context->getSettingsRef()); } class ScopeStack::Index @@ -1291,14 +1300,10 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & String result_name = lambda->arguments->children.at(1)->getColumnName(); lambda_dag.removeUnusedActions(Names(1, result_name)); - auto lambda_actions = std::make_shared( - std::move(lambda_dag), - ExpressionActionsSettings::fromContext(data.getContext(), CompileExpressions::yes)); - - DataTypePtr result_type = lambda_actions->getSampleBlock().getByName(result_name).type; + DataTypePtr result_type = lambda_dag.findInOutputs(result_name).result_type; Names captured; - Names required = lambda_actions->getRequiredColumns(); + Names required = lambda_dag.getRequiredColumnsNames(); for (const auto & required_arg : required) if (findColumn(required_arg, lambda_arguments) == lambda_arguments.end()) captured.push_back(required_arg); @@ -1307,8 +1312,9 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & /// because it does not uniquely define the expression (the types of arguments can be different). String lambda_name = data.getUniqueName("__lambda"); + auto actions_settings = ExpressionActionsSettings::fromContext(data.getContext(), CompileExpressions::yes); auto function_capture = std::make_shared( - lambda_actions, captured, lambda_arguments, result_type, result_name, false); + std::move(lambda_dag), actions_settings, captured, lambda_arguments, result_type, result_name, false); data.addFunction(function_capture, captured, lambda_name); argument_types[i] = std::make_shared(lambda_type->getArgumentTypes(), result_type); diff --git a/src/Interpreters/AggregateDescription.cpp b/src/Interpreters/AggregateDescription.cpp index d4c09995b56..504ed4b8361 100644 --- a/src/Interpreters/AggregateDescription.cpp +++ b/src/Interpreters/AggregateDescription.cpp @@ -1,13 +1,21 @@ #include +#include #include #include #include #include +#include +#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + void AggregateDescription::explain(WriteBuffer & out, size_t indent) const { String prefix(indent, ' '); @@ -121,4 +129,77 @@ void AggregateDescription::explain(JSONBuilder::JSONMap & map) const map.add("Arguments", std::move(args_array)); } +void serializeAggregateDescriptions(const AggregateDescriptions & aggregates, WriteBuffer & out) +{ + writeVarUInt(aggregates.size(), out); + for (const auto & aggregate : aggregates) + { + writeStringBinary(aggregate.column_name, out); + + UInt64 num_args = aggregate.argument_names.size(); + const auto & argument_types = aggregate.function->getArgumentTypes(); + + if (argument_types.size() != num_args) + { + WriteBufferFromOwnString buf; + aggregate.explain(buf, 0); + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Invalid number of for aggregate function. Expected {}, got {}. Description:\n{}", + argument_types.size(), num_args, buf.str()); + } + + writeVarUInt(num_args, out); + for (size_t i = 0; i < num_args; ++i) + { + writeStringBinary(aggregate.argument_names[i], out); + encodeDataType(argument_types[i], out); + } + + writeStringBinary(aggregate.function->getName(), out); + + writeVarUInt(aggregate.parameters.size(), out); + for (const auto & param : aggregate.parameters) + writeFieldBinary(param, out); + } +} + +void deserializeAggregateDescriptions(AggregateDescriptions & aggregates, ReadBuffer & in) +{ + UInt64 num_aggregates; + readVarUInt(num_aggregates, in); + aggregates.resize(num_aggregates); + for (auto & aggregate : aggregates) + { + readStringBinary(aggregate.column_name, in); + + UInt64 num_args; + readVarUInt(num_args, in); + aggregate.argument_names.resize(num_args); + + DataTypes argument_types; + argument_types.reserve(num_args); + + for (auto & arg_name : aggregate.argument_names) + { + readStringBinary(arg_name, in); + argument_types.emplace_back(decodeDataType(in)); + } + + String function_name; + readStringBinary(function_name, in); + + UInt64 num_params; + readVarUInt(num_params, in); + aggregate.parameters.resize(num_params); + for (auto & param : aggregate.parameters) + param = readFieldBinary(in); + + auto action = NullsAction::EMPTY; /// As I understand, it should be resolved to function name. + AggregateFunctionProperties properties; + aggregate.function = AggregateFunctionFactory::instance().get( + function_name, action, argument_types, aggregate.parameters, properties); + } + +} + } diff --git a/src/Interpreters/AggregateDescription.h b/src/Interpreters/AggregateDescription.h index 0f1c0ce67ae..19c999650a8 100644 --- a/src/Interpreters/AggregateDescription.h +++ b/src/Interpreters/AggregateDescription.h @@ -25,4 +25,8 @@ struct AggregateDescription }; using AggregateDescriptions = std::vector; + +void serializeAggregateDescriptions(const AggregateDescriptions & aggregates, WriteBuffer & out); +void deserializeAggregateDescriptions(AggregateDescriptions & aggregates, ReadBuffer & in); + } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 453309ca37d..58aaf1cc92c 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -76,22 +76,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -namespace Setting -{ - extern const SettingsFloat min_hit_rate_to_use_consecutive_keys_optimization; - extern const SettingsUInt64 max_rows_to_group_by; - extern const SettingsOverflowModeGroupBy group_by_overflow_mode; - extern const SettingsUInt64 max_bytes_before_external_group_by; - extern const SettingsDouble max_bytes_ratio_before_external_group_by; - extern const SettingsMaxThreads max_threads; - extern const SettingsUInt64 min_free_disk_space_for_temporary_data; - extern const SettingsBool compile_aggregate_expressions; - extern const SettingsUInt64 min_count_to_compile_aggregate_expression; - extern const SettingsUInt64 max_block_size; - extern const SettingsBool enable_software_prefetch_in_aggregation; - extern const SettingsBool optimize_group_by_constant_keys; -}; - } namespace @@ -195,45 +179,59 @@ Block Aggregator::getHeader(bool final) const } Aggregator::Params::Params( - const Settings & settings, const Names & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, + size_t max_rows_to_group_by_, + OverflowMode group_by_overflow_mode_, size_t group_by_two_level_threshold_, size_t group_by_two_level_threshold_bytes_, + size_t max_bytes_before_external_group_by_, bool empty_result_for_aggregation_by_empty_set_, TemporaryDataOnDiskScopePtr tmp_data_scope_, + size_t max_threads_, + size_t min_free_disk_space_, + bool compile_aggregate_expressions_, + size_t min_count_to_compile_aggregate_expression_, + size_t max_block_size_, + bool enable_prefetch_, bool only_merge_, // true for projections + bool optimize_group_by_constant_keys_, + float min_hit_rate_to_use_consecutive_keys_optimization_, const StatsCollectingParams & stats_collecting_params_) : keys(keys_) , keys_size(keys.size()) , aggregates(aggregates_) , aggregates_size(aggregates.size()) , overflow_row(overflow_row_) - , max_rows_to_group_by(settings[Setting::max_rows_to_group_by]) - , group_by_overflow_mode(settings[Setting::group_by_overflow_mode]) + , max_rows_to_group_by(max_rows_to_group_by_) + , group_by_overflow_mode(group_by_overflow_mode_) , group_by_two_level_threshold(group_by_two_level_threshold_) , group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_) - , max_bytes_before_external_group_by(settings[Setting::max_bytes_before_external_group_by]) + , max_bytes_before_external_group_by(max_bytes_before_external_group_by_) , empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_) , tmp_data_scope(std::move(tmp_data_scope_)) - , max_threads(settings[Setting::max_threads]) - , min_free_disk_space(settings[Setting::min_free_disk_space_for_temporary_data]) - , compile_aggregate_expressions(settings[Setting::compile_aggregate_expressions]) - , min_count_to_compile_aggregate_expression(settings[Setting::min_count_to_compile_aggregate_expression]) - , max_block_size(settings[Setting::max_block_size]) + , max_threads(max_threads_) + , min_free_disk_space(min_free_disk_space_) + , compile_aggregate_expressions(compile_aggregate_expressions_) + , min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_) + , max_block_size(max_block_size_) , only_merge(only_merge_) - , enable_prefetch(settings[Setting::enable_software_prefetch_in_aggregation]) - , optimize_group_by_constant_keys(settings[Setting::optimize_group_by_constant_keys]) - , min_hit_rate_to_use_consecutive_keys_optimization(settings[Setting::min_hit_rate_to_use_consecutive_keys_optimization]) + , enable_prefetch(enable_prefetch_) + , optimize_group_by_constant_keys(optimize_group_by_constant_keys_) + , min_hit_rate_to_use_consecutive_keys_optimization(min_hit_rate_to_use_consecutive_keys_optimization_) , stats_collecting_params(stats_collecting_params_) { - if (settings[Setting::max_bytes_ratio_before_external_group_by] != 0.) +} + +size_t Aggregator::Params::getMaxBytesBeforeExternalGroupBy(size_t max_bytes_before_external_group_by, double max_bytes_ratio_before_external_group_by) +{ + if (max_bytes_ratio_before_external_group_by != 0.) { - if (settings[Setting::max_bytes_before_external_group_by] > 0) + if (max_bytes_before_external_group_by > 0) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Settings max_bytes_ratio_before_external_group_by and max_bytes_before_external_group_by cannot be set simultaneously"); - double ratio = settings[Setting::max_bytes_ratio_before_external_group_by]; + double ratio = max_bytes_ratio_before_external_group_by; if (ratio < 0 || ratio >= 1.) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Setting max_bytes_ratio_before_external_group_by should be >= 0 and < 1 ({})", ratio); @@ -251,6 +249,8 @@ Aggregator::Params::Params( LOG_WARNING(getLogger("Aggregator"), "No system memory limits configured. Ignoring max_bytes_ratio_before_external_group_by"); } } + + return max_bytes_before_external_group_by; } Aggregator::Params::Params( @@ -259,7 +259,7 @@ Aggregator::Params::Params( bool overflow_row_, size_t max_threads_, size_t max_block_size_, - double min_hit_rate_to_use_consecutive_keys_optimization_) + float min_hit_rate_to_use_consecutive_keys_optimization_) : keys(keys_) , keys_size(keys.size()) , aggregates(aggregates_) diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 6691f6d2645..2088e4f7aad 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -131,19 +131,31 @@ public: bool only_merge = false; bool enable_prefetch = false; bool optimize_group_by_constant_keys = false; - const double min_hit_rate_to_use_consecutive_keys_optimization = 0.; + const float min_hit_rate_to_use_consecutive_keys_optimization = 0.; StatsCollectingParams stats_collecting_params; + static size_t getMaxBytesBeforeExternalGroupBy(size_t max_bytes_before_external_group_by, double max_bytes_ratio_before_external_group_by); + Params( - const Settings & settings, const Names & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, + size_t max_rows_to_group_by_, + OverflowMode group_by_overflow_mode_, size_t group_by_two_level_threshold_, size_t group_by_two_level_threshold_bytes_, + size_t max_bytes_before_external_group_by_, bool empty_result_for_aggregation_by_empty_set_, TemporaryDataOnDiskScopePtr tmp_data_scope_, + size_t max_threads_, + size_t min_free_disk_space_, + bool compile_aggregate_expressions_, + size_t min_count_to_compile_aggregate_expression_, + size_t max_block_size_, + bool enable_prefetch_, bool only_merge_, // true for projections + bool optimize_group_by_constant_keys_, + float min_hit_rate_to_use_consecutive_keys_optimization_, const StatsCollectingParams & stats_collecting_params_); /// Only parameters that matter during merge. @@ -153,7 +165,7 @@ public: bool overflow_row_, size_t max_threads_, size_t max_block_size_, - double min_hit_rate_to_use_consecutive_keys_optimization_); + float min_hit_rate_to_use_consecutive_keys_optimization_); Params cloneWithKeys(const Names & keys_, bool only_merge_ = false) { diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 8e85f81b6c6..e4e44e9cbad 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -131,7 +131,6 @@ namespace Setting extern const SettingsBool extremes; extern const SettingsBool final; extern const SettingsBool force_aggregation_in_order; - extern const SettingsOverflowModeGroupBy group_by_overflow_mode; extern const SettingsUInt64 group_by_two_level_threshold; extern const SettingsUInt64 group_by_two_level_threshold_bytes; extern const SettingsBool group_by_use_nulls; @@ -149,14 +148,12 @@ namespace Setting extern const SettingsUInt64 max_result_rows; extern const SettingsUInt64 max_rows_in_distinct; extern const SettingsUInt64 max_rows_in_set_to_optimize_join; - extern const SettingsUInt64 max_rows_to_group_by; extern const SettingsUInt64 max_rows_to_read; extern const SettingsUInt64 max_size_to_preallocate_for_aggregation; extern const SettingsFloat max_streams_to_max_threads_ratio; extern const SettingsUInt64 max_subquery_depth; extern const SettingsMaxThreads max_threads; extern const SettingsUInt64 min_count_to_compile_sort_description; - extern const SettingsFloat min_hit_rate_to_use_consecutive_keys_optimization; extern const SettingsBool multiple_joins_try_to_keep_original_names; extern const SettingsBool optimize_aggregation_in_order; extern const SettingsBool optimize_move_to_prewhere; @@ -178,6 +175,16 @@ namespace Setting extern const SettingsTotalsMode totals_mode; extern const SettingsBool use_concurrency_control; extern const SettingsBool use_with_fill_by_sorting_prefix; + extern const SettingsFloat min_hit_rate_to_use_consecutive_keys_optimization; + extern const SettingsUInt64 max_rows_to_group_by; + extern const SettingsOverflowModeGroupBy group_by_overflow_mode; + extern const SettingsUInt64 max_bytes_before_external_group_by; + extern const SettingsDouble max_bytes_ratio_before_external_group_by; + extern const SettingsUInt64 min_free_disk_space_for_temporary_data; + extern const SettingsBool compile_aggregate_expressions; + extern const SettingsUInt64 min_count_to_compile_aggregate_expression; + extern const SettingsBool enable_software_prefetch_in_aggregation; + extern const SettingsBool optimize_group_by_constant_keys; } namespace ServerSetting @@ -2751,16 +2758,29 @@ static Aggregator::Params getAggregatorParams( return Aggregator::Params { - settings, keys, aggregates, overflow_row, + settings[Setting::max_rows_to_group_by], + settings[Setting::group_by_overflow_mode], group_by_two_level_threshold, group_by_two_level_threshold_bytes, - settings[Setting::empty_result_for_aggregation_by_empty_set] || (settings[Setting::empty_result_for_aggregation_by_constant_keys_on_empty_set] && keys.empty() && query_analyzer.hasConstAggregationKeys()), + Aggregator::Params::getMaxBytesBeforeExternalGroupBy(settings[Setting::max_bytes_before_external_group_by], settings[Setting::max_bytes_ratio_before_external_group_by]), + settings[Setting::empty_result_for_aggregation_by_empty_set] + || (settings[Setting::empty_result_for_aggregation_by_constant_keys_on_empty_set] && keys.empty() + && query_analyzer.hasConstAggregationKeys()), context.getTempDataOnDisk(), - /* only_merge_= */ false, - stats_collecting_params}; + settings[Setting::max_threads], + settings[Setting::min_free_disk_space_for_temporary_data], + settings[Setting::compile_aggregate_expressions], + settings[Setting::min_count_to_compile_aggregate_expression], + settings[Setting::max_block_size], + settings[Setting::enable_software_prefetch_in_aggregation], + /* only_merge */ false, + settings[Setting::optimize_group_by_constant_keys], + settings[Setting::min_hit_rate_to_use_consecutive_keys_optimization], + stats_collecting_params + }; } void InterpreterSelectQuery::executeAggregation( diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index c69e2f84d42..8de6edc5d21 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -38,7 +38,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -static SizeLimits getSizeLimitsForSet(const Settings & settings) +SizeLimits PreparedSets::getSizeLimitsForSet(const Settings & settings) { return SizeLimits(settings[Setting::max_rows_in_set], settings[Setting::max_bytes_in_set], settings[Setting::set_overflow_mode]); } @@ -59,8 +59,9 @@ static bool equals(const DataTypes & lhs, const DataTypes & rhs) } -FutureSetFromStorage::FutureSetFromStorage(SetPtr set_) : set(std::move(set_)) {} +FutureSetFromStorage::FutureSetFromStorage(Hash hash_, SetPtr set_) : hash(hash_), set(std::move(set_)) {} SetPtr FutureSetFromStorage::get() const { return set; } +FutureSet::Hash FutureSetFromStorage::getHash() const { return hash; } DataTypes FutureSetFromStorage::getTypes() const { return set->getElementsTypes(); } SetPtr FutureSetFromStorage::buildOrderedSetInplace(const ContextPtr &) @@ -69,24 +70,41 @@ SetPtr FutureSetFromStorage::buildOrderedSetInplace(const ContextPtr &) } -FutureSetFromTuple::FutureSetFromTuple(Block block, const Settings & settings) +FutureSetFromTuple::FutureSetFromTuple( + Hash hash_, ColumnsWithTypeAndName block, + bool transform_null_in, SizeLimits size_limits) + : hash(hash_) { - auto size_limits = getSizeLimitsForSet(settings); - set = std::make_shared(size_limits, settings[Setting::use_index_for_in_with_subqueries_max_values], settings[Setting::transform_null_in]); - set->setHeader(block.cloneEmpty().getColumnsWithTypeAndName()); + ColumnsWithTypeAndName header = block; + for (auto & elem : header) + elem.column = elem.column->cloneEmpty(); + set = std::make_shared(size_limits, 0, transform_null_in); + + set->setHeader(header); Columns columns; - columns.reserve(block.columns()); + columns.reserve(block.size()); + size_t num_rows = 0; for (const auto & column : block) + { columns.emplace_back(column.column); + num_rows = column.column->size(); + } - set_key_columns.filter = ColumnUInt8::create(block.rows()); + set_key_columns.filter = ColumnUInt8::create(num_rows); set->insertFromColumns(columns, set_key_columns); set->finishInsert(); + + for (const auto & type : set->getElementsTypes()) + { + auto name = type->getName(); + hash = CityHash_v1_0_2::CityHash128WithSeed(name.data(), name.size(), hash); + } } DataTypes FutureSetFromTuple::getTypes() const { return set->getElementsTypes(); } +FutureSet::Hash FutureSetFromTuple::getHash() const { return hash; } SetPtr FutureSetFromTuple::buildOrderedSetInplace(const ContextPtr & context) { @@ -107,34 +125,33 @@ SetPtr FutureSetFromTuple::buildOrderedSetInplace(const ContextPtr & context) FutureSetFromSubquery::FutureSetFromSubquery( - String key, + Hash hash_, std::unique_ptr source_, StoragePtr external_table_, std::shared_ptr external_table_set_, - const Settings & settings) - : external_table(std::move(external_table_)), external_table_set(std::move(external_table_set_)), source(std::move(source_)) + bool transform_null_in, + SizeLimits size_limits, + size_t max_size_for_index) + : hash(hash_), external_table(std::move(external_table_)), external_table_set(std::move(external_table_set_)), source(std::move(source_)) { set_and_key = std::make_shared(); - set_and_key->key = std::move(key); + set_and_key->key = PreparedSets::toString(hash_, {}); - auto size_limits = getSizeLimitsForSet(settings); - set_and_key->set - = std::make_shared(size_limits, settings[Setting::use_index_for_in_with_subqueries_max_values], settings[Setting::transform_null_in]); + set_and_key->set = std::make_shared(size_limits, max_size_for_index, transform_null_in); set_and_key->set->setHeader(source->getCurrentHeader().getColumnsWithTypeAndName()); } FutureSetFromSubquery::FutureSetFromSubquery( - String key, + Hash hash_, QueryTreeNodePtr query_tree_, - const Settings & settings) - : query_tree(std::move(query_tree_)) + bool transform_null_in, + SizeLimits size_limits, + size_t max_size_for_index) + : hash(hash_), query_tree(std::move(query_tree_)) { set_and_key = std::make_shared(); - set_and_key->key = std::move(key); - - auto size_limits = getSizeLimitsForSet(settings); - set_and_key->set - = std::make_shared(size_limits, settings[Setting::use_index_for_in_with_subqueries_max_values], settings[Setting::transform_null_in]); + set_and_key->key = PreparedSets::toString(hash_, {}); + set_and_key->set = std::make_shared(size_limits, max_size_for_index, transform_null_in); } FutureSetFromSubquery::~FutureSetFromSubquery() = default; @@ -158,6 +175,8 @@ DataTypes FutureSetFromSubquery::getTypes() const return set_and_key->set->getElementsTypes(); } +FutureSet::Hash FutureSetFromSubquery::getHash() const { return hash; } + std::unique_ptr FutureSetFromSubquery::build(const ContextPtr & context) { if (set_and_key->set->isCreated()) @@ -266,9 +285,12 @@ String PreparedSets::toString(const PreparedSets::Hash & key, const DataTypes & return buf.str(); } -FutureSetFromTuplePtr PreparedSets::addFromTuple(const Hash & key, Block block, const Settings & settings) +FutureSetFromTuplePtr PreparedSets::addFromTuple(const Hash & key, ColumnsWithTypeAndName block, const Settings & settings) { - auto from_tuple = std::make_shared(std::move(block), settings); + auto size_limits = getSizeLimitsForSet(settings); + auto from_tuple = std::make_shared( + key, std::move(block), + settings[Setting::transform_null_in], size_limits); const auto & set_types = from_tuple->getTypes(); auto & sets_by_hash = sets_from_tuple[key]; @@ -282,7 +304,7 @@ FutureSetFromTuplePtr PreparedSets::addFromTuple(const Hash & key, Block block, FutureSetFromStoragePtr PreparedSets::addFromStorage(const Hash & key, SetPtr set_) { - auto from_storage = std::make_shared(std::move(set_)); + auto from_storage = std::make_shared(key, std::move(set_)); auto [it, inserted] = sets_from_storage.emplace(key, from_storage); if (!inserted) @@ -298,8 +320,10 @@ FutureSetFromSubqueryPtr PreparedSets::addFromSubquery( FutureSetFromSubqueryPtr external_table_set, const Settings & settings) { + auto size_limits = getSizeLimitsForSet(settings); auto from_subquery = std::make_shared( - toString(key, {}), std::move(source), std::move(external_table), std::move(external_table_set), settings); + key, std::move(source), std::move(external_table), std::move(external_table_set), + settings[Setting::transform_null_in], size_limits, settings[Setting::use_index_for_in_with_subqueries_max_values]); auto [it, inserted] = sets_from_subqueries.emplace(key, from_subquery); @@ -314,10 +338,10 @@ FutureSetFromSubqueryPtr PreparedSets::addFromSubquery( QueryTreeNodePtr query_tree, const Settings & settings) { + auto size_limits = getSizeLimitsForSet(settings); auto from_subquery = std::make_shared( - toString(key, {}), - std::move(query_tree), - settings); + key, std::move(query_tree), + settings[Setting::transform_null_in], size_limits, settings[Setting::use_index_for_in_with_subqueries_max_values]); auto [it, inserted] = sets_from_subqueries.emplace(key, from_subquery); diff --git a/src/Interpreters/PreparedSets.h b/src/Interpreters/PreparedSets.h index a6aee974d0e..c3004e94fa8 100644 --- a/src/Interpreters/PreparedSets.h +++ b/src/Interpreters/PreparedSets.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include namespace DB { @@ -50,6 +52,9 @@ public: virtual DataTypes getTypes() const = 0; /// If possible, return set with stored elements useful for PK analysis. virtual SetPtr buildOrderedSetInplace(const ContextPtr & context) = 0; + + using Hash = CityHash_v1_0_2::uint128; + virtual Hash getHash() const = 0; }; using FutureSetPtr = std::shared_ptr; @@ -59,13 +64,15 @@ using FutureSetPtr = std::shared_ptr; class FutureSetFromStorage final : public FutureSet { public: - explicit FutureSetFromStorage(SetPtr set_); + explicit FutureSetFromStorage(Hash hash_, SetPtr set_); SetPtr get() const override; DataTypes getTypes() const override; SetPtr buildOrderedSetInplace(const ContextPtr &) override; + Hash getHash() const override; private: + Hash hash; SetPtr set; }; @@ -76,14 +83,16 @@ using FutureSetFromStoragePtr = std::shared_ptr; class FutureSetFromTuple final : public FutureSet { public: - FutureSetFromTuple(Block block, const Settings & settings); + FutureSetFromTuple(Hash hash_, ColumnsWithTypeAndName block, bool transform_null_in, SizeLimits size_limits); SetPtr get() const override { return set; } SetPtr buildOrderedSetInplace(const ContextPtr & context) override; DataTypes getTypes() const override; + Hash getHash() const override; private: + Hash hash; SetPtr set; SetKeyColumns set_key_columns; }; @@ -106,21 +115,26 @@ class FutureSetFromSubquery final : public FutureSet { public: FutureSetFromSubquery( - String key, + Hash hash_, std::unique_ptr source_, StoragePtr external_table_, std::shared_ptr external_table_set_, - const Settings & settings); + bool transform_null_in, + SizeLimits size_limits, + size_t max_size_for_index); FutureSetFromSubquery( - String key, + Hash hash_, QueryTreeNodePtr query_tree_, - const Settings & settings); + bool transform_null_in, + SizeLimits size_limits, + size_t max_size_for_index); ~FutureSetFromSubquery() override; SetPtr get() const override; DataTypes getTypes() const override; + Hash getHash() const override; SetPtr buildOrderedSetInplace(const ContextPtr & context) override; std::unique_ptr build(const ContextPtr & context); @@ -130,6 +144,7 @@ public: void setQueryPlan(std::unique_ptr source_); private: + Hash hash; SetAndKeyPtr set_and_key; StoragePtr external_table; std::shared_ptr external_table_set; @@ -156,7 +171,7 @@ public: using SetsFromSubqueries = std::unordered_map; FutureSetFromStoragePtr addFromStorage(const Hash & key, SetPtr set_); - FutureSetFromTuplePtr addFromTuple(const Hash & key, Block block, const Settings & settings); + FutureSetFromTuplePtr addFromTuple(const Hash & key, ColumnsWithTypeAndName block, const Settings & settings); FutureSetFromSubqueryPtr addFromSubquery( const Hash & key, @@ -183,6 +198,7 @@ public: // const SetsFromSubqueries & getSetsFromSubquery() const { return sets_from_subqueries; } static String toString(const Hash & key, const DataTypes & types); + static SizeLimits getSizeLimitsForSet(const Settings & settings); private: SetsFromTuple sets_from_tuple; diff --git a/src/Interpreters/SetSerialization.h b/src/Interpreters/SetSerialization.h new file mode 100644 index 00000000000..1413aa556da --- /dev/null +++ b/src/Interpreters/SetSerialization.h @@ -0,0 +1,32 @@ +#pragma once +#include + +namespace DB +{ + +class FutureSet; +using FutureSetPtr = std::shared_ptr; + +struct SerializedSetsRegistry +{ + struct Hashing + { + UInt64 operator()(const FutureSet::Hash & key) const { return key.low64 ^ key.high64; } + }; + + std::unordered_map sets; +}; + +class ColumnSet; + +struct DeserializedSetsRegistry +{ + struct Hashing + { + UInt64 operator()(const FutureSet::Hash & key) const { return key.low64 ^ key.high64; } + }; + + std::unordered_map, Hashing> sets; +}; + +} diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 4e40c31dd78..19ba725523f 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -106,7 +106,6 @@ namespace Setting extern const SettingsBool exact_rows_before_limit; extern const SettingsBool extremes; extern const SettingsBool force_aggregation_in_order; - extern const SettingsOverflowModeGroupBy group_by_overflow_mode; extern const SettingsUInt64 group_by_two_level_threshold; extern const SettingsUInt64 group_by_two_level_threshold_bytes; extern const SettingsBool group_by_use_nulls; @@ -115,9 +114,7 @@ namespace Setting extern const SettingsUInt64 max_size_to_preallocate_for_aggregation; extern const SettingsUInt64 max_subquery_depth; extern const SettingsUInt64 max_rows_in_distinct; - extern const SettingsUInt64 max_rows_to_group_by; extern const SettingsMaxThreads max_threads; - extern const SettingsFloat min_hit_rate_to_use_consecutive_keys_optimization; extern const SettingsBool parallel_replicas_allow_in_with_subquery; extern const SettingsString parallel_replicas_custom_key; extern const SettingsUInt64 parallel_replicas_min_number_of_rows_per_replica; @@ -126,6 +123,16 @@ namespace Setting extern const SettingsFloat totals_auto_threshold; extern const SettingsTotalsMode totals_mode; extern const SettingsBool use_with_fill_by_sorting_prefix; + extern const SettingsFloat min_hit_rate_to_use_consecutive_keys_optimization; + extern const SettingsUInt64 max_rows_to_group_by; + extern const SettingsOverflowModeGroupBy group_by_overflow_mode; + extern const SettingsUInt64 max_bytes_before_external_group_by; + extern const SettingsDouble max_bytes_ratio_before_external_group_by; + extern const SettingsUInt64 min_free_disk_space_for_temporary_data; + extern const SettingsBool compile_aggregate_expressions; + extern const SettingsUInt64 min_count_to_compile_aggregate_expression; + extern const SettingsBool enable_software_prefetch_in_aggregation; + extern const SettingsBool optimize_group_by_constant_keys; } namespace ServerSetting @@ -429,15 +436,27 @@ Aggregator::Params getAggregatorParams(const PlannerContextPtr & planner_context } Aggregator::Params aggregator_params = Aggregator::Params( - settings, aggregation_analysis_result.aggregation_keys, aggregate_descriptions, query_analysis_result.aggregate_overflow_row, + settings[Setting::max_rows_to_group_by], + settings[Setting::group_by_overflow_mode], settings[Setting::group_by_two_level_threshold], settings[Setting::group_by_two_level_threshold_bytes], - settings[Setting::empty_result_for_aggregation_by_empty_set] || (settings[Setting::empty_result_for_aggregation_by_constant_keys_on_empty_set] && aggregation_analysis_result.aggregation_keys.empty() && aggregation_analysis_result.group_by_with_constant_keys), + Aggregator::Params::getMaxBytesBeforeExternalGroupBy(settings[Setting::max_bytes_before_external_group_by], settings[Setting::max_bytes_ratio_before_external_group_by]), + settings[Setting::empty_result_for_aggregation_by_empty_set] + || (settings[Setting::empty_result_for_aggregation_by_constant_keys_on_empty_set] && aggregation_analysis_result.aggregation_keys.empty() + && aggregation_analysis_result.group_by_with_constant_keys), query_context->getTempDataOnDisk(), + settings[Setting::max_threads], + settings[Setting::min_free_disk_space_for_temporary_data], + settings[Setting::compile_aggregate_expressions], + settings[Setting::min_count_to_compile_aggregate_expression], + settings[Setting::max_block_size], + settings[Setting::enable_software_prefetch_in_aggregation], /* only_merge */ false, + settings[Setting::optimize_group_by_constant_keys], + settings[Setting::min_hit_rate_to_use_consecutive_keys_optimization], stats_collecting_params); return aggregator_params; diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 46b1db3f660..8f8584418ea 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -778,12 +778,9 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi lambda_actions_dag.getOutputs().push_back(actions_stack.back().getNodeOrThrow(lambda_expression_node_name)); lambda_actions_dag.removeUnusedActions(Names(1, lambda_expression_node_name)); - auto expression_actions_settings = ExpressionActionsSettings::fromContext(planner_context->getQueryContext(), CompileExpressions::yes); - auto lambda_actions = std::make_shared(std::move(lambda_actions_dag), expression_actions_settings); - Names captured_column_names; ActionsDAG::NodeRawConstPtrs lambda_children; - Names required_column_names = lambda_actions->getRequiredColumns(); + Names required_column_names = lambda_actions_dag.getRequiredColumnsNames(); actions_stack.pop_back(); levels.reset(actions_stack.size()); @@ -802,9 +799,10 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi } } + auto expression_actions_settings = ExpressionActionsSettings::fromContext(planner_context->getQueryContext(), CompileExpressions::yes); auto lambda_node_name = calculateActionNodeName(node, *planner_context); auto function_capture = std::make_shared( - lambda_actions, captured_column_names, lambda_arguments_names_and_types, lambda_node.getExpression()->getResultType(), lambda_expression_node_name, true); + std::move(lambda_actions_dag), expression_actions_settings, captured_column_names, lambda_arguments_names_and_types, lambda_node.getExpression()->getResultType(), lambda_expression_node_name, true); // TODO: Pass IFunctionBase here not FunctionCaptureOverloadResolver. const auto * actions_node = actions_stack[level].addFunctionIfNecessary(lambda_node_name, std::move(lambda_children), function_capture); diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 9fc88be6398..ae1cd83aa4e 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -13,7 +13,9 @@ #include #include #include -#include +#include +#include +#include #include #include #include @@ -30,6 +32,8 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; + extern const int INCORRECT_DATA; } static bool memoryBoundMergingWillBeUsed( @@ -147,6 +151,25 @@ const SortDescription & AggregatingStep::getSortDescription() const return IQueryPlanStep::getSortDescription(); } +static void updateThreadsValues( + size_t & new_merge_threads, + size_t & new_temporary_data_merge_threads, + Aggregator::Params & params, + const BuildQueryPipelineSettings & settings) +{ + /// Update values from settings if plan was deserialized. + if (new_merge_threads == 0) + new_merge_threads = settings.max_threads; + + if (new_temporary_data_merge_threads == 0) + new_temporary_data_merge_threads = settings.aggregation_memory_efficient_merge_threads; + if (new_temporary_data_merge_threads == 0) + new_temporary_data_merge_threads = new_merge_threads; + + if (params.max_threads == 0) + params.max_threads = settings.max_threads; +} + ActionsDAG AggregatingStep::makeCreatingMissingKeysForGroupingSetDAG( const Block & in_header, const Block & out_header, @@ -204,6 +227,10 @@ ActionsDAG AggregatingStep::makeCreatingMissingKeysForGroupingSetDAG( void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) { + size_t new_merge_threads = merge_threads; + size_t new_temporary_data_merge_threads = temporary_data_merge_threads; + updateThreadsValues(new_merge_threads, new_temporary_data_merge_threads, params, settings); + QueryPipelineProcessorsCollector collector(pipeline, this); /// Forget about current totals and extremes. They will be calculated again after aggregation if needed. @@ -279,8 +306,8 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B transform_params_for_set, many_data, j, - merge_threads, - temporary_data_merge_threads, + new_merge_threads, + new_temporary_data_merge_threads, should_produce_results_in_order_of_bucket_number, skip_merging); // For each input stream we have `grouping_sets_size` copies, so port index @@ -373,7 +400,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B return std::make_shared( header, transform_params, sort_description_for_merging, group_by_sort_description, - max_block_size, aggregation_in_order_max_block_bytes / merge_threads, + max_block_size, aggregation_in_order_max_block_bytes / new_merge_threads, many_data, counter++); }); @@ -399,7 +426,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B pipeline.addTransform(std::move(transform)); /// Do merge of aggregated data in parallel. - pipeline.resize(merge_threads); + pipeline.resize(new_merge_threads); const auto & required_sort_description = memoryBoundMergingWillBeUsed() ? group_by_sort_description : SortDescription{}; pipeline.addSimpleTransform( @@ -455,8 +482,8 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B transform_params, many_data, counter++, - merge_threads, - temporary_data_merge_threads, + new_merge_threads, + new_temporary_data_merge_threads, should_produce_results_in_order_of_bucket_number, skip_merging); }); @@ -484,6 +511,7 @@ void AggregatingStep::describeActions(FormatSettings & settings) const settings.out << prefix << "Order: " << dumpSortDescription(sort_description_for_merging) << '\n'; } settings.out << prefix << "Skip merging: " << skip_merging << '\n'; + // settings.out << prefix << "Memory bound merging: " << memory_bound_merging_of_aggregation_results_enabled << '\n'; } void AggregatingStep::describeActions(JSONBuilder::JSONMap & map) const @@ -588,8 +616,12 @@ void AggregatingProjectionStep::updateOutputHeader() QueryPipelineBuilderPtr AggregatingProjectionStep::updatePipeline( QueryPipelineBuilders pipelines, - const BuildQueryPipelineSettings &) + const BuildQueryPipelineSettings & settings) { + size_t new_merge_threads = merge_threads; + size_t new_temporary_data_merge_threads = temporary_data_merge_threads; + updateThreadsValues(new_merge_threads, new_temporary_data_merge_threads, params, settings); + auto & normal_parts_pipeline = pipelines.front(); auto & projection_parts_pipeline = pipelines.back(); @@ -624,7 +656,7 @@ QueryPipelineBuilderPtr AggregatingProjectionStep::updatePipeline( pipeline.addSimpleTransform([&](const Block & header) { return std::make_shared( - header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads); + header, transform_params, many_data, counter++, new_merge_threads, new_temporary_data_merge_threads); }); }; @@ -641,4 +673,173 @@ QueryPipelineBuilderPtr AggregatingProjectionStep::updatePipeline( return pipeline; } + +void AggregatingStep::serializeSettings(QueryPlanSerializationSettings & settings) const +{ + settings.max_block_size = max_block_size; + settings.aggregation_in_order_max_block_bytes = aggregation_in_order_max_block_bytes; + + settings.aggregation_in_order_memory_bound_merging = should_produce_results_in_order_of_bucket_number; + settings.aggregation_sort_result_by_bucket_number = memory_bound_merging_of_aggregation_results_enabled; + + settings.max_rows_to_group_by = params.max_rows_to_group_by; + settings.group_by_overflow_mode = params.group_by_overflow_mode; + + settings.group_by_two_level_threshold = params.group_by_two_level_threshold; + settings.group_by_two_level_threshold_bytes = params.group_by_two_level_threshold_bytes; + + settings.max_bytes_before_external_group_by = params.max_bytes_before_external_group_by; + settings.empty_result_for_aggregation_by_empty_set = params.empty_result_for_aggregation_by_empty_set; + + settings.min_free_disk_space_for_temporary_data = params.min_free_disk_space; + + settings.compile_aggregate_expressions = params.compile_aggregate_expressions; + settings.min_count_to_compile_aggregate_expression = params.min_count_to_compile_aggregate_expression; + + settings.enable_software_prefetch_in_aggregation = params.enable_prefetch; + settings.optimize_group_by_constant_keys = params.optimize_group_by_constant_keys; + settings.min_hit_rate_to_use_consecutive_keys_optimization = params.min_hit_rate_to_use_consecutive_keys_optimization; + + settings.collect_hash_table_stats_during_aggregation = params.stats_collecting_params.isCollectionAndUseEnabled(); + settings.max_entries_for_hash_table_stats = params.stats_collecting_params.max_entries_for_hash_table_stats; + settings.max_size_to_preallocate_for_aggregation = params.stats_collecting_params.max_size_to_preallocate; +} + +void AggregatingStep::serialize(Serialization & ctx) const +{ + if (!sort_description_for_merging.empty()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Serialization of AggregatingStep optimized for in-order is not supported."); + + if (!grouping_sets_params.empty()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Serialization of AggregatingStep with grouping sets is not supported."); + + if (explicit_sorting_required_for_aggregation_in_order) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Serialization of AggregatingStep explicit_sorting_required_for_aggregation_in_order is not supported."); + + /// If you wonder why something is serialized using settings, and other is serialized using flags, considerations are following: + /// * flags are something that may change data format returning from the step + /// * settings are something which already was in Settings.h and, usually, is passed to Aggregator unchanged + /// Flags `final` and `group_by_use_nulls` change types, and `overflow_row` appends additional block to results. + /// Settings like `max_rows_to_group_by` or `empty_result_for_aggregation_by_empty_set` affect the result, + /// but does not change data format. + /// Overall, the rule is not strict. + + UInt8 flags = 0; + if (final) + flags |= 1; + if (params.overflow_row) + flags |= 2; + if (group_by_use_nulls) + flags |= 4; + if (!grouping_sets_params.empty()) + flags |= 8; + /// Ideally, key should be calculated from QueryPlan on the follower. + /// So, let's have a flag to disable sending/reading pre-calculated value. + if (params.stats_collecting_params.isCollectionAndUseEnabled()) + flags |= 16; + + writeIntBinary(flags, ctx.out); + + if (explicit_sorting_required_for_aggregation_in_order) + serializeSortDescription(group_by_sort_description, ctx.out); + + writeVarUInt(params.keys.size(), ctx.out); + for (const auto & key : params.keys) + writeStringBinary(key, ctx.out); + + serializeAggregateDescriptions(params.aggregates, ctx.out); + + if (params.stats_collecting_params.isCollectionAndUseEnabled()) + writeIntBinary(params.stats_collecting_params.key, ctx.out); +} + +std::unique_ptr AggregatingStep::deserialize(Deserialization & ctx) +{ + if (ctx.input_headers.size() != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "AggregatingStep must have one input stream"); + + UInt8 flags; + readIntBinary(flags, ctx.in); + + bool final = bool(flags & 1); + bool overflow_row = bool(flags & 2); + bool group_by_use_nulls = bool(flags & 4); + bool has_grouping_sets = bool(flags & 8); + bool has_stats_key = bool(flags & 16); + + if (has_grouping_sets) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Serialization of AggregatingStep with grouping sets is not supported."); + + UInt64 num_keys; + readVarUInt(num_keys, ctx.in); + Names keys(num_keys); + for (auto & key : keys) + readStringBinary(key, ctx.in); + + AggregateDescriptions aggregates; + deserializeAggregateDescriptions(aggregates, ctx.in); + + UInt64 stats_key = 0; + if (has_stats_key) + readIntBinary(stats_key, ctx.in); + + StatsCollectingParams stats_collecting_params( + stats_key, + ctx.settings.collect_hash_table_stats_during_aggregation, + ctx.settings.max_entries_for_hash_table_stats, + ctx.settings.max_size_to_preallocate_for_aggregation); + + Aggregator::Params params + { + keys, + aggregates, + overflow_row, + ctx.settings.max_rows_to_group_by, + ctx.settings.group_by_overflow_mode, + ctx.settings.group_by_two_level_threshold, + ctx.settings.group_by_two_level_threshold_bytes, + ctx.settings.max_bytes_before_external_group_by, + ctx.settings.empty_result_for_aggregation_by_empty_set, + Context::getGlobalContextInstance()->getTempDataOnDisk(), + 0, //settings.max_threads, + ctx.settings.min_free_disk_space_for_temporary_data, + ctx.settings.compile_aggregate_expressions, + ctx.settings.min_count_to_compile_aggregate_expression, + ctx.settings.max_block_size, + ctx.settings.enable_software_prefetch_in_aggregation, + /* only_merge */ false, + ctx.settings.optimize_group_by_constant_keys, + ctx.settings.min_hit_rate_to_use_consecutive_keys_optimization, + stats_collecting_params + }; + + SortDescription sort_description_for_merging; + GroupingSetsParamsList grouping_sets_params; + + auto aggregating_step = std::make_unique( + ctx.input_headers.front(), + std::move(params), + std::move(grouping_sets_params), + final, + ctx.settings.max_block_size, + ctx.settings.aggregation_in_order_max_block_bytes, + 0, //merge_threads, + 0, //temporary_data_merge_threads, + false, // storage_has_evenly_distributed_read, TODO: later + group_by_use_nulls, + std::move(sort_description_for_merging), + SortDescription{}, + ctx.settings.aggregation_in_order_memory_bound_merging, + ctx.settings.aggregation_sort_result_by_bucket_number, + false); + + return aggregating_step; +} + +void registerAggregatingStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("Aggregating", AggregatingStep::deserialize); +} + + } diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index d76764f05ba..ad06e1feffc 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -74,6 +74,13 @@ public: UInt64 group, bool group_by_use_nulls); + void serializeSettings(QueryPlanSerializationSettings & settings) const override; + void serialize(Serialization & ctx) const override; + + static std::unique_ptr deserialize(Deserialization & ctx); + + void enableMemoryBoundMerging() { memory_bound_merging_of_aggregation_results_enabled = true; } + private: void updateOutputHeader() override; @@ -98,7 +105,7 @@ private: /// These settings are used to determine if we should resize pipeline to 1 at the end. const bool should_produce_results_in_order_of_bucket_number; - const bool memory_bound_merging_of_aggregation_results_enabled; + bool memory_bound_merging_of_aggregation_results_enabled; bool explicit_sorting_required_for_aggregation_in_order; Processors aggregating_in_order; @@ -120,7 +127,7 @@ public: ); String getName() const override { return "AggregatingProjection"; } - QueryPipelineBuilderPtr updatePipeline(QueryPipelineBuilders pipelines, const BuildQueryPipelineSettings &) override; + QueryPipelineBuilderPtr updatePipeline(QueryPipelineBuilders pipelines, const BuildQueryPipelineSettings & settings) override; private: void updateOutputHeader() override; diff --git a/src/Processors/QueryPlan/ArrayJoinStep.cpp b/src/Processors/QueryPlan/ArrayJoinStep.cpp index 4ba53480b67..d969a3bb23a 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.cpp +++ b/src/Processors/QueryPlan/ArrayJoinStep.cpp @@ -1,4 +1,7 @@ #include +#include +#include +#include #include #include #include @@ -79,4 +82,50 @@ void ArrayJoinStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Columns", std::move(columns_array)); } +void ArrayJoinStep::serializeSettings(QueryPlanSerializationSettings & settings) const +{ + settings.max_block_size = max_block_size; +} + +void ArrayJoinStep::serialize(Serialization & ctx) const +{ + UInt8 flags = 0; + if (array_join.is_left) + flags |= 1; + if (is_unaligned) + flags |= 2; + + writeIntBinary(flags, ctx.out); + + writeVarUInt(array_join.columns.size(), ctx.out); + for (const auto & column : array_join.columns) + writeStringBinary(column, ctx.out); +} + +std::unique_ptr ArrayJoinStep::deserialize(Deserialization & ctx) +{ + UInt8 flags; + readIntBinary(flags, ctx.in); + + bool is_left = bool(flags & 1); + bool is_unaligned = bool(flags & 2); + + UInt64 num_columns; + readVarUInt(num_columns, ctx.in); + + ArrayJoin array_join; + array_join.is_left = is_left; + array_join.columns.resize(num_columns); + + for (auto & column : array_join.columns) + readStringBinary(column, ctx.in); + + return std::make_unique(ctx.input_headers.front(), std::move(array_join), is_unaligned, ctx.settings.max_block_size); +} + +void registerArrayJoinStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("ArrayJoin", ArrayJoinStep::deserialize); +} + } diff --git a/src/Processors/QueryPlan/ArrayJoinStep.h b/src/Processors/QueryPlan/ArrayJoinStep.h index 34eb34b5b25..add6212253b 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.h +++ b/src/Processors/QueryPlan/ArrayJoinStep.h @@ -22,6 +22,11 @@ public: const Names & getColumns() const { return array_join.columns; } bool isLeft() const { return array_join.is_left; } + void serializeSettings(QueryPlanSerializationSettings & settings) const override; + void serialize(Serialization & ctx) const override; + + static std::unique_ptr deserialize(Deserialization & ctx); + private: void updateOutputHeader() override; diff --git a/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp b/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp index 1832cc2ad42..c1ba13c77c8 100644 --- a/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp +++ b/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp @@ -9,6 +9,8 @@ namespace DB namespace Setting { extern const SettingsBool query_plan_merge_filters; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 aggregation_memory_efficient_merge_threads; } BuildQueryPipelineSettings BuildQueryPipelineSettings::fromContext(ContextPtr from) @@ -19,6 +21,9 @@ BuildQueryPipelineSettings BuildQueryPipelineSettings::fromContext(ContextPtr fr settings.process_list_element = from->getProcessListElement(); settings.progress_callback = from->getProgressCallback(); + settings.max_threads = from->getSettingsRef()[Setting::max_threads]; + settings.aggregation_memory_efficient_merge_threads = from->getSettingsRef()[Setting::aggregation_memory_efficient_merge_threads]; + /// Setting query_plan_merge_filters is enabled by default. /// But it can brake short-circuit without splitting filter step into smaller steps. /// So, enable and disable this optimizations together. diff --git a/src/Processors/QueryPlan/BuildQueryPipelineSettings.h b/src/Processors/QueryPlan/BuildQueryPipelineSettings.h index 6219e37db58..74ddd789ab1 100644 --- a/src/Processors/QueryPlan/BuildQueryPipelineSettings.h +++ b/src/Processors/QueryPlan/BuildQueryPipelineSettings.h @@ -24,6 +24,9 @@ struct BuildQueryPipelineSettings ProgressCallback progress_callback = nullptr; TemporaryFileLookupPtr temporary_file_lookup; + size_t max_threads; + size_t aggregation_memory_efficient_merge_threads; + const ExpressionActionsSettings & getActionsSettings() const { return actions_settings; } static BuildQueryPipelineSettings fromContext(ContextPtr from); }; diff --git a/src/Processors/QueryPlan/CreatingSetsStep.h b/src/Processors/QueryPlan/CreatingSetsStep.h index 0495ca2e638..5501cbd9bde 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.h +++ b/src/Processors/QueryPlan/CreatingSetsStep.h @@ -66,6 +66,8 @@ public: ContextPtr getContext() const { return context; } PreparedSets::Subqueries detachSets() { return std::move(subqueries); } + void serialize(Serialization &) const override {} + private: void updateOutputHeader() override { output_header = getInputHeaders().front(); } diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index 42f7f8d6e66..b7ce2ab612c 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -1,4 +1,7 @@ #include +#include +#include +#include #include #include #include @@ -13,6 +16,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; } static ITransformingStep::Traits getTraits(bool pre_distinct) @@ -48,6 +52,16 @@ DistinctStep::DistinctStep( { } +void DistinctStep::updateLimitHint(UInt64 hint) +{ + if (hint && limit_hint) + /// Both limits are set - take the min + limit_hint = std::min(hint, limit_hint); + else + /// Some limit is not set - take the other one + limit_hint = std::max(hint, limit_hint); +} + void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { if (!pre_distinct) @@ -158,4 +172,58 @@ void DistinctStep::updateOutputHeader() output_header = input_headers.front(); } +void DistinctStep::serializeSettings(QueryPlanSerializationSettings & settings) const +{ + settings.max_rows_in_distinct = set_size_limits.max_rows; + settings.max_bytes_in_distinct = set_size_limits.max_bytes; + settings.distinct_overflow_mode = set_size_limits.overflow_mode; +} + +void DistinctStep::serialize(Serialization & ctx) const +{ + /// Let's not serialize limit_hint. + /// Ideally, we can get if from a query plan optimization on the follower. + + writeVarUInt(columns.size(), ctx.out); + for (const auto & column : columns) + writeStringBinary(column, ctx.out); +} + +std::unique_ptr DistinctStep::deserialize(Deserialization & ctx, bool pre_distinct_) +{ + if (ctx.input_headers.size() != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "DistinctStep must have one input stream"); + + size_t columns_size; + readVarUInt(columns_size, ctx.in); + Names column_names(columns_size); + for (size_t i = 0; i < columns_size; ++i) + readStringBinary(column_names[i], ctx.in); + + SizeLimits size_limits; + size_limits.max_rows = ctx.settings.max_rows_in_distinct; + size_limits.max_bytes = ctx.settings.max_bytes_in_distinct; + size_limits.overflow_mode = ctx.settings.distinct_overflow_mode; + + return std::make_unique( + ctx.input_headers.front(), size_limits, 0, column_names, pre_distinct_); +} + +std::unique_ptr DistinctStep::deserializeNormal(Deserialization & ctx) +{ + return DistinctStep::deserialize(ctx, false); +} +std::unique_ptr DistinctStep::deserializePre(Deserialization & ctx) +{ + return DistinctStep::deserialize(ctx, true); +} + +void registerDistinctStep(QueryPlanStepRegistry & registry) +{ + /// Preliminary distinct probably can be a query plan optimization. + /// It's easier to serialize it using different names, so that pre-distinct can be potentially removed later. + registry.registerStep("Distinct", DistinctStep::deserializeNormal); + registry.registerStep("PreDistinct", DistinctStep::deserializePre); +} + } diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index d6caf92d072..4b70b67b5e0 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -20,6 +20,8 @@ public: String getName() const override { return "Distinct"; } const Names & getColumnNames() const { return columns; } + String getSerializationName() const override { return pre_distinct ? "PreDistinct" : "Distinct"; } + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; void describeActions(JSONBuilder::JSONMap & map) const override; @@ -28,6 +30,14 @@ public: bool isPreliminary() const { return pre_distinct; } UInt64 getLimitHint() const { return limit_hint; } + void updateLimitHint(UInt64 hint); + + void serializeSettings(QueryPlanSerializationSettings & settings) const override; + void serialize(Serialization & ctx) const override; + + static std::unique_ptr deserialize(Deserialization & ctx, bool pre_distinct_); + static std::unique_ptr deserializeNormal(Deserialization & ctx); + static std::unique_ptr deserializePre(Deserialization & ctx); const SizeLimits & getSetSizeLimits() const { return set_size_limits; } diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp index 963932072da..32f80769b04 100644 --- a/src/Processors/QueryPlan/ExpressionStep.cpp +++ b/src/Processors/QueryPlan/ExpressionStep.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -10,6 +12,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} + static ITransformingStep::Traits getTraits(const ActionsDAG & actions) { return ITransformingStep::Traits @@ -109,4 +116,23 @@ void ExpressionStep::updateOutputHeader() output_header = ExpressionTransform::transformHeader(input_headers.front(), actions_dag); } +void ExpressionStep::serialize(Serialization & ctx) const +{ + actions_dag.serialize(ctx.out, ctx.registry); +} + +std::unique_ptr ExpressionStep::deserialize(Deserialization & ctx) +{ + ActionsDAG actions_dag = ActionsDAG::deserialize(ctx.in, ctx.registry, ctx.context); + if (ctx.input_headers.size() != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "ExpressionStep must have one input stream"); + + return std::make_unique(ctx.input_headers.front(), std::move(actions_dag)); +} + +void registerExpressionStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("Expression", ExpressionStep::deserialize); +} + } diff --git a/src/Processors/QueryPlan/ExpressionStep.h b/src/Processors/QueryPlan/ExpressionStep.h index 234552e5445..15fbbdad807 100644 --- a/src/Processors/QueryPlan/ExpressionStep.h +++ b/src/Processors/QueryPlan/ExpressionStep.h @@ -25,6 +25,9 @@ public: void describeActions(JSONBuilder::JSONMap & map) const override; + void serialize(Serialization & ctx) const override; + static std::unique_ptr deserialize(Deserialization & ctx); + private: void updateOutputHeader() override; diff --git a/src/Processors/QueryPlan/ExtremesStep.cpp b/src/Processors/QueryPlan/ExtremesStep.cpp index 1eb593df2ab..06f6fd80bc4 100644 --- a/src/Processors/QueryPlan/ExtremesStep.cpp +++ b/src/Processors/QueryPlan/ExtremesStep.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include namespace DB @@ -29,4 +31,19 @@ void ExtremesStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil pipeline.addExtremesTransform(); } +void ExtremesStep::serialize(Serialization & ctx) const +{ + (void)ctx; +} + +std::unique_ptr ExtremesStep::deserialize(Deserialization & ctx) +{ + return std::make_unique(ctx.input_headers.front()); +} + +void registerExtremesStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("Extremes", ExtremesStep::deserialize); +} + } diff --git a/src/Processors/QueryPlan/ExtremesStep.h b/src/Processors/QueryPlan/ExtremesStep.h index 363f0a2f6d4..a41e33f38cb 100644 --- a/src/Processors/QueryPlan/ExtremesStep.h +++ b/src/Processors/QueryPlan/ExtremesStep.h @@ -13,6 +13,9 @@ public: void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; + void serialize(Serialization & ctx) const override; + static std::unique_ptr deserialize(Deserialization & ctx); + private: void updateOutputHeader() override { diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index 49190701553..6fb1e2cb73c 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -16,6 +18,12 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} + static ITransformingStep::Traits getTraits() { return ITransformingStep::Traits @@ -240,4 +248,39 @@ void FilterStep::updateOutputHeader() return; } +void FilterStep::serialize(Serialization & ctx) const +{ + UInt8 flags = 0; + if (remove_filter_column) + flags |= 1; + writeIntBinary(flags, ctx.out); + + writeStringBinary(filter_column_name, ctx.out); + + actions_dag.serialize(ctx.out, ctx.registry); +} + +std::unique_ptr FilterStep::deserialize(Deserialization & ctx) +{ + if (ctx.input_headers.size() != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "FilterStep must have one input stream"); + + UInt8 flags; + readIntBinary(flags, ctx.in); + + bool remove_filter_column = bool(flags & 1); + + String filter_column_name; + readStringBinary(filter_column_name, ctx.in); + + ActionsDAG actions_dag = ActionsDAG::deserialize(ctx.in, ctx.registry, ctx.context); + + return std::make_unique(ctx.input_headers.front(), std::move(actions_dag), std::move(filter_column_name), remove_filter_column); +} + +void registerFilterStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("Filter", FilterStep::deserialize); +} + } diff --git a/src/Processors/QueryPlan/FilterStep.h b/src/Processors/QueryPlan/FilterStep.h index cb90459f0ab..ceefd67f8a6 100644 --- a/src/Processors/QueryPlan/FilterStep.h +++ b/src/Processors/QueryPlan/FilterStep.h @@ -26,6 +26,10 @@ public: const String & getFilterColumnName() const { return filter_column_name; } bool removesFilterColumn() const { return remove_filter_column; } + void serialize(Serialization & ctx) const override; + + static std::unique_ptr deserialize(Deserialization & ctx); + private: void updateOutputHeader() override; diff --git a/src/Processors/QueryPlan/IQueryPlanStep.cpp b/src/Processors/QueryPlan/IQueryPlanStep.cpp index fdb1690bc6d..a6770c4347e 100644 --- a/src/Processors/QueryPlan/IQueryPlanStep.cpp +++ b/src/Processors/QueryPlan/IQueryPlanStep.cpp @@ -8,6 +8,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; } IQueryPlanStep::IQueryPlanStep() @@ -146,4 +147,11 @@ void IQueryPlanStep::appendExtraProcessors(const Processors & extra_processors) processors.insert(processors.end(), extra_processors.begin(), extra_processors.end()); } +void IQueryPlanStep::serialize(Serialization & /*ctx*/) const +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method serialize is not implemented for {}", getName()); +} + +void IQueryPlanStep::updateOutputHeader() { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented"); } + } diff --git a/src/Processors/QueryPlan/IQueryPlanStep.h b/src/Processors/QueryPlan/IQueryPlanStep.h index d38b36a3a78..2f5c63ab36f 100644 --- a/src/Processors/QueryPlan/IQueryPlanStep.h +++ b/src/Processors/QueryPlan/IQueryPlanStep.h @@ -24,6 +24,8 @@ namespace JSONBuilder { class JSONMap; } class QueryPlan; using QueryPlanRawPtrs = std::list; +struct QueryPlanSerializationSettings; + using Header = Block; using Headers = std::vector
; @@ -36,6 +38,7 @@ public: virtual ~IQueryPlanStep() = default; virtual String getName() const = 0; + virtual String getSerializationName() const { return getName(); } /// Add processors from current step to QueryPipeline. /// Calling this method, we assume and don't check that: @@ -54,6 +57,11 @@ public: const std::string & getStepDescription() const { return step_description; } void setStepDescription(std::string description) { step_description = std::move(description); } + struct Serialization; + struct Deserialization; + + virtual void serializeSettings(QueryPlanSerializationSettings & /*settings*/) const {} + virtual void serialize(Serialization & /*ctx*/) const; virtual const SortDescription & getSortDescription() const; struct FormatSettings diff --git a/src/Processors/QueryPlan/LimitByStep.cpp b/src/Processors/QueryPlan/LimitByStep.cpp index 92900b8153f..cb14f6b2ef0 100644 --- a/src/Processors/QueryPlan/LimitByStep.cpp +++ b/src/Processors/QueryPlan/LimitByStep.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -24,11 +26,11 @@ static ITransformingStep::Traits getTraits() LimitByStep::LimitByStep( const Header & input_header_, - size_t group_length_, size_t group_offset_, const Names & columns_) + size_t group_length_, size_t group_offset_, Names columns_) : ITransformingStep(input_header_, input_header_, getTraits()) , group_length(group_length_) , group_offset(group_offset_) - , columns(columns_) + , columns(std::move(columns_)) { } @@ -83,4 +85,37 @@ void LimitByStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Offset", group_offset); } +void LimitByStep::serialize(Serialization & ctx) const +{ + writeVarUInt(group_length, ctx.out); + writeVarUInt(group_offset, ctx.out); + + + writeVarUInt(columns.size(), ctx.out); + for (const auto & column : columns) + writeStringBinary(column, ctx.out); +} + +std::unique_ptr LimitByStep::deserialize(Deserialization & ctx) +{ + UInt64 group_length; + UInt64 group_offset; + + readVarUInt(group_length, ctx.in); + readVarUInt(group_offset, ctx.in); + + UInt64 num_columns; + readVarUInt(num_columns, ctx.in); + Names columns(num_columns); + for (auto & column : columns) + readStringBinary(column, ctx.in); + + return std::make_unique(ctx.input_headers.front(), group_length, group_offset, std::move(columns)); +} + +void registerLimitByStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("LimitBy", LimitByStep::deserialize); +} + } diff --git a/src/Processors/QueryPlan/LimitByStep.h b/src/Processors/QueryPlan/LimitByStep.h index e34d1d5327d..f382ebdbd44 100644 --- a/src/Processors/QueryPlan/LimitByStep.h +++ b/src/Processors/QueryPlan/LimitByStep.h @@ -10,7 +10,7 @@ class LimitByStep : public ITransformingStep public: explicit LimitByStep( const Header & input_header_, - size_t group_length_, size_t group_offset_, const Names & columns_); + size_t group_length_, size_t group_offset_, Names columns_); String getName() const override { return "LimitBy"; } @@ -19,6 +19,10 @@ public: void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; + void serialize(Serialization & ctx) const override; + + static std::unique_ptr deserialize(Deserialization & ctx); + private: void updateOutputHeader() override { diff --git a/src/Processors/QueryPlan/LimitStep.cpp b/src/Processors/QueryPlan/LimitStep.cpp index a186e1f7965..18c401e68c1 100644 --- a/src/Processors/QueryPlan/LimitStep.cpp +++ b/src/Processors/QueryPlan/LimitStep.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -76,4 +78,47 @@ void LimitStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Reads All Data", always_read_till_end); } +void LimitStep::serialize(Serialization & ctx) const +{ + UInt8 flags = 0; + if (always_read_till_end) + flags |= 1; + if (with_ties) + flags |= 2; + + writeIntBinary(flags, ctx.out); + + writeVarUInt(limit, ctx.out); + writeVarUInt(offset, ctx.out); + + if (with_ties) + serializeSortDescription(description, ctx.out); +} + +std::unique_ptr LimitStep::deserialize(Deserialization & ctx) +{ + UInt8 flags; + readIntBinary(flags, ctx.in); + + bool always_read_till_end = bool(flags & 1); + bool with_ties = bool(flags & 2); + + UInt64 limit; + UInt64 offset; + + readVarUInt(limit, ctx.in); + readVarUInt(offset, ctx.in); + + SortDescription description; + if (with_ties) + deserializeSortDescription(description, ctx.in); + + return std::make_unique(ctx.input_headers.front(), limit, offset, always_read_till_end, with_ties, std::move(description)); +} + +void registerLimitStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("Limit", LimitStep::deserialize); +} + } diff --git a/src/Processors/QueryPlan/LimitStep.h b/src/Processors/QueryPlan/LimitStep.h index 4a779259681..e1696c6165e 100644 --- a/src/Processors/QueryPlan/LimitStep.h +++ b/src/Processors/QueryPlan/LimitStep.h @@ -33,6 +33,10 @@ public: bool withTies() const { return with_ties; } + void serialize(Serialization & ctx) const override; + + static std::unique_ptr deserialize(Deserialization & ctx); + private: void updateOutputHeader() override { diff --git a/src/Processors/QueryPlan/OffsetStep.cpp b/src/Processors/QueryPlan/OffsetStep.cpp index 1d25414a44c..05b516a5971 100644 --- a/src/Processors/QueryPlan/OffsetStep.cpp +++ b/src/Processors/QueryPlan/OffsetStep.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -46,4 +48,22 @@ void OffsetStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Offset", offset); } +void OffsetStep::serialize(Serialization & ctx) const +{ + writeVarUInt(offset, ctx.out); +} + +std::unique_ptr OffsetStep::deserialize(Deserialization & ctx) +{ + UInt64 offset; + readVarUInt(offset, ctx.in); + + return std::make_unique(ctx.input_headers.front(), offset); +} + +void registerOffsetStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("Offset", OffsetStep::deserialize); +} + } diff --git a/src/Processors/QueryPlan/OffsetStep.h b/src/Processors/QueryPlan/OffsetStep.h index ae6bc1c66c0..c8b755d15fb 100644 --- a/src/Processors/QueryPlan/OffsetStep.h +++ b/src/Processors/QueryPlan/OffsetStep.h @@ -18,6 +18,10 @@ public: void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; + void serialize(Serialization & ctx) const override; + + static std::unique_ptr deserialize(Deserialization & ctx); + private: void updateOutputHeader() override { diff --git a/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp b/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp index 10cc78da33a..aee7ab23683 100644 --- a/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/limitPushDown.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace DB::QueryPlanOptimizations @@ -63,6 +64,12 @@ size_t tryPushDownLimit(QueryPlan::Node * parent_node, QueryPlan::Nodes &) if (tryUpdateLimitForSortingSteps(child_node, limit->getLimitForSorting())) return 0; + if (auto * distinct = typeid_cast(child.get())) + { + distinct->updateLimitHint(limit->getLimitForSorting()); + return 0; + } + if (typeid_cast(child.get())) return 0; diff --git a/src/Processors/QueryPlan/QueryPlanSerializationSettings.cpp b/src/Processors/QueryPlan/QueryPlanSerializationSettings.cpp new file mode 100644 index 00000000000..9b11132e27c --- /dev/null +++ b/src/Processors/QueryPlan/QueryPlanSerializationSettings.cpp @@ -0,0 +1,8 @@ +#include + +namespace DB +{ + +IMPLEMENT_SETTINGS_TRAITS(QueryPlanSerializationSettingsTraits, PLAN_SERIALIZATION_SETTINGS); + +} diff --git a/src/Processors/QueryPlan/QueryPlanSerializationSettings.h b/src/Processors/QueryPlan/QueryPlanSerializationSettings.h new file mode 100644 index 00000000000..4e119051a51 --- /dev/null +++ b/src/Processors/QueryPlan/QueryPlanSerializationSettings.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +#define PLAN_SERIALIZATION_SETTINGS(M, ALIAS) \ + M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size in rows for reading", 0) \ + \ + M(UInt64, max_rows_in_distinct, 0, "Maximum number of elements during execution of DISTINCT.", 0) \ + M(UInt64, max_bytes_in_distinct, 0, "Maximum total size of state (in uncompressed bytes) in memory for the execution of DISTINCT.", 0) \ + M(OverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + \ + M(UInt64, max_rows_to_sort, 0, "If more than the specified amount of records have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \ + M(UInt64, max_bytes_to_sort, 0, "If more than the specified amount of (uncompressed) bytes have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \ + M(OverflowMode, sort_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + \ + M(UInt64, prefer_external_sort_block_bytes, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging.", 0) \ + M(UInt64, max_bytes_before_external_sort, 0, "If memory usage during ORDER BY operation is exceeding this threshold in bytes, activate the 'external sorting' mode (spill data to disk). Recommended value is half of available system memory.", 0) \ + M(UInt64, max_bytes_before_remerge_sort, 1000000000, "In case of ORDER BY with LIMIT, when memory usage is higher than specified threshold, perform additional steps of merging blocks before final merge to keep just top LIMIT rows.", 0) \ + M(Float, remerge_sort_lowered_memory_bytes_ratio, 2., "If memory usage after remerge does not reduced by this ratio, remerge will be disabled.", 0) \ + M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \ + \ + M(UInt64, aggregation_in_order_max_block_bytes, 50000000, "Maximal size of block in bytes accumulated during aggregation in order of primary key. Lower block size allows to parallelize more final merge stage of aggregation.", 0) \ + M(Bool, aggregation_in_order_memory_bound_merging, true, "Enable memory bound merging strategy when in-order is applied.", 0) \ + M(Bool, aggregation_sort_result_by_bucket_number, true, "Send intermediate aggregation result in order of bucket number.", 0) \ + \ + M(UInt64, max_rows_to_group_by, 0, "If aggregation during GROUP BY is generating more than the specified number of rows (unique GROUP BY keys), the behavior will be determined by the 'group_by_overflow_mode' which by default is - throw an exception, but can be also switched to an approximate GROUP BY mode.", 0) \ + M(OverflowModeGroupBy, group_by_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \ + M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ + M(UInt64, max_bytes_before_external_group_by, 0, "If memory usage during GROUP BY operation is exceeding this threshold in bytes, activate the 'external aggregation' mode (spill data to disk). Recommended value is half of available system memory.", 0) \ + M(Bool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \ + M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \ + M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \ + M(Bool, enable_software_prefetch_in_aggregation, true, "Enable use of software prefetch in aggregation", 0) \ + M(Bool, optimize_group_by_constant_keys, true, "Optimize GROUP BY when all keys in block are constant", 0) \ + M(Float, min_hit_rate_to_use_consecutive_keys_optimization, 0.5, "Minimal hit rate of a cache which is used for consecutive keys optimization in aggregation to keep it enabled", 0) \ + M(Bool, collect_hash_table_stats_during_aggregation, true, "Enable collecting hash table statistics to optimize memory allocation", 0) \ + M(UInt64, max_entries_for_hash_table_stats, 10'000, "How many entries hash table statistics collected during aggregation is allowed to have", 0) \ + M(UInt64, max_size_to_preallocate_for_aggregation, 100'000'000, "For how many elements it is allowed to preallocate space in all hash tables in total before aggregation", 0) \ + \ + M(TotalsMode, totals_mode, TotalsMode::AFTER_HAVING_EXCLUSIVE, "How to calculate TOTALS when HAVING is present, as well as when max_rows_to_group_by and group_by_overflow_mode = ‘any’ are present.", IMPORTANT) \ + M(Float, totals_auto_threshold, 0.5, "The threshold for totals_mode = 'auto'.", 0) \ + \ + + +DECLARE_SETTINGS_TRAITS(QueryPlanSerializationSettingsTraits, PLAN_SERIALIZATION_SETTINGS) + +struct QueryPlanSerializationSettings : public BaseSettings +{ + QueryPlanSerializationSettings() = default; +}; + +} diff --git a/src/Processors/QueryPlan/QueryPlanStepRegistry.cpp b/src/Processors/QueryPlan/QueryPlanStepRegistry.cpp new file mode 100644 index 00000000000..0df21ff9d05 --- /dev/null +++ b/src/Processors/QueryPlan/QueryPlanStepRegistry.cpp @@ -0,0 +1,70 @@ +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_IDENTIFIER; + extern const int LOGICAL_ERROR; +} + +QueryPlanStepRegistry & QueryPlanStepRegistry::instance() +{ + static QueryPlanStepRegistry registry; + return registry; +} + +void QueryPlanStepRegistry::registerStep(const std::string & name, StepCreateFunction && create_function) +{ + if (steps.contains(name)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Query plan step '{}' is already registered", name); + steps[name] = std::move(create_function); +} + +QueryPlanStepPtr QueryPlanStepRegistry::createStep( + const std::string & name, + IQueryPlanStep::Deserialization & ctx) const +{ + StepCreateFunction create_function; + { + auto it = steps.find(name); + if (it == steps.end()) + throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "Unknown query plan step: {}", name); + create_function = it->second; + } + return create_function(ctx); +} + +void registerExpressionStep(QueryPlanStepRegistry & registry); +void registerUnionStep(QueryPlanStepRegistry & registry); +void registerDistinctStep(QueryPlanStepRegistry & registry); +void registerSortingStep(QueryPlanStepRegistry & registry); +void registerAggregatingStep(QueryPlanStepRegistry & registry); +void registerArrayJoinStep(QueryPlanStepRegistry & registry); +void registerLimitByStep(QueryPlanStepRegistry & registry); +void registerLimitStep(QueryPlanStepRegistry & registry); +void registerOffsetStep(QueryPlanStepRegistry & registry); +void registerFilterStep(QueryPlanStepRegistry & registry); +void registerTotalsHavingStep(QueryPlanStepRegistry & registry); +void registerExtremesStep(QueryPlanStepRegistry & registry); + +void QueryPlanStepRegistry::registerPlanSteps() +{ + QueryPlanStepRegistry & registry = QueryPlanStepRegistry::instance(); + + registerExpressionStep(registry); + registerUnionStep(registry); + registerDistinctStep(registry); + registerSortingStep(registry); + registerAggregatingStep(registry); + registerArrayJoinStep(registry); + registerLimitByStep(registry); + registerLimitStep(registry); + registerOffsetStep(registry); + registerFilterStep(registry); + registerTotalsHavingStep(registry); + registerExtremesStep(registry); +} + +} diff --git a/src/Processors/QueryPlan/QueryPlanStepRegistry.h b/src/Processors/QueryPlan/QueryPlanStepRegistry.h new file mode 100644 index 00000000000..fdd1232e102 --- /dev/null +++ b/src/Processors/QueryPlan/QueryPlanStepRegistry.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace DB +{ + +class QueryPlanStepRegistry +{ +public: + using StepCreateFunction = std::function; + + QueryPlanStepRegistry() = default; + QueryPlanStepRegistry(const QueryPlanStepRegistry &) = delete; + QueryPlanStepRegistry & operator=(const QueryPlanStepRegistry &) = delete; + + static QueryPlanStepRegistry & instance(); + + static void registerPlanSteps(); + + void registerStep(const std::string & name, StepCreateFunction && create_function); + + QueryPlanStepPtr createStep( + const std::string & name, + IQueryPlanStep::Deserialization & ctx) const; + +private: + std::unordered_map steps; + +}; + +} diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 6899dc7f5d6..6317cce83bd 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1959,9 +1959,10 @@ bool ReadFromMergeTree::isQueryWithSampling() const if (context->getSettingsRef()[Setting::parallel_replicas_count] > 1 && data.supportsSampling()) return true; - const auto & select = query_info.query->as(); if (query_info.table_expression_modifiers) return query_info.table_expression_modifiers->getSampleSizeRatio() != std::nullopt; + + const auto & select = query_info.query->as(); return select.sampleSize() != nullptr; } diff --git a/src/Processors/QueryPlan/Serialization.h b/src/Processors/QueryPlan/Serialization.h new file mode 100644 index 00000000000..80c7dee0252 --- /dev/null +++ b/src/Processors/QueryPlan/Serialization.h @@ -0,0 +1,30 @@ +#pragma once +#include +#include + +namespace DB +{ + +struct SerializedSetsRegistry; +struct DeserializedSetsRegistry; + +struct IQueryPlanStep::Serialization +{ + WriteBuffer & out; + SerializedSetsRegistry & registry; +}; + +struct SerializedSetsRegistry; + +struct IQueryPlanStep::Deserialization +{ + ReadBuffer & in; + DeserializedSetsRegistry & registry; + const ContextPtr & context; + + const Headers & input_headers; + const Header * output_header; + const QueryPlanSerializationSettings & settings; +}; + +} diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index 4d671e325f6..c488610a4c3 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -2,6 +2,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -43,6 +46,8 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; + extern const int NOT_IMPLEMENTED; + extern const int INCORRECT_DATA; } SortingStep::Settings::Settings(const Context & context) @@ -88,6 +93,33 @@ SortingStep::Settings::Settings(size_t max_block_size_) max_block_size = max_block_size_; } +SortingStep::Settings::Settings(const QueryPlanSerializationSettings & settings) +{ + max_block_size = settings.max_block_size; + size_limits = SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode); + max_bytes_before_remerge = settings.max_bytes_before_remerge_sort; + remerge_lowered_memory_bytes_ratio = settings.remerge_sort_lowered_memory_bytes_ratio; + max_bytes_before_external_sort = settings.max_bytes_before_external_sort; + tmp_data = Context::getGlobalContextInstance()->getTempDataOnDisk(); + min_free_disk_space = settings.min_free_disk_space_for_temporary_data; + max_block_bytes = settings.prefer_external_sort_block_bytes; + read_in_order_use_buffering = false; //settings.read_in_order_use_buffering; +} + +void SortingStep::Settings::updatePlanSettings(QueryPlanSerializationSettings & settings) const +{ + settings.max_block_size = max_block_size; + settings.max_rows_to_sort = size_limits.max_rows; + settings.max_bytes_to_sort = size_limits.max_bytes; + settings.sort_overflow_mode = size_limits.overflow_mode; + + settings.max_bytes_before_remerge_sort = max_bytes_before_remerge; + settings.remerge_sort_lowered_memory_bytes_ratio = remerge_lowered_memory_bytes_ratio; + settings.max_bytes_before_external_sort = max_bytes_before_external_sort; + settings.min_free_disk_space_for_temporary_data = min_free_disk_space; + settings.prefer_external_sort_block_bytes = max_block_bytes; +} + static ITransformingStep::Traits getTraits(size_t limit) { return ITransformingStep::Traits @@ -452,4 +484,52 @@ void SortingStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Limit", limit); } +void SortingStep::serializeSettings(QueryPlanSerializationSettings & settings) const +{ + sort_settings.updatePlanSettings(settings); +} + +void SortingStep::serialize(Serialization & ctx) const +{ + if (type != Type::Full) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Serialization of SortingStep is implemented only for Full sorting"); + + /// Do not serialize type here; Later we can use different names if needed.\ + + /// Do not serialize limit for now; it is expected to be pushed down from plan optimization. + + serializeSortDescription(result_description, ctx.out); + + /// Later + if (!partition_by_description.empty()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Serialization of partitioned sorting is not implemented for SortingStep"); + + writeVarUInt(partition_by_description.size(), ctx.out); +} + +std::unique_ptr SortingStep::deserialize(Deserialization & ctx) +{ + if (ctx.input_headers.size() != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "SortingStep must have one input stream"); + + SortingStep::Settings sort_settings(ctx.settings); + + SortDescription result_description; + deserializeSortDescription(result_description, ctx.in); + + UInt64 partition_desc_size; + readVarUInt(partition_desc_size, ctx.in); + + if (partition_desc_size) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Deserialization of partitioned sorting is not implemented for SortingStep"); + + return std::make_unique( + ctx.input_headers.front(), std::move(result_description), 0, std::move(sort_settings)); +} + +void registerSortingStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("Sorting", SortingStep::deserialize); +} + } diff --git a/src/Processors/QueryPlan/SortingStep.h b/src/Processors/QueryPlan/SortingStep.h index d831fe0234f..5eb9fbf1db3 100644 --- a/src/Processors/QueryPlan/SortingStep.h +++ b/src/Processors/QueryPlan/SortingStep.h @@ -23,7 +23,7 @@ public: size_t max_block_size; SizeLimits size_limits; size_t max_bytes_before_remerge = 0; - double remerge_lowered_memory_bytes_ratio = 0; + float remerge_lowered_memory_bytes_ratio = 0; size_t max_bytes_before_external_sort = 0; TemporaryDataOnDiskScopePtr tmp_data = nullptr; size_t min_free_disk_space = 0; @@ -32,6 +32,9 @@ public: explicit Settings(const Context & context); explicit Settings(size_t max_block_size_); + explicit Settings(const QueryPlanSerializationSettings & settings); + + void updatePlanSettings(QueryPlanSerializationSettings & settings) const; }; /// Full @@ -96,6 +99,11 @@ public: UInt64 limit_, bool skip_partial_sort = false); + void serializeSettings(QueryPlanSerializationSettings & settings) const override; + void serialize(Serialization & ctx) const override; + + static std::unique_ptr deserialize(Deserialization & ctx); + private: void scatterByPartitionIfNeeded(QueryPipelineBuilder& pipeline); void updateOutputHeader() override; diff --git a/src/Processors/QueryPlan/SourceStepWithFilter.h b/src/Processors/QueryPlan/SourceStepWithFilter.h index be6585b8755..a650cdc3211 100644 --- a/src/Processors/QueryPlan/SourceStepWithFilter.h +++ b/src/Processors/QueryPlan/SourceStepWithFilter.h @@ -53,6 +53,9 @@ public: void setLimit(size_t limit_value) { + if (limit) + limit_value = std::min(limit_value, *limit); + limit = limit_value; } diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp index 5cd483862ff..df741e01f9b 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.cpp +++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp @@ -1,4 +1,7 @@ #include +#include +#include +#include #include #include #include @@ -10,6 +13,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} + static ITransformingStep::Traits getTraits(bool has_filter) { return ITransformingStep::Traits @@ -33,7 +41,7 @@ TotalsHavingStep::TotalsHavingStep( const std::string & filter_column_, bool remove_filter_, TotalsMode totals_mode_, - double auto_include_threshold_, + float auto_include_threshold_, bool final_) : ITransformingStep( input_header_, @@ -141,5 +149,75 @@ void TotalsHavingStep::updateOutputHeader() getAggregatesMask(input_headers.front(), aggregates)); } +void TotalsHavingStep::serializeSettings(QueryPlanSerializationSettings & settings) const +{ + settings.totals_mode = totals_mode; + settings.totals_auto_threshold = auto_include_threshold; +} + +void TotalsHavingStep::serialize(Serialization & ctx) const +{ + UInt8 flags = 0; + if (final) + flags |= 1; + if (overflow_row) + flags |= 2; + if (actions_dag) + flags |= 4; + if (actions_dag && remove_filter) + flags |= 8; + + writeIntBinary(flags, ctx.out); + + serializeAggregateDescriptions(aggregates, ctx.out); + + if (actions_dag) + { + writeStringBinary(filter_column_name, ctx.out); + actions_dag->serialize(ctx.out, ctx.registry); + } +} + +std::unique_ptr TotalsHavingStep::deserialize(Deserialization & ctx) +{ + if (ctx.input_headers.size() != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "TotalsHaving must have one input stream"); + + UInt8 flags; + readIntBinary(flags, ctx.in); + + bool final = bool(flags & 1); + bool overflow_row = bool(flags & 2); + bool has_actions_dag = bool(flags & 4); + bool remove_filter_column = bool(flags & 8); + + AggregateDescriptions aggregates; + deserializeAggregateDescriptions(aggregates, ctx.in); + + std::optional actions_dag; + String filter_column_name; + if (has_actions_dag) + { + readStringBinary(filter_column_name, ctx.in); + + actions_dag = ActionsDAG::deserialize(ctx.in, ctx.registry, ctx.context); + } + + return std::make_unique( + ctx.input_headers.front(), + std::move(aggregates), + overflow_row, + std::move(actions_dag), + std::move(filter_column_name), + remove_filter_column, + ctx.settings.totals_mode, + ctx.settings.totals_auto_threshold, + final); +} + +void registerTotalsHavingStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("TotalsHaving", TotalsHavingStep::deserialize); +} } diff --git a/src/Processors/QueryPlan/TotalsHavingStep.h b/src/Processors/QueryPlan/TotalsHavingStep.h index 3bcd6360c80..a68eb4c75ca 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.h +++ b/src/Processors/QueryPlan/TotalsHavingStep.h @@ -20,7 +20,7 @@ public: const std::string & filter_column_, bool remove_filter_, TotalsMode totals_mode_, - double auto_include_threshold_, + float auto_include_threshold_, bool final_); String getName() const override { return "TotalsHaving"; } @@ -32,6 +32,11 @@ public: const ActionsDAG * getActions() const { return actions_dag ? &*actions_dag : nullptr; } + void serializeSettings(QueryPlanSerializationSettings & settings) const override; + void serialize(Serialization & ctx) const override; + + static std::unique_ptr deserialize(Deserialization & ctx); + private: void updateOutputHeader() override; @@ -42,7 +47,7 @@ private: String filter_column_name; bool remove_filter; TotalsMode totals_mode; - double auto_include_threshold; + float auto_include_threshold; bool final; }; diff --git a/src/Processors/QueryPlan/UnionStep.cpp b/src/Processors/QueryPlan/UnionStep.cpp index d5c2469629b..0808b833d01 100644 --- a/src/Processors/QueryPlan/UnionStep.cpp +++ b/src/Processors/QueryPlan/UnionStep.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include @@ -37,7 +39,7 @@ void UnionStep::updateOutputHeader() output_header = checkHeaders(input_headers); } -QueryPipelineBuilderPtr UnionStep::updatePipeline(QueryPipelineBuilders pipelines, const BuildQueryPipelineSettings &) +QueryPipelineBuilderPtr UnionStep::updatePipeline(QueryPipelineBuilders pipelines, const BuildQueryPipelineSettings & settings) { auto pipeline = std::make_unique(); @@ -49,6 +51,8 @@ QueryPipelineBuilderPtr UnionStep::updatePipeline(QueryPipelineBuilders pipeline return pipeline; } + size_t new_max_threads = max_threads ? max_threads : settings.max_threads; + for (auto & cur_pipeline : pipelines) { #if !defined(NDEBUG) @@ -75,7 +79,7 @@ QueryPipelineBuilderPtr UnionStep::updatePipeline(QueryPipelineBuilders pipeline } } - *pipeline = QueryPipelineBuilder::unitePipelines(std::move(pipelines), max_threads, &processors); + *pipeline = QueryPipelineBuilder::unitePipelines(std::move(pipelines), new_max_threads, &processors); return pipeline; } @@ -84,4 +88,19 @@ void UnionStep::describePipeline(FormatSettings & settings) const IQueryPlanStep::describePipeline(processors, settings); } +void UnionStep::serialize(Serialization & ctx) const +{ + (void)ctx; +} + +std::unique_ptr UnionStep::deserialize(Deserialization & ctx) +{ + return std::make_unique(ctx.input_headers); +} + +void registerUnionStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("Union", &UnionStep::deserialize); +} + } diff --git a/src/Processors/QueryPlan/UnionStep.h b/src/Processors/QueryPlan/UnionStep.h index efb8f51c7a4..87a3a064eaf 100644 --- a/src/Processors/QueryPlan/UnionStep.h +++ b/src/Processors/QueryPlan/UnionStep.h @@ -13,12 +13,15 @@ public: String getName() const override { return "Union"; } - QueryPipelineBuilderPtr updatePipeline(QueryPipelineBuilders pipelines, const BuildQueryPipelineSettings &) override; + QueryPipelineBuilderPtr updatePipeline(QueryPipelineBuilders pipelines, const BuildQueryPipelineSettings & settings) override; void describePipeline(FormatSettings & settings) const override; size_t getMaxThreads() const { return max_threads; } + void serialize(Serialization & ctx) const override; + static std::unique_ptr deserialize(Deserialization & ctx); + private: void updateOutputHeader() override; diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.cpp b/src/Processors/TTL/TTLAggregationAlgorithm.cpp index 5fc76db2eec..a86f24b06c4 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.cpp +++ b/src/Processors/TTL/TTLAggregationAlgorithm.cpp @@ -43,15 +43,25 @@ TTLAggregationAlgorithm::TTLAggregationAlgorithm( const Settings & settings = storage_.getContext()->getSettingsRef(); Aggregator::Params params( - settings, keys, aggregates, /*overflow_row_=*/ false, - /*group_by_two_level_threshold_=*/ 0, - /*group_by_two_level_threshold_bytes_=*/ 0, + settings[Setting::max_rows_to_group_by], + settings[Setting::group_by_overflow_mode], + /*group_by_two_level_threshold*/ 0, + /*group_by_two_level_threshold_bytes*/ 0, + Aggregator::Params::getMaxBytesBeforeExternalGroupBy(settings[Setting::max_bytes_before_external_group_by], settings[Setting::max_bytes_ratio_before_external_group_by]), settings[Setting::empty_result_for_aggregation_by_empty_set], storage_.getContext()->getTempDataOnDisk(), - /*only_merge_=*/false, + settings[Setting::max_threads], + settings[Setting::min_free_disk_space_for_temporary_data], + settings[Setting::compile_aggregate_expressions], + settings[Setting::min_count_to_compile_aggregate_expression], + settings[Setting::max_block_size], + settings[Setting::enable_software_prefetch_in_aggregation], + /*only_merge=*/false, + settings[Setting::optimize_group_by_constant_keys], + settings[Setting::min_chunk_bytes_for_parallel_parsing], /*stats_collecting_params_=*/{}); aggregator = std::make_unique(header, params); diff --git a/src/QueryPipeline/SizeLimits.cpp b/src/QueryPipeline/SizeLimits.cpp index 4161f3f365f..3e86ae7e9f9 100644 --- a/src/QueryPipeline/SizeLimits.cpp +++ b/src/QueryPipeline/SizeLimits.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include namespace ProfileEvents @@ -14,6 +16,12 @@ namespace ProfileEvents namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; +} + bool SizeLimits::check(UInt64 rows, UInt64 bytes, const char * what, int too_many_rows_exception_code, int too_many_bytes_exception_code) const { if (overflow_mode == OverflowMode::THROW) @@ -64,4 +72,25 @@ bool SizeLimits::check(UInt64 rows, UInt64 bytes, const char * what, int excepti return check(rows, bytes, what, exception_code, exception_code); } +static void checkAllowedOwerflowMode(OverflowMode mode, int code) +{ + if (!(mode == OverflowMode::BREAK || mode == OverflowMode::THROW)) + throw Exception(code, "Unexpected overflow mode {}", mode); +} + +void SizeLimits::serialize(WriteBuffer & out) const +{ + checkAllowedOwerflowMode(overflow_mode, ErrorCodes::LOGICAL_ERROR); + writeVarUInt(max_rows, out); + writeVarUInt(max_bytes, out); + writeIntBinary(overflow_mode, out); +} +void SizeLimits::deserialize(ReadBuffer & in) +{ + checkAllowedOwerflowMode(overflow_mode, ErrorCodes::INCORRECT_DATA); + readVarUInt(max_rows, in); + readVarUInt(max_bytes, in); + readIntBinary(overflow_mode, in); +} + } diff --git a/src/QueryPipeline/SizeLimits.h b/src/QueryPipeline/SizeLimits.h index 1c84f81a127..04293ee8966 100644 --- a/src/QueryPipeline/SizeLimits.h +++ b/src/QueryPipeline/SizeLimits.h @@ -18,6 +18,9 @@ enum class OverflowMode : uint8_t ANY = 2, }; +class WriteBuffer; +class ReadBuffer; + struct SizeLimits { @@ -38,6 +41,9 @@ struct SizeLimits bool softCheck(UInt64 rows, UInt64 bytes) const; bool hasLimits() const { return max_rows || max_bytes; } + + void serialize(WriteBuffer & out) const; + void deserialize(ReadBuffer & in); }; } From 5144fec1b00a2e31bae48a3e91e93b80125c30d4 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 3 Dec 2024 16:18:29 +0100 Subject: [PATCH 286/502] Fix typo --- src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp index 8034c9bf8ae..09af871a7cd 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp @@ -137,7 +137,7 @@ ObjectStorageQueueSource::Source::ObjectInfoPtr ObjectStorageQueueSource::FileIt /// so replica B would successfully set itself as processor of this file in keeper /// and face "The specified key does not exist" after that. /// - /// This existance check here is enough, + /// This existence check here is enough, /// only because we do applyActionAfterProcessing BEFORE setting file as processed /// and because at this exact place we already successfully set file as processing, /// e.g. file deletion and marking file as processed in keeper already took place. From 9ba29534722042195238e1917e2a22d1b070d476 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 3 Dec 2024 17:10:43 +0100 Subject: [PATCH 287/502] More accurate PK size calculation --- src/Storages/MergeTree/PrimaryIndexCache.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/PrimaryIndexCache.h b/src/Storages/MergeTree/PrimaryIndexCache.h index 5ec185dcf58..c960b3349f5 100644 --- a/src/Storages/MergeTree/PrimaryIndexCache.h +++ b/src/Storages/MergeTree/PrimaryIndexCache.h @@ -24,9 +24,10 @@ struct PrimaryIndexWeightFunction size_t operator()(const PrimaryIndex & index) const { - size_t res = 0; + size_t res = PRIMARY_INDEX_CACHE_OVERHEAD; + res += index.capacity() * sizeof(PrimaryIndex::value_type); for (const auto & column : index) - res += column->byteSize(); + res += column->allocatedBytes(); return res; } }; From c04a28d0095dc0296e1af4f08072826adfc37019 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:31:45 +0100 Subject: [PATCH 288/502] Change the logic to use analyzer --- .../AggregateFunctionGroupConcat.cpp | 340 ++++++++---------- .../AggregateFunctionGroupConcat.h | 78 ++++ src/Analyzer/QueryTreeBuilder.cpp | 94 +++-- src/Parsers/ASTFunction.cpp | 42 +-- src/Parsers/ASTFunction.h | 2 - 5 files changed, 304 insertions(+), 252 deletions(-) create mode 100644 src/AggregateFunctions/AggregateFunctionGroupConcat.h diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp index 8cf5ec5705a..81dd7541e3a 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp @@ -1,27 +1,7 @@ -#include -#include -#include - -#include -#include +#include #include - -#include -#include - -#include -#include #include -#include -#include -#include -#include - -#include -#include - - namespace DB { struct Settings; @@ -33,209 +13,193 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -namespace +void GroupConcatDataBase::checkAndUpdateSize(UInt64 add, Arena * arena) { + if (data_size + add >= allocated_size) + { + auto old_size = allocated_size; + allocated_size = std::max(2 * allocated_size, data_size + add); + data = arena->realloc(data, old_size, allocated_size); + } +} -struct GroupConcatDataBase +void GroupConcatDataBase::insertChar(const char * str, UInt64 str_size, Arena * arena) { - UInt64 data_size = 0; - UInt64 allocated_size = 0; - char * data = nullptr; + checkAndUpdateSize(str_size, arena); + memcpy(data + data_size, str, str_size); + data_size += str_size; +} - void checkAndUpdateSize(UInt64 add, Arena * arena) - { - if (data_size + add >= allocated_size) - { - auto old_size = allocated_size; - allocated_size = std::max(2 * allocated_size, data_size + add); - data = arena->realloc(data, old_size, allocated_size); - } - } - - void insertChar(const char * str, UInt64 str_size, Arena * arena) - { - checkAndUpdateSize(str_size, arena); - memcpy(data + data_size, str, str_size); - data_size += str_size; - } - - void insert(const IColumn * column, const SerializationPtr & serialization, size_t row_num, Arena * arena) - { - WriteBufferFromOwnString buff; - serialization->serializeText(*column, row_num, buff, FormatSettings{}); - auto string = buff.stringView(); - insertChar(string.data(), string.size(), arena); - } - -}; +void GroupConcatDataBase::insert(const IColumn * column, const SerializationPtr & serialization, size_t row_num, Arena * arena) +{ + WriteBufferFromOwnString buff; + serialization->serializeText(*column, row_num, buff, FormatSettings{}); + auto string = buff.stringView(); + insertChar(string.data(), string.size(), arena); +} template -struct GroupConcatData; - -template<> -struct GroupConcatData final : public GroupConcatDataBase +UInt64 GroupConcatData::getSize(size_t i) const { -}; - -template<> -struct GroupConcatData final : public GroupConcatDataBase -{ - using Offset = UInt64; - using Allocator = MixedAlignedArenaAllocator; - using Offsets = PODArray; - - /// offset[i * 2] - beginning of the i-th row, offset[i * 2 + 1] - end of the i-th row - Offsets offsets; - UInt64 num_rows = 0; - - UInt64 getSize(size_t i) const { return offsets[i * 2 + 1] - offsets[i * 2]; } - - UInt64 getString(size_t i) const { return offsets[i * 2]; } - - void insert(const IColumn * column, const SerializationPtr & serialization, size_t row_num, Arena * arena) - { - WriteBufferFromOwnString buff; - serialization->serializeText(*column, row_num, buff, {}); - auto string = buff.stringView(); - - checkAndUpdateSize(string.size(), arena); - memcpy(data + data_size, string.data(), string.size()); - offsets.push_back(data_size, arena); - data_size += string.size(); - offsets.push_back(data_size, arena); - num_rows++; - } -}; + return offsets[i * 2 + 1] - offsets[i * 2]; +} template -class GroupConcatImpl final - : public IAggregateFunctionDataHelper, GroupConcatImpl> +UInt64 GroupConcatData::getString(size_t i) const { - static constexpr auto name = "groupConcat"; + return offsets[i * 2]; +} - SerializationPtr serialization; - UInt64 limit; - const String delimiter; - const DataTypePtr type; +template +void GroupConcatData::insert(const IColumn * column, const SerializationPtr & serialization, size_t row_num, Arena * arena) +{ + WriteBufferFromOwnString buff; + serialization->serializeText(*column, row_num, buff, {}); + auto string = buff.stringView(); -public: - GroupConcatImpl(const DataTypePtr & data_type_, const Array & parameters_, UInt64 limit_, const String & delimiter_) - : IAggregateFunctionDataHelper, GroupConcatImpl>( - {data_type_}, parameters_, std::make_shared()) - , limit(limit_) - , delimiter(delimiter_) - , type(data_type_) - { - serialization = isFixedString(type) ? std::make_shared()->getDefaultSerialization() : this->argument_types[0]->getDefaultSerialization(); - } + checkAndUpdateSize(string.size(), arena); + memcpy(data + data_size, string.data(), string.size()); + offsets.push_back(data_size, arena); + data_size += string.size(); + offsets.push_back(data_size, arena); + num_rows++; +} - String getName() const override { return name; } +template +GroupConcatImpl::GroupConcatImpl( + const DataTypePtr & data_type_, const Array & parameters_, UInt64 limit_, const String & delimiter_) + : IAggregateFunctionDataHelper, GroupConcatImpl>( + {data_type_}, parameters_, std::make_shared()) + , limit(limit_) + , delimiter(delimiter_) + , type(data_type_) +{ + serialization = isFixedString(type) ? std::make_shared()->getDefaultSerialization() : this->argument_types[0]->getDefaultSerialization(); +} - void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override - { - auto & cur_data = this->data(place); +template +String GroupConcatImpl::getName() const +{ + return name; +} - if constexpr (has_limit) - if (cur_data.num_rows >= limit) - return; - if (cur_data.data_size != 0) - cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); +template +void GroupConcatImpl::add( + AggregateDataPtr __restrict place, + const IColumn ** columns, + size_t row_num, + Arena * arena) const +{ + auto & cur_data = this->data(place); - if (isFixedString(type)) - { - ColumnWithTypeAndName col = {columns[0]->getPtr(), type, "column"}; - const auto & col_str = castColumn(col, std::make_shared()); - cur_data.insert(col_str.get(), serialization, row_num, arena); - } - else - cur_data.insert(columns[0], serialization, row_num, arena); - } - - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override - { - auto & cur_data = this->data(place); - auto & rhs_data = this->data(rhs); - - if (rhs_data.data_size == 0) + if constexpr (has_limit) + if (cur_data.num_rows >= limit) return; - if constexpr (has_limit) - { - UInt64 new_elems_count = std::min(rhs_data.num_rows, limit - cur_data.num_rows); - for (UInt64 i = 0; i < new_elems_count; ++i) - { - if (cur_data.data_size != 0) - cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); + if (cur_data.data_size != 0) + cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); - cur_data.offsets.push_back(cur_data.data_size, arena); - cur_data.insertChar(rhs_data.data + rhs_data.getString(i), rhs_data.getSize(i), arena); - cur_data.num_rows++; - cur_data.offsets.push_back(cur_data.data_size, arena); - } - } - else + if (isFixedString(type)) + { + ColumnWithTypeAndName col = {columns[0]->getPtr(), type, "column"}; + const auto & col_str = castColumn(col, std::make_shared()); + cur_data.insert(col_str.get(), serialization, row_num, arena); + } + else + cur_data.insert(columns[0], serialization, row_num, arena); +} + +template +void GroupConcatImpl::merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const +{ + auto & cur_data = this->data(place); + auto & rhs_data = this->data(rhs); + + if (rhs_data.data_size == 0) + return; + + if constexpr (has_limit) + { + UInt64 new_elems_count = std::min(rhs_data.num_rows, limit - cur_data.num_rows); + for (UInt64 i = 0; i < new_elems_count; ++i) { if (cur_data.data_size != 0) cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); - cur_data.insertChar(rhs_data.data, rhs_data.data_size, arena); + cur_data.offsets.push_back(cur_data.data_size, arena); + cur_data.insertChar(rhs_data.data + rhs_data.getString(i), rhs_data.getSize(i), arena); + cur_data.num_rows++; + cur_data.offsets.push_back(cur_data.data_size, arena); } } - - void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override + else { - auto & cur_data = this->data(place); + if (cur_data.data_size != 0) + cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); - writeVarUInt(cur_data.data_size, buf); - - buf.write(cur_data.data, cur_data.data_size); - - if constexpr (has_limit) - { - writeVarUInt(cur_data.num_rows, buf); - for (const auto & offset : cur_data.offsets) - writeVarUInt(offset, buf); - } + cur_data.insertChar(rhs_data.data, rhs_data.data_size, arena); } +} - void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override +template +void GroupConcatImpl::serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const +{ + auto & cur_data = this->data(place); + + writeVarUInt(cur_data.data_size, buf); + + buf.write(cur_data.data, cur_data.data_size); + + if constexpr (has_limit) { - auto & cur_data = this->data(place); - - UInt64 temp_size = 0; - readVarUInt(temp_size, buf); - - cur_data.checkAndUpdateSize(temp_size, arena); - - buf.readStrict(cur_data.data + cur_data.data_size, temp_size); - cur_data.data_size = temp_size; - - if constexpr (has_limit) - { - readVarUInt(cur_data.num_rows, buf); - cur_data.offsets.resize_exact(cur_data.num_rows * 2, arena); - for (auto & offset : cur_data.offsets) - readVarUInt(offset, buf); - } + writeVarUInt(cur_data.num_rows, buf); + for (const auto & offset : cur_data.offsets) + writeVarUInt(offset, buf); } +} - void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override +template +void GroupConcatImpl::deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const +{ + auto & cur_data = this->data(place); + + UInt64 temp_size = 0; + readVarUInt(temp_size, buf); + + cur_data.checkAndUpdateSize(temp_size, arena); + + buf.readStrict(cur_data.data + cur_data.data_size, temp_size); + cur_data.data_size = temp_size; + + if constexpr (has_limit) { - auto & cur_data = this->data(place); + readVarUInt(cur_data.num_rows, buf); + cur_data.offsets.resize_exact(cur_data.num_rows * 2, arena); + for (auto & offset : cur_data.offsets) + readVarUInt(offset, buf); + } +} - if (cur_data.data_size == 0) - { - to.insertDefault(); - return; - } +template +void GroupConcatImpl::insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const +{ + auto & cur_data = this->data(place); - auto & column_string = assert_cast(to); - column_string.insertData(cur_data.data, cur_data.data_size); + if (cur_data.data_size == 0) + { + to.insertDefault(); + return; } - bool allocatesMemoryInArena() const override { return true; } -}; + auto & column_string = assert_cast(to); + column_string.insertData(cur_data.data, cur_data.data_size); +} + +template +bool GroupConcatImpl::allocatesMemoryInArena() const { return true; } + +// Implementation of add, merge, serialize, deserialize, insertResultInto, etc. remains unchanged. AggregateFunctionPtr createAggregateFunctionGroupConcat( const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) @@ -278,14 +242,12 @@ AggregateFunctionPtr createAggregateFunctionGroupConcat( return std::make_shared>(argument_types[0], parameters, limit, delimiter); } -} - void registerAggregateFunctionGroupConcat(AggregateFunctionFactory & factory) { AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true }; factory.registerFunction("groupConcat", { createAggregateFunctionGroupConcat, properties }); - factory.registerAlias("group_concat", "groupConcat", AggregateFunctionFactory::Case::Insensitive); + factory.registerAlias(GroupConcatImpl::getNameAndAliases().at(1), GroupConcatImpl::getNameAndAliases().at(0), AggregateFunctionFactory::Case::Insensitive); } } diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.h b/src/AggregateFunctions/AggregateFunctionGroupConcat.h new file mode 100644 index 00000000000..290b8d80050 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.h @@ -0,0 +1,78 @@ +#ifndef DB_GROUP_CONCAT_H +#define DB_GROUP_CONCAT_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +struct Settings; + +struct GroupConcatDataBase +{ + UInt64 data_size = 0; + UInt64 allocated_size = 0; + char * data = nullptr; + + void checkAndUpdateSize(UInt64 add, Arena * arena); + void insertChar(const char * str, UInt64 str_size, Arena * arena); + void insert(const IColumn * column, const SerializationPtr & serialization, size_t row_num, Arena * arena); +}; + +template +struct GroupConcatData : public GroupConcatDataBase +{ + using Offset = UInt64; + using Allocator = MixedAlignedArenaAllocator; + using Offsets = PODArray; + + Offsets offsets; + UInt64 num_rows = 0; + + UInt64 getSize(size_t i) const; + UInt64 getString(size_t i) const; + + void insert(const IColumn * column, const SerializationPtr & serialization, size_t row_num, Arena * arena); +}; + +template +class GroupConcatImpl : public IAggregateFunctionDataHelper, GroupConcatImpl> +{ + static constexpr auto name = "groupConcat"; + + constexpr static std::array names_and_aliases = { "groupConcat", "group_concat" }; + + SerializationPtr serialization; + UInt64 limit; + const String delimiter; + const DataTypePtr type; + +public: + GroupConcatImpl(const DataTypePtr & data_type_, const Array & parameters_, UInt64 limit_, const String & delimiter_); + + String getName() const override; + + static std::array getNameAndAliases() + { + return names_and_aliases; + } + + void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena * arena) const override; + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override; + void serialize(ConstAggregateDataPtr place, WriteBuffer & buf, std::optional version) const override; + void deserialize(AggregateDataPtr place, ReadBuffer & buf, std::optional version, Arena * arena) const override; + void insertResultInto(AggregateDataPtr place, IColumn & to, Arena * arena) const override; + + bool allocatesMemoryInArena() const override; +}; + +} // namespace DB + +#endif // DB_GROUP_CONCAT_H diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index d3c88d39213..69e4598a941 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -122,6 +122,8 @@ private: ColumnTransformersNodes buildColumnTransformers(const ASTPtr & matcher_expression, const ContextPtr & context) const; + QueryTreeNodePtr setFirstArgumentAsParameter(const ASTFunction * function, const ContextPtr & context) const; + ASTPtr query; QueryTreeNodePtr query_tree_node; }; @@ -643,32 +645,44 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co } else { - auto function_node = std::make_shared(function->name); - function_node->setNullsAction(function->nulls_action); - - if (function->parameters) + const char * name = function->name.c_str(); + // Check if the function is groupConcat with exactly two arguments + if (std::any_of(GroupConcatImpl::getNameAndAliases().begin(), + GroupConcatImpl::getNameAndAliases().end(), + [name](const char *alias) { return std::strcmp(name, alias) == 0; }) + && function->arguments && function->arguments->children.size() == 2) { - const auto & function_parameters_list = function->parameters->as()->children; - for (const auto & argument : function_parameters_list) - function_node->getParameters().getNodes().push_back(buildExpression(argument, context)); + result = setFirstArgumentAsParameter(function, context); } - - if (function->arguments) + else { - const auto & function_arguments_list = function->arguments->as()->children; - for (const auto & argument : function_arguments_list) - function_node->getArguments().getNodes().push_back(buildExpression(argument, context)); - } + auto function_node = std::make_shared(function->name); + function_node->setNullsAction(function->nulls_action); - if (function->is_window_function) - { - if (function->window_definition) - function_node->getWindowNode() = buildWindow(function->window_definition, context); - else - function_node->getWindowNode() = std::make_shared(Identifier(function->window_name)); - } + if (function->parameters) + { + const auto & function_parameters_list = function->parameters->as()->children; + for (const auto & argument : function_parameters_list) + function_node->getParameters().getNodes().push_back(buildExpression(argument, context)); + } - result = std::move(function_node); + if (function->arguments) + { + const auto & function_arguments_list = function->arguments->as()->children; + for (const auto & argument : function_arguments_list) + function_node->getArguments().getNodes().push_back(buildExpression(argument, context)); + } + + if (function->is_window_function) + { + if (function->window_definition) + function_node->getWindowNode() = buildWindow(function->window_definition, context); + else + function_node->getWindowNode() = std::make_shared(Identifier(function->window_name)); + } + + result = std::move(function_node); + } } } else if (const auto * subquery = expression->as()) @@ -1071,4 +1085,42 @@ QueryTreeNodePtr buildQueryTree(ASTPtr query, ContextPtr context) return builder.getQueryTreeNode(); } +QueryTreeNodePtr QueryTreeBuilder::setFirstArgumentAsParameter(const ASTFunction * function, const ContextPtr & context) const +{ + const auto * first_arg_ast = function->arguments->children[0].get(); + const auto * first_arg_literal = first_arg_ast->as(); + + if (!first_arg_literal) + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "If groupConcat is used with two arguments, the first argument must be a constant String"); + } + + if (first_arg_literal->value.getType() != Field::Types::String) + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "If groupConcat is used with two arguments, the first argument must be a constant String"); + } + + std::string separator = first_arg_literal->value.safeGet(); + + ASTPtr second_arg = function->arguments->children[1]->clone(); + + auto function_node = std::make_shared(function->name); + function_node->setNullsAction(function->nulls_action); + + function_node->getParameters().getNodes().push_back(buildExpression(function->arguments->children[0], context)); // Separator + function_node->getArguments().getNodes().push_back(buildExpression(second_arg, context)); // Column to concatenate + + if (function->is_window_function) + { + if (function->window_definition) + function_node->getWindowNode() = buildWindow(function->window_definition, context); + else + function_node->getWindowNode() = std::make_shared(Identifier(function->window_name)); + } + + return std::move(function_node); +} + } diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index bac512545d9..11cfe2e584e 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -130,51 +130,13 @@ String ASTFunction::getID(char delim) const return "Function" + (delim + name); } -void ASTFunction::groupConcatArgumentOverride(std::shared_ptr res) const -{ - // Clone the first argument to be used as a parameter - ASTPtr first_arg = arguments->children[0]->clone(); - - // Clone the second argument to remain as the function argument - ASTPtr second_arg = arguments->children[1]->clone(); - - // Initialize or clear parameters - if (!res->parameters) - res->parameters = std::make_shared(); - else - res->parameters->children.clear(); - - // Add the first argument as a parameter - res->parameters->children.emplace_back(first_arg); - res->children.emplace_back(res->parameters); - - // Initialize arguments with the second argument only - res->arguments = std::make_shared(); - res->arguments->children.emplace_back(second_arg); - res->children.emplace_back(res->arguments); -} - ASTPtr ASTFunction::clone() const { auto res = std::make_shared(*this); res->children.clear(); - // Special handling for groupConcat with two arguments - if ((name == "groupConcat" || Poco::toLower(name) == "group_concat") && arguments && arguments->children.size() == 2) - groupConcatArgumentOverride(res); - else - { - if (arguments) - { - res->arguments = arguments->clone(); - res->children.push_back(res->arguments); - } - if (parameters) - { - res->parameters = parameters->clone(); - res->children.push_back(res->parameters); - } - } + if (arguments) { res->arguments = arguments->clone(); res->children.push_back(res->arguments); } + if (parameters) { res->parameters = parameters->clone(); res->children.push_back(res->parameters); } if (window_definition) { diff --git a/src/Parsers/ASTFunction.h b/src/Parsers/ASTFunction.h index b6aae46d21e..1b4a5928d1c 100644 --- a/src/Parsers/ASTFunction.h +++ b/src/Parsers/ASTFunction.h @@ -81,8 +81,6 @@ public: bool hasSecretParts() const override; - void groupConcatArgumentOverride(std::shared_ptr res) const; - protected: void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; From 4dffdf525dedb07b82ac7aa729de1214e1e8d061 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 3 Dec 2024 16:40:56 +0000 Subject: [PATCH 289/502] Update test result. --- .../0_stateless/03247_generic_arrayMin_arrayMax_fixes.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03247_generic_arrayMin_arrayMax_fixes.reference b/tests/queries/0_stateless/03247_generic_arrayMin_arrayMax_fixes.reference index 8a143e535e2..7b431abfdae 100644 --- a/tests/queries/0_stateless/03247_generic_arrayMin_arrayMax_fixes.reference +++ b/tests/queries/0_stateless/03247_generic_arrayMin_arrayMax_fixes.reference @@ -1,7 +1,7 @@ -- { echoOn } -- https://github.com/ClickHouse/ClickHouse/issues/68895 SELECT arrayMax(x -> toFixedString('.', 1), []); -. +\0 -- https://github.com/ClickHouse/ClickHouse/issues/69600 SELECT arrayMax(x -> (-x), [1, 2, 4]) AS res; -1 From baa65c86cf4179c8c56f2a6f06e231baf052523f Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 3 Dec 2024 16:27:52 +0100 Subject: [PATCH 290/502] Support parsing GCS S3 error "AuthenticationRequired". --- contrib/aws | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/aws b/contrib/aws index d5450d76abd..71169aeec91 160000 --- a/contrib/aws +++ b/contrib/aws @@ -1 +1 @@ -Subproject commit d5450d76abda556ce145ddabe7e0cc6a7644ec59 +Subproject commit 71169aeec91b41c1bd5cf78fad6158dacdcde9d5 From 9dc3703bce3f005fb3b818b8ce45e22c9f18f584 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:55:29 +0100 Subject: [PATCH 291/502] Add missing errorCodes --- src/Analyzer/QueryTreeBuilder.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 69e4598a941..53a39fdf729 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -70,6 +70,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int BAD_ARGUMENTS; extern const int UNKNOWN_QUERY_PARAMETER; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; } namespace From 525a2de94ffa513ac566f7f700bba2b1540bd4d9 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 3 Dec 2024 18:09:58 +0100 Subject: [PATCH 292/502] Add pragma once --- src/AggregateFunctions/AggregateFunctionGroupConcat.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.h b/src/AggregateFunctions/AggregateFunctionGroupConcat.h index 290b8d80050..1d1f404f99c 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupConcat.h +++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.h @@ -1,3 +1,5 @@ +#pragma once + #ifndef DB_GROUP_CONCAT_H #define DB_GROUP_CONCAT_H From bd284fbb0d5cf5f2eab1a384be66effe99b4a575 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 3 Dec 2024 18:17:25 +0100 Subject: [PATCH 293/502] Add a test --- .../ObjectStorageQueueMetadata.cpp | 24 ++-- .../ObjectStorageQueueSource.cpp | 2 +- .../StorageObjectStorageQueue.cpp | 3 +- .../integration/test_storage_s3_queue/test.py | 105 +++++++++++++++++- 4 files changed, 117 insertions(+), 17 deletions(-) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp index 6aac853b011..85930cef1c4 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp @@ -466,10 +466,22 @@ void ObjectStorageQueueMetadata::cleanupThreadFunc() void ObjectStorageQueueMetadata::cleanupThreadFuncImpl() { auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::ObjectStorageQueueCleanupMaxSetSizeOrTTLMicroseconds); + + const fs::path zookeeper_cleanup_lock_path = zookeeper_path / "cleanup_lock"; const auto zk_client = getZooKeeper(); + + /// Create a lock so that with distributed processing + /// multiple nodes do not execute cleanup in parallel. + auto ephemeral_node = zkutil::EphemeralNodeHolder::tryCreate(zookeeper_cleanup_lock_path, *zk_client, toString(getCurrentTime())); + if (!ephemeral_node) + { + LOG_TEST(log, "Cleanup is already being executed by another node"); + return; + } + /// TODO because of this lock we might not update local file statuses on time on one of the nodes. + const fs::path zookeeper_processed_path = zookeeper_path / "processed"; const fs::path zookeeper_failed_path = zookeeper_path / "failed"; - const fs::path zookeeper_cleanup_lock_path = zookeeper_path / "cleanup_lock"; Strings processed_nodes; auto code = zk_client->tryGetChildren(zookeeper_processed_path, processed_nodes); @@ -516,16 +528,6 @@ void ObjectStorageQueueMetadata::cleanupThreadFuncImpl() LOG_TRACE(log, "Will check limits for {} nodes", nodes_num); - /// Create a lock so that with distributed processing - /// multiple nodes do not execute cleanup in parallel. - auto ephemeral_node = zkutil::EphemeralNodeHolder::tryCreate(zookeeper_cleanup_lock_path, *zk_client, toString(getCurrentTime())); - if (!ephemeral_node) - { - LOG_TEST(log, "Cleanup is already being executed by another node"); - return; - } - /// TODO because of this lock we might not update local file statuses on time on one of the nodes. - struct Node { std::string zk_path; diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp index 09af871a7cd..8b5241dddb6 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp @@ -120,7 +120,7 @@ ObjectStorageQueueSource::Source::ObjectInfoPtr ObjectStorageQueueSource::FileIt if (file_metadata->setProcessing()) { if (file_deletion_on_processed_enabled - && !object_storage->exists(StoredObject(object_info->getPath()))) + && !object_storage->exists(StoredObject(object_info->relative_path))) { /// Imagine the following case: /// Replica A processed fileA and deletes it afterwards. diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 4ac96b3367a..8f26a8149ed 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -741,8 +741,7 @@ StorageObjectStorageQueue::createFileIterator(ContextPtr local_context, const Ac const auto & table_metadata = getTableMetadata(); bool file_deletion_enabled = table_metadata.getMode() == ObjectStorageQueueMode::UNORDERED - && table_metadata.tracked_files_ttl_sec - && table_metadata.tracked_files_limit; + && (table_metadata.tracked_files_ttl_sec || table_metadata.tracked_files_limit); return std::make_shared( files_metadata, std::move(glob_iterator), object_storage, file_deletion_enabled, shutdown_called, log); diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 81f935b1ee9..5aaf4eaac54 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -5,6 +5,7 @@ import random import string import time import uuid +from multiprocessing.dummy import Pool import pytest @@ -205,6 +206,11 @@ def run_query(instance, query, stdin=None, settings=None): return result +def random_str(length=6): + alphabet = string.ascii_lowercase + string.digits + return "".join(random.SystemRandom().choice(alphabet) for _ in range(length)) + + def generate_random_files( started_cluster, files_path, @@ -214,10 +220,18 @@ def generate_random_files( row_num=10, start_ind=0, bucket=None, + use_random_names=False, ): - files = [ - (f"{files_path}/test_{i}.csv", i) for i in range(start_ind, start_ind + count) - ] + if use_random_names: + files = [ + (f"{files_path}/{random_str(10)}.csv", i) + for i in range(start_ind, start_ind + count) + ] + else: + files = [ + (f"{files_path}/test_{i}.csv", i) + for i in range(start_ind, start_ind + count) + ] files.sort(key=lambda x: x[0]) print(f"Generating files: {files}") @@ -2301,3 +2315,88 @@ def test_alter_settings(started_cluster): check_int_settings(node, int_settings) check_string_settings(node, string_settings) + + +def test_list_and_delete_race(started_cluster): + node = started_cluster.instances["instance"] + node_2 = started_cluster.instances["instance2"] + table_name = f"list_and_delete_race_{generate_random_string()}" + dst_table_name = f"{table_name}_dst" + keeper_path = f"/clickhouse/test_{table_name}" + files_path = f"{table_name}_data" + files_to_generate = 1000 + row_num = 10 + + for instance in [node, node_2]: + create_table( + started_cluster, + instance, + table_name, + "unordered", + files_path, + additional_settings={ + "keeper_path": keeper_path, + "tracked_files_limit": 1, + "polling_max_timeout_ms": 0, + "processing_threads_num": 1, + "polling_min_timeout_ms": 200, + "cleanup_interval_min_ms": 0, + "cleanup_interval_max_ms": 0, + "polling_backoff_ms": 100, + "after_processing": "delete", + }, + ) + + threads = 10 + total_rows = row_num * files_to_generate * (threads + 1) + + busy_pool = Pool(10) + + def generate(_): + generate_random_files( + started_cluster, + files_path, + files_to_generate, + row_num=row_num, + use_random_names=True, + ) + + generate(0) + + p = busy_pool.map_async(generate, range(10)) + + create_mv(node, table_name, dst_table_name) + time.sleep(2) + create_mv(node_2, table_name, dst_table_name) + + p.wait() + + def get_count(node, table_name): + return int(run_query(node, f"SELECT count() FROM {table_name}")) + + for _ in range(150): + if ( + get_count(node, dst_table_name) + get_count(node_2, dst_table_name) + ) == total_rows: + break + time.sleep(1) + + assert ( + get_count(node, dst_table_name) + get_count(node_2, dst_table_name) + == total_rows + ) + + get_query = f"SELECT column1, column2, column3 FROM {dst_table_name}" + res1 = [list(map(int, l.split())) for l in run_query(node, get_query).splitlines()] + res2 = [ + list(map(int, l.split())) for l in run_query(node_2, get_query).splitlines() + ] + + logging.debug( + f"res1 size: {len(res1)}, res2 size: {len(res2)}, total_rows: {total_rows}" + ) + + assert len(res1) + len(res2) == total_rows + assert node.contains_in_log( + "because of the race with list & delete" + ) or node_2.contains_in_log("because of the race with list & delete") From ecd6282af56e83955cebda50dd25e82d5e72da84 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 3 Dec 2024 18:18:59 +0100 Subject: [PATCH 294/502] Update AggregateFunctionGroupConcat.h --- src/AggregateFunctions/AggregateFunctionGroupConcat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.h b/src/AggregateFunctions/AggregateFunctionGroupConcat.h index 1d1f404f99c..3e5f824f8f5 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupConcat.h +++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.h @@ -75,6 +75,6 @@ public: bool allocatesMemoryInArena() const override; }; -} // namespace DB +} -#endif // DB_GROUP_CONCAT_H +#endif From c5779654e807848757331a1c394660590d9c1e16 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 3 Dec 2024 18:23:37 +0100 Subject: [PATCH 295/502] Update size in test --- tests/queries/0_stateless/03273_primary_index_cache.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03273_primary_index_cache.reference b/tests/queries/0_stateless/03273_primary_index_cache.reference index 611787366ee..1c11a9b1fd4 100644 --- a/tests/queries/0_stateless/03273_primary_index_cache.reference +++ b/tests/queries/0_stateless/03273_primary_index_cache.reference @@ -3,14 +3,14 @@ PrimaryIndexCacheBytes 0 PrimaryIndexCacheFiles 0 99 0 -PrimaryIndexCacheBytes 1280 +PrimaryIndexCacheBytes 1808 PrimaryIndexCacheFiles 2 0 PrimaryIndexCacheBytes 0 PrimaryIndexCacheFiles 0 49 0 -PrimaryIndexCacheBytes 640 +PrimaryIndexCacheBytes 904 PrimaryIndexCacheFiles 1 2 160 1280 1 80 640 From 4405416ee377c863b63d7d081c16be6f3323f0f4 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 3 Dec 2024 18:24:36 +0100 Subject: [PATCH 296/502] Add tests for aliases --- tests/queries/0_stateless/03156_group_concat.reference | 2 ++ tests/queries/0_stateless/03156_group_concat.sql | 3 +++ 2 files changed, 5 insertions(+) diff --git a/tests/queries/0_stateless/03156_group_concat.reference b/tests/queries/0_stateless/03156_group_concat.reference index 23c5d43bd60..6f4b84fbc1d 100644 --- a/tests/queries/0_stateless/03156_group_concat.reference +++ b/tests/queries/0_stateless/03156_group_concat.reference @@ -20,4 +20,6 @@ TESTING GroupConcat second argument overload 95,123 abc.a.makson95 [1,2,3]/[993,986,979,972]/[] +[1,2,3]/[993,986,979,972]/[] +[1,2,3]/[993,986,979,972]/[] 488890 diff --git a/tests/queries/0_stateless/03156_group_concat.sql b/tests/queries/0_stateless/03156_group_concat.sql index c6fea68551a..d64d382bd74 100644 --- a/tests/queries/0_stateless/03156_group_concat.sql +++ b/tests/queries/0_stateless/03156_group_concat.sql @@ -48,6 +48,9 @@ SELECT groupConcat(',', p_int) FROM test_groupConcat; SELECT groupConcat('.')(p_string) FROM test_groupConcat; SELECT groupConcat('/', p_array) FROM test_groupConcat; +SELECT group_concat('/', p_array) FROM test_groupConcat; +SELECT grouP_CONcat('/', p_array) FROM test_groupConcat; + DROP TABLE IF EXISTS test_groupConcat; CREATE TABLE test_groupConcat From 7bee0a487285ecfdecdafdeb4de463b6d93b867c Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 3 Dec 2024 18:29:50 +0100 Subject: [PATCH 297/502] Speed up the test --- tests/integration/test_storage_s3_queue/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 5aaf4eaac54..39780c4913c 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -2347,10 +2347,10 @@ def test_list_and_delete_race(started_cluster): }, ) - threads = 10 + threads = 4 total_rows = row_num * files_to_generate * (threads + 1) - busy_pool = Pool(10) + busy_pool = Pool(threads) def generate(_): generate_random_files( @@ -2363,7 +2363,7 @@ def test_list_and_delete_race(started_cluster): generate(0) - p = busy_pool.map_async(generate, range(10)) + p = busy_pool.map_async(generate, range(threads)) create_mv(node, table_name, dst_table_name) time.sleep(2) From b8f1110455fc2446d4a5aed55f14d412b2bd6a99 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 3 Dec 2024 18:36:08 +0100 Subject: [PATCH 298/502] Update QueryTreeBuilder.cpp --- src/Analyzer/QueryTreeBuilder.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 53a39fdf729..6a761befe2b 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -40,6 +40,8 @@ #include #include +#include + #include #include From a71c00000b1fb66e9c5806b081d889751fc51868 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 3 Dec 2024 19:20:07 +0100 Subject: [PATCH 299/502] impl --- docs/en/engines/table-engines/special/memory.md | 2 ++ src/Columns/ColumnArray.cpp | 6 +++--- src/Columns/ColumnArray.h | 2 +- src/Columns/ColumnCompressed.cpp | 5 +++-- src/Columns/ColumnCompressed.h | 2 +- src/Columns/ColumnDecimal.cpp | 4 ++-- src/Columns/ColumnDecimal.h | 2 +- src/Columns/ColumnDynamic.cpp | 4 ++-- src/Columns/ColumnDynamic.h | 2 +- src/Columns/ColumnFixedString.cpp | 4 ++-- src/Columns/ColumnFixedString.h | 2 +- src/Columns/ColumnMap.cpp | 4 ++-- src/Columns/ColumnMap.h | 2 +- src/Columns/ColumnNullable.cpp | 6 +++--- src/Columns/ColumnNullable.h | 2 +- src/Columns/ColumnObject.cpp | 8 ++++---- src/Columns/ColumnObject.h | 2 +- src/Columns/ColumnSparse.cpp | 6 +++--- src/Columns/ColumnSparse.h | 2 +- src/Columns/ColumnString.cpp | 6 +++--- src/Columns/ColumnString.h | 2 +- src/Columns/ColumnTuple.cpp | 4 ++-- src/Columns/ColumnTuple.h | 2 +- src/Columns/ColumnVariant.cpp | 8 ++++---- src/Columns/ColumnVariant.h | 2 +- src/Columns/ColumnVector.cpp | 4 ++-- src/Columns/ColumnVector.h | 2 +- src/Columns/IColumn.h | 3 ++- src/Core/Block.cpp | 2 +- src/Interpreters/Cache/QueryCache.cpp | 2 +- src/Storages/StorageMemory.cpp | 7 +++---- 31 files changed, 57 insertions(+), 54 deletions(-) diff --git a/docs/en/engines/table-engines/special/memory.md b/docs/en/engines/table-engines/special/memory.md index f28157ebde2..3eb3e617ff9 100644 --- a/docs/en/engines/table-engines/special/memory.md +++ b/docs/en/engines/table-engines/special/memory.md @@ -36,6 +36,8 @@ Upper and lower bounds can be specified to limit Memory engine table size, effec - Requires `max_rows_to_keep` - `max_rows_to_keep` — Maximum rows to keep within memory table where oldest rows are deleted on each insertion (i.e circular buffer). Max rows can exceed the stated limit if the oldest batch of rows to remove falls under the `min_rows_to_keep` limit when adding a large block. - Default value: `0` +- `compress` - Whether to compress data in memory. + - Default value: `false` ## Usage {#usage} diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 3f88ca93a97..013821db2c9 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -1024,10 +1024,10 @@ void ColumnArray::updatePermutationWithCollation(const Collator & collator, Perm DefaultPartialSort()); } -ColumnPtr ColumnArray::compress() const +ColumnPtr ColumnArray::compress(bool force_compression) const { - ColumnPtr data_compressed = data->compress(); - ColumnPtr offsets_compressed = offsets->compress(); + ColumnPtr data_compressed = data->compress(force_compression); + ColumnPtr offsets_compressed = offsets->compress(force_compression); size_t byte_size = data_compressed->byteSize() + offsets_compressed->byteSize(); diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index a66f9041213..dee6ae931f2 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -159,7 +159,7 @@ public: /// For example, `getDataInRange(0, size())` is the same as `getDataPtr()->clone()`. MutableColumnPtr getDataInRange(size_t start, size_t length) const; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnCompressed.cpp b/src/Columns/ColumnCompressed.cpp index 3bdc514d6d8..adb2a5f391d 100644 --- a/src/Columns/ColumnCompressed.cpp +++ b/src/Columns/ColumnCompressed.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes } -std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, size_t data_size, bool always_compress) +std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, size_t data_size, bool force_compression) { size_t max_dest_size = LZ4_COMPRESSBOUND(data_size); @@ -35,7 +35,8 @@ std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, si throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress column"); /// If compression is inefficient. - if (!always_compress && static_cast(compressed_size) * 2 > data_size) + const size_t threshold = force_compression ? 1 : 2; + if (static_cast(compressed_size) * threshold > data_size) return {}; /// Shrink to fit. diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h index c4270e8216b..b030e762acd 100644 --- a/src/Columns/ColumnCompressed.h +++ b/src/Columns/ColumnCompressed.h @@ -72,7 +72,7 @@ public: /// If data is not worth to be compressed and not 'always_compress' - returns nullptr. /// Note: shared_ptr is to allow to be captured by std::function. - static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool always_compress); + static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool force_compression); static void decompressBuffer( const void * compressed_data, void * decompressed_data, size_t compressed_size, size_t decompressed_size); diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 73366150e7d..c286c54198a 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -478,7 +478,7 @@ ColumnPtr ColumnDecimal::replicate(const IColumn::Offsets & offsets) const } template -ColumnPtr ColumnDecimal::compress() const +ColumnPtr ColumnDecimal::compress(bool force_compression) const { const size_t data_size = data.size(); const size_t source_size = data_size * sizeof(T); @@ -487,7 +487,7 @@ ColumnPtr ColumnDecimal::compress() const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, false); + auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, force_compression); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 690549e4a56..3e5c189b731 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -140,7 +140,7 @@ public: return false; } - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void insertValue(const T value) { data.push_back(value); } Container & getData() { return data; } diff --git a/src/Columns/ColumnDynamic.cpp b/src/Columns/ColumnDynamic.cpp index a4c932eafdd..2d05701c57b 100644 --- a/src/Columns/ColumnDynamic.cpp +++ b/src/Columns/ColumnDynamic.cpp @@ -991,9 +991,9 @@ void ColumnDynamic::updatePermutation(IColumn::PermutationSortDirection directio updatePermutationImpl(limit, res, equal_ranges, ComparatorDescendingStable(*this, nan_direction_hint), comparator_equal, DefaultSort(), DefaultPartialSort()); } -ColumnPtr ColumnDynamic::compress() const +ColumnPtr ColumnDynamic::compress(bool force_compression) const { - ColumnPtr variant_compressed = variant_column_ptr->compress(); + ColumnPtr variant_compressed = variant_column_ptr->compress(force_compression); size_t byte_size = variant_compressed->byteSize(); return ColumnCompressed::create(size(), byte_size, [my_variant_compressed = std::move(variant_compressed), my_variant_info = variant_info, my_max_dynamic_types = max_dynamic_types, my_global_max_dynamic_types = global_max_dynamic_types, my_statistics = statistics]() mutable diff --git a/src/Columns/ColumnDynamic.h b/src/Columns/ColumnDynamic.h index bdbad99519f..093aaaf2793 100644 --- a/src/Columns/ColumnDynamic.h +++ b/src/Columns/ColumnDynamic.h @@ -335,7 +335,7 @@ public: return false; } - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; double getRatioOfDefaultRows(double sample_ratio) const override { diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 04e894ee5ab..f076f904768 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -419,7 +419,7 @@ void ColumnFixedString::getExtremes(Field & min, Field & max) const get(max_idx, max); } -ColumnPtr ColumnFixedString::compress() const +ColumnPtr ColumnFixedString::compress(bool force_compression) const { size_t source_size = chars.size(); @@ -427,7 +427,7 @@ ColumnPtr ColumnFixedString::compress() const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(chars.data(), source_size, false); + auto compressed = ColumnCompressed::compressBuffer(chars.data(), source_size, force_compression); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index 8cf0a6a57da..f55fb60a976 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -175,7 +175,7 @@ public: ColumnPtr replicate(const Offsets & offsets) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void reserve(size_t size) override { diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp index a5511dfeeb4..fb9c8c9fbaf 100644 --- a/src/Columns/ColumnMap.cpp +++ b/src/Columns/ColumnMap.cpp @@ -352,9 +352,9 @@ bool ColumnMap::dynamicStructureEquals(const IColumn & rhs) const return false; } -ColumnPtr ColumnMap::compress() const +ColumnPtr ColumnMap::compress(bool force_compression) const { - auto compressed = nested->compress(); + auto compressed = nested->compress(force_compression); const auto byte_size = compressed->byteSize(); /// The order of evaluation of function arguments is unspecified /// and could cause interacting with object in moved-from state diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h index 8dfa5bb5845..31404a3e152 100644 --- a/src/Columns/ColumnMap.h +++ b/src/Columns/ColumnMap.h @@ -120,7 +120,7 @@ public: const ColumnTuple & getNestedData() const { return assert_cast(getNestedColumn().getData()); } ColumnTuple & getNestedData() { return assert_cast(getNestedColumn().getData()); } - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; bool hasDynamicStructure() const override { return nested->hasDynamicStructure(); } bool dynamicStructureEquals(const IColumn & rhs) const override; diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 6e8bd3fc70c..640550fcf9a 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -773,10 +773,10 @@ void ColumnNullable::protect() getNullMapColumn().protect(); } -ColumnPtr ColumnNullable::compress() const +ColumnPtr ColumnNullable::compress(bool force_compression) const { - ColumnPtr nested_compressed = nested_column->compress(); - ColumnPtr null_map_compressed = null_map->compress(); + ColumnPtr nested_compressed = nested_column->compress(force_compression); + ColumnPtr null_map_compressed = null_map->compress(force_compression); size_t byte_size = nested_column->byteSize() + null_map->byteSize(); diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 32ce66c5965..3a0be008cc2 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -141,7 +141,7 @@ public: // Special function for nullable minmax index void getExtremesNullLast(Field & min, Field & max) const; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 064145c5d4f..17a90cc9b50 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -1224,14 +1224,14 @@ bool ColumnObject::structureEquals(const IColumn & rhs) const return true; } -ColumnPtr ColumnObject::compress() const +ColumnPtr ColumnObject::compress(bool force_compression) const { std::unordered_map compressed_typed_paths; compressed_typed_paths.reserve(typed_paths.size()); size_t byte_size = 0; for (const auto & [path, column] : typed_paths) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(force_compression); byte_size += compressed_column->byteSize(); compressed_typed_paths[path] = std::move(compressed_column); } @@ -1240,12 +1240,12 @@ ColumnPtr ColumnObject::compress() const compressed_dynamic_paths.reserve(dynamic_paths_ptrs.size()); for (const auto & [path, column] : dynamic_paths_ptrs) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(force_compression); byte_size += compressed_column->byteSize(); compressed_dynamic_paths[path] = std::move(compressed_column); } - auto compressed_shared_data = shared_data->compress(); + auto compressed_shared_data = shared_data->compress(force_compression); byte_size += compressed_shared_data->byteSize(); auto decompress = diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index 7b8a381d571..3160b66cd20 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -171,7 +171,7 @@ public: bool structureEquals(const IColumn & rhs) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void finalize() override; bool isFinalized() const override; diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp index a0e47e65fc6..b7d82ed8a09 100644 --- a/src/Columns/ColumnSparse.cpp +++ b/src/Columns/ColumnSparse.cpp @@ -774,10 +774,10 @@ UInt64 ColumnSparse::getNumberOfDefaultRows() const return _size - offsets->size(); } -ColumnPtr ColumnSparse::compress() const +ColumnPtr ColumnSparse::compress(bool force_compression) const { - auto values_compressed = values->compress(); - auto offsets_compressed = offsets->compress(); + auto values_compressed = values->compress(force_compression); + auto offsets_compressed = offsets->compress(force_compression); size_t byte_size = values_compressed->byteSize() + offsets_compressed->byteSize(); diff --git a/src/Columns/ColumnSparse.h b/src/Columns/ColumnSparse.h index 619dce63c1e..f95752cd546 100644 --- a/src/Columns/ColumnSparse.h +++ b/src/Columns/ColumnSparse.h @@ -147,7 +147,7 @@ public: double getRatioOfDefaultRows(double sample_ratio) const override; UInt64 getNumberOfDefaultRows() const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 269c20397b4..0ed4f5f432d 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -627,7 +627,7 @@ void ColumnString::getExtremes(Field & min, Field & max) const get(max_idx, max); } -ColumnPtr ColumnString::compress() const +ColumnPtr ColumnString::compress(bool force_compression) const { const size_t source_chars_size = chars.size(); const size_t source_offsets_elements = offsets.size(); @@ -637,13 +637,13 @@ ColumnPtr ColumnString::compress() const if (source_chars_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto chars_compressed = ColumnCompressed::compressBuffer(chars.data(), source_chars_size, false); + auto chars_compressed = ColumnCompressed::compressBuffer(chars.data(), source_chars_size, force_compression); /// Return original column if not compressible. if (!chars_compressed) return ColumnCompressed::wrap(this->getPtr()); - auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, true); + auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, /*force_compression=*/true); const size_t chars_compressed_size = chars_compressed->size(); const size_t offsets_compressed_size = offsets_compressed->size(); diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index c2371412437..b2e340be61b 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -284,7 +284,7 @@ public: ColumnPtr replicate(const Offsets & replicate_offsets) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void reserve(size_t n) override; size_t capacity() const override; diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 28e5f03cc3c..9bb377f56ae 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -796,7 +796,7 @@ void ColumnTuple::takeDynamicStructureFromSourceColumns(const Columns & source_c } -ColumnPtr ColumnTuple::compress() const +ColumnPtr ColumnTuple::compress(bool force_compression) const { if (columns.empty()) { @@ -812,7 +812,7 @@ ColumnPtr ColumnTuple::compress() const compressed.reserve(columns.size()); for (const auto & column : columns) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(force_compression); byte_size += compressed_column->byteSize(); compressed.emplace_back(std::move(compressed_column)); } diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index d5eee911edc..b8b3697b84d 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -125,7 +125,7 @@ public: void forEachSubcolumnRecursively(RecursiveMutableColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; bool isCollationSupported() const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void finalize() override; bool isFinalized() const override; diff --git a/src/Columns/ColumnVariant.cpp b/src/Columns/ColumnVariant.cpp index 2fa59b8e33c..38d3bac3c10 100644 --- a/src/Columns/ColumnVariant.cpp +++ b/src/Columns/ColumnVariant.cpp @@ -1426,16 +1426,16 @@ bool ColumnVariant::dynamicStructureEquals(const IColumn & rhs) const return true; } -ColumnPtr ColumnVariant::compress() const +ColumnPtr ColumnVariant::compress(bool force_compression) const { - ColumnPtr local_discriminators_compressed = local_discriminators->compress(); - ColumnPtr offsets_compressed = offsets->compress(); + ColumnPtr local_discriminators_compressed = local_discriminators->compress(force_compression); + ColumnPtr offsets_compressed = offsets->compress(force_compression); size_t byte_size = local_discriminators_compressed->byteSize() + offsets_compressed->byteSize(); Columns compressed; compressed.reserve(variants.size()); for (const auto & variant : variants) { - auto compressed_variant = variant->compress(); + auto compressed_variant = variant->compress(force_compression); byte_size += compressed_variant->byteSize(); compressed.emplace_back(std::move(compressed_variant)); } diff --git a/src/Columns/ColumnVariant.h b/src/Columns/ColumnVariant.h index a68a961169c..c7e37517004 100644 --- a/src/Columns/ColumnVariant.h +++ b/src/Columns/ColumnVariant.h @@ -254,7 +254,7 @@ public: void forEachSubcolumn(MutableColumnCallback callback) override; void forEachSubcolumnRecursively(RecursiveMutableColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; double getRatioOfDefaultRows(double sample_ratio) const override; UInt64 getNumberOfDefaultRows() const override; void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override; diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 3c7727f37c4..62f6c23c4f8 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -951,7 +951,7 @@ void ColumnVector::getExtremes(Field & min, Field & max) const } template -ColumnPtr ColumnVector::compress() const +ColumnPtr ColumnVector::compress(bool force_compression) const { const size_t data_size = data.size(); const size_t source_size = data_size * sizeof(T); @@ -960,7 +960,7 @@ ColumnPtr ColumnVector::compress() const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, false); + auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, force_compression); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 1387cca1ece..22b064ae053 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -286,7 +286,7 @@ public: ColumnPtr createWithOffsets(const IColumn::Offsets & offsets, const ColumnConst & column_with_default_value, size_t total_rows, size_t shift) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; /// Replace elements that match the filter with zeroes. If inverted replaces not matched elements. void applyZeroMap(const IColumn::Filter & filt, bool inverted = false); diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 9d1b42d2bc1..e2099ac34b9 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -601,7 +601,8 @@ public: /// Compress column in memory to some representation that allows to decompress it back. /// Return itself if compression is not applicable for this column type. - [[nodiscard]] virtual Ptr compress() const + /// The flag `force_compression` indicates that compression should be performed even if it's not efficient (if only compression factor < 1). + [[nodiscard]] virtual Ptr compress([[maybe_unused]] bool force_compression) const { /// No compression by default. return getPtr(); diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 02176a6b77a..0efb4596dcd 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -608,7 +608,7 @@ Block Block::compress() const size_t num_columns = data.size(); Columns new_columns(num_columns); for (size_t i = 0; i < num_columns; ++i) - new_columns[i] = data[i].column->compress(); + new_columns[i] = data[i].column->compress(/*force_compression=*/false); return cloneWithColumns(new_columns); } diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp index 7dbee567c5b..de3d720fc35 100644 --- a/src/Interpreters/Cache/QueryCache.cpp +++ b/src/Interpreters/Cache/QueryCache.cpp @@ -469,7 +469,7 @@ void QueryCache::Writer::finalizeWrite() Columns compressed_columns; for (const auto & column : columns) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(/*force_compression=*/false); compressed_columns.push_back(compressed_column); } Chunk compressed_chunk(compressed_columns, chunk.getNumRows()); diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index d798e1b4fb5..55574e7b1db 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -91,8 +91,7 @@ public: { Block compressed_block; for (const auto & elem : block) - compressed_block.insert({ elem.column->compress(), elem.type, elem.name }); - + compressed_block.insert({elem.column->compress(/*force_compression=*/true), elem.type, elem.name}); new_blocks.push_back(std::move(compressed_block)); } else @@ -259,7 +258,7 @@ void StorageMemory::mutate(const MutationCommands & commands, ContextPtr context { if ((*memory_settings)[MemorySetting::compress]) for (auto & elem : block) - elem.column = elem.column->compress(); + elem.column = elem.column->compress(/*force_compression=*/true); out.push_back(block); } @@ -574,7 +573,7 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat { Block compressed_block; for (const auto & elem : block) - compressed_block.insert({ elem.column->compress(), elem.type, elem.name }); + compressed_block.insert({elem.column->compress(/*force_compression=*/true), elem.type, elem.name}); new_blocks.push_back(std::move(compressed_block)); } From d6480b846c070cd5953dc4839142756de472551f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 3 Dec 2024 20:30:06 +0100 Subject: [PATCH 300/502] Fix run issues --- src/AggregateFunctions/AggregateFunctionGroupConcat.h | 7 +++---- src/Analyzer/QueryTreeBuilder.cpp | 9 ++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.h b/src/AggregateFunctions/AggregateFunctionGroupConcat.h index 3e5f824f8f5..585dec8e90f 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupConcat.h +++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.h @@ -49,8 +49,6 @@ class GroupConcatImpl : public IAggregateFunctionDataHelper names_and_aliases = { "groupConcat", "group_concat" }; - SerializationPtr serialization; UInt64 limit; const String delimiter; @@ -61,9 +59,10 @@ public: String getName() const override; - static std::array getNameAndAliases() + static const std::vector& getNameAndAliases() { - return names_and_aliases; + static const std::vector aliases = {"groupConcat", "group_concat"}; + return aliases; } void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena * arena) const override; diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 6a761befe2b..3f4ef587cbc 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -648,11 +648,10 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co } else { - const char * name = function->name.c_str(); - // Check if the function is groupConcat with exactly two arguments - if (std::any_of(GroupConcatImpl::getNameAndAliases().begin(), - GroupConcatImpl::getNameAndAliases().end(), - [name](const char *alias) { return std::strcmp(name, alias) == 0; }) + const auto & group_concat_aliases = GroupConcatImpl::getNameAndAliases(); + if (!function->name.empty() && std::any_of( + group_concat_aliases.begin(), group_concat_aliases.end(), + [&](const std::string &s) { return s == function->name; }) && function->arguments && function->arguments->children.size() == 2) { result = setFirstArgumentAsParameter(function, context); From 4df0603c58a2ecdbdf4b858681f183ba187147ed Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 3 Dec 2024 19:41:27 +0000 Subject: [PATCH 301/502] Fix --- src/Core/SettingsChangesHistory.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index a99d114f0d8..4fcee0f45dd 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -68,6 +68,7 @@ static std::initializer_list Date: Tue, 3 Dec 2024 20:53:39 +0100 Subject: [PATCH 302/502] Add lowercase support. --- src/Analyzer/QueryTreeBuilder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 3f4ef587cbc..975c30f4dd7 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -651,7 +651,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co const auto & group_concat_aliases = GroupConcatImpl::getNameAndAliases(); if (!function->name.empty() && std::any_of( group_concat_aliases.begin(), group_concat_aliases.end(), - [&](const std::string &s) { return s == function->name; }) + [&](const std::string &s) { return s == Poco::toLower(function->name); }) && function->arguments && function->arguments->children.size() == 2) { result = setFirstArgumentAsParameter(function, context); From 72e19cbbcb23e8ef66f42c93422c7b9f93cc32f1 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 3 Dec 2024 21:12:23 +0100 Subject: [PATCH 303/502] Update QueryTreeBuilder.cpp --- src/Analyzer/QueryTreeBuilder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 975c30f4dd7..0d44cf76462 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -651,7 +651,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co const auto & group_concat_aliases = GroupConcatImpl::getNameAndAliases(); if (!function->name.empty() && std::any_of( group_concat_aliases.begin(), group_concat_aliases.end(), - [&](const std::string &s) { return s == Poco::toLower(function->name); }) + [&](const std::string &s) { return Poco::toLower(s) == Poco::toLower(function->name); }) && function->arguments && function->arguments->children.size() == 2) { result = setFirstArgumentAsParameter(function, context); From 9e730e4206da16fb319c9746965d73bbb66a8a9e Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 3 Dec 2024 20:41:25 +0000 Subject: [PATCH 304/502] Support Dynamic type in functions ifNull and coalesce --- src/Functions/coalesce.cpp | 6 +- src/Functions/if.cpp | 4 +- src/Functions/ifNull.cpp | 2 +- .../03280_dynamic_if_null.reference | 10 ++ .../0_stateless/03280_dynamic_if_null.sql | 4 + .../03281_dynamic_coalesce.reference | 100 ++++++++++++++++++ .../0_stateless/03281_dynamic_coalesce.sql | 15 +++ 7 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/03280_dynamic_if_null.reference create mode 100644 tests/queries/0_stateless/03280_dynamic_if_null.sql create mode 100644 tests/queries/0_stateless/03281_dynamic_coalesce.reference create mode 100644 tests/queries/0_stateless/03281_dynamic_coalesce.sql diff --git a/src/Functions/coalesce.cpp b/src/Functions/coalesce.cpp index 2a8cad386d4..cc2ce2eaf8d 100644 --- a/src/Functions/coalesce.cpp +++ b/src/Functions/coalesce.cpp @@ -67,7 +67,7 @@ public: filtered_args.push_back(arg); - if (!arg->isNullable()) + if (!canContainNull(*arg)) break; } @@ -90,7 +90,7 @@ public: auto res = getLeastSupertype(new_args); /// if last argument is not nullable, result should be also not nullable - if (!new_args.back()->isNullable() && res->isNullable()) + if (!canContainNull(*new_args.back()) && res->isNullable()) res = removeNullable(res); return res; @@ -113,7 +113,7 @@ public: filtered_args.push_back(arg); - if (!type->isNullable()) + if (!canContainNull(*type)) break; } diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index ef358780b20..84926f5efed 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -949,8 +949,8 @@ private: ColumnPtr executeForNullableThenElse(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const { - /// If result type is Variant, we don't need to remove Nullable. - if (isVariant(result_type)) + /// If result type is Variant/Dynamic, we don't need to remove Nullable. + if (isVariant(result_type) || isDynamic(result_type)) return nullptr; const ColumnWithTypeAndName & arg_cond = arguments[0]; diff --git a/src/Functions/ifNull.cpp b/src/Functions/ifNull.cpp index f1186a9abdc..10b79b01d5c 100644 --- a/src/Functions/ifNull.cpp +++ b/src/Functions/ifNull.cpp @@ -57,7 +57,7 @@ public: return arguments[1].column; /// Could not contain nulls, so nullIf makes no sense. - if (!arguments[0].type->isNullable()) + if (!canContainNull(*arguments[0].type)) return arguments[0].column; /// ifNull(col1, col2) == if(isNotNull(col1), assumeNotNull(col1), col2) diff --git a/tests/queries/0_stateless/03280_dynamic_if_null.reference b/tests/queries/0_stateless/03280_dynamic_if_null.reference new file mode 100644 index 00000000000..48f569502c5 --- /dev/null +++ b/tests/queries/0_stateless/03280_dynamic_if_null.reference @@ -0,0 +1,10 @@ +0 +42 +2 +42 +4 +0 +42 +2 +42 +4 diff --git a/tests/queries/0_stateless/03280_dynamic_if_null.sql b/tests/queries/0_stateless/03280_dynamic_if_null.sql new file mode 100644 index 00000000000..1fb956467f8 --- /dev/null +++ b/tests/queries/0_stateless/03280_dynamic_if_null.sql @@ -0,0 +1,4 @@ +set enable_dynamic_type = 1; + +select ifNull(number % 2 ? NULL : number::Dynamic, 42) from numbers(5); + diff --git a/tests/queries/0_stateless/03281_dynamic_coalesce.reference b/tests/queries/0_stateless/03281_dynamic_coalesce.reference new file mode 100644 index 00000000000..93156fb2da9 --- /dev/null +++ b/tests/queries/0_stateless/03281_dynamic_coalesce.reference @@ -0,0 +1,100 @@ +0 +42 +2 +42 +4 +0 +\N +2 +42 +4 +0 +\N +2 +42 +4 +0 +\N +2 +42 +4 +0 +\N +2 +42 +4 +\N +6 +43 +8 +42 +0 +\N +2 +42 +4 +\N +6 +43 +8 +42 +0 +\N +2 +42 +4 +\N +6 +43 +8 +42 +0 +\N +2 +42 +4 +\N +6 +43 +8 +42 +0 +\N +2 +42 +4 +\N +6 +43 +8 +42 +0 +\N +2 +42 +4 +\N +6 +43 +8 +42 +0 +\N +2 +42 +4 +\N +6 +43 +8 +42 +0 +\N +2 +42 +4 +\N +6 +43 +8 +42 diff --git a/tests/queries/0_stateless/03281_dynamic_coalesce.sql b/tests/queries/0_stateless/03281_dynamic_coalesce.sql new file mode 100644 index 00000000000..1eb25d3d85f --- /dev/null +++ b/tests/queries/0_stateless/03281_dynamic_coalesce.sql @@ -0,0 +1,15 @@ +set enable_dynamic_type=1; + +select coalesce(number % 2 ? NULL : number::Dynamic, 42) as res from numbers(5); +select coalesce(number % 2 ? NULL : number::Dynamic, number % 3 ? NULL : 42) as res from numbers(5); +select coalesce(number % 2 ? NULL : number, number % 3 ? NULL : 42::Dynamic) as res from numbers(5); +select coalesce(number % 2 ? NULL : number::Dynamic, number % 3 ? NULL : 42::Dynamic) as res from numbers(5); +select coalesce(number % 2 ? NULL : number::Dynamic, number % 3 ? NULL : 42, number % 4 == 1 ? NULL : 43) as res from numbers(10); +select coalesce(number % 2 ? NULL : number, number % 3 ? NULL : 42::Dynamic, number % 4 == 1 ? NULL : 43) as res from numbers(10); +select coalesce(number % 2 ? NULL : number, number % 3 ? NULL : 42, number % 4 == 1 ? NULL : 43::Dynamic) as res from numbers(10); +select coalesce(number % 2 ? NULL : number, number % 3 ? NULL : 42, number % 4 == 1 ? NULL : 43::Dynamic) as res from numbers(10); +select coalesce(number % 2 ? NULL : number::Dynamic, number % 3 ? NULL : 42::Dynamic, number % 4 == 1 ? NULL : 43) as res from numbers(10); +select coalesce(number % 2 ? NULL : number, number % 3 ? NULL : 42::Dynamic, number % 4 == 1 ? NULL : 43::Dynamic) as res from numbers(10); +select coalesce(number % 2 ? NULL : number::Dynamic, number % 3 ? NULL : 42, number % 4 == 1 ? NULL : 43::Dynamic) as res from numbers(10); +select coalesce(number % 2 ? NULL : number::Dynamic, number % 3 ? NULL : 42::Dynamic, number % 4 == 1 ? NULL : 43::Dynamic) as res from numbers(10); + From 4ccbe8f1cc9eff2e7e87d44d1a3d2f4bbf032c4b Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 3 Dec 2024 22:27:33 +0100 Subject: [PATCH 305/502] Test for primary index cache size with LowCardinality columns --- ...mary_index_cache_low_cardinality.reference | 8 ++++ ...73_primary_index_cache_low_cardinality.sql | 41 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 tests/queries/0_stateless/03273_primary_index_cache_low_cardinality.reference create mode 100644 tests/queries/0_stateless/03273_primary_index_cache_low_cardinality.sql diff --git a/tests/queries/0_stateless/03273_primary_index_cache_low_cardinality.reference b/tests/queries/0_stateless/03273_primary_index_cache_low_cardinality.reference new file mode 100644 index 00000000000..17ae5eb983d --- /dev/null +++ b/tests/queries/0_stateless/03273_primary_index_cache_low_cardinality.reference @@ -0,0 +1,8 @@ +PrimaryIndexCacheBytes 5556 +PrimaryIndexCacheFiles 1 +PrimaryIndexCacheBytes 0 +PrimaryIndexCacheFiles 0 +3 +PrimaryIndexCacheBytes 5556 +PrimaryIndexCacheFiles 1 +1 3 6 diff --git a/tests/queries/0_stateless/03273_primary_index_cache_low_cardinality.sql b/tests/queries/0_stateless/03273_primary_index_cache_low_cardinality.sql new file mode 100644 index 00000000000..0f6fccbc62d --- /dev/null +++ b/tests/queries/0_stateless/03273_primary_index_cache_low_cardinality.sql @@ -0,0 +1,41 @@ +-- Tags: no-parallel + +DROP TABLE IF EXISTS t_primary_index_cache; + +SYSTEM DROP PRIMARY INDEX CACHE; + +CREATE TABLE t_primary_index_cache (a LowCardinality(String), b LowCardinality(String)) +ENGINE = MergeTree ORDER BY (a, b) +SETTINGS use_primary_key_cache = 1, prewarm_primary_key_cache = 1, index_granularity = 8192, index_granularity_bytes = '10M', min_bytes_for_wide_part = 0; + +-- Insert will prewarm primary index cache +INSERT INTO t_primary_index_cache SELECT number%10, number%11 FROM numbers(10000); + +-- Check cache size +SYSTEM RELOAD ASYNCHRONOUS METRICS; +SELECT metric, value FROM system.asynchronous_metrics WHERE metric IN ('PrimaryIndexCacheFiles', 'PrimaryIndexCacheBytes') ORDER BY metric; + +SYSTEM DROP PRIMARY INDEX CACHE; + +-- Check that cache is empty +SYSTEM RELOAD ASYNCHRONOUS METRICS; +SELECT metric, value FROM system.asynchronous_metrics WHERE metric IN ('PrimaryIndexCacheFiles', 'PrimaryIndexCacheBytes') ORDER BY metric; + +-- Trigger index reload +SELECT max(length(a || b)) FROM t_primary_index_cache WHERE a > '1' AND b < '99' SETTINGS log_comment = '03273_reload_query'; + +-- Check that cache size is the same as after prewarm +SYSTEM RELOAD ASYNCHRONOUS METRICS; +SELECT metric, value FROM system.asynchronous_metrics WHERE metric IN ('PrimaryIndexCacheFiles', 'PrimaryIndexCacheBytes') ORDER BY metric; + +SYSTEM FLUSH LOGS; + +SELECT + ProfileEvents['LoadedPrimaryIndexFiles'], + ProfileEvents['LoadedPrimaryIndexRows'], + ProfileEvents['LoadedPrimaryIndexBytes'] +FROM system.query_log +WHERE log_comment = '03273_reload_query' AND current_database = currentDatabase() AND type = 'QueryFinish' +ORDER BY event_time_microseconds; + +DROP TABLE t_primary_index_cache; From 86f657ffc6762f3d813937620fc57247deab9841 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 8 Oct 2024 13:45:37 +0200 Subject: [PATCH 306/502] Adjust error message about intersecting parts (remove ZooKeeper note from common code) It could happen for MergeTree as well. Signed-off-by: Azat Khuzhin --- src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 8278a26cb30..bfe5c84725a 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1342,7 +1342,7 @@ void MergeTreeData::PartLoadingTree::add(const MergeTreePartInfo & info, const S if (!prev_info.isDisjoint(info)) { throw Exception(ErrorCodes::LOGICAL_ERROR, - "Part {} intersects previous part {}. It is a bug or a result of manual intervention in the server or ZooKeeper data", + "Part {} intersects previous part {}. It is a bug or a result of manual intervention", name, prev->second->name); } } @@ -1359,7 +1359,7 @@ void MergeTreeData::PartLoadingTree::add(const MergeTreePartInfo & info, const S if (!next_info.isDisjoint(info)) { throw Exception(ErrorCodes::LOGICAL_ERROR, - "Part {} intersects next part {}. It is a bug or a result of manual intervention in the server or ZooKeeper data", + "Part {} intersects next part {}. It is a bug or a result of manual intervention", name, it->second->name); } } From 12bb273beb953693c572c737b7a1436d8ff07007 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 8 Oct 2024 13:54:23 +0200 Subject: [PATCH 307/502] Do not ignore error during moving parts to detached for MergeTree MergeTree is fragile in this case, since the source of truth for it is the filesystem and it will be left in an inconsistent state. Image the following: - during startup you got some broken part, that should be moved to detached - this rename throws exception (i.e. permission error) - but this error is ignored, server continues, and tries to merge something that produces intersecting parts for that broken part - later on restart you will got intersecting parts (since at the time of scanning for intersecting parts, the part is not checked) Signed-off-by: Azat Khuzhin Co-authored-by: Alexander Tokmakov --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 32 ++++++++++--------- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 15 +++++---- src/Storages/StorageReplicatedMergeTree.cpp | 4 +-- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index e880c70ba8d..3ec6340f11a 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -2007,7 +2007,6 @@ bool IMergeTreeDataPart::shallParticipateInMerges(const StoragePolicyPtr & stora } void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_new_dir_if_exists) -try { assertOnDisk(); @@ -2033,18 +2032,6 @@ try for (const auto & [_, part] : projection_parts) part->getDataPartStorage().changeRootPath(old_projection_root_path, new_projection_root_path); } -catch (...) -{ - if (startsWith(new_relative_path, fs::path(MergeTreeData::DETACHED_DIR_NAME) / "")) - { - // Don't throw when the destination is to the detached folder. It might be able to - // recover in some cases, such as fetching parts into multi-disks while some of the - // disks are broken. - tryLogCurrentException(__PRETTY_FUNCTION__); - } - else - throw; -} std::pair IMergeTreeDataPart::canRemovePart() const { @@ -2159,11 +2146,26 @@ String IMergeTreeDataPart::getRelativePathOfActivePart() const return fs::path(getDataPartStorage().getFullRootPath()) / name / ""; } -void IMergeTreeDataPart::renameToDetached(const String & prefix) +void IMergeTreeDataPart::renameToDetached(const String & prefix, bool ignore_error) { auto path_to_detach = getRelativePathForDetachedPart(prefix, /* broken */ false); assert(path_to_detach); - renameTo(path_to_detach.value(), true); + try + { + renameTo(path_to_detach.value(), true); + } + catch (...) + { + if (ignore_error) + { + // Don't throw when the destination is to the detached folder. It might be able to + // recover in some cases, such as fetching parts into multi-disks while some of the + // disks are broken. + tryLogCurrentException(__PRETTY_FUNCTION__); + } + else + throw; + } part_is_probably_removed_from_disk = true; } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index e563a4e1906..ad20d5a5d0c 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -413,7 +413,7 @@ public: auto getFilesChecksums() const { return checksums.files; } /// Moves a part to detached/ directory and adds prefix to its name - void renameToDetached(const String & prefix); + void renameToDetached(const String & prefix, bool ignore_error = false); /// Makes checks and move part to new directory /// Changes only relative_dir_name, you need to update other metadata (name, is_temp) explicitly diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index bfe5c84725a..e1c4303bee3 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2022,9 +2022,10 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks, std::optional(this) != nullptr; if (!is_static_storage) for (auto & part : broken_parts_to_detach) - part->renameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes + part->renameToDetached("broken-on-start", /*ignore_error=*/ replicated); /// detached parts must not have '_' in prefixes resetObjectColumnsFromActiveParts(part_lock); resetSerializationHints(part_lock); @@ -2112,6 +2113,7 @@ try ThreadPoolCallbackRunnerLocal runner(getUnexpectedPartsLoadingThreadPool().get(), "UnexpectedParts"); + bool replicated = dynamic_cast(this) != nullptr; for (auto & load_state : unexpected_data_parts) { std::lock_guard lock(unexpected_data_parts_mutex); @@ -2127,9 +2129,7 @@ try chassert(load_state.part); if (load_state.is_broken) - { - load_state.part->renameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes - } + load_state.part->renameToDetached("broken-on-start", /*ignore_error=*/ replicated); /// detached parts must not have '_' in prefixes }, Priority{}); } runner.waitForAllToFinishAndRethrowFirstError(); @@ -2180,6 +2180,7 @@ try ThreadPoolCallbackRunnerLocal runner(getOutdatedPartsLoadingThreadPool().get(), "OutdatedParts"); + bool replicated = dynamic_cast(this) != nullptr; while (true) { ThreadFuzzer::maybeInjectSleep(); @@ -2220,7 +2221,7 @@ try if (res.is_broken) { forcefullyRemoveBrokenOutdatedPartFromZooKeeperBeforeDetaching(res.part->name); - res.part->renameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes + res.part->renameToDetached("broken-on-start", /*ignore_error=*/ replicated); /// detached parts must not have '_' in prefixes } else if (res.part->is_duplicate) res.part->remove(); @@ -4593,6 +4594,7 @@ void MergeTreeData::forcefullyMovePartToDetachedAndRemoveFromMemory(const MergeT auto lock = lockParts(); bool removed_active_part = false; bool restored_active_part = false; + bool replicated = dynamic_cast(this) != nullptr; auto it_part = data_parts_by_info.find(part_to_detach->info); if (it_part == data_parts_by_info.end()) @@ -4611,7 +4613,7 @@ void MergeTreeData::forcefullyMovePartToDetachedAndRemoveFromMemory(const MergeT } modifyPartState(it_part, DataPartState::Deleting); - asMutableDeletingPart(part)->renameToDetached(prefix); + asMutableDeletingPart(part)->renameToDetached(prefix, /*ignore_error=*/ replicated); LOG_TEST(log, "forcefullyMovePartToDetachedAndRemoveFromMemory: removing {} from data_parts_indexes", part->getNameWithState()); data_parts_indexes.erase(it_part); @@ -6002,6 +6004,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartRestoredFromBackup(cons .withPartType(MergeTreeDataPartType::Wide) .build(); } + /// Errors should not be ignored in RESTORE, since you should not restore to broken disks. part->renameToDetached("broken-from-backup"); }; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ac1fbed2ec7..b7aad02a29f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1851,9 +1851,7 @@ bool StorageReplicatedMergeTree::checkPartsImpl(bool skip_sanity_checks) /// detached all unexpected data parts after sanity check. for (auto & part_state : unexpected_data_parts) - { - part_state.part->renameToDetached("ignored"); - } + part_state.part->renameToDetached("ignored", /* ignore_error= */ true); unexpected_data_parts.clear(); return true; From 50fc0359b99f1d4a77cd5d0bcf1f088215a0dcdd Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 8 Oct 2024 13:57:44 +0200 Subject: [PATCH 308/502] Ignore only ErrnoException/fs::filesystem_error while moving parts to detached Signed-off-by: Azat Khuzhin --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 3ec6340f11a..fe43e2a1ac8 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -2154,7 +2154,23 @@ void IMergeTreeDataPart::renameToDetached(const String & prefix, bool ignore_err { renameTo(path_to_detach.value(), true); } - catch (...) + /// This exceptions majority of cases: + /// - fsync + /// - mtime adjustment + catch (const ErrnoException &) + { + if (ignore_error) + { + // Don't throw when the destination is to the detached folder. It might be able to + // recover in some cases, such as fetching parts into multi-disks while some of the + // disks are broken. + tryLogCurrentException(__PRETTY_FUNCTION__); + } + else + throw; + } + /// - rename + catch (const fs::filesystem_error &) { if (ignore_error) { From d90364e0119cf20a06b4bb3c68e4dcb2292d87e5 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 4 Dec 2024 00:19:21 +0100 Subject: [PATCH 309/502] Poco Net: make MessageHeader limits configurable (cherry-pick of https://github.com/pocoproject/poco/pull/3404) --- .../poco/Net/include/Poco/Net/MessageHeader.h | 26 +++++++++++- base/poco/Net/src/MessageHeader.cpp | 40 ++++++++++++++++--- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/base/poco/Net/include/Poco/Net/MessageHeader.h b/base/poco/Net/include/Poco/Net/MessageHeader.h index bbfd849b304..851eae57436 100644 --- a/base/poco/Net/include/Poco/Net/MessageHeader.h +++ b/base/poco/Net/include/Poco/Net/MessageHeader.h @@ -103,6 +103,26 @@ namespace Net /// /// The default limit is 100. + int getNameLengthLimit() const; + /// Returns the maximum length of a field name. + /// + /// See setNameLengthLimit() for more information. + + void setNameLengthLimit(int limit); + /// Sets the maximum length of a field name. + /// + /// The default limit is 256. + + int getValueLengthLimit() const; + /// Returns the maximum length of a field value. + /// + /// See setValueLengthLimit() for more information. + + void setValueLengthLimit(int limit); + /// Sets the maximum length of a field value. + /// + /// The default limit is 8192. + bool hasToken(const std::string & fieldName, const std::string & token) const; /// Returns true iff the field with the given fieldName contains /// the given token. Tokens in a header field are expected to be @@ -157,12 +177,14 @@ namespace Net enum Limits /// Limits for basic sanity checks when reading a header { - MAX_NAME_LENGTH = 256, - MAX_VALUE_LENGTH = 8192, + DFL_NAME_LENGTH_LIMIT = 256, + DFL_VALUE_LENGTH_LIMIT = 8192, DFL_FIELD_LIMIT = 100 }; int _fieldLimit; + int _nameLengthLimit; + int _valueLengthLimit; }; diff --git a/base/poco/Net/src/MessageHeader.cpp b/base/poco/Net/src/MessageHeader.cpp index 0dbfa3e5cbd..1fcd78a9068 100644 --- a/base/poco/Net/src/MessageHeader.cpp +++ b/base/poco/Net/src/MessageHeader.cpp @@ -28,14 +28,18 @@ namespace Net { MessageHeader::MessageHeader(): - _fieldLimit(DFL_FIELD_LIMIT) + _fieldLimit(DFL_FIELD_LIMIT), + _nameLengthLimit(DFL_NAME_LENGTH_LIMIT), + _valueLengthLimit(DFL_VALUE_LENGTH_LIMIT) { } MessageHeader::MessageHeader(const MessageHeader& messageHeader): NameValueCollection(messageHeader), - _fieldLimit(DFL_FIELD_LIMIT) + _fieldLimit(DFL_FIELD_LIMIT), + _nameLengthLimit(DFL_NAME_LENGTH_LIMIT), + _valueLengthLimit(DFL_VALUE_LENGTH_LIMIT) { } @@ -80,12 +84,12 @@ void MessageHeader::read(std::istream& istr) throw MessageException("Too many header fields"); name.clear(); value.clear(); - while (ch != eof && ch != ':' && ch != '\n' && name.length() < MAX_NAME_LENGTH) { name += ch; ch = buf.sbumpc(); } + while (ch != eof && ch != ':' && ch != '\n' && name.length() < _nameLengthLimit) { name += ch; ch = buf.sbumpc(); } if (ch == '\n') { ch = buf.sbumpc(); continue; } // ignore invalid header lines if (ch != ':') throw MessageException("Field name too long/no colon found"); if (ch != eof) ch = buf.sbumpc(); // ':' while (ch != eof && Poco::Ascii::isSpace(ch) && ch != '\r' && ch != '\n') ch = buf.sbumpc(); - while (ch != eof && ch != '\r' && ch != '\n' && value.length() < MAX_VALUE_LENGTH) { value += ch; ch = buf.sbumpc(); } + while (ch != eof && ch != '\r' && ch != '\n' && value.length() < _valueLengthLimit) { value += ch; ch = buf.sbumpc(); } if (ch == '\r') ch = buf.sbumpc(); if (ch == '\n') ch = buf.sbumpc(); @@ -93,7 +97,7 @@ void MessageHeader::read(std::istream& istr) throw MessageException("Field value too long/no CRLF found"); while (ch == ' ' || ch == '\t') // folding { - while (ch != eof && ch != '\r' && ch != '\n' && value.length() < MAX_VALUE_LENGTH) { value += ch; ch = buf.sbumpc(); } + while (ch != eof && ch != '\r' && ch != '\n' && value.length() < _valueLengthLimit) { value += ch; ch = buf.sbumpc(); } if (ch == '\r') ch = buf.sbumpc(); if (ch == '\n') ch = buf.sbumpc(); @@ -122,6 +126,32 @@ void MessageHeader::setFieldLimit(int limit) } +int MessageHeader::getNameLengthLimit() const +{ + return _nameLengthLimit; +} + +void MessageHeader::setNameLengthLimit(int limit) +{ + poco_assert(limit >= 0); + + _nameLengthLimit = limit; +} + + +int MessageHeader::getValueLengthLimit() const +{ + return _valueLengthLimit; +} + +void MessageHeader::setValueLengthLimit(int limit) +{ + poco_assert(limit >= 0); + + _valueLengthLimit = limit; +} + + bool MessageHeader::hasToken(const std::string& fieldName, const std::string& token) const { std::string field = get(fieldName, ""); From bd00110e9c89dd066d313258059f608a1d948666 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Wed, 4 Dec 2024 09:09:08 +0800 Subject: [PATCH 310/502] Update tests/queries/0_stateless/03278_enum_string_functions.sql Co-authored-by: Vladimir Cherkasov --- tests/queries/0_stateless/03278_enum_string_functions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03278_enum_string_functions.sql b/tests/queries/0_stateless/03278_enum_string_functions.sql index e5d0f65b52b..ccb102887e5 100644 --- a/tests/queries/0_stateless/03278_enum_string_functions.sql +++ b/tests/queries/0_stateless/03278_enum_string_functions.sql @@ -24,7 +24,7 @@ SELECT hasToken(e, 'a') FROM test_enum_string_functions; -- hasTokenOrNull -- SELECT hasTokenOrNull(e, 'a') FROM test_enum_string_functions; -DROP TABLE jsons; +DROP TABLE IF EXISTS jsons; CREATE TABLE jsons ( `json` Enum('a', '{"a":1}') From ce6a116fdb229476ce0cfa086798a21d5cc6a213 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Wed, 4 Dec 2024 12:31:45 +0800 Subject: [PATCH 311/502] fix vector out of range --- src/Interpreters/convertFieldToType.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 3fabedba873..99f01a6f120 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -585,6 +585,8 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID throw; } + if (!col->size()) + return Field(Null()); Field parsed = (*col)[0]; return convertFieldToType(parsed, type, from_type_hint, format_settings); } From 2bfe8ad1d81eb8c9a9eec291f6f7e3eebaf9f97f Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Tue, 3 Dec 2024 18:57:09 +0000 Subject: [PATCH 312/502] fix throw on non-existent user in remotequeryexecutor --- src/Core/SettingsChangesHistory.cpp | 2 +- src/QueryPipeline/RemoteQueryExecutor.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 4c64e2fe6d5..c6a5b85ac0e 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -97,7 +97,7 @@ static std::initializer_list local_granted_roles; if (context->getSettingsRef()[Setting::push_external_roles_in_interserver_queries] && !modified_client_info.initial_user.empty()) { - auto user = context->getAccessControl().read(modified_client_info.initial_user, true); + auto user = context->getAccessControl().read(modified_client_info.initial_user, false); boost::container::flat_set granted_roles; if (user) { From 7e5aa5f1bd5b0f814b7ee49a23bffeac95dd5d03 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Wed, 4 Dec 2024 17:12:55 +0800 Subject: [PATCH 313/502] fix enum comparition --- tests/queries/0_stateless/01310_enum_comparison.reference | 1 + tests/queries/0_stateless/01310_enum_comparison.sql | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01310_enum_comparison.reference b/tests/queries/0_stateless/01310_enum_comparison.reference index b261da18d51..e22493782f0 100644 --- a/tests/queries/0_stateless/01310_enum_comparison.reference +++ b/tests/queries/0_stateless/01310_enum_comparison.reference @@ -1,2 +1,3 @@ 1 0 +0 diff --git a/tests/queries/0_stateless/01310_enum_comparison.sql b/tests/queries/0_stateless/01310_enum_comparison.sql index 50752392bb3..f67fdaf7711 100644 --- a/tests/queries/0_stateless/01310_enum_comparison.sql +++ b/tests/queries/0_stateless/01310_enum_comparison.sql @@ -3,4 +3,4 @@ INSERT INTO enum VALUES ('hello'); SELECT count() FROM enum WHERE x = 'hello'; SELECT count() FROM enum WHERE x = 'world'; -SELECT count() FROM enum WHERE x = 'xyz'; -- { serverError UNKNOWN_ELEMENT_OF_ENUM } +SELECT count() FROM enum WHERE x = 'xyz'; From c3f7cba0ebfe4b29699cfea47d5436c6680e321f Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Wed, 4 Dec 2024 17:33:05 +0800 Subject: [PATCH 314/502] fix tests --- .../03278_enum_string_functions.reference | 47 +++++++++++++++++-- .../03278_enum_string_functions.sql | 23 ++------- 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/tests/queries/0_stateless/03278_enum_string_functions.reference b/tests/queries/0_stateless/03278_enum_string_functions.reference index 1bc1be5cc77..1c7679c6e63 100644 --- a/tests/queries/0_stateless/03278_enum_string_functions.reference +++ b/tests/queries/0_stateless/03278_enum_string_functions.reference @@ -1,36 +1,73 @@ +-- { echoOn } + +DROP TABLE IF EXISTS test_enum_string_functions; +CREATE TABLE test_enum_string_functions(e Enum('a'=1, 'b'=2)) ENGINE=TinyLog; +INSERT INTO test_enum_string_functions VALUES ('a'); +SELECT * from test_enum_string_functions WHERE e LIKE '%abc%'; +SELECT * from test_enum_string_functions WHERE e NOT LIKE '%abc%'; a +SELECT * from test_enum_string_functions WHERE e iLike '%a%'; a +SELECT position(e, 'a') FROM test_enum_string_functions; 1 +SELECT match(e, 'a') FROM test_enum_string_functions; 1 +SELECT locate('a', e) FROM test_enum_string_functions; 1 +SELECT countSubstrings(e, 'a') FROM test_enum_string_functions; 1 +SELECT countSubstringsCaseInsensitive(e, 'a') FROM test_enum_string_functions; 1 +SELECT countSubstringsCaseInsensitiveUTF8(e, 'a') FROM test_enum_string_functions; 1 +SELECT hasToken(e, 'a') FROM test_enum_string_functions; 1 +SELECT hasTokenOrNull(e, 'a') FROM test_enum_string_functions; +1 +DROP TABLE IF EXISTS jsons; +CREATE TABLE jsons +( + `json` Enum('a', '{"a":1}') +) +ENGINE = Memory; +INSERT INTO jsons VALUES ('{"a":1}'); +INSERT INTO jsons VALUES ('a'); +SELECT simpleJSONHas(json, 'foo') FROM jsons; +0 +0 +SELECT simpleJSONHas(json, 'a') FROM jsons; 1 0 -0 +SELECT simpleJSONExtractUInt(json, 'a') FROM jsons; 1 0 +SELECT simpleJSONExtractUInt(json, 'not exsits') FROM jsons; +0 +0 +SELECT simpleJSONExtractInt(json, 'a') FROM jsons; 1 0 +SELECT simpleJSONExtractInt(json, 'not exsits') FROM jsons; 0 0 -0 -1 -0 -0 +SELECT simpleJSONExtractFloat(json, 'a') FROM jsons; 1 0 +SELECT simpleJSONExtractFloat(json, 'not exsits') FROM jsons; 0 0 +SELECT simpleJSONExtractBool(json, 'a') FROM jsons; 0 0 +SELECT simpleJSONExtractBool(json, 'not exsits') FROM jsons; 0 0 +SELECT positionUTF8(json, 'a') FROM jsons; 3 1 +SELECT positionCaseInsensitiveUTF8(json, 'A') FROM jsons; 3 1 +SELECT positionCaseInsensitive(json, 'A') FROM jsons; 3 1 diff --git a/tests/queries/0_stateless/03278_enum_string_functions.sql b/tests/queries/0_stateless/03278_enum_string_functions.sql index e5d0f65b52b..eda46704d75 100644 --- a/tests/queries/0_stateless/03278_enum_string_functions.sql +++ b/tests/queries/0_stateless/03278_enum_string_functions.sql @@ -1,30 +1,21 @@ +-- { echoOn } + DROP TABLE IF EXISTS test_enum_string_functions; CREATE TABLE test_enum_string_functions(e Enum('a'=1, 'b'=2)) ENGINE=TinyLog; INSERT INTO test_enum_string_functions VALUES ('a'); --- like -- SELECT * from test_enum_string_functions WHERE e LIKE '%abc%'; --- not like -- SELECT * from test_enum_string_functions WHERE e NOT LIKE '%abc%'; --- ilike -- SELECT * from test_enum_string_functions WHERE e iLike '%a%'; --- position -- SELECT position(e, 'a') FROM test_enum_string_functions; --- match -- SELECT match(e, 'a') FROM test_enum_string_functions; --- locate -- SELECT locate('a', e) FROM test_enum_string_functions; --- countsubstrings -- SELECT countSubstrings(e, 'a') FROM test_enum_string_functions; --- countSburstringsCaseInsensitive -- SELECT countSubstringsCaseInsensitive(e, 'a') FROM test_enum_string_functions; --- countSubstringsCaseInsensitiveUTF8 -- SELECT countSubstringsCaseInsensitiveUTF8(e, 'a') FROM test_enum_string_functions; --- hasToken -- SELECT hasToken(e, 'a') FROM test_enum_string_functions; --- hasTokenOrNull -- SELECT hasTokenOrNull(e, 'a') FROM test_enum_string_functions; -DROP TABLE jsons; +DROP TABLE IF EXISTS jsons; CREATE TABLE jsons ( `json` Enum('a', '{"a":1}') @@ -32,24 +23,16 @@ CREATE TABLE jsons ENGINE = Memory; INSERT INTO jsons VALUES ('{"a":1}'); INSERT INTO jsons VALUES ('a'); --- simpleJSONHas -- SELECT simpleJSONHas(json, 'foo') FROM jsons; SELECT simpleJSONHas(json, 'a') FROM jsons; --- simpleJSONExtractString -- SELECT simpleJSONExtractUInt(json, 'a') FROM jsons; SELECT simpleJSONExtractUInt(json, 'not exsits') FROM jsons; --- simpleJSONExtractInt -- SELECT simpleJSONExtractInt(json, 'a') FROM jsons; SELECT simpleJSONExtractInt(json, 'not exsits') FROM jsons; --- simpleJSONExtractFloat -- SELECT simpleJSONExtractFloat(json, 'a') FROM jsons; SELECT simpleJSONExtractFloat(json, 'not exsits') FROM jsons; --- simpleJSONExtractBool -- SELECT simpleJSONExtractBool(json, 'a') FROM jsons; SELECT simpleJSONExtractBool(json, 'not exsits') FROM jsons; --- positionUTF8 -- SELECT positionUTF8(json, 'a') FROM jsons; --- positionCaseInsensitiveUTF8 -- SELECT positionCaseInsensitiveUTF8(json, 'A') FROM jsons; --- positionCaseInsensitive -- SELECT positionCaseInsensitive(json, 'A') FROM jsons; From 51b5f43de227c67b57b4b0d0e63ac14aafdb09e6 Mon Sep 17 00:00:00 2001 From: Francesco Ciocchetti Date: Wed, 4 Dec 2024 11:25:17 +0100 Subject: [PATCH 315/502] Make the List Blob Azure work regardless if the endpoint has a trailing slash or not --- .../AzureBlobStorageCommon.cpp | 2 +- ...n_rewritable_with_trailing_slash.reference | 11 ++++ ...re_plain_rewritable_with_trailing_slash.sh | 53 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.reference create mode 100755 tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.sh diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index c1db590f1e8..5b38e7e53c6 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -108,7 +108,7 @@ ListBlobsPagedResponse ContainerClientWrapper::ListBlobs(const ListBlobsOptions new_options.Prefix = blob_prefix / options.Prefix.ValueOr(""); auto response = client.ListBlobs(new_options); - auto blob_prefix_str = blob_prefix.empty() ? "" : blob_prefix.string() + "/"; + auto blob_prefix_str = blob_prefix.empty() ? "" : (blob_prefix.string() + (blob_prefix.string().back() == '/' ? "" : "/")); for (auto & blob : response.Blobs) { diff --git a/tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.reference b/tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.reference new file mode 100644 index 00000000000..9aa9873514a --- /dev/null +++ b/tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.reference @@ -0,0 +1,11 @@ +10006 +0 0 0 +1 1 1 +1 2 0 +2 2 2 +2 2 2 +3 1 9 +3 3 3 +4 4 4 +4 7 7 +5 5 5 diff --git a/tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.sh b/tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.sh new file mode 100755 index 00000000000..528a4364c98 --- /dev/null +++ b/tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-shared-merge-tree +# Tag no-fasttest: requires Azure +# Tag no-shared-merge-tree: does not support replication + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +container="cont-$(echo "${CLICKHOUSE_TEST_UNIQUE_NAME}" | tr _ -)" + +${CLICKHOUSE_CLIENT} --query "drop table if exists test_azure_mt" + +${CLICKHOUSE_CLIENT} -nm --query " +create table test_azure_mt (a Int32, b Int64, c Int64) engine = MergeTree() partition by intDiv(a, 1000) order by tuple(a, b) +settings disk = disk( + type = object_storage, + metadata_type = plain_rewritable, + object_storage_type = azure_blob_storage, + name = '${container}', + path='/var/lib/clickhouse/disks/${container}/tables', + container_name = '${container}', + endpoint = 'http://localhost:10000/devstoreaccount1/${container}/plain-tables/', + account_name = 'devstoreaccount1', + account_key = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='); +" + +${CLICKHOUSE_CLIENT} -nm --query " +insert into test_azure_mt (*) values (1, 2, 0), (2, 2, 2), (3, 1, 9), (4, 7, 7), (5, 10, 2), (6, 12, 5); +insert into test_azure_mt (*) select number, number, number from numbers_mt(10000); +select count(*) from test_azure_mt; +select (*) from test_azure_mt order by tuple(a, b) limit 10; +" + +${CLICKHOUSE_CLIENT} --query "optimize table test_azure_mt final" + +${CLICKHOUSE_CLIENT} -m --query " +alter table test_azure_mt add projection test_azure_mt_projection (select * order by b)" 2>&1 | grep -Fq "SUPPORT_IS_DISABLED" + +${CLICKHOUSE_CLIENT} -nm --query " +alter table test_azure_mt update c = 0 where a % 2 = 1; +alter table test_azure_mt add column d Int64 after c; +alter table test_azure_mt drop column c; +" 2>&1 | grep -Fq "SUPPORT_IS_DISABLED" + +${CLICKHOUSE_CLIENT} -nm --query " +detach table test_azure_mt; +attach table test_azure_mt; +" + +${CLICKHOUSE_CLIENT} --query "drop table test_azure_mt sync" From 050bf762c2eb199193c6da1685cb92f6b71a3624 Mon Sep 17 00:00:00 2001 From: Francesco Ciocchetti Date: Wed, 4 Dec 2024 11:35:00 +0100 Subject: [PATCH 316/502] use fs::path to simplify the statement --- .../ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index 5b38e7e53c6..e32cedad895 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace ProfileEvents { @@ -108,7 +109,7 @@ ListBlobsPagedResponse ContainerClientWrapper::ListBlobs(const ListBlobsOptions new_options.Prefix = blob_prefix / options.Prefix.ValueOr(""); auto response = client.ListBlobs(new_options); - auto blob_prefix_str = blob_prefix.empty() ? "" : (blob_prefix.string() + (blob_prefix.string().back() == '/' ? "" : "/")); + auto blob_prefix_str = fs::path(blob_prefix / "").string(); for (auto & blob : response.Blobs) { From e94a42c576d5299aa3bc6f76724b3f750ce46e8f Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 4 Dec 2024 11:38:01 +0100 Subject: [PATCH 317/502] Update test.py --- tests/integration/test_storage_s3_queue/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 39780c4913c..af39e4725b3 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -2347,7 +2347,7 @@ def test_list_and_delete_race(started_cluster): }, ) - threads = 4 + threads = 6 total_rows = row_num * files_to_generate * (threads + 1) busy_pool = Pool(threads) From d0aaea9e821a0b10659404f14581e982a26be427 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 3 Dec 2024 12:56:23 +0100 Subject: [PATCH 318/502] Add dmesg outptut for the case of build failure --- tests/ci/build_check.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 77d91c8400b..c28aba049d9 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -209,6 +209,9 @@ def main(): # We check if docker works, because if it's down, it's infrastructure try: subprocess.check_call("docker info", shell=True) + logging.warning("Collecting 'dmesg -T' content") + with TeePopen("sudo dmesg -T", build_output_path / "dmesg.log") as process: + process.wait() except subprocess.CalledProcessError: logging.error( "The dockerd looks down, won't upload anything and generate report" From cedc08be915c11dbb7a21976f230dc2176d6fd90 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 3 Dec 2024 13:04:17 +0100 Subject: [PATCH 319/502] Rebuild clickhouse on the build script changes --- tests/ci/ci_definitions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index d55e347c3ee..0e2ca626edc 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -612,6 +612,7 @@ class CommonJobConfigs: "./docker/packager/packager", "./rust", "./tests/ci/version_helper.py", + "./tests/ci/build_check.py", # FIXME: This is a WA to rebuild the CH and recreate the Performance.tar.zst artifact # when there are changes in performance test scripts. # Due to the current design of the perf test we need to rebuild CH when the performance test changes, From 7dc9c8c79c7b6ec43ffd9adccb8c50e72d194caa Mon Sep 17 00:00:00 2001 From: Francesco Ciocchetti Date: Wed, 4 Dec 2024 11:44:21 +0100 Subject: [PATCH 320/502] Trigger CI after updating PR Description From 9172d572c64c8ddadd9e3a599fd9f9f7eb5cd83c Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 4 Dec 2024 11:57:43 +0100 Subject: [PATCH 321/502] Update src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp --- .../ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index e32cedad895..fc16fd2c0fe 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -109,7 +109,7 @@ ListBlobsPagedResponse ContainerClientWrapper::ListBlobs(const ListBlobsOptions new_options.Prefix = blob_prefix / options.Prefix.ValueOr(""); auto response = client.ListBlobs(new_options); - auto blob_prefix_str = fs::path(blob_prefix / "").string(); + String blob_prefix_str = blob_prefix / ""; for (auto & blob : response.Blobs) { From 70fce88bde64c65d6f02ffef57a85fba9375d163 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:11:31 +0100 Subject: [PATCH 322/502] Fix tests --- tests/queries/0_stateless/03156_group_concat.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/03156_group_concat.sql b/tests/queries/0_stateless/03156_group_concat.sql index d64d382bd74..719d271f4b5 100644 --- a/tests/queries/0_stateless/03156_group_concat.sql +++ b/tests/queries/0_stateless/03156_group_concat.sql @@ -44,6 +44,10 @@ SELECT length(groupConcat(number)) FROM numbers(100000); SELECT 'TESTING GroupConcat second argument overload'; +TRUNCATE TABLE test_groupConcat; + +INSERT INTO test_groupConcat VALUES (0, 95, 'abc', [1, 2, 3]), (1, NULL, 'a', [993, 986, 979, 972]), (2, 123, 'makson95', []); + SELECT groupConcat(',', p_int) FROM test_groupConcat; SELECT groupConcat('.')(p_string) FROM test_groupConcat; SELECT groupConcat('/', p_array) FROM test_groupConcat; From 86602c00b679b3d5aafd5a60d4539ed66ffb951a Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 4 Dec 2024 11:13:44 +0000 Subject: [PATCH 323/502] Merge master and remove setting --- src/Core/Settings.cpp | 8 +- .../DataLakes/IcebergMetadata.cpp | 97 ++++++++++++++++--- .../StorageObjectStorageSettings.cpp | 8 +- .../StorageObjectStorageSettings.h | 8 -- 4 files changed, 92 insertions(+), 29 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 19ec3821d2d..d41c020bf87 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -5559,13 +5559,6 @@ Only available in ClickHouse Cloud. Exclude new data parts from SELECT queries u )", 0) \ DECLARE(Int64, prefer_warmed_unmerged_parts_seconds, 0, R"( Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm. -)", 0) \ - DECLARE(Bool, iceberg_engine_ignore_schema_evolution, false, R"( -Allow to ignore schema evolution in Iceberg table engine and read all data using schema specified by the user on table creation or latest schema parsed from metadata on table creation. - -:::note -Enabling this setting can lead to incorrect result as in case of evolved schema all data files will be read using the same schema. -::: )", 0) \ DECLARE(Bool, allow_deprecated_error_prone_window_functions, false, R"( Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference) @@ -5999,6 +5992,7 @@ Experimental data deduplication for SELECT queries based on part UUIDs MAKE_OBSOLETE(M, Bool, optimize_monotonous_functions_in_order_by, false) \ MAKE_OBSOLETE(M, UInt64, http_max_chunk_size, 100_GiB) \ MAKE_OBSOLETE(M, Bool, enable_deflate_qpl_codec, false) \ + MAKE_OBSOLETE(M, Bool, iceberg_engine_ignore_schema_evolution, false) \ /** The section above is for obsolete settings. Do not add anything there. */ #endif /// __CLION_IDE__ diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 9d029794df7..1b3ea2c76ef 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -40,7 +40,7 @@ namespace DB { namespace Setting { - extern const SettingsBool iceberg_engine_ignore_schema_evolution; +extern const SettingsBool allow_data_lake_dynamic_schema; } namespace ErrorCodes @@ -52,27 +52,98 @@ extern const int UNSUPPORTED_METHOD; extern const int LOGICAL_ERROR; } -Int32 parseTableSchema(const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor) +std::pair parseTableSchemaV2Method(const Poco::JSON::Object::Ptr & metadata_object) +{ + Poco::JSON::Object::Ptr schema; + if (!metadata_object->has("current-schema-id")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'current-schema-id' field is missing in metadata"); + auto current_schema_id = metadata_object->getValue("current-schema-id"); + if (!metadata_object->has("schemas")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schemas' field is missing in metadata"); + auto schemas = metadata_object->get("schemas").extract(); + if (schemas->size() == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: schemas field is empty"); + for (uint32_t i = 0; i != schemas->size(); ++i) + { + auto current_schema = schemas->getObject(i); + if (!current_schema->has("schema-id")) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schema-id' field is missing in schema"); + } + if (current_schema->getValue("schema-id") == current_schema_id) + { + schema = current_schema; + break; + } + } + + if (!schema) + throw Exception(ErrorCodes::BAD_ARGUMENTS, R"(There is no schema with "schema-id" that matches "current-schema-id" in metadata)"); + if (schema->getValue("schema-id") != current_schema_id) + throw Exception(ErrorCodes::BAD_ARGUMENTS, R"(Field "schema-id" of the schema doesn't match "current-schema-id" in metadata)"); + return {schema, current_schema_id}; +} + +std::pair parseTableSchemaV1Method(const Poco::JSON::Object::Ptr & metadata_object) +{ + if (!metadata_object->has("schema")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schema' field is missing in metadata"); + Poco::JSON::Object::Ptr schema = metadata_object->getObject("schema"); + if (!metadata_object->has("schema")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schema-id' field is missing in schema"); + auto current_schema_id = schema->getValue("schema-id"); + return {schema, current_schema_id}; +} + +Int32 parseTableSchema( + const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor, LoggerPtr metadata_logger) { Int32 format_version = metadata_object->getValue("format-version"); if (format_version == 2) { - auto fields = metadata_object->get("schemas").extract(); - for (size_t i = 0; i != fields->size(); ++i) - { - auto field = fields->getObject(static_cast(i)); - schema_processor.addIcebergTableSchema(field); - } - return metadata_object->getValue("current-schema-id"); + auto [schema, current_schema_id] = parseTableSchemaV2Method(metadata_object); + schema_processor.addIcebergTableSchema(schema); + return current_schema_id; } else { - auto schema = metadata_object->getObject("schema"); - schema_processor.addIcebergTableSchema(schema); - return schema->getValue("schema-id"); + try + { + auto [schema, current_schema_id] = parseTableSchemaV1Method(metadata_object); + schema_processor.addIcebergTableSchema(schema); + return current_schema_id; + } + catch (const Exception & first_error) + { + if (first_error.code() != ErrorCodes::BAD_ARGUMENTS) + throw; + try + { + auto [schema, current_schema_id] = parseTableSchemaV2Method(metadata_object); + schema_processor.addIcebergTableSchema(schema); + LOG_WARNING( + metadata_logger, + "Iceberg table schema was parsed using v2 specification, but it was impossible to parse it using v1 " + "specification. Be " + "aware that you Iceberg writing engine violates Iceberg specification. Error during parsing {}", + first_error.displayText()); + return current_schema_id; + } + catch (const Exception & second_error) + { + if (first_error.code() != ErrorCodes::BAD_ARGUMENTS) + throw; + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot parse Iceberg table schema both with v1 and v2 methods. Old method error: {}. New method error: {}", + first_error.displayText(), + second_error.displayText()); + } + } } } + IcebergMetadata::IcebergMetadata( ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, @@ -90,7 +161,7 @@ IcebergMetadata::IcebergMetadata( , schema_processor(IcebergSchemaProcessor()) , log(getLogger("IcebergMetadata")) { - auto schema_id = parseTableSchema(object, schema_processor); + auto schema_id = parseTableSchema(object, schema_processor, log); schema = *(schema_processor.getClickhouseTableSchemaById(schema_id)); current_schema_id = schema_id; } diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp index b280086c28b..383f5434203 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp @@ -11,7 +11,13 @@ namespace DB { -#define STORAGE_OBJECT_STORAGE_RELATED_SETTINGS(DECLARE, ALIAS) DECLARE(Bool, allow_dynamic_metadata_for_data_lakes, false, "", 0) +#define STORAGE_OBJECT_STORAGE_RELATED_SETTINGS(DECLARE, ALIAS) \ + DECLARE( \ + Bool, \ + allow_dynamic_metadata_for_data_lakes, \ + false, \ + "If enabled, indicates that metadata is taken from iceberg specification that is pulled from cloud before each query.", \ + 0) #define LIST_OF_STORAGE_OBJECT_STORAGE_SETTINGS(M, ALIAS) \ STORAGE_OBJECT_STORAGE_RELATED_SETTINGS(M, ALIAS) \ diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSettings.h b/src/Storages/ObjectStorage/StorageObjectStorageSettings.h index e58a7e3c184..7baf5c4f29d 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSettings.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageSettings.h @@ -54,16 +54,8 @@ struct StorageObjectStorageSettings STORAGE_OBJECT_STORAGE_SETTINGS_SUPPORTED_TYPES(StorageObjectStorageSettings, DECLARE_SETTING_SUBSCRIPT_OPERATOR) - void dumpToSystemEngineSettingsColumns( - MutableColumnsAndConstraints & params, - const std::string & table_name, - const std::string & database_name, - const StorageObjectStorage & storage) const; - void loadFromQuery(ASTStorage & storage_def); - void applyChanges(const SettingsChanges & changes); - Field get(const std::string & name); private: From df3e4cbfe6883b6e1d8f0ddb2a773090fd12f189 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 4 Dec 2024 12:23:52 +0100 Subject: [PATCH 324/502] Output logs for integration tests, add TeePopen.terminate --- tests/ci/integration_tests_runner.py | 17 ++++++++--------- tests/ci/tee_popen.py | 5 ++++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py index 35f7309fbe2..612be7f8f36 100755 --- a/tests/ci/integration_tests_runner.py +++ b/tests/ci/integration_tests_runner.py @@ -655,13 +655,12 @@ class ClickhouseIntegrationTestsRunner: log_basename = test_group_str + "_" + str(i) + ".log" log_path = os.path.join(self.repo_path, "tests/integration", log_basename) - with open(log_path, "w", encoding="utf-8") as log: - logging.info("Executing cmd: %s", cmd) - # ignore retcode, since it meaningful due to pipe to tee - with subprocess.Popen(cmd, shell=True, stderr=log, stdout=log) as proc: - global runner_subprocess # pylint:disable=global-statement - runner_subprocess = proc - proc.wait() + logging.info("Executing cmd: %s", cmd) + # ignore retcode, since it meaningful due to pipe to tee + with TeePopen(cmd, log_path) as proc: + global runner_subprocess # pylint:disable=global-statement + runner_subprocess = proc + proc.wait() extra_logs_names = [log_basename] log_result_path = os.path.join( @@ -1089,7 +1088,7 @@ def run(): timeout_expired = False -runner_subprocess = None # type:Optional[subprocess.Popen] +runner_subprocess = None # type:Optional[TeePopen] def handle_sigterm(signum, _frame): @@ -1098,7 +1097,7 @@ def handle_sigterm(signum, _frame): global timeout_expired # pylint:disable=global-statement timeout_expired = True if runner_subprocess: - runner_subprocess.send_signal(signal.SIGTERM) + runner_subprocess.terminate() if __name__ == "__main__": diff --git a/tests/ci/tee_popen.py b/tests/ci/tee_popen.py index e5802d006f8..26d1d4b9c00 100644 --- a/tests/ci/tee_popen.py +++ b/tests/ci/tee_popen.py @@ -44,9 +44,12 @@ class TeePopen: self.timeout, ) self.send_signal(signal.SIGTERM) + self.timeout_exceeded = True + self.terminate() + + def terminate(self) -> None: time_wait = 0 self.terminated_by_sigterm = True - self.timeout_exceeded = True while self.process.poll() is None and time_wait < 100: print("wait...") wait = 5 From 62f2eda0f13331935732ff1c5bb69da0a9eb14b0 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 4 Dec 2024 13:08:58 +0100 Subject: [PATCH 325/502] Fix TeePopen.terminate by moving send_signal there --- tests/ci/tee_popen.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/ci/tee_popen.py b/tests/ci/tee_popen.py index 26d1d4b9c00..e9e00704f9a 100644 --- a/tests/ci/tee_popen.py +++ b/tests/ci/tee_popen.py @@ -43,25 +43,24 @@ class TeePopen: self.process.pid, self.timeout, ) - self.send_signal(signal.SIGTERM) self.timeout_exceeded = True self.terminate() - def terminate(self) -> None: + def terminate(self, wait_before_kill: int = 100) -> None: time_wait = 0 + time_sleep = 5 self.terminated_by_sigterm = True - while self.process.poll() is None and time_wait < 100: - print("wait...") - wait = 5 - sleep(wait) - time_wait += wait + self.send_signal(signal.SIGTERM) + while self.process.poll() is None and time_wait < wait_before_kill: + logging.warning("Wait the process %s to terminate", self.process.pid) + sleep(time_sleep) + time_wait += time_sleep + + self.terminated_by_sigkill = True while self.process.poll() is None: - logging.error( - "Process is still running. Send SIGKILL", - ) + logging.error("Process is still running. Send SIGKILL") self.send_signal(signal.SIGKILL) - self.terminated_by_sigkill = True - sleep(5) + sleep(time_sleep) def __enter__(self) -> "TeePopen": self.process = Popen( From 7dad309f47162a628d2e19893bced79336fd6f5a Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 4 Dec 2024 12:36:18 +0000 Subject: [PATCH 326/502] Reduce changes in Iceberg Metadata File --- .../DataLakes/IcebergMetadata.cpp | 191 +++++++++--------- 1 file changed, 96 insertions(+), 95 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 1b3ea2c76ef..980d2f479cb 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -45,104 +45,15 @@ extern const SettingsBool allow_data_lake_dynamic_schema; namespace ErrorCodes { -extern const int FILE_DOESNT_EXIST; -extern const int ILLEGAL_COLUMN; -extern const int BAD_ARGUMENTS; -extern const int UNSUPPORTED_METHOD; -extern const int LOGICAL_ERROR; -} - -std::pair parseTableSchemaV2Method(const Poco::JSON::Object::Ptr & metadata_object) -{ - Poco::JSON::Object::Ptr schema; - if (!metadata_object->has("current-schema-id")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'current-schema-id' field is missing in metadata"); - auto current_schema_id = metadata_object->getValue("current-schema-id"); - if (!metadata_object->has("schemas")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schemas' field is missing in metadata"); - auto schemas = metadata_object->get("schemas").extract(); - if (schemas->size() == 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: schemas field is empty"); - for (uint32_t i = 0; i != schemas->size(); ++i) - { - auto current_schema = schemas->getObject(i); - if (!current_schema->has("schema-id")) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schema-id' field is missing in schema"); - } - if (current_schema->getValue("schema-id") == current_schema_id) - { - schema = current_schema; - break; - } - } - - if (!schema) - throw Exception(ErrorCodes::BAD_ARGUMENTS, R"(There is no schema with "schema-id" that matches "current-schema-id" in metadata)"); - if (schema->getValue("schema-id") != current_schema_id) - throw Exception(ErrorCodes::BAD_ARGUMENTS, R"(Field "schema-id" of the schema doesn't match "current-schema-id" in metadata)"); - return {schema, current_schema_id}; -} - -std::pair parseTableSchemaV1Method(const Poco::JSON::Object::Ptr & metadata_object) -{ - if (!metadata_object->has("schema")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schema' field is missing in metadata"); - Poco::JSON::Object::Ptr schema = metadata_object->getObject("schema"); - if (!metadata_object->has("schema")) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schema-id' field is missing in schema"); - auto current_schema_id = schema->getValue("schema-id"); - return {schema, current_schema_id}; + extern const int FILE_DOESNT_EXIST; + extern const int ILLEGAL_COLUMN; + extern const int BAD_ARGUMENTS; + extern const int UNSUPPORTED_METHOD; + extern const int LOGICAL_ERROR; } Int32 parseTableSchema( - const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor, LoggerPtr metadata_logger) -{ - Int32 format_version = metadata_object->getValue("format-version"); - if (format_version == 2) - { - auto [schema, current_schema_id] = parseTableSchemaV2Method(metadata_object); - schema_processor.addIcebergTableSchema(schema); - return current_schema_id; - } - else - { - try - { - auto [schema, current_schema_id] = parseTableSchemaV1Method(metadata_object); - schema_processor.addIcebergTableSchema(schema); - return current_schema_id; - } - catch (const Exception & first_error) - { - if (first_error.code() != ErrorCodes::BAD_ARGUMENTS) - throw; - try - { - auto [schema, current_schema_id] = parseTableSchemaV2Method(metadata_object); - schema_processor.addIcebergTableSchema(schema); - LOG_WARNING( - metadata_logger, - "Iceberg table schema was parsed using v2 specification, but it was impossible to parse it using v1 " - "specification. Be " - "aware that you Iceberg writing engine violates Iceberg specification. Error during parsing {}", - first_error.displayText()); - return current_schema_id; - } - catch (const Exception & second_error) - { - if (first_error.code() != ErrorCodes::BAD_ARGUMENTS) - throw; - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Cannot parse Iceberg table schema both with v1 and v2 methods. Old method error: {}. New method error: {}", - first_error.displayText(), - second_error.displayText()); - } - } - } -} - + const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor, LoggerPtr metadata_logger); IcebergMetadata::IcebergMetadata( ObjectStoragePtr object_storage_, @@ -340,6 +251,96 @@ bool IcebergSchemaProcessor::allowPrimitiveTypeConversion(const String & old_typ } return allowed_type_conversion; } +std::pair parseTableSchemaV2Method(const Poco::JSON::Object::Ptr & metadata_object) +{ + Poco::JSON::Object::Ptr schema; + if (!metadata_object->has("current-schema-id")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'current-schema-id' field is missing in metadata"); + auto current_schema_id = metadata_object->getValue("current-schema-id"); + if (!metadata_object->has("schemas")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schemas' field is missing in metadata"); + auto schemas = metadata_object->get("schemas").extract(); + if (schemas->size() == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: schemas field is empty"); + for (uint32_t i = 0; i != schemas->size(); ++i) + { + auto current_schema = schemas->getObject(i); + if (!current_schema->has("schema-id")) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schema-id' field is missing in schema"); + } + if (current_schema->getValue("schema-id") == current_schema_id) + { + schema = current_schema; + break; + } + } + + if (!schema) + throw Exception(ErrorCodes::BAD_ARGUMENTS, R"(There is no schema with "schema-id" that matches "current-schema-id" in metadata)"); + if (schema->getValue("schema-id") != current_schema_id) + throw Exception(ErrorCodes::BAD_ARGUMENTS, R"(Field "schema-id" of the schema doesn't match "current-schema-id" in metadata)"); + return {schema, current_schema_id}; +} + +std::pair parseTableSchemaV1Method(const Poco::JSON::Object::Ptr & metadata_object) +{ + if (!metadata_object->has("schema")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schema' field is missing in metadata"); + Poco::JSON::Object::Ptr schema = metadata_object->getObject("schema"); + if (!metadata_object->has("schema")) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse Iceberg table schema: 'schema-id' field is missing in schema"); + auto current_schema_id = schema->getValue("schema-id"); + return {schema, current_schema_id}; +} + +Int32 parseTableSchema( + const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor, LoggerPtr metadata_logger) +{ + Int32 format_version = metadata_object->getValue("format-version"); + if (format_version == 2) + { + auto [schema, current_schema_id] = parseTableSchemaV2Method(metadata_object); + schema_processor.addIcebergTableSchema(schema); + return current_schema_id; + } + else + { + try + { + auto [schema, current_schema_id] = parseTableSchemaV1Method(metadata_object); + schema_processor.addIcebergTableSchema(schema); + return current_schema_id; + } + catch (const Exception & first_error) + { + if (first_error.code() != ErrorCodes::BAD_ARGUMENTS) + throw; + try + { + auto [schema, current_schema_id] = parseTableSchemaV2Method(metadata_object); + schema_processor.addIcebergTableSchema(schema); + LOG_WARNING( + metadata_logger, + "Iceberg table schema was parsed using v2 specification, but it was impossible to parse it using v1 " + "specification. Be " + "aware that you Iceberg writing engine violates Iceberg specification. Error during parsing {}", + first_error.displayText()); + return current_schema_id; + } + catch (const Exception & second_error) + { + if (first_error.code() != ErrorCodes::BAD_ARGUMENTS) + throw; + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot parse Iceberg table schema both with v1 and v2 methods. Old method error: {}. New method error: {}", + first_error.displayText(), + second_error.displayText()); + } + } + } +} // Ids are passed only for error logging purposes From 6ce9d45392d08be1e7fc7846e3a9afc77f2d4924 Mon Sep 17 00:00:00 2001 From: Michael Stetsyuk Date: Wed, 4 Dec 2024 13:06:41 +0000 Subject: [PATCH 327/502] better --- programs/server/Server.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index a21658b9795..cabac3017c4 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -366,11 +366,11 @@ namespace ErrorCodes } -namespace StartupScriptsExecutionState +enum StartupScriptsExecutionState : CurrentMetrics::Value { - [[maybe_unused]] const CurrentMetrics::Value NotFinished = 0; - const CurrentMetrics::Value Success = 1; - const CurrentMetrics::Value Failure = 2; + NotFinished = 0, + Success = 1, + Failure = 2, }; From 56f040734d08a5a00d5de361b8f12400d834751c Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Wed, 4 Dec 2024 10:15:53 -0300 Subject: [PATCH 328/502] fix docs --- docs/en/sql-reference/functions/array-functions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index 02506e0c56e..b3a589c76d9 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -2187,9 +2187,9 @@ select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); Result: ``` text -┌─arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1])─┐ -│ 0.8333333333333333 │ -└───────────────────────────────────────────────┘ +┌─arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1])─┐ +│ 0.8333333333333333 │ +└─────────────────────────────────────────────────┘ ``` ## arrayMap(func, arr1, ...) From 8ecdfb9bc2d606cadd8fe1200ed8903d90ada4d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 13:23:07 +0100 Subject: [PATCH 329/502] Minor include --- src/AggregateFunctions/AggregateFunctionFactory.cpp | 8 +++++++- src/AggregateFunctions/AggregateFunctionFactory.h | 8 +++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index ac969f2ebfd..a28fc36cdd6 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -1,12 +1,13 @@ #include #include +#include #include #include #include #include #include +#include #include -#include static constexpr size_t MAX_AGGREGATE_FUNCTION_NAME_LENGTH = 1000; @@ -349,4 +350,9 @@ AggregateFunctionFactory & AggregateFunctionFactory::instance() return ret; } + +bool AggregateUtils::isAggregateFunction(const ASTFunction & node) +{ + return AggregateFunctionFactory::instance().isAggregateFunctionName(node.name); +} } diff --git a/src/AggregateFunctions/AggregateFunctionFactory.h b/src/AggregateFunctions/AggregateFunctionFactory.h index a5fa3424543..9a7e4240dd4 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.h +++ b/src/AggregateFunctions/AggregateFunctionFactory.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include @@ -23,6 +22,8 @@ class IDataType; using DataTypePtr = std::shared_ptr; using DataTypes = std::vector; +class ASTFunction; + /** * The invoker has arguments: name of aggregate function, types of arguments, values of parameters. * Parameters are for "parametric" aggregate functions. @@ -114,10 +115,7 @@ private: struct AggregateUtils { - static bool isAggregateFunction(const ASTFunction & node) - { - return AggregateFunctionFactory::instance().isAggregateFunctionName(node.name); - } + static bool isAggregateFunction(const ASTFunction & node); }; const String & getAggregateFunctionCanonicalNameIfAny(const String & name); From 4f22469537771074e97b77fa790d26a8e74fbb8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 13:45:49 +0100 Subject: [PATCH 330/502] Remove heavy header --- src/DataTypes/Serializations/SerializationInfo.cpp | 7 +++---- src/DataTypes/Serializations/SerializationInfo.h | 8 +++++--- .../Serializations/SerializationInfoTuple.cpp | 14 +++++++++----- .../Serializations/SerializationInfoTuple.h | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/DataTypes/Serializations/SerializationInfo.cpp b/src/DataTypes/Serializations/SerializationInfo.cpp index 172a71204f9..16d14a483c2 100644 --- a/src/DataTypes/Serializations/SerializationInfo.cpp +++ b/src/DataTypes/Serializations/SerializationInfo.cpp @@ -162,13 +162,11 @@ void SerializationInfo::deserializeFromKindsBinary(ReadBuffer & in) kind = *maybe_kind; } -Poco::JSON::Object SerializationInfo::toJSON() const +void SerializationInfo::toJSON(Poco::JSON::Object & object) const { - Poco::JSON::Object object; object.set(KEY_KIND, ISerialization::kindToString(kind)); object.set(KEY_NUM_DEFAULTS, data.num_defaults); object.set(KEY_NUM_ROWS, data.num_rows); - return object; } void SerializationInfo::fromJSON(const Poco::JSON::Object & object) @@ -276,7 +274,8 @@ void SerializationInfoByName::writeJSON(WriteBuffer & out) const Poco::JSON::Array column_infos; for (const auto & [name, info] : *this) { - auto info_json = info->toJSON(); + Poco::JSON::Object info_json; + info->toJSON(info_json); info_json.set(KEY_NAME, name); column_infos.add(std::move(info_json)); /// NOLINT } diff --git a/src/DataTypes/Serializations/SerializationInfo.h b/src/DataTypes/Serializations/SerializationInfo.h index c30e50ab12c..d1b5e595e6b 100644 --- a/src/DataTypes/Serializations/SerializationInfo.h +++ b/src/DataTypes/Serializations/SerializationInfo.h @@ -4,8 +4,10 @@ #include #include -#include - +namespace Poco::JSON +{ +class Object; +} namespace DB { @@ -67,7 +69,7 @@ public: virtual void serialializeKindBinary(WriteBuffer & out) const; virtual void deserializeFromKindsBinary(ReadBuffer & in); - virtual Poco::JSON::Object toJSON() const; + virtual void toJSON(Poco::JSON::Object & object) const; virtual void fromJSON(const Poco::JSON::Object & object); void setKind(ISerialization::Kind kind_) { kind = kind_; } diff --git a/src/DataTypes/Serializations/SerializationInfoTuple.cpp b/src/DataTypes/Serializations/SerializationInfoTuple.cpp index b7449be3cc5..7f36ecd64de 100644 --- a/src/DataTypes/Serializations/SerializationInfoTuple.cpp +++ b/src/DataTypes/Serializations/SerializationInfoTuple.cpp @@ -3,6 +3,8 @@ #include #include +#include + namespace DB { @@ -151,15 +153,17 @@ void SerializationInfoTuple::deserializeFromKindsBinary(ReadBuffer & in) elem->deserializeFromKindsBinary(in); } -Poco::JSON::Object SerializationInfoTuple::toJSON() const +void SerializationInfoTuple::toJSON(Poco::JSON::Object & object) const { - auto object = SerializationInfo::toJSON(); + SerializationInfo::toJSON(object); Poco::JSON::Array subcolumns; for (const auto & elem : elems) - subcolumns.add(elem->toJSON()); - + { + Poco::JSON::Object sub_column_json; + elem->toJSON(sub_column_json); + subcolumns.add(sub_column_json); + } object.set("subcolumns", subcolumns); - return object; } void SerializationInfoTuple::fromJSON(const Poco::JSON::Object & object) diff --git a/src/DataTypes/Serializations/SerializationInfoTuple.h b/src/DataTypes/Serializations/SerializationInfoTuple.h index a6b9c89166f..c0050df9b6e 100644 --- a/src/DataTypes/Serializations/SerializationInfoTuple.h +++ b/src/DataTypes/Serializations/SerializationInfoTuple.h @@ -29,7 +29,7 @@ public: void serialializeKindBinary(WriteBuffer & out) const override; void deserializeFromKindsBinary(ReadBuffer & in) override; - Poco::JSON::Object toJSON() const override; + void toJSON(Poco::JSON::Object & object) const override; void fromJSON(const Poco::JSON::Object & object) override; const MutableSerializationInfoPtr & getElementInfo(size_t i) const { return elems[i]; } From 8c23d9ae585241237a8990d448ada4a8c813fd86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 14:06:24 +0100 Subject: [PATCH 331/502] More unnecessary headers --- src/Common/LocalDateTime.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Common/LocalDateTime.h b/src/Common/LocalDateTime.h index 74c51a74a16..21f75b43061 100644 --- a/src/Common/LocalDateTime.h +++ b/src/Common/LocalDateTime.h @@ -1,11 +1,9 @@ #pragma once -#include -#include -#include #include #include +#include /** Stores calendar date and time in broken-down form. * Could be initialized from date and time in text form like '2011-01-01 00:00:00' or from time_t. From 4bdefc91ce805a7cf22946c87c0955aaee5edfc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 14:10:07 +0100 Subject: [PATCH 332/502] Remove more trash --- src/IO/ReadHelpers.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 867c985934b..77c72d7bc29 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -1,12 +1,9 @@ #pragma once -#include #include #include #include #include -#include -#include #include #include @@ -20,12 +17,10 @@ #include #include #include -#include #include #include #include -#include #include #include @@ -35,7 +30,6 @@ #include -#include #include #include #include From f96c3e9ca8b1b9a0fe1c0cfa10273bd7b9f1fab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 14:44:56 +0100 Subject: [PATCH 333/502] Remove more unnecessary nested headers --- src/Backups/RestorerFromBackup.cpp | 1 + src/Backups/RestorerFromBackup.h | 2 ++ src/Bridge/IBridge.cpp | 1 + .../NamedCollectionsMetadataStorage.cpp | 1 + src/Common/SignalHandlers.cpp | 2 ++ src/Common/mysqlxx/Value.cpp | 2 ++ src/Common/mysqlxx/mysqlxx/Value.h | 16 ++++------------ src/Compression/CompressionCodecEncrypted.cpp | 12 ++++++------ src/Coordination/KeeperSnapshotManager.cpp | 1 + src/IO/ReadBuffer.h | 2 -- src/Interpreters/TransactionLog.cpp | 16 ++++++++-------- src/Server/PrometheusRequestHandlerFactory.cpp | 1 + .../HDFS/AsynchronousReadBufferFromHDFS.h | 15 ++++++--------- 13 files changed, 35 insertions(+), 37 deletions(-) diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index f907d80a64a..dd362c898e5 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -29,6 +29,7 @@ #include #include +#include #include namespace fs = std::filesystem; diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index aa5288643a0..de070d72e77 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -8,7 +8,9 @@ #include #include #include + #include +#include namespace DB diff --git a/src/Bridge/IBridge.cpp b/src/Bridge/IBridge.cpp index 0f26c753bcb..bf6ef1426c9 100644 --- a/src/Bridge/IBridge.cpp +++ b/src/Bridge/IBridge.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include diff --git a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp index 8bb411f1437..ead463e5887 100644 --- a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp +++ b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Common/SignalHandlers.cpp b/src/Common/SignalHandlers.cpp index aec00fbdc0b..5f1acbe7ee3 100644 --- a/src/Common/SignalHandlers.cpp +++ b/src/Common/SignalHandlers.cpp @@ -16,6 +16,8 @@ #include #include +#include + #pragma clang diagnostic ignored "-Wreserved-identifier" namespace DB diff --git a/src/Common/mysqlxx/Value.cpp b/src/Common/mysqlxx/Value.cpp index 41d717669b9..85f8ee8156c 100644 --- a/src/Common/mysqlxx/Value.cpp +++ b/src/Common/mysqlxx/Value.cpp @@ -3,6 +3,8 @@ #include #include +#include + namespace mysqlxx { diff --git a/src/Common/mysqlxx/mysqlxx/Value.h b/src/Common/mysqlxx/mysqlxx/Value.h index 6fa2fed03d4..147356d21f7 100644 --- a/src/Common/mysqlxx/mysqlxx/Value.h +++ b/src/Common/mysqlxx/mysqlxx/Value.h @@ -1,19 +1,11 @@ #pragma once -#include -#include -#include -#include - -#include -#include - -#include -#include #include - -#include #include +#include + +#include +#include namespace mysqlxx diff --git a/src/Compression/CompressionCodecEncrypted.cpp b/src/Compression/CompressionCodecEncrypted.cpp index b41abefa852..7e466a3e602 100644 --- a/src/Compression/CompressionCodecEncrypted.cpp +++ b/src/Compression/CompressionCodecEncrypted.cpp @@ -1,14 +1,14 @@ -#include "config.h" #include -#include -#include -#include -#include #include +#include +#include #include -#include +#include +#include +#include #include #include +#include "config.h" #if USE_SSL # include diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp index 1568b616b44..495c6c174ce 100644 --- a/src/Coordination/KeeperSnapshotManager.cpp +++ b/src/Coordination/KeeperSnapshotManager.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/src/IO/ReadBuffer.h b/src/IO/ReadBuffer.h index 1bddd555c6e..3adb5174ca9 100644 --- a/src/IO/ReadBuffer.h +++ b/src/IO/ReadBuffer.h @@ -1,13 +1,11 @@ #pragma once -#include #include #include #include #include #include -#include namespace DB diff --git a/src/Interpreters/TransactionLog.cpp b/src/Interpreters/TransactionLog.cpp index 69ef01d5f04..4146e8c2a58 100644 --- a/src/Interpreters/TransactionLog.cpp +++ b/src/Interpreters/TransactionLog.cpp @@ -1,17 +1,17 @@ -#include -#include -#include -#include +#include +#include #include #include -#include +#include +#include +#include +#include +#include #include #include -#include -#include #include #include - +#include namespace DB { diff --git a/src/Server/PrometheusRequestHandlerFactory.cpp b/src/Server/PrometheusRequestHandlerFactory.cpp index 1dcdc71e46f..98fda55fa88 100644 --- a/src/Server/PrometheusRequestHandlerFactory.cpp +++ b/src/Server/PrometheusRequestHandlerFactory.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.h b/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.h index 9846d74453b..e01586c4c35 100644 --- a/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.h +++ b/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.h @@ -6,15 +6,12 @@ #include #include -#include -#include - -#include -#include -#include -#include -#include -#include +# include +# include +# include +# include +# include +# include namespace DB { From d585d2fe72ddea542121a4a1e951d380d51b936f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 16:34:27 +0100 Subject: [PATCH 334/502] Less functions --- src/AggregateFunctions/AggregateFunctionAvg.h | 22 +++++++++++++++++-- .../AggregateFunctionAvgWeighted.cpp | 16 +++++++++----- src/DataTypes/Native.h | 2 +- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionAvg.h b/src/AggregateFunctions/AggregateFunctionAvg.h index 8d53a081ee0..3635f8e2f40 100644 --- a/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/src/AggregateFunctions/AggregateFunctionAvg.h @@ -141,6 +141,9 @@ public: bool isCompilable() const override { + if constexpr (!canBeNativeType() || !canBeNativeType()) + return false; + bool can_be_compiled = true; for (const auto & argument : this->argument_types) @@ -158,7 +161,8 @@ public: b.CreateMemSet(aggregate_data_ptr, llvm::ConstantInt::get(b.getInt8Ty(), 0), sizeof(Fraction), llvm::assumeAligned(this->alignOfData())); } - void compileMerge(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_dst_ptr, llvm::Value * aggregate_data_src_ptr) const override + void compileMergeImpl(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_dst_ptr, llvm::Value * aggregate_data_src_ptr) const + requires(canBeNativeType() && canBeNativeType()) { llvm::IRBuilder<> & b = static_cast &>(builder); @@ -185,7 +189,15 @@ public: b.CreateStore(denominator_result_value, denominator_dst_ptr); } - llvm::Value * compileGetResult(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr) const override + void + compileMerge(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_dst_ptr, llvm::Value * aggregate_data_src_ptr) const override + { + if constexpr (canBeNativeType() && canBeNativeType()) + compileMergeImpl(builder, aggregate_data_dst_ptr, aggregate_data_src_ptr); + } + + llvm::Value * compileGetResultImpl(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr) const + requires(canBeNativeType() && canBeNativeType()) { llvm::IRBuilder<> & b = static_cast &>(builder); @@ -204,6 +216,12 @@ public: return b.CreateFDiv(double_numerator, double_denominator); } + llvm::Value * compileGetResult(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr) const override + { + if constexpr (canBeNativeType() && canBeNativeType()) + return compileGetResultImpl(builder, aggregate_data_ptr); + } + #endif private: diff --git a/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp b/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp index d33e843fac9..bcd295d72d8 100644 --- a/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp +++ b/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp @@ -59,13 +59,13 @@ public: bool isCompilable() const override { - bool can_be_compiled = Base::isCompilable(); - can_be_compiled &= canBeNativeType(); - - return can_be_compiled; + if constexpr (!canBeNativeType() || !canBeNativeType() || !canBeNativeType()) + return false; + return Base::isCompilable(); } - void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const ValuesWithType & arguments) const override + void compileAddImpl(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const ValuesWithType & arguments) const + requires(canBeNativeType() && canBeNativeType() && canBeNativeType()) { llvm::IRBuilder<> & b = static_cast &>(builder); @@ -94,6 +94,12 @@ public: b.CreateStore(denominator_value_updated, denominator_ptr); } + void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const ValuesWithType & arguments) const override + { + if constexpr (canBeNativeType() && canBeNativeType() && canBeNativeType()) + compileAddImpl(builder, aggregate_data_ptr, arguments); + } + #endif }; diff --git a/src/DataTypes/Native.h b/src/DataTypes/Native.h index 7fee452b1f0..3edce1bb9cf 100644 --- a/src/DataTypes/Native.h +++ b/src/DataTypes/Native.h @@ -30,7 +30,7 @@ bool canBeNativeType(const IDataType & type); bool canBeNativeType(const DataTypePtr & type); template -static inline bool canBeNativeType() +static constexpr bool canBeNativeType() { if constexpr (std::is_same_v || std::is_same_v) return true; From e41a018a5fef5cbbf82a03270a26b4f1d0ee2f81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 17:11:29 +0100 Subject: [PATCH 335/502] Less nested includes --- src/AggregateFunctions/AggregateFunctionAvg.h | 2 +- src/Formats/EscapingRuleUtils.cpp | 2 +- src/IO/ReadHelpers.cpp | 22 +++---- src/IO/ReadHelpers.h | 5 +- src/IO/VarInt.h | 31 ---------- src/IO/examples/CMakeLists.txt | 3 - src/IO/examples/var_uint.cpp | 58 ------------------- .../Impl/JSONEachRowRowInputFormat.cpp | 2 +- 8 files changed, 16 insertions(+), 109 deletions(-) delete mode 100644 src/IO/examples/var_uint.cpp diff --git a/src/AggregateFunctions/AggregateFunctionAvg.h b/src/AggregateFunctions/AggregateFunctionAvg.h index 3635f8e2f40..79f46e79992 100644 --- a/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/src/AggregateFunctions/AggregateFunctionAvg.h @@ -10,7 +10,6 @@ #include #include #include -#include #include "config.h" @@ -220,6 +219,7 @@ public: { if constexpr (canBeNativeType() && canBeNativeType()) return compileGetResultImpl(builder, aggregate_data_ptr); + return nullptr; } #endif diff --git a/src/Formats/EscapingRuleUtils.cpp b/src/Formats/EscapingRuleUtils.cpp index 687814baaa6..efa9ccb6cb7 100644 --- a/src/Formats/EscapingRuleUtils.cpp +++ b/src/Formats/EscapingRuleUtils.cpp @@ -82,7 +82,7 @@ void skipFieldByEscapingRule(ReadBuffer & buf, FormatSettings::EscapingRule esca readCSVStringInto(out, buf, format_settings.csv); break; case FormatSettings::EscapingRule::JSON: - skipJSONField(buf, StringRef(field_name, field_name_len), format_settings.json); + skipJSONField(buf, std::string_view(field_name, field_name_len), format_settings.json); break; case FormatSettings::EscapingRule::Raw: readStringInto(out, buf); diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index caaab68ba1a..194aa90542e 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -1585,14 +1585,14 @@ template bool readDateTimeTextFallback(time_t &, ReadBuffer &, const template -ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings, size_t current_depth) +ReturnType skipJSONFieldImpl(ReadBuffer & buf, std::string_view name_of_field, const FormatSettings::JSON & settings, size_t current_depth) { static constexpr bool throw_exception = std::is_same_v; if (unlikely(current_depth > settings.max_depth)) { if constexpr (throw_exception) - throw Exception(ErrorCodes::TOO_DEEP_RECURSION, "JSON is too deep for key '{}'", name_of_field.toString()); + throw Exception(ErrorCodes::TOO_DEEP_RECURSION, "JSON is too deep for key '{}'", name_of_field); return ReturnType(false); } @@ -1602,7 +1602,7 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo if (buf.eof()) { if constexpr (throw_exception) - throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected EOF for key '{}'", name_of_field.toString()); + throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected EOF for key '{}'", name_of_field); return ReturnType(false); } if (*buf.position() == '"') /// skip double-quoted string @@ -1622,7 +1622,7 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo if (!tryReadFloatText(v, buf)) { if constexpr (throw_exception) - throw Exception(ErrorCodes::INCORRECT_DATA, "Expected a number field for key '{}'", name_of_field.toString()); + throw Exception(ErrorCodes::INCORRECT_DATA, "Expected a number field for key '{}'", name_of_field); return ReturnType(false); } } @@ -1680,7 +1680,7 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo else { if constexpr (throw_exception) - throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected symbol for key '{}'", name_of_field.toString()); + throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected symbol for key '{}'", name_of_field); return ReturnType(false); } } @@ -1704,7 +1704,7 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo else { if constexpr (throw_exception) - throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected symbol for key '{}'", name_of_field.toString()); + throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected symbol for key '{}'", name_of_field); return ReturnType(false); } @@ -1713,7 +1713,7 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo if (buf.eof() || !(*buf.position() == ':')) { if constexpr (throw_exception) - throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected symbol for key '{}'", name_of_field.toString()); + throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected symbol for key '{}'", name_of_field); return ReturnType(false); } ++buf.position(); @@ -1737,7 +1737,7 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo if (buf.eof()) { if constexpr (throw_exception) - throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected EOF for key '{}'", name_of_field.toString()); + throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected EOF for key '{}'", name_of_field); return ReturnType(false); } ++buf.position(); @@ -1750,7 +1750,7 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo "Cannot read JSON field here: '{}'. Unexpected symbol '{}'{}", String(buf.position(), std::min(buf.available(), size_t(10))), std::string(1, *buf.position()), - name_of_field.empty() ? "" : " for key " + name_of_field.toString()); + name_of_field.empty() ? "" : " for key " + String{name_of_field}); return ReturnType(false); } @@ -1758,12 +1758,12 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo return ReturnType(true); } -void skipJSONField(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings) +void skipJSONField(ReadBuffer & buf, std::string_view name_of_field, const FormatSettings::JSON & settings) { skipJSONFieldImpl(buf, name_of_field, settings, 0); } -bool trySkipJSONField(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings) +bool trySkipJSONField(ReadBuffer & buf, std::string_view name_of_field, const FormatSettings::JSON & settings) { return skipJSONFieldImpl(buf, name_of_field, settings, 0); } diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 77c72d7bc29..0d96fb0f928 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -1709,8 +1708,8 @@ inline void skipWhitespaceIfAny(ReadBuffer & buf, bool one_line = false) } /// Skips json value. -void skipJSONField(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings); -bool trySkipJSONField(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings); +void skipJSONField(ReadBuffer & buf, std::string_view name_of_field, const FormatSettings::JSON & settings); +bool trySkipJSONField(ReadBuffer & buf, std::string_view name_of_field, const FormatSettings::JSON & settings); /** Read serialized exception. diff --git a/src/IO/VarInt.h b/src/IO/VarInt.h index 2c41ae0a9c5..237f9d81e4d 100644 --- a/src/IO/VarInt.h +++ b/src/IO/VarInt.h @@ -5,10 +5,6 @@ #include #include -#include -#include - - namespace DB { @@ -37,20 +33,6 @@ inline void writeVarUInt(UInt64 x, WriteBuffer & ostr) ++ostr.position(); } -inline void writeVarUInt(UInt64 x, std::ostream & ostr) -{ - while (x > 0x7F) - { - uint8_t byte = 0x80 | (x & 0x7F); - ostr.put(byte); - - x >>= 7; - } - - uint8_t final_byte = static_cast(x); - ostr.put(final_byte); -} - inline char * writeVarUInt(UInt64 x, char * ostr) { while (x > 0x7F) @@ -114,19 +96,6 @@ inline void readVarUInt(UInt64 & x, ReadBuffer & istr) varint_impl::readVarUInt(x, istr); } -inline void readVarUInt(UInt64 & x, std::istream & istr) -{ - x = 0; - for (size_t i = 0; i < 10; ++i) - { - UInt64 byte = istr.get(); - x |= (byte & 0x7F) << (7 * i); - - if (!(byte & 0x80)) - return; - } -} - inline const char * readVarUInt(UInt64 & x, const char * istr, size_t size) { const char * end = istr + size; diff --git a/src/IO/examples/CMakeLists.txt b/src/IO/examples/CMakeLists.txt index 8dea3a84cc4..677cc1d397c 100644 --- a/src/IO/examples/CMakeLists.txt +++ b/src/IO/examples/CMakeLists.txt @@ -19,9 +19,6 @@ target_link_libraries (valid_utf8_perf PRIVATE clickhouse_common_io clickhouse_c clickhouse_add_executable (valid_utf8 valid_utf8.cpp) target_link_libraries (valid_utf8 PRIVATE clickhouse_common_io clickhouse_common_config) -clickhouse_add_executable (var_uint var_uint.cpp) -target_link_libraries (var_uint PRIVATE clickhouse_common_io clickhouse_common_config) - clickhouse_add_executable (read_escaped_string read_escaped_string.cpp) target_link_libraries (read_escaped_string PRIVATE clickhouse_common_io clickhouse_common_config) diff --git a/src/IO/examples/var_uint.cpp b/src/IO/examples/var_uint.cpp deleted file mode 100644 index eacc1631ffd..00000000000 --- a/src/IO/examples/var_uint.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include - - -int main(int argc, char ** argv) -{ - if (argc != 2) - { - std::cerr << "Usage: " << std::endl - << argv[0] << " unsigned_number" << std::endl; - return 1; - } - - UInt64 x = DB::parse(argv[1]); - - std::cout << std::hex << std::showbase << "Input: " << x << std::endl; - - Poco::HexBinaryEncoder hex(std::cout); - std::cout << "writeVarUInt(std::ostream): 0x"; - DB::writeVarUInt(x, hex); - std::cout << std::endl; - - std::string s; - - { - DB::WriteBufferFromString wb(s); - DB::writeVarUInt(x, wb); - wb.next(); - } - - std::cout << "writeVarUInt(WriteBuffer): 0x"; - hex << s; - std::cout << std::endl; - - s.clear(); - s.resize(9); - - s.resize(DB::writeVarUInt(x, s.data()) - s.data()); - - std::cout << "writeVarUInt(char *): 0x"; - hex << s; - std::cout << std::endl; - - UInt64 y = 0; - - DB::ReadBufferFromString rb(s); - DB::readVarUInt(y, rb); - - std::cerr << "Input: " << x << ", readVarUInt(writeVarUInt()): " << y << std::endl; - - return 0; -} diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index 77e75efd06f..27412a55eb8 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -121,7 +121,7 @@ void JSONEachRowRowInputFormat::skipUnknownField(StringRef name_ref) if (!format_settings.skip_unknown_fields) throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown field found while parsing JSONEachRow format: {}", name_ref.toString()); - skipJSONField(*in, name_ref, format_settings.json); + skipJSONField(*in, std::string_view(name_ref.data, name_ref.size), format_settings.json); } void JSONEachRowRowInputFormat::readField(size_t index, MutableColumns & columns) From e2b8186d4b2fe9fea0873c4b073d0dc968ec9cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 2 Dec 2024 17:50:57 +0100 Subject: [PATCH 336/502] Less instantiations --- src/AggregateFunctions/AggregateFunctionAvg.h | 9 ++++++++- .../AggregateFunctionAvgWeighted.cpp | 16 +++++++++++++++- src/DataTypes/Native.h | 8 +++----- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionAvg.h b/src/AggregateFunctions/AggregateFunctionAvg.h index 79f46e79992..d16824b71dc 100644 --- a/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/src/AggregateFunctions/AggregateFunctionAvg.h @@ -326,7 +326,8 @@ public: #if USE_EMBEDDED_COMPILER - void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const ValuesWithType & arguments) const override + void compileAddImpl(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const ValuesWithType & arguments) const + requires(canBeNativeType() && canBeNativeType()) { llvm::IRBuilder<> & b = static_cast &>(builder); @@ -345,6 +346,12 @@ public: b.CreateStore(denominator_value_updated, denominator_ptr); } + void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const ValuesWithType & arguments) const override + { + if constexpr(canBeNativeType() && canBeNativeType()) + compileAddImpl(builder, aggregate_data_ptr, arguments); + } + #endif private: diff --git a/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp b/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp index bcd295d72d8..7a01e34c919 100644 --- a/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp +++ b/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp @@ -94,6 +94,20 @@ public: b.CreateStore(denominator_value_updated, denominator_ptr); } + void + compileMerge(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_dst_ptr, llvm::Value * aggregate_data_src_ptr) const override + { + if constexpr (canBeNativeType() && canBeNativeType() && canBeNativeType()) + Base::compileMergeImpl(builder, aggregate_data_dst_ptr, aggregate_data_src_ptr); + } + + llvm::Value * compileGetResult(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr) const override + { + if constexpr (canBeNativeType() && canBeNativeType() && canBeNativeType()) + return Base::compileGetResultImpl(builder, aggregate_data_ptr); + return nullptr; + } + void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const ValuesWithType & arguments) const override { if constexpr (canBeNativeType() && canBeNativeType() && canBeNativeType()) @@ -110,7 +124,7 @@ bool allowTypes(const DataTypePtr& left, const DataTypePtr& right) noexcept constexpr auto allow = [](WhichDataType t) { - return t.isInt() || t.isUInt() || t.isFloat(); + return t.isInt() || t.isUInt() || t.isNativeFloat(); }; return allow(l_dt) && allow(r_dt); diff --git a/src/DataTypes/Native.h b/src/DataTypes/Native.h index 3edce1bb9cf..1900906204f 100644 --- a/src/DataTypes/Native.h +++ b/src/DataTypes/Native.h @@ -74,14 +74,12 @@ static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder) template static inline DataTypePtr toNativeDataType() { - if constexpr (std::is_same_v || std::is_same_v || + static_assert(std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || std::is_same_v) - return std::make_shared>(); - - throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid cast to native data type"); + std::is_same_v || std::is_same_v); + return std::make_shared>(); } /// Cast LLVM value with type to bool From 035b8a3b20b809cba48c681ae452e560d181d73a Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 4 Dec 2024 15:07:54 +0100 Subject: [PATCH 337/502] impl --- .../02812_from_to_utc_timestamp.sh | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh b/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh index 9eb4484ace0..cdebcc5de2c 100755 --- a/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh +++ b/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh @@ -6,6 +6,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh + +client_opts=( + --session_timezone Europe/Moscow +) + $CLICKHOUSE_CLIENT -q "drop table if exists test_tbl" $CLICKHOUSE_CLIENT -q "create table test_tbl (x UInt32, y DateTime, z DateTime64) engine=MergeTree order by x" ${CLICKHOUSE_CLIENT} -q "INSERT INTO test_tbl values(1, '2023-03-16', '2023-03-16 11:22:33')" @@ -13,15 +18,15 @@ ${CLICKHOUSE_CLIENT} -q "INSERT INTO test_tbl values(2, '2023-03-16 11:22:33', ' ${CLICKHOUSE_CLIENT} -q "INSERT INTO test_tbl values(3, '2023-03-16 11:22:33', '2023-03-16 11:22:33.123456')" $CLICKHOUSE_CLIENT -q "select x, to_utc_timestamp(toDateTime('2023-03-16 11:22:33'), 'Etc/GMT+1'), from_utc_timestamp(toDateTime64('2023-03-16 11:22:33', 3), 'Etc/GMT+1'), to_utc_timestamp(y, 'Asia/Shanghai'), from_utc_timestamp(z, 'Asia/Shanghai') from test_tbl order by x" # timestamp convert between DST timezone and UTC -$CLICKHOUSE_CLIENT -q "select to_utc_timestamp(toDateTime('2024-02-24 11:22:33'), 'Europe/Madrid'), from_utc_timestamp(toDateTime('2024-02-24 11:22:33'), 'Europe/Madrid')" -$CLICKHOUSE_CLIENT -q "select to_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'Europe/Madrid'), from_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'Europe/Madrid')" -$CLICKHOUSE_CLIENT -q "select to_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'EST')" +$CLICKHOUSE_CLIENT "${client_opts[@]}" -q "select to_utc_timestamp(toDateTime('2024-02-24 11:22:33'), 'Europe/Madrid'), from_utc_timestamp(toDateTime('2024-02-24 11:22:33'), 'Europe/Madrid')" +$CLICKHOUSE_CLIENT "${client_opts[@]}" -q "select to_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'Europe/Madrid'), from_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'Europe/Madrid')" +$CLICKHOUSE_CLIENT "${client_opts[@]}" -q "select to_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'EST')" -$CLICKHOUSE_CLIENT -q "select 'leap year:', to_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'EST')" -$CLICKHOUSE_CLIENT -q "select 'non-leap year:', to_utc_timestamp(toDateTime('2023-02-29 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2023-02-29 11:22:33'), 'EST')" -$CLICKHOUSE_CLIENT -q "select 'leap year:', to_utc_timestamp(toDateTime('2024-02-28 23:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-03-01 00:22:33'), 'EST')" -$CLICKHOUSE_CLIENT -q "select 'non-leap year:', to_utc_timestamp(toDateTime('2023-02-28 23:22:33'), 'EST'), from_utc_timestamp(toDateTime('2023-03-01 00:22:33'), 'EST')" -$CLICKHOUSE_CLIENT -q "select 'timezone with half-hour offset:', to_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'Australia/Adelaide'), from_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'Australia/Adelaide')" -$CLICKHOUSE_CLIENT -q "select 'jump over a year:', to_utc_timestamp(toDateTime('2023-12-31 23:01:01'), 'EST'), from_utc_timestamp(toDateTime('2024-01-01 01:01:01'), 'EST')" +$CLICKHOUSE_CLIENT "${client_opts[@]}" -q "select 'leap year:', to_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'EST')" +$CLICKHOUSE_CLIENT "${client_opts[@]}" -q "select 'non-leap year:', to_utc_timestamp(toDateTime('2023-02-29 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2023-02-29 11:22:33'), 'EST')" +$CLICKHOUSE_CLIENT "${client_opts[@]}" -q "select 'leap year:', to_utc_timestamp(toDateTime('2024-02-28 23:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-03-01 00:22:33'), 'EST')" +$CLICKHOUSE_CLIENT "${client_opts[@]}" -q "select 'non-leap year:', to_utc_timestamp(toDateTime('2023-02-28 23:22:33'), 'EST'), from_utc_timestamp(toDateTime('2023-03-01 00:22:33'), 'EST')" +$CLICKHOUSE_CLIENT "${client_opts[@]}" -q "select 'timezone with half-hour offset:', to_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'Australia/Adelaide'), from_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'Australia/Adelaide')" +$CLICKHOUSE_CLIENT "${client_opts[@]}" -q "select 'jump over a year:', to_utc_timestamp(toDateTime('2023-12-31 23:01:01'), 'EST'), from_utc_timestamp(toDateTime('2024-01-01 01:01:01'), 'EST')" $CLICKHOUSE_CLIENT -q "drop table test_tbl" From 19bc382978bc0f865a2c5779fbc4bbd4cdc2bc18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 3 Dec 2024 16:47:54 +0100 Subject: [PATCH 338/502] Remove some nested deps on SipHash --- .../Combinators/AggregateFunctionDistinct.h | 1 - src/Analyzer/Passes/CNF.h | 4 ---- src/Columns/ColumnVector.h | 3 ++- src/Compression/CompressionCodecDoubleDelta.cpp | 1 + src/Compression/CompressionCodecGorilla.cpp | 1 + src/Compression/CompressionCodecT64.cpp | 1 + src/Compression/ICompressionCodec.cpp | 1 + src/Compression/ICompressionCodec.h | 2 +- src/Coordination/RocksDBContainer.h | 1 - src/Parsers/ASTFunctionWithKeyValueArguments.h | 2 ++ src/Parsers/ASTInsertQuery.h | 2 ++ 11 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h b/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h index 658d53abb2f..820917c1b4d 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h @@ -8,7 +8,6 @@ #include #include #include -#include #include diff --git a/src/Analyzer/Passes/CNF.h b/src/Analyzer/Passes/CNF.h index 9325d97d2f2..470ead2ab66 100644 --- a/src/Analyzer/Passes/CNF.h +++ b/src/Analyzer/Passes/CNF.h @@ -3,12 +3,8 @@ #include #include -#include - #include -#include - namespace DB::Analyzer { diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 22b064ae053..e5ece863a3b 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -13,6 +13,8 @@ #include "config.h" +class SipHash; + namespace DB { @@ -21,7 +23,6 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } - /** A template for columns that use a simple array to store. */ template diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index 722e23f51e6..8a634203686 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -1,5 +1,6 @@ #pragma clang diagnostic ignored "-Wreserved-identifier" +#include #include #include #include diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index 8c7cd37f17c..2f27f0b5d66 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -1,5 +1,6 @@ #pragma clang diagnostic ignored "-Wreserved-identifier" +#include #include #include #include diff --git a/src/Compression/CompressionCodecT64.cpp b/src/Compression/CompressionCodecT64.cpp index 634142503f5..c6af1055bc8 100644 --- a/src/Compression/CompressionCodecT64.cpp +++ b/src/Compression/CompressionCodecT64.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Compression/ICompressionCodec.cpp b/src/Compression/ICompressionCodec.cpp index a31d0485982..ed768440aa3 100644 --- a/src/Compression/ICompressionCodec.cpp +++ b/src/Compression/ICompressionCodec.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Compression/ICompressionCodec.h b/src/Compression/ICompressionCodec.h index 549817cb0b9..b410708b3ed 100644 --- a/src/Compression/ICompressionCodec.h +++ b/src/Compression/ICompressionCodec.h @@ -5,8 +5,8 @@ #include #include #include -#include +class SipHash; namespace DB { diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h index 7b079d0adc0..df7643430ed 100644 --- a/src/Coordination/RocksDBContainer.h +++ b/src/Coordination/RocksDBContainer.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include #include #include #include diff --git a/src/Parsers/ASTFunctionWithKeyValueArguments.h b/src/Parsers/ASTFunctionWithKeyValueArguments.h index 8e2d790c36a..bd389ca8ee6 100644 --- a/src/Parsers/ASTFunctionWithKeyValueArguments.h +++ b/src/Parsers/ASTFunctionWithKeyValueArguments.h @@ -3,6 +3,8 @@ #include #include +class SipHash; + namespace DB { diff --git a/src/Parsers/ASTInsertQuery.h b/src/Parsers/ASTInsertQuery.h index 8b3a4a0bde0..0734329c5f7 100644 --- a/src/Parsers/ASTInsertQuery.h +++ b/src/Parsers/ASTInsertQuery.h @@ -3,6 +3,8 @@ #include #include +class SipHash; + namespace DB { From fd4795c00f6905457889b515ae78250d2fce18ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 3 Dec 2024 17:34:20 +0100 Subject: [PATCH 339/502] Remove some nested deps on SipHash --- src/Columns/ColumnObject.cpp | 1 + src/Columns/ColumnString.cpp | 16 +++++++ src/Columns/ColumnString.h | 18 ++------ src/Columns/ColumnUnique.cpp | 43 +++++++++++++++++++ src/Columns/ColumnUnique.h | 39 ++++------------- .../SerializationLowCardinality.cpp | 1 + src/Functions/transform.cpp | 1 + src/Processors/Merges/Algorithms/Graphite.cpp | 16 +++++++ src/Processors/Merges/Algorithms/Graphite.h | 20 ++------- src/Storages/ColumnDependency.cpp | 15 +++++++ src/Storages/ColumnDependency.h | 9 +--- src/Storages/Kafka/KafkaConsumer2.cpp | 10 +++++ src/Storages/Kafka/KafkaConsumer2.h | 9 +--- src/Storages/MarkCache.cpp | 10 +++++ src/Storages/MarkCache.h | 8 +--- src/Storages/MergeTree/PrimaryIndexCache.cpp | 9 ++++ src/Storages/MergeTree/PrimaryIndexCache.h | 8 +--- 17 files changed, 141 insertions(+), 92 deletions(-) create mode 100644 src/Storages/ColumnDependency.cpp diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 17a90cc9b50..f110e8b442b 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 0ed4f5f432d..4bdc253bfc4 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -695,4 +696,19 @@ void ColumnString::validate() const last_offset, chars.size()); } +void ColumnString::updateHashWithValue(size_t n, SipHash & hash) const +{ + size_t string_size = sizeAt(n); + size_t offset = offsetAt(n); + + hash.update(reinterpret_cast(&string_size), sizeof(string_size)); + hash.update(reinterpret_cast(&chars[offset]), string_size); +} + +void ColumnString::updateHashFast(SipHash & hash) const +{ + hash.update(reinterpret_cast(offsets.data()), offsets.size() * sizeof(offsets[0])); + hash.update(reinterpret_cast(chars.data()), chars.size() * sizeof(chars[0])); +} + } diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index b2e340be61b..4bf24217383 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -15,7 +14,7 @@ class Collator; - +class SipHash; namespace DB { @@ -207,22 +206,11 @@ public: const char * skipSerializedInArena(const char * pos) const override; - void updateHashWithValue(size_t n, SipHash & hash) const override - { - size_t string_size = sizeAt(n); - size_t offset = offsetAt(n); - - hash.update(reinterpret_cast(&string_size), sizeof(string_size)); - hash.update(reinterpret_cast(&chars[offset]), string_size); - } + void updateHashWithValue(size_t n, SipHash & hash) const override; WeakHash32 getWeakHash32() const override; - void updateHashFast(SipHash & hash) const override - { - hash.update(reinterpret_cast(offsets.data()), offsets.size() * sizeof(offsets[0])); - hash.update(reinterpret_cast(chars.data()), chars.size() * sizeof(chars[0])); - } + void updateHashFast(SipHash & hash) const override; #if !defined(DEBUG_OR_SANITIZER_BUILD) void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; diff --git a/src/Columns/ColumnUnique.cpp b/src/Columns/ColumnUnique.cpp index 773edbfd590..33c608cd74a 100644 --- a/src/Columns/ColumnUnique.cpp +++ b/src/Columns/ColumnUnique.cpp @@ -1,8 +1,41 @@ #include +#include namespace DB { +template +void ColumnUnique::updateHashWithValue(size_t n, SipHash & hash_func) const +{ + return getNestedColumn()->updateHashWithValue(n, hash_func); +} + +template +UInt128 ColumnUnique::IncrementalHash::getHash(const ColumnType & column) +{ + size_t column_size = column.size(); + UInt128 cur_hash; + + if (column_size != num_added_rows.load()) + { + SipHash sip_hash; + for (size_t i = 0; i < column_size; ++i) + column.updateHashWithValue(i, sip_hash); + + std::lock_guard lock(mutex); + hash = sip_hash.get128(); + cur_hash = hash; + num_added_rows.store(column_size); + } + else + { + std::lock_guard lock(mutex); + cur_hash = hash; + } + + return cur_hash; +} + /// Explicit template instantiations. template class ColumnUnique; template class ColumnUnique; @@ -25,5 +58,15 @@ template class ColumnUnique; template class ColumnUnique; template class ColumnUnique; template class ColumnUnique; +template class ColumnUnique>; +template class ColumnUnique>; +template class ColumnUnique>; +template class ColumnUnique>; + + // template class IColumnHelper, ColumnFixedSizeHelper>; + // template class IColumnHelper, ColumnFixedSizeHelper>; + // template class IColumnHelper, ColumnFixedSizeHelper>; + // template class IColumnHelper, ColumnFixedSizeHelper>; + // template class IColumnHelper, ColumnFixedSizeHelper>; } diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index ce7bbf0766f..53d0b8bc376 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -87,10 +87,7 @@ public: StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override; char * serializeValueIntoMemory(size_t n, char * memory) const override; const char * skipSerializedInArena(const char * pos) const override; - void updateHashWithValue(size_t n, SipHash & hash_func) const override - { - return getNestedColumn()->updateHashWithValue(n, hash_func); - } + void updateHashWithValue(size_t n, SipHash & hash_func) const override; #if !defined(DEBUG_OR_SANITIZER_BUILD) int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override; @@ -721,33 +718,6 @@ IColumnUnique::IndexesWithOverflow ColumnUnique::uniqueInsertRangeWi return indexes_with_overflow; } -template -UInt128 ColumnUnique::IncrementalHash::getHash(const ColumnType & column) -{ - size_t column_size = column.size(); - UInt128 cur_hash; - - if (column_size != num_added_rows.load()) - { - SipHash sip_hash; - for (size_t i = 0; i < column_size; ++i) - column.updateHashWithValue(i, sip_hash); - - std::lock_guard lock(mutex); - hash = sip_hash.get128(); - cur_hash = hash; - num_added_rows.store(column_size); - } - else - { - std::lock_guard lock(mutex); - cur_hash = hash; - } - - return cur_hash; -} - - extern template class ColumnUnique; extern template class ColumnUnique; extern template class ColumnUnique; @@ -766,5 +736,12 @@ extern template class ColumnUnique; extern template class ColumnUnique; extern template class ColumnUnique; extern template class ColumnUnique; +extern template class ColumnUnique; +extern template class ColumnUnique; +extern template class ColumnUnique; +extern template class ColumnUnique>; +extern template class ColumnUnique>; +extern template class ColumnUnique>; +extern template class ColumnUnique>; } diff --git a/src/DataTypes/Serializations/SerializationLowCardinality.cpp b/src/DataTypes/Serializations/SerializationLowCardinality.cpp index 248fe2681b0..eeca2206b4c 100644 --- a/src/DataTypes/Serializations/SerializationLowCardinality.cpp +++ b/src/DataTypes/Serializations/SerializationLowCardinality.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace DB diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp index 8f2f5d760f8..5c82c8abf9a 100644 --- a/src/Functions/transform.cpp +++ b/src/Functions/transform.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Processors/Merges/Algorithms/Graphite.cpp b/src/Processors/Merges/Algorithms/Graphite.cpp index 718de3655af..275b9202317 100644 --- a/src/Processors/Merges/Algorithms/Graphite.cpp +++ b/src/Processors/Merges/Algorithms/Graphite.cpp @@ -500,4 +500,20 @@ void setGraphitePatternsFromConfig(ContextPtr context, const String & config_ele } } +void Params::updateHash(SipHash & hash) const +{ + hash.update(path_column_name); + hash.update(time_column_name); + hash.update(value_column_name); + hash.update(value_column_name); + hash.update(version_column_name); + hash.update(patterns_typed); + for (const auto & p : patterns) + p.updateHash(hash); + for (const auto & p : patterns_plain) + p.updateHash(hash); + for (const auto & p : patterns_tagged) + p.updateHash(hash); +} + } diff --git a/src/Processors/Merges/Algorithms/Graphite.h b/src/Processors/Merges/Algorithms/Graphite.h index ce3331053d1..d05fe385b9a 100644 --- a/src/Processors/Merges/Algorithms/Graphite.h +++ b/src/Processors/Merges/Algorithms/Graphite.h @@ -3,7 +3,6 @@ #include #include #include -#include /** Intended for implementation of "rollup" - aggregation (rounding) of older data * for a table with Graphite data (Graphite is the system for time series monitoring). @@ -90,6 +89,9 @@ * * */ + +class SipHash; + namespace DB::Graphite { @@ -145,21 +147,7 @@ struct Params Graphite::Patterns patterns; Graphite::Patterns patterns_plain; Graphite::Patterns patterns_tagged; - void updateHash(SipHash & hash) const - { - hash.update(path_column_name); - hash.update(time_column_name); - hash.update(value_column_name); - hash.update(value_column_name); - hash.update(version_column_name); - hash.update(patterns_typed); - for (const auto & p : patterns) - p.updateHash(hash); - for (const auto & p : patterns_plain) - p.updateHash(hash); - for (const auto & p : patterns_tagged) - p.updateHash(hash); - } + void updateHash(SipHash & hash) const; }; using RollupRule = std::pair; diff --git a/src/Storages/ColumnDependency.cpp b/src/Storages/ColumnDependency.cpp new file mode 100644 index 00000000000..1c7f4e74ab7 --- /dev/null +++ b/src/Storages/ColumnDependency.cpp @@ -0,0 +1,15 @@ +#include +#include + +namespace DB +{ + +UInt64 ColumnDependency::Hash::operator() (const ColumnDependency & dependency) const +{ + SipHash hash; + hash.update(dependency.column_name); + hash.update(dependency.kind); + return hash.get64(); +} + +} diff --git a/src/Storages/ColumnDependency.h b/src/Storages/ColumnDependency.h index dcbda7a4b86..241a09abfca 100644 --- a/src/Storages/ColumnDependency.h +++ b/src/Storages/ColumnDependency.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include @@ -48,13 +47,7 @@ struct ColumnDependency struct Hash { - UInt64 operator()(const ColumnDependency & dependency) const - { - SipHash hash; - hash.update(dependency.column_name); - hash.update(dependency.kind); - return hash.get64(); - } + UInt64 operator()(const ColumnDependency & dependency) const; }; }; diff --git a/src/Storages/Kafka/KafkaConsumer2.cpp b/src/Storages/Kafka/KafkaConsumer2.cpp index 4f3d2369328..a70a73f9be5 100644 --- a/src/Storages/Kafka/KafkaConsumer2.cpp +++ b/src/Storages/Kafka/KafkaConsumer2.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -379,4 +380,13 @@ void KafkaConsumer2::resetIfStopped() stalled_status = StalledStatus::CONSUMER_STOPPED; } } + +std::size_t KafkaConsumer2::OnlyTopicNameAndPartitionIdHash::operator()(const TopicPartition & tp) const +{ + SipHash s; + s.update(tp.topic); + s.update(tp.partition_id); + return s.get64(); +} + } diff --git a/src/Storages/Kafka/KafkaConsumer2.h b/src/Storages/Kafka/KafkaConsumer2.h index dd2cfe87aa0..79a1ea7ad9f 100644 --- a/src/Storages/Kafka/KafkaConsumer2.h +++ b/src/Storages/Kafka/KafkaConsumer2.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -46,13 +45,7 @@ public: struct OnlyTopicNameAndPartitionIdHash { - std::size_t operator()(const TopicPartition & tp) const - { - SipHash s; - s.update(tp.topic); - s.update(tp.partition_id); - return s.get64(); - } + std::size_t operator()(const TopicPartition & tp) const; }; struct OnlyTopicNameAndPartitionIdEquality diff --git a/src/Storages/MarkCache.cpp b/src/Storages/MarkCache.cpp index 85f0aeac692..799205ab6e2 100644 --- a/src/Storages/MarkCache.cpp +++ b/src/Storages/MarkCache.cpp @@ -1,3 +1,4 @@ +#include #include namespace DB @@ -8,4 +9,13 @@ MarkCache::MarkCache(const String & cache_policy, size_t max_size_in_bytes, doub : Base(cache_policy, max_size_in_bytes, 0, size_ratio) { } + +UInt128 MarkCache::hash(const String& path_to_file) +{ + SipHash hash; + hash.update(path_to_file.data(), path_to_file.size() + 1); + return hash.get128(); +} + + } diff --git a/src/Storages/MarkCache.h b/src/Storages/MarkCache.h index 311fccdf810..d338ec5ea22 100644 --- a/src/Storages/MarkCache.h +++ b/src/Storages/MarkCache.h @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -44,12 +43,7 @@ public: MarkCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); /// Calculate key from path to file and offset. - static UInt128 hash(const String & path_to_file) - { - SipHash hash; - hash.update(path_to_file.data(), path_to_file.size() + 1); - return hash.get128(); - } + static UInt128 hash(const String & path_to_file); template MappedPtr getOrSet(const Key & key, LoadFunc && load) diff --git a/src/Storages/MergeTree/PrimaryIndexCache.cpp b/src/Storages/MergeTree/PrimaryIndexCache.cpp index aeb9969f578..224047a449f 100644 --- a/src/Storages/MergeTree/PrimaryIndexCache.cpp +++ b/src/Storages/MergeTree/PrimaryIndexCache.cpp @@ -1,3 +1,4 @@ +#include #include namespace DB @@ -5,4 +6,12 @@ namespace DB template class CacheBase; + +UInt128 PrimaryIndexCache::hash(const String & part_path) +{ + SipHash hash; + hash.update(part_path.data(), part_path.size() + 1); + return hash.get128(); +} + } diff --git a/src/Storages/MergeTree/PrimaryIndexCache.h b/src/Storages/MergeTree/PrimaryIndexCache.h index 5ec185dcf58..e4c03050488 100644 --- a/src/Storages/MergeTree/PrimaryIndexCache.h +++ b/src/Storages/MergeTree/PrimaryIndexCache.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include #include #include @@ -49,12 +48,7 @@ public: } /// Calculate key from path to file and offset. - static UInt128 hash(const String & part_path) - { - SipHash hash; - hash.update(part_path.data(), part_path.size() + 1); - return hash.get128(); - } + static UInt128 hash(const String & part_path); template MappedPtr getOrSet(const Key & key, LoadFunc && load) From 90a03e149ace82b7d55ab6753dfbaf61fd330f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 3 Dec 2024 18:48:16 +0100 Subject: [PATCH 340/502] Remove deps from SipHash --- src/Analyzer/ArrayJoinNode.cpp | 1 + src/Analyzer/JoinNode.cpp | 1 + src/Analyzer/QueryTreeBuilder.cpp | 1 + src/Analyzer/TableFunctionNode.cpp | 2 + src/Analyzer/TableNode.cpp | 1 + src/Backups/BackupEntriesCollector.cpp | 1 + src/Common/QueryFuzzer.cpp | 1 + src/Common/computeMaxTableNameLength.cpp | 2 + src/Common/quoteString.h | 1 - src/Coordination/KeeperStateManager.cpp | 1 + src/Coordination/KeeperStorage.cpp | 8 ++++ src/Coordination/KeeperStorage.h | 8 +--- src/Core/QualifiedTableName.cpp | 25 +++++++++++ src/Core/QualifiedTableName.h | 41 ++----------------- src/DataTypes/DataTypeObject.cpp | 1 + src/Databases/DDLDependencyVisitor.cpp | 3 +- src/Databases/DDLRenamingVisitor.cpp | 1 + src/Databases/DatabaseFilesystem.cpp | 1 + src/Databases/DatabaseHDFS.cpp | 1 + src/Databases/DatabaseMemory.cpp | 1 + src/Databases/DatabaseReplicated.cpp | 1 + src/Databases/DatabasesCommon.cpp | 1 + src/Databases/DatabasesOverlay.cpp | 1 + src/Disks/DiskFomAST.cpp | 1 + .../ObjectStorages/S3/S3ObjectStorage.cpp | 1 + .../UserDefinedExecutableFunctionFactory.cpp | 1 + src/Interpreters/AsynchronousInsertQueue.cpp | 1 + .../DatabaseAndTableWithAlias.cpp | 1 + src/Interpreters/IInterpreter.cpp | 1 + src/Interpreters/InterpreterSelectQuery.cpp | 1 + src/Interpreters/MutationsInterpreter.cpp | 1 + .../ServerAsynchronousMetrics.cpp | 1 + src/Interpreters/Session.cpp | 1 + src/Interpreters/StorageID.cpp | 9 ++++ src/Interpreters/StorageID.h | 8 +--- src/Parsers/ASTIdentifier.cpp | 1 + src/Parsers/ASTInsertQuery.cpp | 2 + src/Parsers/ASTSelectQuery.cpp | 1 + src/Parsers/ASTViewTargets.cpp | 1 + src/Parsers/ExpressionElementParsers.cpp | 1 + src/Planner/PlannerActionsVisitor.cpp | 1 + src/Planner/PlannerJoinTree.cpp | 1 + .../Impl/NativeORCBlockInputFormat.cpp | 1 + src/Processors/Merges/Algorithms/Graphite.cpp | 1 + .../Transforms/buildPushingToViewsChain.cpp | 1 + src/Server/ReplicasStatusHandler.cpp | 1 + src/Storages/AlterCommands.cpp | 1 + .../DistributedAsyncInsertBatch.cpp | 1 + src/Storages/MergeTree/IMergeTreeDataPart.cpp | 1 + .../MergeTree/MergeFromLogEntryTask.cpp | 1 + .../MergeTree/MergeTreeDataPartWriterWide.cpp | 1 + .../MergeTree/MergeTreeDataSelectExecutor.cpp | 1 + .../MergeTreeIndexBloomFilterText.cpp | 1 + .../MergeTree/MergeTreeIndexFullText.cpp | 1 + .../MergeTree/MergeTreeIndexMinMax.cpp | 1 + src/Storages/MergeTree/MergeTreeIndexSet.cpp | 2 + .../MergeTreeIndexVectorSimilarity.cpp | 1 + src/Storages/StorageMerge.cpp | 1 + src/Storages/StorageNull.cpp | 1 + src/Storages/StorageSQLite.cpp | 1 + src/Storages/StorageSnapshot.cpp | 2 + .../TimeSeriesDefinitionNormalizer.cpp | 1 + 62 files changed, 109 insertions(+), 53 deletions(-) create mode 100644 src/Core/QualifiedTableName.cpp diff --git a/src/Analyzer/ArrayJoinNode.cpp b/src/Analyzer/ArrayJoinNode.cpp index b37f35fd84c..47e291ed280 100644 --- a/src/Analyzer/ArrayJoinNode.cpp +++ b/src/Analyzer/ArrayJoinNode.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Analyzer/JoinNode.cpp b/src/Analyzer/JoinNode.cpp index 722c1e19b7e..bce71b24028 100644 --- a/src/Analyzer/JoinNode.cpp +++ b/src/Analyzer/JoinNode.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index d3c88d39213..1b23c2fce10 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include diff --git a/src/Analyzer/TableFunctionNode.cpp b/src/Analyzer/TableFunctionNode.cpp index 87d2fdcffb5..6d1386d874b 100644 --- a/src/Analyzer/TableFunctionNode.cpp +++ b/src/Analyzer/TableFunctionNode.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include diff --git a/src/Analyzer/TableNode.cpp b/src/Analyzer/TableNode.cpp index 506bb1c08ef..89e83670a78 100644 --- a/src/Analyzer/TableNode.cpp +++ b/src/Analyzer/TableNode.cpp @@ -11,6 +11,7 @@ #include #include +#include namespace DB { diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 00a4471d994..a41391b7c97 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Common/QueryFuzzer.cpp b/src/Common/QueryFuzzer.cpp index ce5baad816f..979549c695c 100644 --- a/src/Common/QueryFuzzer.cpp +++ b/src/Common/QueryFuzzer.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include diff --git a/src/Common/computeMaxTableNameLength.cpp b/src/Common/computeMaxTableNameLength.cpp index eb11c43a149..98c930cb201 100644 --- a/src/Common/computeMaxTableNameLength.cpp +++ b/src/Common/computeMaxTableNameLength.cpp @@ -1,6 +1,8 @@ #include #include +#include + namespace DB { diff --git a/src/Common/quoteString.h b/src/Common/quoteString.h index 3f17d6e7621..2a23e76eb5a 100644 --- a/src/Common/quoteString.h +++ b/src/Common/quoteString.h @@ -2,7 +2,6 @@ #include #include -#include namespace DB diff --git a/src/Coordination/KeeperStateManager.cpp b/src/Coordination/KeeperStateManager.cpp index d0356a82ede..51c848f97b0 100644 --- a/src/Coordination/KeeperStateManager.cpp +++ b/src/Coordination/KeeperStateManager.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index b132ee010f4..7836c3fa1c8 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -3622,6 +3622,14 @@ bool KeeperStorageBase::checkDigest(const Digest & first, const Digest & second) return first.value == second.value; } +UInt64 KeeperStorageBase::WatchInfoHash::operator()(WatchInfo info) const +{ + SipHash hash; + hash.update(info.path); + hash.update(info.is_list_watch); + return hash.get64(); +} + template String KeeperStorage::generateDigest(const String & userdata) { diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h index 4df58933102..327d5457451 100644 --- a/src/Coordination/KeeperStorage.h +++ b/src/Coordination/KeeperStorage.h @@ -326,13 +326,7 @@ public: struct WatchInfoHash { - auto operator()(WatchInfo info) const - { - SipHash hash; - hash.update(info.path); - hash.update(info.is_list_watch); - return hash.get64(); - } + UInt64 operator()(WatchInfo info) const; }; using SessionAndWatcher = std::unordered_map>; diff --git a/src/Core/QualifiedTableName.cpp b/src/Core/QualifiedTableName.cpp new file mode 100644 index 00000000000..661cc4227b1 --- /dev/null +++ b/src/Core/QualifiedTableName.cpp @@ -0,0 +1,25 @@ +#include +#include +#include +#include + +namespace DB +{ + +UInt64 QualifiedTableName::hash() const +{ + SipHash hash_state; + hash_state.update(database.data(), database.size()); + hash_state.update(table.data(), table.size()); + return hash_state.get64(); +} + +QualifiedTableName QualifiedTableName::parseFromString(const String & maybe_qualified_name) +{ + auto name = tryParseFromString(maybe_qualified_name); + if (!name) + throw Exception(ErrorCodes::SYNTAX_ERROR, "Invalid qualified name: {}", maybe_qualified_name); + return *name; +} + +} diff --git a/src/Core/QualifiedTableName.h b/src/Core/QualifiedTableName.h index 5899c23ee91..79c101b915e 100644 --- a/src/Core/QualifiedTableName.h +++ b/src/Core/QualifiedTableName.h @@ -1,12 +1,10 @@ #pragma once +#include + #include #include #include -#include -#include -#include -#include namespace DB { @@ -32,13 +30,7 @@ struct QualifiedTableName return std::forward_as_tuple(database, table) < std::forward_as_tuple(other.database, other.table); } - UInt64 hash() const - { - SipHash hash_state; - hash_state.update(database.data(), database.size()); - hash_state.update(table.data(), table.size()); - return hash_state.get64(); - } + UInt64 hash() const; std::vector getParts() const { @@ -86,13 +78,7 @@ struct QualifiedTableName return name; } - static QualifiedTableName parseFromString(const String & maybe_qualified_name) - { - auto name = tryParseFromString(maybe_qualified_name); - if (!name) - throw Exception(ErrorCodes::SYNTAX_ERROR, "Invalid qualified name: {}", maybe_qualified_name); - return *name; - } + static QualifiedTableName parseFromString(const String & maybe_qualified_name); }; } @@ -111,22 +97,3 @@ template <> struct hash } }; } - -namespace fmt -{ - template <> - struct formatter - { - static constexpr auto parse(format_parse_context & ctx) - { - return ctx.begin(); - } - - template - auto format(const DB::QualifiedTableName & name, FormatContext & ctx) const - { - return fmt::format_to(ctx.out(), "{}.{}", DB::backQuoteIfNeed(name.database), DB::backQuoteIfNeed(name.table)); - } - }; -} - diff --git a/src/DataTypes/DataTypeObject.cpp b/src/DataTypes/DataTypeObject.cpp index 9a60d1e55b8..ea0ff4942a4 100644 --- a/src/DataTypes/DataTypeObject.cpp +++ b/src/DataTypes/DataTypeObject.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp index 6cf01e4e87e..bae6d2a0d3e 100644 --- a/src/Databases/DDLDependencyVisitor.cpp +++ b/src/Databases/DDLDependencyVisitor.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -473,7 +474,7 @@ namespace try { ParserSelectWithUnionQuery parser; - String description = fmt::format("Query for ClickHouse dictionary {}", data.table_name); + String description = fmt::format("Query for ClickHouse dictionary {}.{}", backQuoteIfNeed(data.table_name.database), backQuoteIfNeed(data.table_name.table)); String fixed_query = removeWhereConditionPlaceholder(query); const Settings & settings = data.global_context->getSettingsRef(); ASTPtr select = parseQuery( diff --git a/src/Databases/DDLRenamingVisitor.cpp b/src/Databases/DDLRenamingVisitor.cpp index 06d410bddb6..fc05da53f91 100644 --- a/src/Databases/DDLRenamingVisitor.cpp +++ b/src/Databases/DDLRenamingVisitor.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/DatabaseFilesystem.cpp b/src/Databases/DatabaseFilesystem.cpp index f3a280e9a2b..01331ddb87f 100644 --- a/src/Databases/DatabaseFilesystem.cpp +++ b/src/Databases/DatabaseFilesystem.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include diff --git a/src/Databases/DatabaseHDFS.cpp b/src/Databases/DatabaseHDFS.cpp index f072fd33bd8..fb7f410181b 100644 --- a/src/Databases/DatabaseHDFS.cpp +++ b/src/Databases/DatabaseHDFS.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 6159ccb3967..09ab96717d0 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 9ea6ec27df1..b887d14b939 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index 7bb4d23ef92..fb50005d119 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include namespace DB diff --git a/src/Databases/DatabasesOverlay.cpp b/src/Databases/DatabasesOverlay.cpp index 495733e15fd..e5acc48b7db 100644 --- a/src/Databases/DatabasesOverlay.cpp +++ b/src/Databases/DatabasesOverlay.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Disks/DiskFomAST.cpp b/src/Disks/DiskFomAST.cpp index 5329ff8748a..d86310b74df 100644 --- a/src/Disks/DiskFomAST.cpp +++ b/src/Disks/DiskFomAST.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 9fca3cad688..35df967a061 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/UserDefined/UserDefinedExecutableFunctionFactory.cpp b/src/Functions/UserDefined/UserDefinedExecutableFunctionFactory.cpp index 9b236172c43..53dc0702b83 100644 --- a/src/Functions/UserDefined/UserDefinedExecutableFunctionFactory.cpp +++ b/src/Functions/UserDefined/UserDefinedExecutableFunctionFactory.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index ceda7adbcd4..29922cdee14 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/DatabaseAndTableWithAlias.cpp b/src/Interpreters/DatabaseAndTableWithAlias.cpp index 329391b45d7..0a7593443d8 100644 --- a/src/Interpreters/DatabaseAndTableWithAlias.cpp +++ b/src/Interpreters/DatabaseAndTableWithAlias.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include diff --git a/src/Interpreters/IInterpreter.cpp b/src/Interpreters/IInterpreter.cpp index c4255b8a290..b865ff49598 100644 --- a/src/Interpreters/IInterpreter.cpp +++ b/src/Interpreters/IInterpreter.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index d31b941030f..c4245ebe856 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -96,6 +96,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index d95e2392ad6..b61718ac3aa 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -42,6 +42,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index dfaebec03a8..45a0848e001 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -10,6 +10,7 @@ #include #include +#include #include diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 60a5b0a850f..a98a9ba017d 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/StorageID.cpp b/src/Interpreters/StorageID.cpp index f49e5bc28d5..c3d19da056f 100644 --- a/src/Interpreters/StorageID.cpp +++ b/src/Interpreters/StorageID.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -106,4 +107,12 @@ String StorageID::getShortName() const return database_name + "." + table_name; } +size_t StorageID::DatabaseAndTableNameHash::operator()(const StorageID & storage_id) const +{ + SipHash hash_state; + hash_state.update(storage_id.database_name.data(), storage_id.database_name.size()); + hash_state.update(storage_id.table_name.data(), storage_id.table_name.size()); + return hash_state.get64(); +} + } diff --git a/src/Interpreters/StorageID.h b/src/Interpreters/StorageID.h index ad55d16e284..a5ea12e7ffe 100644 --- a/src/Interpreters/StorageID.h +++ b/src/Interpreters/StorageID.h @@ -99,13 +99,7 @@ struct StorageID /// Calculates hash using only the database and table name of a StorageID. struct DatabaseAndTableNameHash { - size_t operator()(const StorageID & storage_id) const - { - SipHash hash_state; - hash_state.update(storage_id.database_name.data(), storage_id.database_name.size()); - hash_state.update(storage_id.table_name.data(), storage_id.table_name.size()); - return hash_state.get64(); - } + size_t operator()(const StorageID & storage_id) const; }; /// Checks if the database and table name of two StorageIDs are equal. diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index ea4180c64e3..54b5f45f20b 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Parsers/ASTInsertQuery.cpp b/src/Parsers/ASTInsertQuery.cpp index 2a073733968..0b236813d8d 100644 --- a/src/Parsers/ASTInsertQuery.cpp +++ b/src/Parsers/ASTInsertQuery.cpp @@ -1,4 +1,6 @@ #include + +#include #include #include #include diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index ed35b9d2c4d..f904bae3fa1 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Parsers/ASTViewTargets.cpp b/src/Parsers/ASTViewTargets.cpp index eba118ba79a..84361e591ae 100644 --- a/src/Parsers/ASTViewTargets.cpp +++ b/src/Parsers/ASTViewTargets.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 80eed615b1d..28760cdaef9 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 8f8584418ea..0c92d28e077 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 48a68d97440..82f8e61c2bf 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp index 93ed4402b4b..2ccb077dd94 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp @@ -36,6 +36,7 @@ # include # include # include +# include # include # include "ArrowBufferedStreams.h" diff --git a/src/Processors/Merges/Algorithms/Graphite.cpp b/src/Processors/Merges/Algorithms/Graphite.cpp index 275b9202317..fb7809adaee 100644 --- a/src/Processors/Merges/Algorithms/Graphite.cpp +++ b/src/Processors/Merges/Algorithms/Graphite.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index a21218dfc88..b48ba3eb8a0 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index 730fdd50930..32a3eca1ee9 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index fbca222b1e7..996f41175d6 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp index 7270e02f506..2488807e700 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index e880c70ba8d..0831529f69e 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index 0b89fd711a8..685365e1ee3 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 13cb42f46f4..349f75c76a9 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 2934dc1c91c..450badb9248 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp index 7c1c425877b..135330def36 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp index f502726763c..10068d28548 100644 --- a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp index c859053b976..ec0a6533784 100644 --- a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp @@ -8,6 +8,7 @@ #include #include +#include namespace DB { diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp index 2b18f5fe1e5..35b6bafc6a5 100644 --- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include diff --git a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp index 0b17fa05072..554768c5371 100644 --- a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index c99593eaa8f..28f58b4bf37 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/StorageNull.cpp b/src/Storages/StorageNull.cpp index 7a8852ca2d6..be81a7bf127 100644 --- a/src/Storages/StorageNull.cpp +++ b/src/Storages/StorageNull.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/StorageSQLite.cpp b/src/Storages/StorageSQLite.cpp index b90b15f3b99..3389174f5a0 100644 --- a/src/Storages/StorageSQLite.cpp +++ b/src/Storages/StorageSQLite.cpp @@ -2,6 +2,7 @@ #if USE_SQLITE #include +#include #include #include #include diff --git a/src/Storages/StorageSnapshot.cpp b/src/Storages/StorageSnapshot.cpp index 13a46990556..51937007966 100644 --- a/src/Storages/StorageSnapshot.cpp +++ b/src/Storages/StorageSnapshot.cpp @@ -3,6 +3,8 @@ #include #include #include +#include + #include namespace DB diff --git a/src/Storages/TimeSeries/TimeSeriesDefinitionNormalizer.cpp b/src/Storages/TimeSeries/TimeSeriesDefinitionNormalizer.cpp index 9f578043e54..561e5b0a927 100644 --- a/src/Storages/TimeSeries/TimeSeriesDefinitionNormalizer.cpp +++ b/src/Storages/TimeSeries/TimeSeriesDefinitionNormalizer.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include From c239adec3643e66bd942eaa2752d473364feea44 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Wed, 4 Dec 2024 14:59:58 +0000 Subject: [PATCH 341/502] add test --- .../test_ldap_external_user_directory/test.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/integration/test_ldap_external_user_directory/test.py b/tests/integration/test_ldap_external_user_directory/test.py index ce16d7ad286..0f2aefbffcd 100644 --- a/tests/integration/test_ldap_external_user_directory/test.py +++ b/tests/integration/test_ldap_external_user_directory/test.py @@ -183,3 +183,23 @@ def test_push_role_to_other_nodes(ldap_cluster): instance2.query("DROP ROLE IF EXISTS role_read") delete_ldap_group(ldap_cluster, group_cn="clickhouse-role_read") + + +def test_remote_query_user_does_not_exist_locally(ldap_cluster): + """ + Check that even if user does not exist locally, using it to execute remote queries is still possible + """ + instance2.query("DROP USER IF EXISTS non_local") + + instance2.query("CREATE USER non_local") + + instance2.query("DROP TABLE IF EXISTS test_table") + instance2.query("CREATE TABLE test_table (id Int16) ENGINE=Memory") + instance2.query("INSERT INTO test_table VALUES (123)") + instance2.query("GRANT SELECT ON default.test_table TO non_local") + + result = instance1.query("SELECT * FROM remote('instance2', 'default.test_table', 'non_local')") + assert result.strip() == "123" + + instance2.query("DROP USER IF EXISTS non_local") + instance2.query("DROP TABLE IF EXISTS test_table") From dc49651ae7023bcdc6ea9b776a9666c232156237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 4 Dec 2024 16:01:29 +0100 Subject: [PATCH 342/502] Disable 03008_azure_plain_rewritable with distributed cache --- tests/queries/0_stateless/03008_azure_plain_rewritable.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03008_azure_plain_rewritable.sh b/tests/queries/0_stateless/03008_azure_plain_rewritable.sh index fe56b5cedd3..c148afb70f1 100755 --- a/tests/queries/0_stateless/03008_azure_plain_rewritable.sh +++ b/tests/queries/0_stateless/03008_azure_plain_rewritable.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-shared-merge-tree +# Tags: no-fasttest, no-shared-merge-tree, no-distributed-cache # Tag no-fasttest: requires Azure # Tag no-shared-merge-tree: does not support replication +# Tag no-distributed-cache: Not supported auth type set -e From 35e67393bd2318435de854c3676ee0ba8f43b3bd Mon Sep 17 00:00:00 2001 From: Francesco Ciocchetti Date: Wed, 4 Dec 2024 16:15:40 +0100 Subject: [PATCH 343/502] Name of the container, generated form the test name, is 67 chars which is too long. shrinking it --- ...eference => 03008_azure_plain_rewritable_with_slash.reference} | 0 ...ailing_slash.sh => 03008_azure_plain_rewritable_with_slash.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{03008_azure_plain_rewritable_with_trailing_slash.reference => 03008_azure_plain_rewritable_with_slash.reference} (100%) rename tests/queries/0_stateless/{03008_azure_plain_rewritable_with_trailing_slash.sh => 03008_azure_plain_rewritable_with_slash.sh} (100%) diff --git a/tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.reference b/tests/queries/0_stateless/03008_azure_plain_rewritable_with_slash.reference similarity index 100% rename from tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.reference rename to tests/queries/0_stateless/03008_azure_plain_rewritable_with_slash.reference diff --git a/tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.sh b/tests/queries/0_stateless/03008_azure_plain_rewritable_with_slash.sh similarity index 100% rename from tests/queries/0_stateless/03008_azure_plain_rewritable_with_trailing_slash.sh rename to tests/queries/0_stateless/03008_azure_plain_rewritable_with_slash.sh From d9da4e42356f720cf5662c67b764bda5d95668c7 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:27:11 +0100 Subject: [PATCH 344/502] Update test.py --- tests/integration/test_storage_s3_queue/test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index af39e4725b3..732ab6ce4a7 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -2318,6 +2318,9 @@ def test_alter_settings(started_cluster): def test_list_and_delete_race(started_cluster): + if node.is_built_with_sanitizer(): + # Issue does not reproduce under sanitizer + return node = started_cluster.instances["instance"] node_2 = started_cluster.instances["instance2"] table_name = f"list_and_delete_race_{generate_random_string()}" From bd74ac343b7767b2f72c779b87a75d455a446ba6 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Wed, 4 Dec 2024 15:40:43 +0000 Subject: [PATCH 345/502] fix black --- tests/integration/test_ldap_external_user_directory/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_ldap_external_user_directory/test.py b/tests/integration/test_ldap_external_user_directory/test.py index 0f2aefbffcd..efda6ac7831 100644 --- a/tests/integration/test_ldap_external_user_directory/test.py +++ b/tests/integration/test_ldap_external_user_directory/test.py @@ -198,7 +198,9 @@ def test_remote_query_user_does_not_exist_locally(ldap_cluster): instance2.query("INSERT INTO test_table VALUES (123)") instance2.query("GRANT SELECT ON default.test_table TO non_local") - result = instance1.query("SELECT * FROM remote('instance2', 'default.test_table', 'non_local')") + result = instance1.query( + "SELECT * FROM remote('instance2', 'default.test_table', 'non_local')" + ) assert result.strip() == "123" instance2.query("DROP USER IF EXISTS non_local") From 2c68653d0a4d012bcf81f5a890ef78e21dfe4402 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 4 Dec 2024 00:21:33 +0100 Subject: [PATCH 346/502] Add S3 request settings http_max_fields, http_max_field_name_size, http_max_field_value_size for parsing S3 API responses. --- src/Backups/BackupIO_S3.cpp | 6 ++++++ src/IO/S3/PocoHTTPClient.cpp | 6 ++++++ src/IO/S3/PocoHTTPClient.h | 7 +++++++ src/IO/S3RequestSettings.cpp | 5 ++++- 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 7dacd8102cc..e0df983c8b9 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -52,6 +52,9 @@ namespace S3RequestSetting { extern const S3RequestSettingsBool allow_native_copy; extern const S3RequestSettingsString storage_class_name; + extern const S3RequestSettingsUInt64 http_max_fields; + extern const S3RequestSettingsUInt64 http_max_field_name_size; + extern const S3RequestSettingsUInt64 http_max_field_value_size; } namespace ErrorCodes @@ -100,6 +103,9 @@ namespace client_configuration.requestTimeoutMs = 60 * 60 * 1000; client_configuration.http_keep_alive_timeout = S3::DEFAULT_KEEP_ALIVE_TIMEOUT; client_configuration.http_keep_alive_max_requests = S3::DEFAULT_KEEP_ALIVE_MAX_REQUESTS; + client_configuration.http_max_fields = request_settings[S3RequestSetting::http_max_fields]; + client_configuration.http_max_field_name_size = request_settings[S3RequestSetting::http_max_field_name_size]; + client_configuration.http_max_field_value_size = request_settings[S3RequestSetting::http_max_field_value_size]; S3::ClientSettings client_settings{ .use_virtual_addressing = s3_uri.is_virtual_hosted_style, diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 3e060e21c51..b53058a42da 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -163,6 +163,9 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_config , remote_host_filter(client_configuration.remote_host_filter) , s3_max_redirects(client_configuration.s3_max_redirects) , s3_use_adaptive_timeouts(client_configuration.s3_use_adaptive_timeouts) + , http_max_fields(client_configuration.http_max_fields) + , http_max_field_name_size(client_configuration.http_max_field_name_size) + , http_max_field_value_size(client_configuration.http_max_field_value_size) , enable_s3_requests_logging(client_configuration.enable_s3_requests_logging) , for_disk_s3(client_configuration.for_disk_s3) , get_request_throttler(client_configuration.get_request_throttler) @@ -466,6 +469,9 @@ void PocoHTTPClient::makeRequestInternalImpl( } Poco::Net::HTTPResponse poco_response; + poco_response.setFieldLimit(static_cast(http_max_fields)); + poco_response.setNameLengthLimit(static_cast(http_max_field_name_size)); + poco_response.setValueLengthLimit(static_cast(http_max_field_value_size)); Stopwatch watch; diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index eb65460ce13..f62371706c3 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -57,6 +57,10 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration size_t http_keep_alive_timeout = DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT; size_t http_keep_alive_max_requests = DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST; + UInt64 http_max_fields = 1000000; + UInt64 http_max_field_name_size = 128 * 1024; + UInt64 http_max_field_value_size = 128 * 1024; + std::function error_report; void updateSchemeAndRegion(); @@ -177,6 +181,9 @@ protected: const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects = 0; bool s3_use_adaptive_timeouts = true; + const UInt64 http_max_fields = 1000000; + const UInt64 http_max_field_name_size = 128 * 1024; + const UInt64 http_max_field_value_size = 128 * 1024; bool enable_s3_requests_logging = false; bool for_disk_s3 = false; diff --git a/src/IO/S3RequestSettings.cpp b/src/IO/S3RequestSettings.cpp index e35bcfdbc4d..08eb9aaec5a 100644 --- a/src/IO/S3RequestSettings.cpp +++ b/src/IO/S3RequestSettings.cpp @@ -37,7 +37,10 @@ namespace ErrorCodes DECLARE(Bool, check_objects_after_upload, S3::DEFAULT_CHECK_OBJECTS_AFTER_UPLOAD, "", 0) \ DECLARE(Bool, throw_on_zero_files_match, false, "", 0) \ DECLARE(UInt64, max_single_operation_copy_size, S3::DEFAULT_MAX_SINGLE_OPERATION_COPY_SIZE, "", 0) \ - DECLARE(String, storage_class_name, "", "", 0) + DECLARE(String, storage_class_name, "", "", 0) \ + DECLARE(UInt64, http_max_fields, 1000000, "", 0) \ + DECLARE(UInt64, http_max_field_name_size, 128 * 1024, "", 0) \ + DECLARE(UInt64, http_max_field_value_size, 128 * 1024, "", 0) #define PART_UPLOAD_SETTINGS(DECLARE, ALIAS) \ DECLARE(UInt64, strict_upload_part_size, 0, "", 0) \ From d77df2b4f278247d30887dd87da6f1b06a6e7840 Mon Sep 17 00:00:00 2001 From: Michael Stetsyuk Date: Wed, 4 Dec 2024 16:05:06 +0000 Subject: [PATCH 347/502] empty From 1dc394c3ab1156e8544241c4fdc2e6d11ba3a453 Mon Sep 17 00:00:00 2001 From: Ladislav Snizek Date: Wed, 4 Dec 2024 17:14:55 +0100 Subject: [PATCH 348/502] Correct documentation for the min_free_disk_space settings --- CHANGELOG.md | 2 +- docs/changelogs/v24.10.1.2812-stable.md | 2 +- docs/changelogs/v24.9.1.3278-stable.md | 4 ++-- docs/en/operations/settings/merge-tree-settings.md | 4 ++-- docs/ja/operations/settings/merge-tree-settings.md | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a915b9be7d5..139446bd5c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -226,7 +226,7 @@ * `CREATE TABLE AS` will copy `PRIMARY KEY`, `ORDER BY`, and similar clauses (of `MergeTree` tables). [#69739](https://github.com/ClickHouse/ClickHouse/pull/69739) ([sakulali](https://github.com/sakulali)). * Support 64-bit XID in Keeper. It can be enabled with the `use_xid_64` configuration value. [#69908](https://github.com/ClickHouse/ClickHouse/pull/69908) ([Antonio Andelic](https://github.com/antonio2368)). * Command-line arguments for Bool settings are set to true when no value is provided for the argument (e.g. `clickhouse-client --optimize_aggregation_in_order --query "SELECT 1"`). [#70459](https://github.com/ClickHouse/ClickHouse/pull/70459) ([davidtsuk](https://github.com/davidtsuk)). -* Added user-level settings `min_free_disk_bytes_to_throw_insert` and `min_free_disk_ratio_to_throw_insert` to prevent insertions on disks that are almost full. [#69755](https://github.com/ClickHouse/ClickHouse/pull/69755) ([Marco Vilas Boas](https://github.com/marco-vb)). +* Added user-level settings `min_free_disk_bytes_to_perform_insert` and `min_free_disk_perform_to_throw_insert` to prevent insertions on disks that are almost full. [#69755](https://github.com/ClickHouse/ClickHouse/pull/69755) ([Marco Vilas Boas](https://github.com/marco-vb)). * Embedded documentation for settings will be strictly more detailed and complete than the documentation on the website. This is the first step before making the website documentation always auto-generated from the source code. This has long-standing implications: - it will be guaranteed to have every setting; - there is no chance of having default values obsolete; - we can generate this documentation for each ClickHouse version; - the documentation can be displayed by the server itself even without Internet access. Generate the docs on the website from the source code. [#70289](https://github.com/ClickHouse/ClickHouse/pull/70289) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Allow empty needle in the function `replace`, the same behavior with PostgreSQL. [#69918](https://github.com/ClickHouse/ClickHouse/pull/69918) ([zhanglistar](https://github.com/zhanglistar)). * Allow empty needle in functions `replaceRegexp*`. [#70053](https://github.com/ClickHouse/ClickHouse/pull/70053) ([zhanglistar](https://github.com/zhanglistar)). diff --git a/docs/changelogs/v24.10.1.2812-stable.md b/docs/changelogs/v24.10.1.2812-stable.md index c26bbf706ff..fdb81401bf5 100644 --- a/docs/changelogs/v24.10.1.2812-stable.md +++ b/docs/changelogs/v24.10.1.2812-stable.md @@ -65,7 +65,7 @@ sidebar_label: 2024 * Follow-up to https://github.com/ClickHouse/ClickHouse/pull/69346 Point 4 described there will work now as well:. [#69563](https://github.com/ClickHouse/ClickHouse/pull/69563) ([Vitaly Baranov](https://github.com/vitlibar)). * Implement generic SerDe between Avro Union and ClickHouse Variant type. Resolves [#69713](https://github.com/ClickHouse/ClickHouse/issues/69713). [#69712](https://github.com/ClickHouse/ClickHouse/pull/69712) ([Jiří Kozlovský](https://github.com/jirislav)). * 1. CREATE TABLE AS will copy PRIMARY KEY, ORDER BY, and similar clauses. Now it is supported only for the MergeTree family of table engines. 2. For example, the follow SQL statements will trigger exception in the past, but this PR fixes it: if the destination table do not provide an `ORDER BY` or `PRIMARY KEY` expression in the table definition, we will copy that from source table. [#69739](https://github.com/ClickHouse/ClickHouse/pull/69739) ([sakulali](https://github.com/sakulali)). -* Added user-level settings `min_free_disk_bytes_to_throw_insert` and `min_free_disk_ratio_to_throw_insert` to prevent insertions on disks that are almost full. [#69755](https://github.com/ClickHouse/ClickHouse/pull/69755) ([Marco Vilas Boas](https://github.com/marco-vb)). +* Added user-level settings `min_free_disk_bytes_to_perform_insert` and `min_free_disk_ratio_to_perform_insert` to prevent insertions on disks that are almost full. [#69755](https://github.com/ClickHouse/ClickHouse/pull/69755) ([Marco Vilas Boas](https://github.com/marco-vb)). * If you run `clickhouse-client` or other CLI application and it starts up slowly due to an overloaded server, and you start typing your query, such as `SELECT`, the previous versions will display the remaining of the terminal echo contents before printing the greetings message, such as `SELECTClickHouse local version 24.10.1.1.` instead of `ClickHouse local version 24.10.1.1.`. Now it is fixed. This closes [#31696](https://github.com/ClickHouse/ClickHouse/issues/31696). [#69856](https://github.com/ClickHouse/ClickHouse/pull/69856) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Add new column readonly_duration to the system.replicas table. Needed to be able to distinguish actual readonly replicas from sentinel ones in alerts. [#69871](https://github.com/ClickHouse/ClickHouse/pull/69871) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)). * Change the join to sort settings type to unsigned int. [#69886](https://github.com/ClickHouse/ClickHouse/pull/69886) ([kevinyhzou](https://github.com/KevinyhZou)). diff --git a/docs/changelogs/v24.9.1.3278-stable.md b/docs/changelogs/v24.9.1.3278-stable.md index 687bbc9e5c4..6a10c557c6c 100644 --- a/docs/changelogs/v24.9.1.3278-stable.md +++ b/docs/changelogs/v24.9.1.3278-stable.md @@ -89,7 +89,7 @@ sidebar_label: 2024 * Restore mode that replaces all external table engines and functions to Null (`restore_replace_external_engines_to_null`, `restore_replace_external_table_functions_to_null` settings) was failing if table had SETTINGS. Now it removes settings from table definition in this case and allows to restore such tables. [#69253](https://github.com/ClickHouse/ClickHouse/pull/69253) ([Ilya Yatsishin](https://github.com/qoega)). * Reduce memory usage of inserts to JSON by using adaptive write buffer size. A lot of files created by JSON column in wide part contains small amount of data and it doesn't make sense to allocate 1MB buffer for them. [#69272](https://github.com/ClickHouse/ClickHouse/pull/69272) ([Pavel Kruglov](https://github.com/Avogar)). * CLICKHOUSE_PASSWORD is escaped for XML in clickhouse image's entrypoint. [#69301](https://github.com/ClickHouse/ClickHouse/pull/69301) ([aohoyd](https://github.com/aohoyd)). -* Added user-level settings `min_free_disk_bytes_to_throw_insert` and `min_free_disk_ratio_to_throw_insert` to prevent insertions on disks that are almost full. [#69376](https://github.com/ClickHouse/ClickHouse/pull/69376) ([Marco Vilas Boas](https://github.com/marco-vb)). +* Added user-level settings `min_free_disk_bytes_to_perform_insert` and `min_free_disk_ratio_to_perform_insert` to prevent insertions on disks that are almost full. [#69376](https://github.com/ClickHouse/ClickHouse/pull/69376) ([Marco Vilas Boas](https://github.com/marco-vb)). * Not retaining thread in concurrent hash join threadpool to avoid query excessively spawn threads. [#69406](https://github.com/ClickHouse/ClickHouse/pull/69406) ([Duc Canh Le](https://github.com/canhld94)). * Allow empty arguments for arrayZip/arrayZipUnaligned, as concat did in https://github.com/ClickHouse/ClickHouse/pull/65887. It is for spark compatiability in Gluten CH Backend. [#69576](https://github.com/ClickHouse/ClickHouse/pull/69576) ([李扬](https://github.com/taiyang-li)). * Support more advanced SSL options for Keeper's internal communication (e.g. private keys with passphrase). [#69582](https://github.com/ClickHouse/ClickHouse/pull/69582) ([Antonio Andelic](https://github.com/antonio2368)). @@ -199,7 +199,7 @@ sidebar_label: 2024 * NO CL ENTRY: 'Revert "Fix prewhere without columns and without adaptive index granularity (almost w/o anything)"'. [#68897](https://github.com/ClickHouse/ClickHouse/pull/68897) ([Alexander Gololobov](https://github.com/davenger)). * NO CL ENTRY: 'Revert "Speed up some Kafka tests with multiprocessing"'. [#69356](https://github.com/ClickHouse/ClickHouse/pull/69356) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * NO CL ENTRY: 'Revert "Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. V"'. [#69393](https://github.com/ClickHouse/ClickHouse/pull/69393) ([Alexander Tokmakov](https://github.com/tavplubix)). -* NO CL ENTRY: 'Revert "Add user-level settings min_free_diskspace_bytes_to_throw_insert and min_free_diskspace_ratio_to_throw_insert"'. [#69705](https://github.com/ClickHouse/ClickHouse/pull/69705) ([Raúl Marín](https://github.com/Algunenano)). +* NO CL ENTRY: 'Revert "Add user-level settings min_free_disk_bytes_to_perform_insert and min_free_disk_ratio_to_perform_insert"'. [#69705](https://github.com/ClickHouse/ClickHouse/pull/69705) ([Raúl Marín](https://github.com/Algunenano)). * NO CL ENTRY: 'Revert "Support more oss endpoints"'. [#69779](https://github.com/ClickHouse/ClickHouse/pull/69779) ([Raúl Marín](https://github.com/Algunenano)). #### NOT FOR CHANGELOG / INSIGNIFICANT diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 8eebaf20617..383023a36db 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -1078,7 +1078,7 @@ Default value: throw ## min_free_disk_bytes_to_perform_insert -The minimum number of bytes that should be free in disk space in order to insert data. If the number of available free bytes is less than `min_free_disk_bytes_to_throw_insert` then an exception is thrown and the insert is not executed. Note that this setting: +The minimum number of bytes that should be free in disk space in order to insert data. If the number of available free bytes is less than `min_free_disk_bytes_to_perform_insert` then an exception is thrown and the insert is not executed. Note that this setting: - takes into account the `keep_free_space_bytes` setting. - does not take into account the amount of data that will be written by the `INSERT` operation. - is only checked if a positive (non-zero) number of bytes is specified @@ -1114,4 +1114,4 @@ When `cache_populated_by_fetch` is disabled (the default setting), new data part If enabled, `cache_populated_by_fetch` will instead cause all nodes to load new data parts from storage into their cache without requiring a query to trigger such an action. -Default value: 0. \ No newline at end of file +Default value: 0. diff --git a/docs/ja/operations/settings/merge-tree-settings.md b/docs/ja/operations/settings/merge-tree-settings.md index d744716a28f..2cfd4773aa9 100644 --- a/docs/ja/operations/settings/merge-tree-settings.md +++ b/docs/ja/operations/settings/merge-tree-settings.md @@ -1058,7 +1058,7 @@ LZ4 または ZSTD の圧縮率は平均20〜40％改善します。 ## min_free_disk_bytes_to_perform_insert -データを挿入するためにディスクスペースに空いているべき最小バイト数。利用可能なバイトが `min_free_disk_bytes_to_throw_insert` 未満の場合は例外がスローされ、挿入が実行されません。この設定は以下を念頭に置いています。 +データを挿入するためにディスクスペースに空いているべき最小バイト数。利用可能なバイトが `min_free_disk_bytes_to_perform_insert` 未満の場合は例外がスローされ、挿入が実行されません。この設定は以下を念頭に置いています。 - `keep_free_space_bytes` 設定を考慮します。 - `INSERT` 操作によって書き込まれるデータ量は考慮しません。 - 正の（ゼロでない）バイト数が指定された場合にのみチェックされます From 136f5c482c01bf9129480bfe93a6d13da89a79e6 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov <48961922+Avogar@users.noreply.github.com> Date: Wed, 4 Dec 2024 17:36:18 +0100 Subject: [PATCH 349/502] Fix style --- src/TableFunctions/ITableFunctionFileLike.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index ef8226ea62e..b7956a74ddc 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -6,6 +6,12 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + class ColumnsDescription; class Context; From 7fc60d84cc9e060428900ed06f74b15db7e8a337 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 4 Dec 2024 17:32:06 +0100 Subject: [PATCH 350/502] Better deletion of keeper metadata in s3queue --- .../ObjectStorageQueueMetadata.cpp | 159 ++++++++++++++++++ .../ObjectStorageQueueMetadata.h | 3 + .../ObjectStorageQueueMetadataFactory.cpp | 27 +-- .../ObjectStorageQueueMetadataFactory.h | 16 +- .../ObjectStorageQueueSettings.cpp | 4 +- .../StorageObjectStorageQueue.cpp | 9 +- .../integration/test_storage_s3_queue/test.py | 124 ++++++++++++++ 7 files changed, 314 insertions(+), 28 deletions(-) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp index 6aac853b011..6f536d78ac0 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -438,6 +439,164 @@ ObjectStorageQueueTableMetadata ObjectStorageQueueMetadata::syncWithKeeper( "of wrong zookeeper path or because of logical error"); } +namespace +{ + struct Info + { + std::string hostname; + std::string table_id; + + bool operator ==(const Info & other) const + { + return hostname == other.hostname && table_id == other.table_id; + } + + static Info create(const StorageID & storage_id) + { + Info self; + self.hostname = DNSResolver::instance().getHostName(); + self.table_id = storage_id.hasUUID() ? toString(storage_id.uuid) : storage_id.getFullTableName(); + return self; + } + + std::string serialize() const + { + WriteBufferFromOwnString buf; + size_t version = 0; + buf << version << "\n"; + buf << hostname << "\n"; + buf << table_id << "\n"; + return buf.str(); + } + + static Info deserialize(const std::string & str) + { + ReadBufferFromString buf(str); + Info info; + size_t version; + buf >> version >> "\n"; + buf >> info.hostname >> "\n"; + buf >> info.table_id >> "\n"; + return info; + } + }; +} + +void ObjectStorageQueueMetadata::registerIfNot(const StorageID & storage_id) +{ + const auto registry_path = zookeeper_path / "registry"; + const auto self = Info::create(storage_id); + + Coordination::Error code; + for (size_t i = 0; i < 1000; ++i) + { + Coordination::Stat stat; + std::string registry_str; + auto zk_client = getZooKeeper(); + + bool node_exists = zk_client->tryGet(registry_path, registry_str, &stat); + if (node_exists) + { + Strings registered; + splitInto<','>(registered, registry_str); + + for (const auto & elem : registered) + { + if (elem.empty()) + continue; + + auto info = Info::deserialize(elem); + if (info == self) + { + LOG_TRACE(log, "Table {} is already registered", self.table_id); + return; + } + } + } + + LOG_TRACE(log, "Adding {} to registry", self.table_id); + + if (node_exists) + { + auto new_registry_str = registry_str + "," + self.serialize(); + code = zk_client->trySet(registry_path, new_registry_str, stat.version); + } + else + { + code = zk_client->tryCreate(registry_path, self.serialize(), zkutil::CreateMode::Persistent); + } + + if (code == Coordination::Error::ZOK) + return; + + if (code == Coordination::Error::ZBADVERSION + || code == Coordination::Error::ZSESSIONEXPIRED) + continue; + + throw zkutil::KeeperException(code); + } + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot register in keeper. Last error: {}", code); +} + +size_t ObjectStorageQueueMetadata::unregister(const StorageID & storage_id) +{ + const auto registry_path = zookeeper_path / "registry"; + const auto self = Info::create(storage_id); + + Coordination::Error code; + for (size_t i = 0; i < 1000; ++i) + { + Coordination::Stat stat; + std::string registry_str; + auto zk_client = getZooKeeper(); + + bool node_exists = zk_client->tryGet(registry_path, registry_str, &stat); + if (!node_exists) + { + LOG_WARNING(log, "Cannot unregister: registry does not exist"); + chassert(false); + return 0; + } + + Strings registered; + splitInto<','>(registered, registry_str); + + bool found = false; + std::string new_registry_str; + size_t count = 0; + for (const auto & elem : registered) + { + if (elem.empty()) + continue; + + auto info = Info::deserialize(elem); + if (info == self) + found = true; + else + { + if (!new_registry_str.empty()) + new_registry_str += ","; + new_registry_str += elem; + count += 1; + } + } + if (!found) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot unregister: not registered"); + + code = zk_client->trySet(registry_path, new_registry_str, stat.version); + + if (code == Coordination::Error::ZOK) + return count; + + if (code == Coordination::Error::ZBADVERSION + || code == Coordination::Error::ZSESSIONEXPIRED) + continue; + + throw zkutil::KeeperException(code); + } + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot unregister in keeper. Last error: {}", code); +} + void ObjectStorageQueueMetadata::cleanupThreadFunc() { /// A background task is responsible for maintaining diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h index 8da8a4c367d..aed610a8230 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h @@ -71,6 +71,9 @@ public: bool is_attach, LoggerPtr log); + void registerIfNot(const StorageID & storage_id); + size_t unregister(const StorageID & storage_id); + void shutdown(); FileMetadataPtr getFileMetadata(const std::string & path, ObjectStorageQueueOrderedFileMetadata::BucketInfoPtr bucket_info = {}); diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp index ba98711eff9..f9902e556f0 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp @@ -16,7 +16,8 @@ ObjectStorageQueueMetadataFactory & ObjectStorageQueueMetadataFactory::instance( ObjectStorageQueueMetadataFactory::FilesMetadataPtr ObjectStorageQueueMetadataFactory::getOrCreate( const std::string & zookeeper_path, - ObjectStorageQueueMetadataPtr metadata) + ObjectStorageQueueMetadataPtr metadata, + const StorageID & storage_id) { std::lock_guard lock(mutex); auto it = metadata_by_path.find(zookeeper_path); @@ -27,16 +28,16 @@ ObjectStorageQueueMetadataFactory::FilesMetadataPtr ObjectStorageQueueMetadataFa else { auto & metadata_from_table = metadata->getTableMetadata(); - auto & metadata_from_keeper = it->second.metadata->getTableMetadata(); + auto & metadata_from_keeper = it->second->getTableMetadata(); metadata_from_table.checkEquals(metadata_from_keeper); - - it->second.ref_count += 1; } - return it->second.metadata; + + it->second->registerIfNot(storage_id); + return it->second; } -void ObjectStorageQueueMetadataFactory::remove(const std::string & zookeeper_path) +void ObjectStorageQueueMetadataFactory::remove(const std::string & zookeeper_path, const StorageID & storage_id) { std::lock_guard lock(mutex); auto it = metadata_by_path.find(zookeeper_path); @@ -44,17 +45,19 @@ void ObjectStorageQueueMetadataFactory::remove(const std::string & zookeeper_pat if (it == metadata_by_path.end()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Metadata with zookeeper path {} does not exist", zookeeper_path); - chassert(it->second.ref_count > 0); - if (--it->second.ref_count == 0) + const size_t registry_size = it->second->unregister(storage_id); + LOG_TRACE(log, "Remaining registry size: {}", registry_size); + + if (registry_size == 0) { try { auto zk_client = Context::getGlobalContextInstance()->getZooKeeper(); - zk_client->tryRemove(it->first); + zk_client->removeRecursive(it->first); } catch (...) { - tryLogCurrentException(__PRETTY_FUNCTION__); + tryLogCurrentException(log); } metadata_by_path.erase(it); @@ -64,8 +67,8 @@ void ObjectStorageQueueMetadataFactory::remove(const std::string & zookeeper_pat std::unordered_map ObjectStorageQueueMetadataFactory::getAll() { std::unordered_map result; - for (const auto & [zk_path, metadata_and_ref_count] : metadata_by_path) - result.emplace(zk_path, metadata_and_ref_count.metadata); + for (const auto & [zk_path, metadata] : metadata_by_path) + result.emplace(zk_path, metadata); return result; } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h index bd2455097ec..0b35d42a9a1 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h @@ -14,25 +14,19 @@ public: FilesMetadataPtr getOrCreate( const std::string & zookeeper_path, - ObjectStorageQueueMetadataPtr metadata); + ObjectStorageQueueMetadataPtr metadata, + const StorageID & storage_id); - void remove(const std::string & zookeeper_path); + void remove(const std::string & zookeeper_path, const StorageID & storage_id); std::unordered_map getAll(); private: - struct Metadata - { - explicit Metadata(std::shared_ptr metadata_) : metadata(metadata_), ref_count(1) {} - - std::shared_ptr metadata; - /// TODO: the ref count should be kept in keeper, because of the case with distributed processing. - size_t ref_count = 0; - }; - using MetadataByPath = std::unordered_map; + using MetadataByPath = std::unordered_map>; MetadataByPath metadata_by_path; std::mutex mutex; + LoggerPtr log = getLogger("QueueMetadataFactory"); }; } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp index 47eb57f918d..060f1cd2dd5 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp @@ -30,8 +30,8 @@ namespace ErrorCodes DECLARE(UInt64, tracked_files_limit, 1000, "For unordered mode. Max set size for tracking processed files in ZooKeeper", 0) \ DECLARE(UInt64, tracked_file_ttl_sec, 0, "Maximum number of seconds to store processed files in ZooKeeper node (store forever by default)", 0) \ DECLARE(UInt64, polling_min_timeout_ms, 1000, "Minimal timeout before next polling", 0) \ - DECLARE(UInt64, polling_max_timeout_ms, 10 * 60 * 1000, "Maximum timeout before next polling", 0) \ - DECLARE(UInt64, polling_backoff_ms, 30 * 1000, "Polling backoff", 0) \ + DECLARE(UInt64, polling_max_timeout_ms, 10000, "Maximum timeout before next polling", 0) \ + DECLARE(UInt64, polling_backoff_ms, 1000, "Polling backoff", 0) \ DECLARE(UInt32, cleanup_interval_min_ms, 60000, "For unordered mode. Polling backoff min for cleanup", 0) \ DECLARE(UInt32, cleanup_interval_max_ms, 60000, "For unordered mode. Polling backoff max for cleanup", 0) \ DECLARE(UInt32, buckets, 0, "Number of buckets for Ordered mode parallel processing", 0) \ diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 6630d5a6c8f..eac03304999 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -214,9 +214,12 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( zk_path, *queue_settings_, storage_metadata.getColumns(), configuration_->format, context_, is_attach, log); auto queue_metadata = std::make_unique( - zk_path, std::move(table_metadata), (*queue_settings_)[ObjectStorageQueueSetting::cleanup_interval_min_ms], (*queue_settings_)[ObjectStorageQueueSetting::cleanup_interval_max_ms]); + zk_path, + std::move(table_metadata), + (*queue_settings_)[ObjectStorageQueueSetting::cleanup_interval_min_ms], + (*queue_settings_)[ObjectStorageQueueSetting::cleanup_interval_max_ms]); - files_metadata = ObjectStorageQueueMetadataFactory::instance().getOrCreate(zk_path, std::move(queue_metadata)); + files_metadata = ObjectStorageQueueMetadataFactory::instance().getOrCreate(zk_path, std::move(queue_metadata), table_id_); task = getContext()->getSchedulePool().createTask("ObjectStorageQueueStreamingTask", [this] { threadFunc(); }); } @@ -248,7 +251,7 @@ void StorageObjectStorageQueue::shutdown(bool is_drop) void StorageObjectStorageQueue::drop() { - ObjectStorageQueueMetadataFactory::instance().remove(zk_path); + ObjectStorageQueueMetadataFactory::instance().remove(zk_path, getStorageID()); } bool StorageObjectStorageQueue::supportsSubsetOfColumns(const ContextPtr & context_) const diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index aecc1df491b..09972872ef3 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -2276,3 +2276,127 @@ def test_alter_settings(started_cluster): check_int_settings(node, int_settings) check_string_settings(node, string_settings) + + +def test_registry(started_cluster): + node1 = started_cluster.instances["node1"] + node2 = started_cluster.instances["node2"] + + table_name = f"test_registry_{uuid.uuid4().hex[:8]}" + db_name = f"db_{table_name}" + dst_table_name = f"{table_name}_dst" + keeper_path = f"/clickhouse/test_{table_name}" + files_path = f"{table_name}_data" + files_to_generate = 1000 + + node1.query(f"DROP DATABASE IF EXISTS {db_name}") + node2.query(f"DROP DATABASE IF EXISTS {db_name}") + + node1.query( + f"CREATE DATABASE {db_name} ENGINE=Replicated('/clickhouse/databases/replicateddb', 'shard1', 'node1')" + ) + node2.query( + f"CREATE DATABASE {db_name} ENGINE=Replicated('/clickhouse/databases/replicateddb', 'shard1', 'node2')" + ) + + create_table( + started_cluster, + node1, + table_name, + "ordered", + files_path, + additional_settings={"keeper_path": keeper_path, "buckets": 3}, + database_name=db_name, + ) + + zk = started_cluster.get_kazoo_client("zoo1") + registry, stat = zk.get(f"{keeper_path}/registry/") + + uuid1 = node1.query( + f"SELECT uuid FROM system.tables WHERE database = '{db_name}' and table = '{table_name}'" + ).strip() + assert uuid1 in str(registry) + + expected = [f"0\\nnode1\\n{uuid1}\\n", f"0\\nnode2\\n{uuid1}\\n"] + + for elem in expected: + assert elem in str(registry) + + total_values = generate_random_files( + started_cluster, files_path, files_to_generate, start_ind=0, row_num=1 + ) + + create_mv(node1, f"{db_name}.{table_name}", dst_table_name) + create_mv(node2, f"{db_name}.{table_name}", dst_table_name) + + def get_count(): + return int( + node1.query( + f"SELECT count() FROM clusterAllReplicas(cluster, default.{dst_table_name})" + ) + ) + + expected_rows = files_to_generate + for _ in range(20): + if expected_rows == get_count(): + break + time.sleep(1) + assert expected_rows == get_count() + + table_name_2 = f"test_registry_{uuid.uuid4().hex[:8]}_2" + create_table( + started_cluster, + node1, + table_name_2, + "ordered", + files_path, + additional_settings={"keeper_path": keeper_path, "buckets": 3}, + database_name=db_name, + ) + + registry, stat = zk.get(f"{keeper_path}/registry/") + + uuid2 = node1.query( + f"SELECT uuid FROM system.tables WHERE database = '{db_name}' and table = '{table_name_2}'" + ).strip() + + assert uuid1 in str(registry) + assert uuid2 in str(registry) + + expected = [ + f"0\\nnode1\\n{uuid1}\\n", + f"0\\nnode2\\n{uuid1}\\n", + f"0\\nnode1\\n{uuid2}\\n", + f"0\\nnode2\\n{uuid2}\\n", + ] + + for elem in expected: + assert elem in str(registry) + + node1.restart_clickhouse() + node2.restart_clickhouse() + + registry, stat = zk.get(f"{keeper_path}/registry/") + + assert uuid1 in str(registry) + assert uuid2 in str(registry) + + node1.query(f"DROP TABLE {db_name}.{table_name_2} SYNC") + + assert zk.exists(keeper_path) is not None + registry, stat = zk.get(f"{keeper_path}/registry/") + + assert uuid1 in str(registry) + assert uuid2 not in str(registry) + + expected = [ + f"0\\nnode1\\n{uuid1}\\n", + f"0\\nnode2\\n{uuid1}\\n", + ] + + for elem in expected: + assert elem in str(registry) + + node1.query(f"DROP TABLE {db_name}.{table_name} SYNC") + + assert zk.exists(keeper_path) is None From ac0fb050f1dac9643d9d67580ccd24efc89e995e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 28 Nov 2024 10:42:59 +0100 Subject: [PATCH 351/502] Add ability to sync ZooKeeper before reloading the config Otherwise it is not deterministic wether ClickHouse will see this change, i.e. if you created znode just before `SYSTEM RELOAD CONFIG` it is not 100% that the ClickHouse will see it. Refs: apache/zookeeper@8ee17324ceb0fd0ace7d87c3bd095ee20c0466aa Signed-off-by: Azat Khuzhin --- programs/server/Server.cpp | 7 ++++++- src/Common/Config/ConfigProcessor.cpp | 6 +++++- src/Common/Config/ConfigProcessor.h | 5 +++-- src/Common/Config/ConfigReloader.cpp | 9 ++++++++- src/Common/ZooKeeper/ZooKeeperNodeCache.cpp | 5 +++++ src/Common/ZooKeeper/ZooKeeperNodeCache.h | 2 ++ src/Core/ServerSettings.cpp | 1 + .../configs/config_zk_include_test.xml | 2 ++ 8 files changed, 32 insertions(+), 5 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 6b5aa1ac0b2..8f97e2929b1 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1245,9 +1245,14 @@ try if (loaded_config.has_zk_includes) { auto old_configuration = loaded_config.configuration; + bool config_reload_sync_zookeeper = old_configuration->getBool("config_reload_sync_zookeeper", false); ConfigProcessor config_processor(config_path); loaded_config = config_processor.loadConfigWithZooKeeperIncludes( - main_config_zk_node_cache, main_config_zk_changed_event, /* fallback_to_preprocessed = */ true); + main_config_zk_node_cache, + main_config_zk_changed_event, + /*fallback_to_preprocessed =*/ true, + /*is_config_changed=*/ true, + config_reload_sync_zookeeper); config_processor.savePreprocessedConfig(loaded_config, path_str); config().removeConfiguration(old_configuration.get()); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index f4dd1cfa3fc..c0691414483 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -811,13 +811,17 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes( zkutil::ZooKeeperNodeCache & zk_node_cache, const zkutil::EventPtr & zk_changed_event, bool fallback_to_preprocessed, - bool is_config_changed) + bool is_config_changed, + bool sync_zookeeper) { XMLDocumentPtr config_xml; bool has_zk_includes; bool processed_successfully = false; try { + if (sync_zookeeper) + zk_node_cache.sync(); + config_xml = processConfig(&has_zk_includes, &zk_node_cache, zk_changed_event, is_config_changed); processed_successfully = true; } diff --git a/src/Common/Config/ConfigProcessor.h b/src/Common/Config/ConfigProcessor.h index 373e0809c5d..87f87a7050a 100644 --- a/src/Common/Config/ConfigProcessor.h +++ b/src/Common/Config/ConfigProcessor.h @@ -96,8 +96,9 @@ public: LoadedConfig loadConfigWithZooKeeperIncludes( zkutil::ZooKeeperNodeCache & zk_node_cache, const zkutil::EventPtr & zk_changed_event, - bool fallback_to_preprocessed = false, - bool is_config_changed = true); + bool fallback_to_preprocessed, + bool is_config_changed, + bool sync_zookeeper); /// Save preprocessed config to specified directory. /// If preprocessed_dir is empty - calculate from loaded_config.path + /preprocessed_configs/ diff --git a/src/Common/Config/ConfigReloader.cpp b/src/Common/Config/ConfigReloader.cpp index a9a02ba6784..84ead4c4c28 100644 --- a/src/Common/Config/ConfigReloader.cpp +++ b/src/Common/Config/ConfigReloader.cpp @@ -123,8 +123,15 @@ std::optional ConfigReloader::reloadIfNewer(bool { loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true, is_config_changed); if (loaded_config.has_zk_includes) + { + bool config_reload_sync_zookeeper = loaded_config.configuration->getBool("config_reload_sync_zookeeper", false); loaded_config = config_processor.loadConfigWithZooKeeperIncludes( - zk_node_cache, zk_changed_event, fallback_to_preprocessed, is_config_changed); + zk_node_cache, + zk_changed_event, + fallback_to_preprocessed, + is_config_changed, + config_reload_sync_zookeeper); + } } catch (const Coordination::Exception & e) { diff --git a/src/Common/ZooKeeper/ZooKeeperNodeCache.cpp b/src/Common/ZooKeeper/ZooKeeperNodeCache.cpp index 36a211b5768..3cad51a7b8a 100644 --- a/src/Common/ZooKeeper/ZooKeeperNodeCache.cpp +++ b/src/Common/ZooKeeper/ZooKeeperNodeCache.cpp @@ -105,4 +105,9 @@ ZooKeeperNodeCache::ZNode ZooKeeperNodeCache::get(const std::string & path, Coor return result; } +void ZooKeeperNodeCache::sync() +{ + get_zookeeper()->sync("/"); +} + } diff --git a/src/Common/ZooKeeper/ZooKeeperNodeCache.h b/src/Common/ZooKeeper/ZooKeeperNodeCache.h index 2167b8bd11b..06633bb87a7 100644 --- a/src/Common/ZooKeeper/ZooKeeperNodeCache.h +++ b/src/Common/ZooKeeper/ZooKeeperNodeCache.h @@ -46,6 +46,8 @@ public: ZNode get(const std::string & path, EventPtr watch_event); ZNode get(const std::string & path, Coordination::WatchCallback watch_callback); + void sync(); + private: GetZooKeeper get_zookeeper; diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 5f48e5ea891..57af6fb9e69 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -195,6 +195,7 @@ namespace DB DECLARE(String, mutation_workload, "default", "Name of workload to be used to access resources for all mutations (may be overridden by a merge tree setting)", 0) \ DECLARE(Bool, prepare_system_log_tables_on_startup, false, "If true, ClickHouse creates all configured `system.*_log` tables before the startup. It can be helpful if some startup scripts depend on these tables.", 0) \ DECLARE(UInt64, config_reload_interval_ms, 2000, "How often clickhouse will reload config and check for new changes", 0) \ + DECLARE(Bool, config_reload_sync_zookeeper, false, "Sync ZooKeeper before reloading config", 0) \ DECLARE(UInt64, memory_worker_period_ms, 0, "Tick period of background memory worker which corrects memory tracker memory usages and cleans up unused pages during higher memory usage. If set to 0, default value will be used depending on the memory usage source", 0) \ DECLARE(Bool, disable_insertion_and_mutation, false, "Disable all insert/alter/delete queries. This setting will be enabled if someone needs read-only nodes to prevent insertion and mutation affect reading performance.", 0) \ DECLARE(UInt64, parts_kill_delay_period, 30, "Period to completely remove parts for SharedMergeTree. Only available in ClickHouse Cloud", 0) \ diff --git a/tests/integration/test_config_substitutions/configs/config_zk_include_test.xml b/tests/integration/test_config_substitutions/configs/config_zk_include_test.xml index 743770c3024..05c3ed332a8 100644 --- a/tests/integration/test_config_substitutions/configs/config_zk_include_test.xml +++ b/tests/integration/test_config_substitutions/configs/config_zk_include_test.xml @@ -1,5 +1,7 @@ + true + 44 From c9291949bab3bd0730d183c60f8811e7422d97b3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 28 Nov 2024 10:54:38 +0100 Subject: [PATCH 352/502] Enable config_reload_sync_zookeeper for test_config_substitutions This should fix flakiness of `test_config_substitutions/test.py::test_config_multiple_zk_substitutions` [1]. [1]: https://s3.amazonaws.com/clickhouse-test-reports/71406/af822de0bf896dcb0a28f55889d12611bd38171a/integration_tests__asan__old_analyzer__[1_6].html Signed-off-by: Azat Khuzhin --- .../configs/config_zk_include_test.xml | 4 +--- .../test_config_substitutions/configs/overrides.xml | 4 ++++ tests/integration/test_config_substitutions/test.py | 5 ++++- 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_config_substitutions/configs/overrides.xml diff --git a/tests/integration/test_config_substitutions/configs/config_zk_include_test.xml b/tests/integration/test_config_substitutions/configs/config_zk_include_test.xml index 05c3ed332a8..500cab81184 100644 --- a/tests/integration/test_config_substitutions/configs/config_zk_include_test.xml +++ b/tests/integration/test_config_substitutions/configs/config_zk_include_test.xml @@ -1,7 +1,5 @@ - true - 44 @@ -11,4 +9,4 @@ - + diff --git a/tests/integration/test_config_substitutions/configs/overrides.xml b/tests/integration/test_config_substitutions/configs/overrides.xml new file mode 100644 index 00000000000..5106f4dc2f7 --- /dev/null +++ b/tests/integration/test_config_substitutions/configs/overrides.xml @@ -0,0 +1,4 @@ + + + true + diff --git a/tests/integration/test_config_substitutions/test.py b/tests/integration/test_config_substitutions/test.py index bb208bcdd86..fee68d028f4 100644 --- a/tests/integration/test_config_substitutions/test.py +++ b/tests/integration/test_config_substitutions/test.py @@ -17,7 +17,10 @@ node3 = cluster.add_instance( user_configs=[ "configs/config_zk.xml", ], - main_configs=["configs/config_zk_include_test.xml"], + main_configs=[ + "configs/config_zk_include_test.xml", + "configs/overrides.xml", + ], with_zookeeper=True, ) node4 = cluster.add_instance( From 86fb8f647fa13f1554b700f24b7d8523f6bc5dea Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 28 Nov 2024 15:09:40 +0100 Subject: [PATCH 353/502] Read config_reload_sync_zookeeper only from the server config Previously it had been read from each config, i.e. for users it should be defined in users.xml/yaml. Signed-off-by: Azat Khuzhin --- programs/keeper/Keeper.cpp | 1 + programs/server/Server.cpp | 5 +++-- src/Access/UsersConfigAccessStorage.cpp | 12 ++++++++++++ src/Common/Config/ConfigReloader.cpp | 16 +++++++--------- src/Common/Config/ConfigReloader.h | 3 +++ 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 936ce15f4c9..2d108a00f50 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -579,6 +579,7 @@ try getKeeperPath(config()), std::move(unused_cache), unused_event, + config().getBool("config_reload_sync_zookeeper", false), [&](ConfigurationPtr config, bool /* initial_loading */) { updateLevels(*config, logger()); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8f97e2929b1..d845eec1762 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -284,6 +284,7 @@ namespace ServerSetting extern const ServerSettingsString primary_index_cache_policy; extern const ServerSettingsUInt64 primary_index_cache_size; extern const ServerSettingsDouble primary_index_cache_size_ratio; + extern const ServerSettingsBool config_reload_sync_zookeeper; } } @@ -1245,14 +1246,13 @@ try if (loaded_config.has_zk_includes) { auto old_configuration = loaded_config.configuration; - bool config_reload_sync_zookeeper = old_configuration->getBool("config_reload_sync_zookeeper", false); ConfigProcessor config_processor(config_path); loaded_config = config_processor.loadConfigWithZooKeeperIncludes( main_config_zk_node_cache, main_config_zk_changed_event, /*fallback_to_preprocessed =*/ true, /*is_config_changed=*/ true, - config_reload_sync_zookeeper); + server_settings[ServerSetting::config_reload_sync_zookeeper]); config_processor.savePreprocessedConfig(loaded_config, path_str); config().removeConfiguration(old_configuration.get()); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); @@ -1686,6 +1686,7 @@ try config().getString("path", DBMS_DEFAULT_PATH), std::move(main_config_zk_node_cache), main_config_zk_changed_event, + server_settings[ServerSetting::config_reload_sync_zookeeper], [&, config_file = config().getString("config-file", "config.xml")](ConfigurationPtr config, bool initial_loading) { if (!initial_loading) diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index eddc7ca1e0e..13533d7fb26 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,11 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; } +namespace ServerSetting +{ + extern const ServerSettingsBool config_reload_sync_zookeeper; +} + namespace { @@ -880,15 +886,21 @@ void UsersConfigAccessStorage::load( const String & preprocessed_dir, const zkutil::GetZooKeeper & get_zookeeper_function) { + bool sync_zookeeper = false; + if (auto context = Context::getGlobalContextInstance()) + sync_zookeeper = context->getServerSettings()[ServerSetting::config_reload_sync_zookeeper]; + std::lock_guard lock{load_mutex}; path = std::filesystem::path{users_config_path}.lexically_normal(); config_reloader.reset(); + config_reloader = std::make_unique( users_config_path, std::vector{{include_from_path}}, preprocessed_dir, zkutil::ZooKeeperNodeCache(get_zookeeper_function), std::make_shared(), + sync_zookeeper, [&](Poco::AutoPtr new_config, bool /*initial_loading*/) { Settings::checkNoSettingNamesAtTopLevel(*new_config, users_config_path); diff --git a/src/Common/Config/ConfigReloader.cpp b/src/Common/Config/ConfigReloader.cpp index 84ead4c4c28..00f5e640fc0 100644 --- a/src/Common/Config/ConfigReloader.cpp +++ b/src/Common/Config/ConfigReloader.cpp @@ -1,11 +1,10 @@ -#include "ConfigReloader.h" - #include #include #include -#include "ConfigProcessor.h" -#include +#include +#include #include +#include namespace fs = std::filesystem; @@ -19,6 +18,7 @@ ConfigReloader::ConfigReloader( const std::string & preprocessed_dir_, zkutil::ZooKeeperNodeCache && zk_node_cache_, const zkutil::EventPtr & zk_changed_event_, + bool sync_zookeeper_, Updater && updater_) : config_path(config_path_) , extra_paths(extra_paths_) @@ -26,6 +26,7 @@ ConfigReloader::ConfigReloader( , zk_node_cache(std::move(zk_node_cache_)) , zk_changed_event(zk_changed_event_) , updater(std::move(updater_)) + , sync_zookeeper(sync_zookeeper_) { auto config = reloadIfNewer(/* force = */ true, /* throw_on_error = */ true, /* fallback_to_preprocessed = */ true, /* initial_loading = */ true); @@ -34,7 +35,7 @@ ConfigReloader::ConfigReloader( else reload_interval = DEFAULT_RELOAD_INTERVAL; - LOG_TRACE(log, "Config reload interval set to {}ms", reload_interval.count()); + LOG_TRACE(log, "Config reload interval set to {}ms (sync_zookeeper: {})", reload_interval.count(), sync_zookeeper); } void ConfigReloader::start() @@ -123,15 +124,12 @@ std::optional ConfigReloader::reloadIfNewer(bool { loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true, is_config_changed); if (loaded_config.has_zk_includes) - { - bool config_reload_sync_zookeeper = loaded_config.configuration->getBool("config_reload_sync_zookeeper", false); loaded_config = config_processor.loadConfigWithZooKeeperIncludes( zk_node_cache, zk_changed_event, fallback_to_preprocessed, is_config_changed, - config_reload_sync_zookeeper); - } + sync_zookeeper); } catch (const Coordination::Exception & e) { diff --git a/src/Common/Config/ConfigReloader.h b/src/Common/Config/ConfigReloader.h index 89ef0fd8a0b..11d5fe246c9 100644 --- a/src/Common/Config/ConfigReloader.h +++ b/src/Common/Config/ConfigReloader.h @@ -35,6 +35,7 @@ public: const std::string & preprocessed_dir, zkutil::ZooKeeperNodeCache && zk_node_cache, const zkutil::EventPtr & zk_changed_event, + bool sync_zookeeper_, Updater && updater); ~ConfigReloader(); @@ -86,6 +87,8 @@ private: /// Locked inside reloadIfNewer. std::mutex reload_mutex; + + bool sync_zookeeper; }; } From f1e909e3096f000ce666f0dcab42bbb376966bda Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 1 Dec 2024 17:12:33 +0100 Subject: [PATCH 354/502] Remove config_reload_sync_zookeeper (issue sync unconditionally) Signed-off-by: Azat Khuzhin --- programs/keeper/Keeper.cpp | 1 - programs/server/Server.cpp | 8 +------- src/Access/UsersConfigAccessStorage.cpp | 12 ------------ src/Common/Config/ConfigProcessor.cpp | 7 ++----- src/Common/Config/ConfigProcessor.h | 5 ++--- src/Common/Config/ConfigReloader.cpp | 17 ++++++----------- src/Common/Config/ConfigReloader.h | 3 --- src/Core/ServerSettings.cpp | 1 - .../configs/overrides.xml | 4 ---- .../test_config_substitutions/test.py | 1 - 10 files changed, 11 insertions(+), 48 deletions(-) delete mode 100644 tests/integration/test_config_substitutions/configs/overrides.xml diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 2d108a00f50..936ce15f4c9 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -579,7 +579,6 @@ try getKeeperPath(config()), std::move(unused_cache), unused_event, - config().getBool("config_reload_sync_zookeeper", false), [&](ConfigurationPtr config, bool /* initial_loading */) { updateLevels(*config, logger()); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index d845eec1762..6b5aa1ac0b2 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -284,7 +284,6 @@ namespace ServerSetting extern const ServerSettingsString primary_index_cache_policy; extern const ServerSettingsUInt64 primary_index_cache_size; extern const ServerSettingsDouble primary_index_cache_size_ratio; - extern const ServerSettingsBool config_reload_sync_zookeeper; } } @@ -1248,11 +1247,7 @@ try auto old_configuration = loaded_config.configuration; ConfigProcessor config_processor(config_path); loaded_config = config_processor.loadConfigWithZooKeeperIncludes( - main_config_zk_node_cache, - main_config_zk_changed_event, - /*fallback_to_preprocessed =*/ true, - /*is_config_changed=*/ true, - server_settings[ServerSetting::config_reload_sync_zookeeper]); + main_config_zk_node_cache, main_config_zk_changed_event, /* fallback_to_preprocessed = */ true); config_processor.savePreprocessedConfig(loaded_config, path_str); config().removeConfiguration(old_configuration.get()); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); @@ -1686,7 +1681,6 @@ try config().getString("path", DBMS_DEFAULT_PATH), std::move(main_config_zk_node_cache), main_config_zk_changed_event, - server_settings[ServerSetting::config_reload_sync_zookeeper], [&, config_file = config().getString("config-file", "config.xml")](ConfigurationPtr config, bool initial_loading) { if (!initial_loading) diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 13533d7fb26..eddc7ca1e0e 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -43,11 +42,6 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; } -namespace ServerSetting -{ - extern const ServerSettingsBool config_reload_sync_zookeeper; -} - namespace { @@ -886,21 +880,15 @@ void UsersConfigAccessStorage::load( const String & preprocessed_dir, const zkutil::GetZooKeeper & get_zookeeper_function) { - bool sync_zookeeper = false; - if (auto context = Context::getGlobalContextInstance()) - sync_zookeeper = context->getServerSettings()[ServerSetting::config_reload_sync_zookeeper]; - std::lock_guard lock{load_mutex}; path = std::filesystem::path{users_config_path}.lexically_normal(); config_reloader.reset(); - config_reloader = std::make_unique( users_config_path, std::vector{{include_from_path}}, preprocessed_dir, zkutil::ZooKeeperNodeCache(get_zookeeper_function), std::make_shared(), - sync_zookeeper, [&](Poco::AutoPtr new_config, bool /*initial_loading*/) { Settings::checkNoSettingNamesAtTopLevel(*new_config, users_config_path); diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index c0691414483..5bf7a8503fd 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -811,17 +811,14 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes( zkutil::ZooKeeperNodeCache & zk_node_cache, const zkutil::EventPtr & zk_changed_event, bool fallback_to_preprocessed, - bool is_config_changed, - bool sync_zookeeper) + bool is_config_changed) { XMLDocumentPtr config_xml; bool has_zk_includes; bool processed_successfully = false; try { - if (sync_zookeeper) - zk_node_cache.sync(); - + zk_node_cache.sync(); config_xml = processConfig(&has_zk_includes, &zk_node_cache, zk_changed_event, is_config_changed); processed_successfully = true; } diff --git a/src/Common/Config/ConfigProcessor.h b/src/Common/Config/ConfigProcessor.h index 87f87a7050a..373e0809c5d 100644 --- a/src/Common/Config/ConfigProcessor.h +++ b/src/Common/Config/ConfigProcessor.h @@ -96,9 +96,8 @@ public: LoadedConfig loadConfigWithZooKeeperIncludes( zkutil::ZooKeeperNodeCache & zk_node_cache, const zkutil::EventPtr & zk_changed_event, - bool fallback_to_preprocessed, - bool is_config_changed, - bool sync_zookeeper); + bool fallback_to_preprocessed = false, + bool is_config_changed = true); /// Save preprocessed config to specified directory. /// If preprocessed_dir is empty - calculate from loaded_config.path + /preprocessed_configs/ diff --git a/src/Common/Config/ConfigReloader.cpp b/src/Common/Config/ConfigReloader.cpp index 00f5e640fc0..a9a02ba6784 100644 --- a/src/Common/Config/ConfigReloader.cpp +++ b/src/Common/Config/ConfigReloader.cpp @@ -1,10 +1,11 @@ +#include "ConfigReloader.h" + #include #include #include -#include -#include -#include +#include "ConfigProcessor.h" #include +#include namespace fs = std::filesystem; @@ -18,7 +19,6 @@ ConfigReloader::ConfigReloader( const std::string & preprocessed_dir_, zkutil::ZooKeeperNodeCache && zk_node_cache_, const zkutil::EventPtr & zk_changed_event_, - bool sync_zookeeper_, Updater && updater_) : config_path(config_path_) , extra_paths(extra_paths_) @@ -26,7 +26,6 @@ ConfigReloader::ConfigReloader( , zk_node_cache(std::move(zk_node_cache_)) , zk_changed_event(zk_changed_event_) , updater(std::move(updater_)) - , sync_zookeeper(sync_zookeeper_) { auto config = reloadIfNewer(/* force = */ true, /* throw_on_error = */ true, /* fallback_to_preprocessed = */ true, /* initial_loading = */ true); @@ -35,7 +34,7 @@ ConfigReloader::ConfigReloader( else reload_interval = DEFAULT_RELOAD_INTERVAL; - LOG_TRACE(log, "Config reload interval set to {}ms (sync_zookeeper: {})", reload_interval.count(), sync_zookeeper); + LOG_TRACE(log, "Config reload interval set to {}ms", reload_interval.count()); } void ConfigReloader::start() @@ -125,11 +124,7 @@ std::optional ConfigReloader::reloadIfNewer(bool loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true, is_config_changed); if (loaded_config.has_zk_includes) loaded_config = config_processor.loadConfigWithZooKeeperIncludes( - zk_node_cache, - zk_changed_event, - fallback_to_preprocessed, - is_config_changed, - sync_zookeeper); + zk_node_cache, zk_changed_event, fallback_to_preprocessed, is_config_changed); } catch (const Coordination::Exception & e) { diff --git a/src/Common/Config/ConfigReloader.h b/src/Common/Config/ConfigReloader.h index 11d5fe246c9..89ef0fd8a0b 100644 --- a/src/Common/Config/ConfigReloader.h +++ b/src/Common/Config/ConfigReloader.h @@ -35,7 +35,6 @@ public: const std::string & preprocessed_dir, zkutil::ZooKeeperNodeCache && zk_node_cache, const zkutil::EventPtr & zk_changed_event, - bool sync_zookeeper_, Updater && updater); ~ConfigReloader(); @@ -87,8 +86,6 @@ private: /// Locked inside reloadIfNewer. std::mutex reload_mutex; - - bool sync_zookeeper; }; } diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 57af6fb9e69..5f48e5ea891 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -195,7 +195,6 @@ namespace DB DECLARE(String, mutation_workload, "default", "Name of workload to be used to access resources for all mutations (may be overridden by a merge tree setting)", 0) \ DECLARE(Bool, prepare_system_log_tables_on_startup, false, "If true, ClickHouse creates all configured `system.*_log` tables before the startup. It can be helpful if some startup scripts depend on these tables.", 0) \ DECLARE(UInt64, config_reload_interval_ms, 2000, "How often clickhouse will reload config and check for new changes", 0) \ - DECLARE(Bool, config_reload_sync_zookeeper, false, "Sync ZooKeeper before reloading config", 0) \ DECLARE(UInt64, memory_worker_period_ms, 0, "Tick period of background memory worker which corrects memory tracker memory usages and cleans up unused pages during higher memory usage. If set to 0, default value will be used depending on the memory usage source", 0) \ DECLARE(Bool, disable_insertion_and_mutation, false, "Disable all insert/alter/delete queries. This setting will be enabled if someone needs read-only nodes to prevent insertion and mutation affect reading performance.", 0) \ DECLARE(UInt64, parts_kill_delay_period, 30, "Period to completely remove parts for SharedMergeTree. Only available in ClickHouse Cloud", 0) \ diff --git a/tests/integration/test_config_substitutions/configs/overrides.xml b/tests/integration/test_config_substitutions/configs/overrides.xml deleted file mode 100644 index 5106f4dc2f7..00000000000 --- a/tests/integration/test_config_substitutions/configs/overrides.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - true - diff --git a/tests/integration/test_config_substitutions/test.py b/tests/integration/test_config_substitutions/test.py index fee68d028f4..80a1fecb4ca 100644 --- a/tests/integration/test_config_substitutions/test.py +++ b/tests/integration/test_config_substitutions/test.py @@ -19,7 +19,6 @@ node3 = cluster.add_instance( ], main_configs=[ "configs/config_zk_include_test.xml", - "configs/overrides.xml", ], with_zookeeper=True, ) From 47397a45c585f1f0b835d5ea5afca592e7f29d9d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 1 Dec 2024 17:06:43 +0100 Subject: [PATCH 355/502] tests: make test_config_substitutions idempotent Now the following passed: $ bin=$(which clickhouse); g=$PWD; docker run --privileged -it --rm --volume=$bin:/clickhouse --volume=$bin:/usr/share/clickhouse_fresh --volume=/dev/null:/usr/bin/clickhouse-odbc-bridge --volume=/dev/null:/usr/share/clickhouse-odbc-bridge_fresh --volume=$g/programs/server:/clickhouse-config --volume=$g:/ClickHouse --volume=$g/docker/test/integration/runner/compose:/compose:ro --volume=clickhouse-integration-tests:/var/lib/docker -e PYTEST_ADDOPTS='-vvv -s --pdb --count 2 test_config_substitutions/test.py' --name ch clickhouse/integration-tests-runner Signed-off-by: Azat Khuzhin --- tests/integration/helpers/cluster.py | 15 +++ .../test_config_substitutions/test.py | 93 ++++++++++--------- 2 files changed, 66 insertions(+), 42 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 504a70d3865..46d2ea3f902 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -17,6 +17,7 @@ import subprocess import time import traceback import urllib.parse +from contextlib import contextmanager from functools import cache from pathlib import Path from typing import Any, List, Sequence, Tuple, Union @@ -4542,6 +4543,20 @@ class ClickHouseInstance: if key != "DSN": f.write(key + "=" + value + "\n") + @contextmanager + def with_replace_config(self, path, replacement): + """Create a copy of existing config (if exists) and revert on leaving the context""" + self.exec_in_container( + ["bash", "-c", f"test ! -f {path} || mv --no-clobber {path} {path}.bak"] + ) + self.exec_in_container( + ["bash", "-c", "echo '{}' > {}".format(replacement, path)] + ) + yield + self.exec_in_container( + ["bash", "-c", f"test ! -f {path}.bak || mv {path}.bak {path}"] + ) + def replace_config(self, path_to_config, replacement): self.exec_in_container( ["bash", "-c", "echo '{}' > {}".format(replacement, path_to_config)] diff --git a/tests/integration/test_config_substitutions/test.py b/tests/integration/test_config_substitutions/test.py index 80a1fecb4ca..659b6ac8e61 100644 --- a/tests/integration/test_config_substitutions/test.py +++ b/tests/integration/test_config_substitutions/test.py @@ -21,6 +21,7 @@ node3 = cluster.add_instance( "configs/config_zk_include_test.xml", ], with_zookeeper=True, + stay_alive=True, ) node4 = cluster.add_instance( "node4", @@ -133,7 +134,7 @@ def test_config(start_cluster): def test_config_from_env_overrides(start_cluster): - node7.replace_config( + with node7.with_replace_config( "/etc/clickhouse-server/users.d/000-users_with_env_subst.xml", """ @@ -155,13 +156,14 @@ def test_config_from_env_overrides(start_cluster): """, - ) - with pytest.raises( - QueryRuntimeException, - match="Failed to preprocess config '/etc/clickhouse-server/users.xml': Exception: Element has value and does not have 'replace' attribute, can't process from_env substitution", ): - node7.query("SYSTEM RELOAD CONFIG") - node7.replace_config( + with pytest.raises( + QueryRuntimeException, + match="Failed to preprocess config '/etc/clickhouse-server/users.xml': Exception: Element has value and does not have 'replace' attribute, can't process from_env substitution", + ): + node7.query("SYSTEM RELOAD CONFIG") + + with node7.with_replace_config( "/etc/clickhouse-server/users.d/000-users_with_env_subst.xml", """ @@ -183,26 +185,27 @@ def test_config_from_env_overrides(start_cluster): """, - ) - node7.query("SYSTEM RELOAD CONFIG") + ): + node7.query("SYSTEM RELOAD CONFIG") def test_config_merge_from_env_overrides(start_cluster): + node7.query("SYSTEM RELOAD CONFIG") assert ( node7.query( "SELECT value FROM system.server_settings WHERE name='max_thread_pool_size'" ) == "10000\n" ) - node7.replace_config( + with node7.with_replace_config( "/etc/clickhouse-server/config.d/010-server_with_env_subst.xml", """ 9000 """, - ) - node7.query("SYSTEM RELOAD CONFIG") + ): + node7.query("SYSTEM RELOAD CONFIG") def test_include_config(start_cluster): @@ -223,6 +226,7 @@ def test_include_config(start_cluster): def test_allow_databases(start_cluster): + node5.query("DROP DATABASE IF EXISTS db1 SYNC") node5.query("CREATE DATABASE db1") node5.query( "CREATE TABLE db1.test_table(date Date, k1 String, v1 Int32) ENGINE = MergeTree(date, (k1, date), 8192)" @@ -293,6 +297,9 @@ def test_allow_databases(start_cluster): def test_config_multiple_zk_substitutions(start_cluster): + # NOTE: background_pool_size cannot be decreased, so let's restart ClickHouse to make the test idempotent (i.e. runned multiple times) + node3.restart_clickhouse() + node3.query("SYSTEM RELOAD CONFIG") assert ( node3.query( "SELECT value FROM system.merge_tree_settings WHERE name='min_bytes_for_wide_part'" @@ -319,34 +326,36 @@ def test_config_multiple_zk_substitutions(start_cluster): ) zk = cluster.get_kazoo_client("zoo1") - zk.create( - path="/background_pool_size", - value=b"72", - makepath=True, - ) - - node3.replace_config( - "/etc/clickhouse-server/config.d/config_zk_include_test.xml", - """ - - - 44 - - - 1 - 1111 - - - - -""", - ) - - node3.query("SYSTEM RELOAD CONFIG") - - assert ( - node3.query( - "SELECT value FROM system.server_settings WHERE name='background_pool_size'" + try: + zk.create( + path="/background_pool_size", + value=b"72", + makepath=True, ) - == "72\n" - ) + + with node3.with_replace_config( + "/etc/clickhouse-server/config.d/config_zk_include_test.xml", + """ + + + 44 + + + 1 + 1111 + + + + + """, + ): + node3.query("SYSTEM RELOAD CONFIG") + + assert ( + node3.query( + "SELECT value FROM system.server_settings WHERE name='background_pool_size'" + ) + == "72\n" + ) + finally: + zk.delete(path="/background_pool_size") From 89dbb42874934ee3d2c934f020df51561139bdc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 4 Dec 2024 17:51:51 +0100 Subject: [PATCH 356/502] Remove more deps from SipHash --- src/Common/ColumnsHashing.h | 14 +++++++++++ src/Functions/PolygonUtils.h | 25 ------------------- src/Functions/array/arrayEnumerateRanked.cpp | 13 ++++++++++ src/Functions/array/arrayEnumerateRanked.h | 14 +---------- src/Functions/pointInPolygon.cpp | 22 ++++++++++++++++ src/IO/MMappedFileCache.cpp | 17 +++++++++++++ src/IO/MMappedFileCache.h | 11 +------- src/IO/UncompressedCache.cpp | 10 ++++++++ src/IO/UncompressedCache.h | 10 +------- src/Interpreters/AggregatedDataVariants.h | 2 +- src/Interpreters/AggregationCommon.h | 14 ----------- src/Interpreters/Aggregator.h | 4 +-- src/Interpreters/HashJoin/AddedColumns.h | 1 + src/Interpreters/HashJoin/HashJoin.h | 1 - src/Interpreters/HashJoin/KeyGetter.h | 1 + .../MergeTree/MergeTreeDataWriter.cpp | 2 +- 16 files changed, 85 insertions(+), 76 deletions(-) create mode 100644 src/IO/MMappedFileCache.cpp diff --git a/src/Common/ColumnsHashing.h b/src/Common/ColumnsHashing.h index cffce29cc75..b191c90136d 100644 --- a/src/Common/ColumnsHashing.h +++ b/src/Common/ColumnsHashing.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -26,6 +27,19 @@ namespace ErrorCodes namespace ColumnsHashing { +/// Hash a set of keys into a UInt128 value. +static inline UInt128 ALWAYS_INLINE hash128( /// NOLINT + size_t i, + size_t keys_size, + const ColumnRawPtrs & key_columns) +{ + SipHash hash; + for (size_t j = 0; j < keys_size; ++j) + key_columns[j]->updateHashWithValue(i, hash); + + return hash.get128(); +} + /// For the case when there is one numeric key. /// UInt8/16/32/64 for any type with corresponding bit width. template diff --git a/src/Functions/PolygonUtils.h b/src/Functions/PolygonUtils.h index 601ffcb00b4..be5cc239ba4 100644 --- a/src/Functions/PolygonUtils.h +++ b/src/Functions/PolygonUtils.h @@ -7,7 +7,6 @@ #include #include #include -#include #include /// Warning in boost::geometry during template strategy substitution. @@ -612,28 +611,4 @@ NO_INLINE ColumnPtr pointInPolygon(const IColumn & x, const IColumn & y, PointIn using Impl = TypeListChangeRoot; return Impl::call(x, y, impl); } - - -template -UInt128 sipHash128(Polygon && polygon) -{ - SipHash hash; - - auto hash_ring = [&hash](const auto & ring) - { - UInt32 size = static_cast(ring.size()); - hash.update(size); - hash.update(reinterpret_cast(ring.data()), size * sizeof(ring[0])); - }; - - hash_ring(polygon.outer()); - - const auto & inners = polygon.inners(); - hash.update(inners.size()); - for (auto & inner : inners) - hash_ring(inner); - - return hash.get128(); -} - } diff --git a/src/Functions/array/arrayEnumerateRanked.cpp b/src/Functions/array/arrayEnumerateRanked.cpp index 69d8954bfcf..a27bc81a99d 100644 --- a/src/Functions/array/arrayEnumerateRanked.cpp +++ b/src/Functions/array/arrayEnumerateRanked.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -12,6 +13,18 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +UInt128 hash128depths(const std::vector & indices, const ColumnRawPtrs & key_columns) +{ + SipHash hash; + for (size_t j = 0, keys_size = key_columns.size(); j < keys_size; ++j) + { + // Debug: const auto & field = (*key_columns[j])[indices[j]]; DUMP(j, indices[j], field); + key_columns[j]->updateHashWithValue(indices[j], hash); + } + + return hash.get128(); +} + ArraysDepths getArraysDepths(const ColumnsWithTypeAndName & arguments, const char * function_name) { const size_t num_arguments = arguments.size(); diff --git a/src/Functions/array/arrayEnumerateRanked.h b/src/Functions/array/arrayEnumerateRanked.h index 269a7db6e92..1694f279ed3 100644 --- a/src/Functions/array/arrayEnumerateRanked.h +++ b/src/Functions/array/arrayEnumerateRanked.h @@ -10,7 +10,6 @@ #include #include #include -#include #include @@ -132,18 +131,7 @@ private: /// Hash a set of keys into a UInt128 value. -static UInt128 hash128depths(const std::vector & indices, const ColumnRawPtrs & key_columns) -{ - SipHash hash; - for (size_t j = 0, keys_size = key_columns.size(); j < keys_size; ++j) - { - // Debug: const auto & field = (*key_columns[j])[indices[j]]; DUMP(j, indices[j], field); - key_columns[j]->updateHashWithValue(indices[j], hash); - } - - return hash.get128(); -} - +UInt128 hash128depths(const std::vector & indices, const ColumnRawPtrs & key_columns); template ColumnPtr FunctionArrayEnumerateRankedExtended::executeImpl( diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 789f6039e03..4d43f90856d 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -54,6 +55,27 @@ using Point = boost::geometry::model::d2::point_xy; using Polygon = boost::geometry::model::polygon; using Box = boost::geometry::model::box; +template +UInt128 sipHash128(Polygon && polygon) +{ + SipHash hash; + + auto hash_ring = [&hash](const auto & ring) + { + UInt32 size = static_cast(ring.size()); + hash.update(size); + hash.update(reinterpret_cast(ring.data()), size * sizeof(ring[0])); + }; + + hash_ring(polygon.outer()); + + const auto & inners = polygon.inners(); + hash.update(inners.size()); + for (auto & inner : inners) + hash_ring(inner); + + return hash.get128(); +} template class FunctionPointInPolygon : public IFunction diff --git a/src/IO/MMappedFileCache.cpp b/src/IO/MMappedFileCache.cpp new file mode 100644 index 00000000000..db84c787ead --- /dev/null +++ b/src/IO/MMappedFileCache.cpp @@ -0,0 +1,17 @@ +#include +#include + +namespace DB +{ + +UInt128 MMappedFileCache::hash(const String & path_to_file, size_t offset, ssize_t length) +{ + SipHash hash; + hash.update(path_to_file.data(), path_to_file.size() + 1); + hash.update(offset); + hash.update(length); + + return hash.get128(); +} + +} diff --git a/src/IO/MMappedFileCache.h b/src/IO/MMappedFileCache.h index bb30829ed69..bbd7443f4ab 100644 --- a/src/IO/MMappedFileCache.h +++ b/src/IO/MMappedFileCache.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -31,15 +30,7 @@ public: : Base(max_size_in_bytes) {} /// Calculate key from path to file and offset. - static UInt128 hash(const String & path_to_file, size_t offset, ssize_t length = -1) - { - SipHash hash; - hash.update(path_to_file.data(), path_to_file.size() + 1); - hash.update(offset); - hash.update(length); - - return hash.get128(); - } + static UInt128 hash(const String & path_to_file, size_t offset, ssize_t length = -1); template MappedPtr getOrSet(const Key & key, LoadFunc && load) diff --git a/src/IO/UncompressedCache.cpp b/src/IO/UncompressedCache.cpp index 7309ef5d2f4..0c4fee53b33 100644 --- a/src/IO/UncompressedCache.cpp +++ b/src/IO/UncompressedCache.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -8,4 +9,13 @@ UncompressedCache::UncompressedCache(const String & cache_policy, size_t max_siz : Base(cache_policy, max_size_in_bytes, 0, size_ratio) { } + +UInt128 UncompressedCache::hash(const String & path_to_file, size_t offset) +{ + SipHash hash; + hash.update(path_to_file.data(), path_to_file.size() + 1); + hash.update(offset); + + return hash.get128(); +} } diff --git a/src/IO/UncompressedCache.h b/src/IO/UncompressedCache.h index aa515eec357..125d58e2eee 100644 --- a/src/IO/UncompressedCache.h +++ b/src/IO/UncompressedCache.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -46,14 +45,7 @@ public: UncompressedCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); /// Calculate key from path to file and offset. - static UInt128 hash(const String & path_to_file, size_t offset) - { - SipHash hash; - hash.update(path_to_file.data(), path_to_file.size() + 1); - hash.update(offset); - - return hash.get128(); - } + static UInt128 hash(const String & path_to_file, size_t offset); template MappedPtr getOrSet(const Key & key, LoadFunc && load) diff --git a/src/Interpreters/AggregatedDataVariants.h b/src/Interpreters/AggregatedDataVariants.h index 99c136735e5..ccd203e7de6 100644 --- a/src/Interpreters/AggregatedDataVariants.h +++ b/src/Interpreters/AggregatedDataVariants.h @@ -1,7 +1,7 @@ #pragma once -#include #include #include +#include #include #include diff --git a/src/Interpreters/AggregationCommon.h b/src/Interpreters/AggregationCommon.h index 8a81f4d4614..6232800fd62 100644 --- a/src/Interpreters/AggregationCommon.h +++ b/src/Interpreters/AggregationCommon.h @@ -2,8 +2,6 @@ #include -#include -#include #include #include #include @@ -248,18 +246,6 @@ static inline T ALWAYS_INLINE packFixed( return key; } - -/// Hash a set of keys into a UInt128 value. -static inline UInt128 ALWAYS_INLINE hash128( /// NOLINT - size_t i, size_t keys_size, const ColumnRawPtrs & key_columns) -{ - SipHash hash; - for (size_t j = 0; j < keys_size; ++j) - key_columns[j]->updateHashWithValue(i, hash); - - return hash.get128(); -} - /** Serialize keys into a continuous chunk of memory. */ static inline StringRef ALWAYS_INLINE serializeKeysToPoolContiguous( /// NOLINT diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 2088e4f7aad..d3461082fd6 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -13,11 +13,11 @@ #include #include +#include +#include #include -#include #include #include -#include #include diff --git a/src/Interpreters/HashJoin/AddedColumns.h b/src/Interpreters/HashJoin/AddedColumns.h index 885c1baca8c..753df0fde04 100644 --- a/src/Interpreters/HashJoin/AddedColumns.h +++ b/src/Interpreters/HashJoin/AddedColumns.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include diff --git a/src/Interpreters/HashJoin/HashJoin.h b/src/Interpreters/HashJoin/HashJoin.h index 85cfb0869e7..8db444a84f1 100644 --- a/src/Interpreters/HashJoin/HashJoin.h +++ b/src/Interpreters/HashJoin/HashJoin.h @@ -24,7 +24,6 @@ #include #include #include -#include #include #include diff --git a/src/Interpreters/HashJoin/KeyGetter.h b/src/Interpreters/HashJoin/KeyGetter.h index 35ff2bb6eb5..8f780f70370 100644 --- a/src/Interpreters/HashJoin/KeyGetter.h +++ b/src/Interpreters/HashJoin/KeyGetter.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace DB diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 897471acee9..4752ecdd124 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -103,7 +103,7 @@ void buildScatterSelector( for (size_t i = 0; i < num_rows; ++i) { - Data::key_type key = hash128(i, columns.size(), columns); + Data::key_type key = ColumnsHashing::hash128(i, columns.size(), columns); typename Data::LookupResult it; bool inserted; partitions_map.emplace(key, it, inserted); From d33c22b3e37f9e7afded3add4577b27d0e069541 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 4 Dec 2024 18:11:54 +0100 Subject: [PATCH 357/502] fix test --- .../0_stateless/03277_async_insert.reference | 2 ++ .../queries/0_stateless/03277_async_insert.sh | 19 +++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/03277_async_insert.reference b/tests/queries/0_stateless/03277_async_insert.reference index e69de29bb2d..6ed281c757a 100644 --- a/tests/queries/0_stateless/03277_async_insert.reference +++ b/tests/queries/0_stateless/03277_async_insert.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/03277_async_insert.sh b/tests/queries/0_stateless/03277_async_insert.sh index 46cfcc38e6a..4b059643a4e 100755 --- a/tests/queries/0_stateless/03277_async_insert.sh +++ b/tests/queries/0_stateless/03277_async_insert.sh @@ -7,18 +7,21 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e +FILE_PREFIX="${CLICKHOUSE_USER_FILES_UNIQUE:?}_test_03277" -mkdir -p "${CLICKHOUSE_USER_FILES_UNIQUE:?}" -rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE:?}"/* +function cleanup() +{ + rm "${FILE_PREFIX}"* +} +trap cleanup EXIT -FILE_CSV="${CLICKHOUSE_USER_FILES_UNIQUE:?}/03277.csv" -FILE_ARROW="${CLICKHOUSE_USER_FILES_UNIQUE:?}/03277.arrow" +FILE_CSV="${FILE_PREFIX}.csv" +FILE_ARROW="${FILE_PREFIX}.arrow" $CLICKHOUSE_CLIENT -q " SET async_insert = 1; - SELECT '${FILE_CSV}'; CREATE TABLE t0 (c0 Int) ENGINE = File(CSV); INSERT INTO TABLE t0 (c0) VALUES (1); INSERT INTO TABLE FUNCTION file('$FILE_CSV', 'CSV', 'c0 Int') SELECT c0 FROM t0; @@ -30,13 +33,9 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE t0" $CLICKHOUSE_CLIENT -q " SET async_insert = 1; - SELECT '${FILE_ARROW}'; CREATE TABLE t0 (c0 Int) ENGINE = Join(ANY, INNER, c0); INSERT INTO TABLE FUNCTION file('$FILE_ARROW', 'Arrow', 'c0 Int') SELECT c0 FROM t0; INSERT INTO TABLE t0 (c0) FROM INFILE '$FILE_ARROW' FORMAT Arrow; " $CLICKHOUSE_CLIENT -q "SELECT * from t0" -$CLICKHOUSE_CLIENT -q "DROP TABLE t0" - - -rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE:?}"/* \ No newline at end of file +$CLICKHOUSE_CLIENT -q "DROP TABLE t0" \ No newline at end of file From 31c5de433bba2a248a3a0c4eac85cac400b9e36e Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 4 Dec 2024 18:12:49 +0100 Subject: [PATCH 358/502] fix style --- tests/queries/0_stateless/03277_async_insert.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03277_async_insert.sh b/tests/queries/0_stateless/03277_async_insert.sh index 4b059643a4e..270a5834d4f 100755 --- a/tests/queries/0_stateless/03277_async_insert.sh +++ b/tests/queries/0_stateless/03277_async_insert.sh @@ -11,7 +11,7 @@ FILE_PREFIX="${CLICKHOUSE_USER_FILES_UNIQUE:?}_test_03277" function cleanup() { - rm "${FILE_PREFIX}"* + rm "${FILE_PREFIX:?}"* } trap cleanup EXIT From 35c0ea345f433db0c6747220c7f8a8051eafb66e Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:15:19 +0100 Subject: [PATCH 359/502] Build fix + reviews --- src/Analyzer/QueryTreeBuilder.cpp | 2 -- .../queries/0_stateless/03156_group_concat.reference | 2 ++ tests/queries/0_stateless/03156_group_concat.sql | 12 +++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 0d44cf76462..a45702cf5b0 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -1104,8 +1104,6 @@ QueryTreeNodePtr QueryTreeBuilder::setFirstArgumentAsParameter(const ASTFunction "If groupConcat is used with two arguments, the first argument must be a constant String"); } - std::string separator = first_arg_literal->value.safeGet(); - ASTPtr second_arg = function->arguments->children[1]->clone(); auto function_node = std::make_shared(function->name); diff --git a/tests/queries/0_stateless/03156_group_concat.reference b/tests/queries/0_stateless/03156_group_concat.reference index 6f4b84fbc1d..8bf47211a1d 100644 --- a/tests/queries/0_stateless/03156_group_concat.reference +++ b/tests/queries/0_stateless/03156_group_concat.reference @@ -22,4 +22,6 @@ abc.a.makson95 [1,2,3]/[993,986,979,972]/[] [1,2,3]/[993,986,979,972]/[] [1,2,3]/[993,986,979,972]/[] +[1,2,3]/[993,986,979,972]/[] +[1,2,3]/[993,986,979,972]/[] 488890 diff --git a/tests/queries/0_stateless/03156_group_concat.sql b/tests/queries/0_stateless/03156_group_concat.sql index 719d271f4b5..eee71d9490f 100644 --- a/tests/queries/0_stateless/03156_group_concat.sql +++ b/tests/queries/0_stateless/03156_group_concat.sql @@ -48,12 +48,14 @@ TRUNCATE TABLE test_groupConcat; INSERT INTO test_groupConcat VALUES (0, 95, 'abc', [1, 2, 3]), (1, NULL, 'a', [993, 986, 979, 972]), (2, 123, 'makson95', []); -SELECT groupConcat(',', p_int) FROM test_groupConcat; -SELECT groupConcat('.')(p_string) FROM test_groupConcat; -SELECT groupConcat('/', p_array) FROM test_groupConcat; +SELECT groupConcat(',', p_int) FROM test_groupConcat SETTINGS enable_analyzer=1; +SELECT groupConcat('.')(p_string) FROM test_groupConcat SETTINGS enable_analyzer=1; +SELECT groupConcat('/', p_array) FROM test_groupConcat SETTINGS enable_analyzer=1; -SELECT group_concat('/', p_array) FROM test_groupConcat; -SELECT grouP_CONcat('/', p_array) FROM test_groupConcat; +SELECT group_concat('/', p_array) FROM test_groupConcat SETTINGS enable_analyzer=1; +SELECT grouP_CONcat('/', p_array) FROM test_groupConcat SETTINGS enable_analyzer=1; +SELECT grouP_CONcat(',')('/', p_array) FROM test_groupConcat SETTINGS enable_analyzer=1; -- overrides current parameter +SELECT grouP_CONcat(',', 2)('/', p_array) FROM test_groupConcat SETTINGS enable_analyzer=1; -- works fine with both arguments DROP TABLE IF EXISTS test_groupConcat; From c434a3f87af75e131bc0bb6e21967f1640ffd159 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 4 Dec 2024 19:04:43 +0100 Subject: [PATCH 360/502] Support oauth_server_uri --- src/Databases/Iceberg/DatabaseIceberg.cpp | 4 +- .../Iceberg/DatabaseIcebergSettings.cpp | 1 + src/Databases/Iceberg/RestCatalog.cpp | 44 +++++++++++++++---- src/Databases/Iceberg/RestCatalog.h | 2 + 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index dd559ba8b92..8544239d6b1 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -36,6 +36,7 @@ namespace DatabaseIcebergSetting extern const DatabaseIcebergSettingsString auth_header; extern const DatabaseIcebergSettingsString auth_scope; extern const DatabaseIcebergSettingsString storage_endpoint; + extern const DatabaseIcebergSettingsString oauth_server_uri; extern const DatabaseIcebergSettingsBool vended_credentials; } namespace Setting @@ -119,6 +120,7 @@ std::shared_ptr DatabaseIceberg::getCatalog(ContextPtr) const settings[DatabaseIcebergSetting::catalog_credential].value, settings[DatabaseIcebergSetting::auth_scope].value, settings[DatabaseIcebergSetting::auth_header], + settings[DatabaseIcebergSetting::oauth_server_uri].value, Context::getGlobalContextInstance()); } } @@ -364,7 +366,7 @@ void registerDatabaseIceberg(DatabaseFactory & factory) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine `{}` must have arguments", database_engine_name); ASTs & engine_args = function_define->arguments->children; - if (engine_args.size() < 1) + if (engine_args.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine `{}` must have arguments", database_engine_name); for (auto & engine_arg : engine_args) diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp index de04fc0bd11..33374edbb6d 100644 --- a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes DECLARE(String, catalog_credential, "", "", 0) \ DECLARE(Bool, vended_credentials, true, "Use vended credentials (storage credentials) from catalog", 0) \ DECLARE(String, auth_scope, "PRINCIPAL_ROLE:ALL", "Authorization scope for client credentials or token exchange", 0) \ + DECLARE(String, oauth_server_uri, "", "OAuth server uri", 0) \ DECLARE(String, warehouse, "", "Warehouse name inside the catalog", 0) \ DECLARE(String, auth_header, "", "Authorization header of format 'Authorization: '", 0) \ DECLARE(String, storage_endpoint, "", "Object storage endpoint", 0) \ diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index 262ac53e99f..b138cc1c14b 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -101,6 +101,13 @@ StorageType parseStorageTypeFromLocation(const std::string & location) return *storage_type; } +std::string correctAPIURI(const std::string & uri) +{ + if (uri.ends_with("v1")) + return uri; + return std::filesystem::path(uri) / "v1"; +} + } std::string RestCatalog::Config::toString() const @@ -122,12 +129,14 @@ RestCatalog::RestCatalog( const std::string & catalog_credential_, const std::string & auth_scope_, const std::string & auth_header_, + const std::string & oauth_server_uri_, DB::ContextPtr context_) : ICatalog(warehouse_) , DB::WithContext(context_) - , base_url(base_url_) + , base_url(correctAPIURI(base_url_)) , log(getLogger("RestCatalog(" + warehouse_ + ")")) , auth_scope(auth_scope_) + , oauth_server_uri(oauth_server_uri_) { if (!catalog_credential_.empty()) { @@ -217,14 +226,30 @@ std::string RestCatalog::retrieveAccessToken() const headers.emplace_back("Content-Type", "application/x-www-form-urlencoded"); headers.emplace_back("Accepts", "application/json; charset=UTF-8"); - Poco::URI url(base_url / oauth_tokens_endpoint); - Poco::URI::QueryParameters params = { - {"grant_type", "client_credentials"}, - {"scope", auth_scope}, - {"client_id", client_id}, - {"client_secret", client_secret}, - }; - url.setQueryParameters(params); + Poco::URI url; + DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; + if (oauth_server_uri.empty()) + { + url = Poco::URI(base_url / oauth_tokens_endpoint); + + Poco::URI::QueryParameters params = { + {"grant_type", "client_credentials"}, + {"scope", auth_scope}, + {"client_id", client_id}, + {"client_secret", client_secret}, + }; + url.setQueryParameters(params); + } + else + { + url = Poco::URI(oauth_server_uri); + out_stream_callback = [&](std::ostream & os) + { + os << fmt::format( + "grant_type=client_credentials&scope={}&client_id={}&client_secret={}", + auth_scope, client_id, client_secret); + }; + } const auto & context = getContext(); auto wb = DB::BuilderRWBufferFromHTTP(url) @@ -233,6 +258,7 @@ std::string RestCatalog::retrieveAccessToken() const .withSettings(context->getReadSettings()) .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) .withHostFilter(&context->getRemoteHostFilter()) + .withOutCallback(std::move(out_stream_callback)) .withSkipNotFound(false) .withHeaders(headers) .create(credentials); diff --git a/src/Databases/Iceberg/RestCatalog.h b/src/Databases/Iceberg/RestCatalog.h index 4505e020580..aab8be6ed8d 100644 --- a/src/Databases/Iceberg/RestCatalog.h +++ b/src/Databases/Iceberg/RestCatalog.h @@ -24,6 +24,7 @@ public: const std::string & catalog_credential_, const std::string & auth_scope_, const std::string & auth_header_, + const std::string & oauth_server_uri_, DB::ContextPtr context_); ~RestCatalog() override = default; @@ -73,6 +74,7 @@ private: std::string client_id; std::string client_secret; std::string auth_scope; + std::string oauth_server_uri; mutable std::optional access_token; Poco::Net::HTTPBasicCredentials credentials{}; From fe01be106c5d6d127bb080ddf5cf86065b8acbc9 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:38:26 +0100 Subject: [PATCH 361/502] Update test.py --- tests/integration/test_storage_s3_queue/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 732ab6ce4a7..e12f23b994c 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -2318,10 +2318,10 @@ def test_alter_settings(started_cluster): def test_list_and_delete_race(started_cluster): + node = started_cluster.instances["instance"] if node.is_built_with_sanitizer(): # Issue does not reproduce under sanitizer return - node = started_cluster.instances["instance"] node_2 = started_cluster.instances["instance2"] table_name = f"list_and_delete_race_{generate_random_string()}" dst_table_name = f"{table_name}_dst" From cd254a6ee81c15890d7ab90106e860b86d74c5d7 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 4 Dec 2024 19:41:32 +0100 Subject: [PATCH 362/502] Remove useless setting --- src/Core/SettingsEnums.cpp | 7 ---- src/Core/SettingsEnums.h | 10 ----- src/Databases/Iceberg/DatabaseIceberg.cpp | 37 ++++++++----------- src/Databases/Iceberg/DatabaseIceberg.h | 7 ++-- .../Iceberg/DatabaseIcebergSettings.cpp | 1 - .../Iceberg/DatabaseIcebergSettings.h | 1 - .../Iceberg/DatabaseIcebergStorageType.h | 14 +++++++ src/Databases/Iceberg/ICatalog.cpp | 32 ++++++++++++++++ src/Databases/Iceberg/ICatalog.h | 4 ++ src/Databases/Iceberg/RestCatalog.cpp | 26 ------------- 10 files changed, 68 insertions(+), 71 deletions(-) create mode 100644 src/Databases/Iceberg/DatabaseIcebergStorageType.h diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 8f7ff6f149c..01b9229ebb7 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -283,11 +283,4 @@ IMPLEMENT_SETTING_ENUM( IMPLEMENT_SETTING_ENUM(DatabaseIcebergCatalogType, ErrorCodes::BAD_ARGUMENTS, {{"rest", DatabaseIcebergCatalogType::REST}}) -IMPLEMENT_SETTING_ENUM(DatabaseIcebergStorageType, ErrorCodes::BAD_ARGUMENTS, - {{"s3", DatabaseIcebergStorageType::S3}, - {"azure", DatabaseIcebergStorageType::Azure}, - {"hdfs", DatabaseIcebergStorageType::HDFS}, - {"local", DatabaseIcebergStorageType::Local}, - }) - } diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 33ef8e9623f..aefbc45d411 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -366,14 +366,4 @@ enum class DatabaseIcebergCatalogType : uint8_t DECLARE_SETTING_ENUM(DatabaseIcebergCatalogType) -enum class DatabaseIcebergStorageType : uint8_t -{ - S3, - Azure, - Local, - HDFS, -}; - -DECLARE_SETTING_ENUM(DatabaseIcebergStorageType) - } diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 8544239d6b1..62af8288933 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -30,7 +30,6 @@ namespace DB namespace DatabaseIcebergSetting { extern const DatabaseIcebergSettingsDatabaseIcebergCatalogType catalog_type; - extern const DatabaseIcebergSettingsDatabaseIcebergStorageType storage_type; extern const DatabaseIcebergSettingsString warehouse; extern const DatabaseIcebergSettingsString catalog_credential; extern const DatabaseIcebergSettingsString auth_header; @@ -72,18 +71,17 @@ DatabaseIceberg::DatabaseIceberg( const std::string & database_name_, const std::string & url_, const DatabaseIcebergSettings & settings_, - ASTPtr database_engine_definition_, - ContextPtr context_) + ASTPtr database_engine_definition_) : IDatabase(database_name_) , url(url_) , settings(settings_) , database_engine_definition(database_engine_definition_) , log(getLogger("DatabaseIceberg(" + database_name_ + ")")) { - validateSettings(context_); + validateSettings(); } -void DatabaseIceberg::validateSettings(const ContextPtr & context_) +void DatabaseIceberg::validateSettings() { if (settings[DatabaseIcebergSetting::warehouse].value.empty()) { @@ -91,18 +89,6 @@ void DatabaseIceberg::validateSettings(const ContextPtr & context_) ErrorCodes::BAD_ARGUMENTS, "`warehouse` setting cannot be empty. " "Please specify 'SETTINGS warehouse=' in the CREATE DATABASE query"); } - - if (!settings[DatabaseIcebergSetting::storage_type].changed) - { - auto catalog = getCatalog(context_); - const auto storage_type = catalog->getStorageType(); - if (!storage_type) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Storage type is not found in catalog config. " - "Please specify it manually via 'SETTINGS storage_type=' in CREATE DATABASE query"); - } - } } std::shared_ptr DatabaseIceberg::getCatalog(ContextPtr) const @@ -127,11 +113,11 @@ std::shared_ptr DatabaseIceberg::getCatalog(ContextPtr) const return catalog_impl; } -std::shared_ptr DatabaseIceberg::getConfiguration() const +std::shared_ptr DatabaseIceberg::getConfiguration(DatabaseIcebergStorageType type) const { /// TODO: add tests for azure, local storage types. - switch (settings[DatabaseIcebergSetting::storage_type].value) + switch (type) { #if USE_AWS_S3 case DB::DatabaseIcebergStorageType::S3: @@ -234,7 +220,15 @@ StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_ LOG_TEST(log, "Using table endpoint: {}", table_endpoint); const auto columns = ColumnsDescription(table_metadata.getSchema()); - const auto configuration = getConfiguration(); + + DatabaseIcebergStorageType storage_type; + auto storage_type_from_catalog = catalog->getStorageType(); + if (storage_type_from_catalog.has_value()) + storage_type = storage_type_from_catalog.value(); + else + storage_type = table_metadata.getStorageType(); + + const auto configuration = getConfiguration(storage_type); /// with_table_structure = false: because there will be /// no table structure in table definition AST. @@ -382,8 +376,7 @@ void registerDatabaseIceberg(DatabaseFactory & factory) args.database_name, url, database_settings, - database_engine_define->clone(), - args.context); + database_engine_define->clone()); }; factory.registerDatabase("Iceberg", create_fn, { .supports_arguments = true, .supports_settings = true }); } diff --git a/src/Databases/Iceberg/DatabaseIceberg.h b/src/Databases/Iceberg/DatabaseIceberg.h index 4b9929a3a76..5138d0f639c 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.h +++ b/src/Databases/Iceberg/DatabaseIceberg.h @@ -18,8 +18,7 @@ public: const std::string & database_name_, const std::string & url_, const DatabaseIcebergSettings & settings_, - ASTPtr database_engine_definition_, - ContextPtr context_); + ASTPtr database_engine_definition_); String getEngineName() const override { return "Iceberg"; } @@ -57,9 +56,9 @@ private: mutable std::shared_ptr catalog_impl; - void validateSettings(const ContextPtr & context_); + void validateSettings(); std::shared_ptr getCatalog(ContextPtr context_) const; - std::shared_ptr getConfiguration() const; + std::shared_ptr getConfiguration(DatabaseIcebergStorageType type) const; std::string getStorageEndpointForTable(const Iceberg::TableMetadata & table_metadata) const; }; diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp index 33374edbb6d..37b4909106b 100644 --- a/src/Databases/Iceberg/DatabaseIcebergSettings.cpp +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.cpp @@ -16,7 +16,6 @@ namespace ErrorCodes #define DATABASE_ICEBERG_RELATED_SETTINGS(DECLARE, ALIAS) \ DECLARE(DatabaseIcebergCatalogType, catalog_type, DatabaseIcebergCatalogType::REST, "Catalog type", 0) \ - DECLARE(DatabaseIcebergStorageType, storage_type, DatabaseIcebergStorageType::S3, "Storage type: S3, Local, Azure, HDFS", 0) \ DECLARE(String, catalog_credential, "", "", 0) \ DECLARE(Bool, vended_credentials, true, "Use vended credentials (storage credentials) from catalog", 0) \ DECLARE(String, auth_scope, "PRINCIPAL_ROLE:ALL", "Authorization scope for client credentials or token exchange", 0) \ diff --git a/src/Databases/Iceberg/DatabaseIcebergSettings.h b/src/Databases/Iceberg/DatabaseIcebergSettings.h index 4e5bc0defba..041a99c6d83 100644 --- a/src/Databases/Iceberg/DatabaseIcebergSettings.h +++ b/src/Databases/Iceberg/DatabaseIcebergSettings.h @@ -18,7 +18,6 @@ class SettingsChanges; M(CLASS_NAME, UInt64) \ M(CLASS_NAME, Bool) \ M(CLASS_NAME, DatabaseIcebergCatalogType) \ - M(CLASS_NAME, DatabaseIcebergStorageType) DATABASE_ICEBERG_SETTINGS_SUPPORTED_TYPES(DatabaseIcebergSettings, DECLARE_SETTING_TRAIT) diff --git a/src/Databases/Iceberg/DatabaseIcebergStorageType.h b/src/Databases/Iceberg/DatabaseIcebergStorageType.h new file mode 100644 index 00000000000..cc3c8f8cb1d --- /dev/null +++ b/src/Databases/Iceberg/DatabaseIcebergStorageType.h @@ -0,0 +1,14 @@ +#include + +namespace DB +{ + +enum class DatabaseIcebergStorageType : uint8_t +{ + S3, + Azure, + Local, + HDFS, +}; + +} diff --git a/src/Databases/Iceberg/ICatalog.cpp b/src/Databases/Iceberg/ICatalog.cpp index 97b2bfeeef9..4568cf95ac0 100644 --- a/src/Databases/Iceberg/ICatalog.cpp +++ b/src/Databases/Iceberg/ICatalog.cpp @@ -12,6 +12,32 @@ namespace DB::ErrorCodes namespace Iceberg { +StorageType parseStorageTypeFromLocation(const std::string & location) +{ + /// Table location in catalog metadata always starts with one of s3://, file://, etc. + /// So just extract this part of the path and deduce storage type from it. + + auto pos = location.find("://"); + if (pos == std::string::npos) + { + throw DB::Exception( + DB::ErrorCodes::NOT_IMPLEMENTED, + "Unexpected path format: {}", location); + } + + auto storage_type_str = location.substr(0, pos); + auto storage_type = magic_enum::enum_cast(Poco::toUpper(storage_type_str)); + + if (!storage_type) + { + throw DB::Exception( + DB::ErrorCodes::NOT_IMPLEMENTED, + "Unsupported storage type: {}", storage_type_str); + } + + return *storage_type; +} + void TableMetadata::setLocation(const std::string & location_) { if (!with_location) @@ -83,4 +109,10 @@ std::shared_ptr TableMetadata::getStorageCredentials() cons return storage_credentials; } + +StorageType TableMetadata::getStorageType() const +{ + return parseStorageTypeFromLocation(location_without_path); +} + } diff --git a/src/Databases/Iceberg/ICatalog.h b/src/Databases/Iceberg/ICatalog.h index 9657ef6ba41..45d9f056477 100644 --- a/src/Databases/Iceberg/ICatalog.h +++ b/src/Databases/Iceberg/ICatalog.h @@ -3,10 +3,12 @@ #include #include #include +#include namespace Iceberg { using StorageType = DB::DatabaseIcebergStorageType; +StorageType parseStorageTypeFromLocation(const std::string & location); /// A class representing table metadata, /// which was received from Catalog. @@ -32,6 +34,8 @@ public: bool requiresSchema() const { return with_schema; } bool requiresCredentials() const { return with_storage_credentials; } + StorageType getStorageType() const; + private: /// Starts with s3://, file://, etc. /// For example, `s3://bucket/` diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index b138cc1c14b..460fce78696 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -75,32 +75,6 @@ DB::HTTPHeaderEntry parseAuthHeader(const std::string & auth_header) return DB::HTTPHeaderEntry(auth_header.substr(0, pos), auth_header.substr(pos + 1)); } -StorageType parseStorageTypeFromLocation(const std::string & location) -{ - /// Table location in catalog metadata always starts with one of s3://, file://, etc. - /// So just extract this part of the path and deduce storage type from it. - - auto pos = location.find("://"); - if (pos == std::string::npos) - { - throw DB::Exception( - DB::ErrorCodes::NOT_IMPLEMENTED, - "Unexpected path format: {}", location); - } - - auto storage_type_str = location.substr(0, pos); - auto storage_type = magic_enum::enum_cast(Poco::toUpper(storage_type_str)); - - if (!storage_type) - { - throw DB::Exception( - DB::ErrorCodes::NOT_IMPLEMENTED, - "Unsupported storage type: {}", storage_type_str); - } - - return *storage_type; -} - std::string correctAPIURI(const std::string & uri) { if (uri.ends_with("v1")) From d8f395c3dee09d2e759df44841fa5aa69613e67a Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Wed, 4 Dec 2024 20:13:46 +0100 Subject: [PATCH 363/502] Cancel async marks load task if it hasn't started --- src/Common/ProfileEvents.cpp | 1 + src/Storages/MergeTree/MergeTreeMarksLoader.cpp | 11 +++++++++++ src/Storages/MergeTree/MergeTreeMarksLoader.h | 3 +++ 3 files changed, 15 insertions(+) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 007c3ab4419..11397835501 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -229,6 +229,7 @@ \ M(WaitMarksLoadMicroseconds, "Time spent loading marks", ValueType::Microseconds) \ M(BackgroundLoadingMarksTasks, "Number of background tasks for loading marks", ValueType::Number) \ + M(LoadingMarksTasksCanceled, "Number of times background tasks for loading marks were canceled", ValueType::Number) \ M(LoadedMarksFiles, "Number of mark files loaded.", ValueType::Number) \ M(LoadedMarksCount, "Number of marks loaded (total across columns).", ValueType::Number) \ M(LoadedMarksMemoryBytes, "Size of in-memory representations of loaded marks.", ValueType::Bytes) \ diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index b4523c9116d..3dfb2065158 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -16,6 +16,7 @@ namespace ProfileEvents { extern const Event WaitMarksLoadMicroseconds; extern const Event BackgroundLoadingMarksTasks; + extern const Event LoadingMarksTasksCanceled; extern const Event LoadedMarksFiles; extern const Event LoadedMarksCount; extern const Event LoadedMarksMemoryBytes; @@ -34,6 +35,7 @@ namespace ErrorCodes extern const int CANNOT_READ_ALL_DATA; extern const int CORRUPTED_DATA; extern const int LOGICAL_ERROR; + extern const int ASYNC_LOAD_CANCELED; } MergeTreeMarksGetter::MergeTreeMarksGetter(MarkCache::MappedPtr marks_, size_t num_columns_in_mark_) @@ -84,7 +86,10 @@ void MergeTreeMarksLoader::startAsyncLoad() MergeTreeMarksLoader::~MergeTreeMarksLoader() { if (future.valid()) + { + is_canceled = true; future.wait(); + } } MergeTreeMarksGetterPtr MergeTreeMarksLoader::loadMarks() @@ -252,6 +257,12 @@ std::future MergeTreeMarksLoader::loadMarksAsync() return scheduleFromThreadPoolUnsafe( [this]() -> MarkCache::MappedPtr { + if (is_canceled) + { + ProfileEvents::increment(ProfileEvents::LoadingMarksTasksCanceled); + throw Exception(ErrorCodes::ASYNC_LOAD_CANCELED, "Background task for loading marks was canceled"); + } + ProfileEvents::increment(ProfileEvents::BackgroundLoadingMarksTasks); return loadMarksSync(); }, diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.h b/src/Storages/MergeTree/MergeTreeMarksLoader.h index de202ef76ba..9f4baaabd77 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -4,6 +4,8 @@ #include #include +#include + namespace DB { @@ -73,6 +75,7 @@ private: std::future future; ThreadPool * load_marks_threadpool; + std::atomic is_canceled{false}; }; using MergeTreeMarksLoaderPtr = std::shared_ptr; From 4e5a67b80cbdb5910b45a5ac357b33e4136e23b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 4 Dec 2024 19:18:13 +0000 Subject: [PATCH 364/502] Fix style --- src/Core/QualifiedTableName.cpp | 5 +++++ src/Core/QualifiedTableName.h | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Core/QualifiedTableName.cpp b/src/Core/QualifiedTableName.cpp index 661cc4227b1..b32764f93a0 100644 --- a/src/Core/QualifiedTableName.cpp +++ b/src/Core/QualifiedTableName.cpp @@ -6,6 +6,11 @@ namespace DB { +namespace ErrorCodes +{ +extern const int SYNTAX_ERROR; +} + UInt64 QualifiedTableName::hash() const { SipHash hash_state; diff --git a/src/Core/QualifiedTableName.h b/src/Core/QualifiedTableName.h index 79c101b915e..86a903be406 100644 --- a/src/Core/QualifiedTableName.h +++ b/src/Core/QualifiedTableName.h @@ -9,11 +9,6 @@ namespace DB { -namespace ErrorCodes -{ -extern const int SYNTAX_ERROR; -} - //TODO replace with StorageID struct QualifiedTableName { From 8bf03e8a212860a86c37edbaeeee4680c7022020 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Wed, 4 Dec 2024 19:53:40 +0000 Subject: [PATCH 365/502] small test update --- tests/integration/test_ldap_external_user_directory/test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_ldap_external_user_directory/test.py b/tests/integration/test_ldap_external_user_directory/test.py index efda6ac7831..e7884b8ae3f 100644 --- a/tests/integration/test_ldap_external_user_directory/test.py +++ b/tests/integration/test_ldap_external_user_directory/test.py @@ -190,10 +190,9 @@ def test_remote_query_user_does_not_exist_locally(ldap_cluster): Check that even if user does not exist locally, using it to execute remote queries is still possible """ instance2.query("DROP USER IF EXISTS non_local") + instance2.query("DROP TABLE IF EXISTS test_table sync") instance2.query("CREATE USER non_local") - - instance2.query("DROP TABLE IF EXISTS test_table") instance2.query("CREATE TABLE test_table (id Int16) ENGINE=Memory") instance2.query("INSERT INTO test_table VALUES (123)") instance2.query("GRANT SELECT ON default.test_table TO non_local") @@ -204,4 +203,4 @@ def test_remote_query_user_does_not_exist_locally(ldap_cluster): assert result.strip() == "123" instance2.query("DROP USER IF EXISTS non_local") - instance2.query("DROP TABLE IF EXISTS test_table") + instance2.query("DROP TABLE IF EXISTS test_table SYNC") From 9290e328ef9ca03c4bb4ff6b627269a36deaa51c Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 4 Dec 2024 20:55:02 +0100 Subject: [PATCH 366/502] Minor changes. --- src/Functions/array/arrayPrAUC.cpp | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/Functions/array/arrayPrAUC.cpp b/src/Functions/array/arrayPrAUC.cpp index 4724bc3cfef..92849968a92 100644 --- a/src/Functions/array/arrayPrAUC.cpp +++ b/src/Functions/array/arrayPrAUC.cpp @@ -78,33 +78,35 @@ public: static FunctionPtr create(ContextPtr) { return std::make_shared(); } private: - static Float64 - apply(const IColumn & scores, const IColumn & labels, ColumnArray::Offset current_offset, ColumnArray::Offset next_offset) + static Float64 apply(const IColumn & scores, const IColumn & labels, ColumnArray::Offset current_offset, ColumnArray::Offset next_offset) { + size_t size = next_offset - current_offset; + if (size == 0) + return 0.0; + struct ScoreLabel { Float64 score; bool label; }; - size_t size = next_offset - current_offset; PODArrayWithStackMemory sorted_labels(size); for (size_t i = 0; i < size; ++i) { - bool label = labels.getFloat64(current_offset + i) > 0; + sorted_labels[i].label = labels.getFloat64(current_offset + i) > 0; sorted_labels[i].score = scores.getFloat64(current_offset + i); - sorted_labels[i].label = label; } /// Sorting scores in descending order to traverse the Precision Recall curve from left to right std::sort(sorted_labels.begin(), sorted_labels.end(), [](const auto & lhs, const auto & rhs) { return lhs.score > rhs.score; }); - size_t prev_tp = 0, curr_tp = 0; + size_t prev_tp = 0; + size_t curr_tp = 0; size_t curr_p = 0; Float64 prev_score = sorted_labels[0].score; - Float64 curr_precision = 1.0; + Float64 curr_precision; Float64 area = 0.0; @@ -123,7 +125,8 @@ private: * * This can be done because (TP + FN) is constant and equal to total positive labels. */ - curr_precision = static_cast(curr_tp) / curr_p; + curr_precision = static_cast(curr_tp) / curr_p; /// curr_p should never be 0 because first element is not executed in this if statement + /// and it is always positive because it has the biggest prediction value. area += curr_precision * (curr_tp - prev_tp); prev_tp = curr_tp; prev_score = sorted_labels[i].score; @@ -134,12 +137,13 @@ private: curr_p += 1; } - curr_precision = curr_p > 0 ? static_cast(curr_tp) / curr_p : 1.0; - area += curr_precision * (curr_tp - prev_tp); - /// If there were no positive labels, Recall did not change and the area is 0 if (curr_tp == 0) return 0.0; + + curr_precision = curr_p > 0 ? static_cast(curr_tp) / curr_p : 1.0; + area += curr_precision * (curr_tp - prev_tp); + /// Finally, we divide by (TP + FN) to obtain the Recall /// At this point we've traversed the whole curve and curr_tp = total positive labels (TP + FN) return area / curr_tp; @@ -167,7 +171,7 @@ public: String getName() const override { return name; } bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } + size_t getNumberOfArguments() const override { return 2; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } From 1aa320c0ca8f3cc26b66d695a41a43364398d66e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 4 Dec 2024 18:05:52 +0100 Subject: [PATCH 367/502] tests: allow nested with_replace_config() Signed-off-by: Azat Khuzhin --- tests/integration/helpers/cluster.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 46d2ea3f902..d47bd5a1c01 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -17,6 +17,7 @@ import subprocess import time import traceback import urllib.parse +import uuid from contextlib import contextmanager from functools import cache from pathlib import Path @@ -4546,15 +4547,19 @@ class ClickHouseInstance: @contextmanager def with_replace_config(self, path, replacement): """Create a copy of existing config (if exists) and revert on leaving the context""" + _directory, filename = os.path.split(path) + basename, extension = os.path.splitext(filename) + id = uuid.uuid4() + backup_path = f"/tmp/{basename}_{id}{extension}" self.exec_in_container( - ["bash", "-c", f"test ! -f {path} || mv --no-clobber {path} {path}.bak"] + ["bash", "-c", f"test ! -f {path} || mv --no-clobber {path} {backup_path}"] ) self.exec_in_container( ["bash", "-c", "echo '{}' > {}".format(replacement, path)] ) yield self.exec_in_container( - ["bash", "-c", f"test ! -f {path}.bak || mv {path}.bak {path}"] + ["bash", "-c", f"test ! -f {backup_path} || mv {backup_path} {path}"] ) def replace_config(self, path_to_config, replacement): From 933418c3ebc3100118ba4b4eb07dddc79f44454e Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 5 Dec 2024 00:21:39 +0000 Subject: [PATCH 368/502] improve performance of deserialization of aggregate functions --- .../SerializationAggregateFunction.cpp | 8 ++++-- src/IO/ReadHelpers.h | 12 ++++++++- .../aggregate_functions_deserialization.xml | 27 +++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 tests/performance/aggregate_functions_deserialization.xml diff --git a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp index 04e61558852..905de3f9e78 100644 --- a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp +++ b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp @@ -79,13 +79,16 @@ void SerializationAggregateFunction::deserializeBinaryBulk(IColumn & column, Rea size_t size_of_state = function->sizeOfData(); size_t align_of_state = function->alignOfData(); + /// Adjust the size of state to make all states aligned in vector. + size_t total_size_of_state = (size_of_state + align_of_state - 1) / align_of_state * align_of_state; + char * memory = arena.alignedAlloc(total_size_of_state * limit, align_of_state); + char * place = memory; + for (size_t i = 0; i < limit; ++i) { if (istr.eof()) break; - AggregateDataPtr place = arena.alignedAlloc(size_of_state, align_of_state); - function->create(place); try @@ -99,6 +102,7 @@ void SerializationAggregateFunction::deserializeBinaryBulk(IColumn & column, Rea } vec.push_back(place); + place += total_size_of_state; } } diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 867c985934b..7e32f163671 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -110,7 +110,17 @@ inline void readChar(char & x, ReadBuffer & buf) template inline void readPODBinary(T & x, ReadBuffer & buf) { - buf.readStrict(reinterpret_cast(&x), sizeof(x)); /// NOLINT + /// If the whole value fits in buffer do not call readStrict and copy with + /// __builtin_memcpy since it is faster than generic memcpy for small copies. + if (buf.position() + sizeof(T) <= buf.buffer().end()) [[likely]] + { + __builtin_memcpy(reinterpret_cast(&x), buf.position(), sizeof(T)); + buf.position() += sizeof(T); + } + else + { + buf.readStrict(reinterpret_cast(&x), sizeof(T)); /// NOLINT + } } inline void readUUIDBinary(UUID & x, ReadBuffer & buf) diff --git a/tests/performance/aggregate_functions_deserialization.xml b/tests/performance/aggregate_functions_deserialization.xml new file mode 100644 index 00000000000..218c19e4e49 --- /dev/null +++ b/tests/performance/aggregate_functions_deserialization.xml @@ -0,0 +1,27 @@ + + + CREATE TABLE agg_deserialize + ( + t DateTime, + v1 AggregateFunction(avgState, UInt64), + v2 AggregateFunction(argMax, UInt64, DateTime) + ) + ENGINE = MergeTree() ORDER BY t + + + + INSERT INTO agg_deserialize SELECT + now() + number AS t, + initializeAggregation('avgState', number), + initializeAggregation('argMaxState', number, t) + FROM numbers(50000000) + + + SELECT v1 FROM agg_deserialize FORMAT Null + SELECT toStartOfHour(t) AS h, avgMerge(v1) FROM agg_deserialize GROUP BY h FORMAT Null + + SELECT v2 FROM agg_deserialize FORMAT Null + SELECT toStartOfHour(t) AS h, argMaxMerge(v2) FROM agg_deserialize GROUP BY h FORMAT Null + + DROP TABLE IF EXISTS agg_deserialize + From c25306320fa2272645f9f7eeb5708dbc5fdc9dbc Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 5 Dec 2024 01:44:16 +0000 Subject: [PATCH 369/502] disable test with distributed cache --- .../0_stateless/03008_azure_plain_rewritable_with_slash.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03008_azure_plain_rewritable_with_slash.sh b/tests/queries/0_stateless/03008_azure_plain_rewritable_with_slash.sh index 528a4364c98..05163999b2b 100755 --- a/tests/queries/0_stateless/03008_azure_plain_rewritable_with_slash.sh +++ b/tests/queries/0_stateless/03008_azure_plain_rewritable_with_slash.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-shared-merge-tree +# Tags: no-fasttest, no-shared-merge-tree, no-distributed-cache # Tag no-fasttest: requires Azure # Tag no-shared-merge-tree: does not support replication +# Tag no-distributed-cache: Not supported auth type set -e From f5c635df3508b87bf830a2d929918b9afa3947d1 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Thu, 5 Dec 2024 10:22:06 +0800 Subject: [PATCH 370/502] fix clang-tidy build --- src/Interpreters/convertFieldToType.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 99f01a6f120..ef70307ccf1 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -585,7 +585,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID throw; } - if (!col->size()) + if (col->empty()) return Field(Null()); Field parsed = (*col)[0]; return convertFieldToType(parsed, type, from_type_hint, format_settings); From 6c8512f12612af4392b9b0f2cb10ff96122ceaab Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Thu, 5 Dec 2024 00:10:32 -0300 Subject: [PATCH 371/502] address comments --- src/Functions/array/arrayPrAUC.cpp | 35 ++++++++++--------- .../0_stateless/03272_array_pr_auc.sql | 8 +++-- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/Functions/array/arrayPrAUC.cpp b/src/Functions/array/arrayPrAUC.cpp index 92849968a92..57f0f447bbd 100644 --- a/src/Functions/array/arrayPrAUC.cpp +++ b/src/Functions/array/arrayPrAUC.cpp @@ -11,7 +11,6 @@ namespace DB namespace ErrorCodes { -extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -19,7 +18,7 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; /** The function takes two arrays: scores and labels. - * Label can be one of two values: positive and negative. + * Label can be one of two values: positive (> 0) and negative (<= 0). * Score can be arbitrary number. * * These values are considered as the output of classifier. We have some true labels for objects. @@ -102,8 +101,8 @@ private: std::sort(sorted_labels.begin(), sorted_labels.end(), [](const auto & lhs, const auto & rhs) { return lhs.score > rhs.score; }); size_t prev_tp = 0; - size_t curr_tp = 0; - size_t curr_p = 0; + size_t curr_tp = 0; /// True positives predictions (positive label and score > threshold) + size_t curr_p = 0; /// Total positive predictions (score > threshold) Float64 prev_score = sorted_labels[0].score; Float64 curr_precision; @@ -125,8 +124,8 @@ private: * * This can be done because (TP + FN) is constant and equal to total positive labels. */ - curr_precision = static_cast(curr_tp) / curr_p; /// curr_p should never be 0 because first element is not executed in this if statement - /// and it is always positive because it has the biggest prediction value. + curr_precision = static_cast(curr_tp) / curr_p; /// curr_p should never be 0 because this if statement isn't executed on the first iteration and the + /// following iterations will have already counted (curr_p += 1) at least one positive prediction area += curr_precision * (curr_tp - prev_tp); prev_tp = curr_tp; prev_score = sorted_labels[i].score; @@ -177,26 +176,28 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - size_t number_of_arguments = arguments.size(); - - if (number_of_arguments < 2 || number_of_arguments > 2) + if (arguments.size() != 2) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Number of arguments for function {} doesn't match: passed {}, should be 2.", getName(), - number_of_arguments); + arguments.size()); for (size_t i = 0; i < 2; ++i) { const DataTypeArray * array_type = checkAndGetDataType(arguments[i].type.get()); if (!array_type) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The two first arguments for function {} must be of type Array.", getName()); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Both arguments for function {} must be of type Array", getName()); const auto & nested_type = array_type->getNestedType(); - if (!isNativeNumber(nested_type) && !isEnum(nested_type)) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{} cannot process values of type {}", getName(), nested_type->getName()); + + /// The first argument (scores) must be an array of numbers + if (i == 0 && !isNativeNumber(nested_type)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{} cannot process values of type {} in its first argument", getName(), nested_type->getName()); + + /// The second argument (labels) must be an array of numbers or enums + if (i == 1 && !isNativeNumber(nested_type) && !isEnum(nested_type)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{} cannot process values of type {} in its second argument", getName(), nested_type->getName()); } return std::make_shared(); @@ -213,7 +214,7 @@ public: if (!col_array1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, - "Illegal column {} of first argument of function {}", + "Illegal column {} of first argument of function {}, should be an Array", arguments[0].column->getName(), getName()); @@ -221,7 +222,7 @@ public: if (!col_array2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, - "Illegal column {} of second argument of function {}", + "Illegal column {} of second argument of function {}, should be an Array", arguments[1].column->getName(), getName()); diff --git a/tests/queries/0_stateless/03272_array_pr_auc.sql b/tests/queries/0_stateless/03272_array_pr_auc.sql index 682a2e2315a..dfdcc5b5fe4 100644 --- a/tests/queries/0_stateless/03272_array_pr_auc.sql +++ b/tests/queries/0_stateless/03272_array_pr_auc.sql @@ -38,8 +38,12 @@ SELECT floor(arrayPrAUC([0, 1, 1], [0, 1, 1]), 10); SELECT floor(arrayPrAUC([0, 1, 1], [0, 0, 1]), 10); -- negative tests -select arrayPrAUC([], []); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select arrayPrAUC([], []); -- { serverError BAD_ARGUMENTS } select arrayPrAUC([0, 0, 1, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select arrayPrAUC([0.1, 0.35], [0, 0, 1, 1]); -- { serverError BAD_ARGUMENTS } -select arrayPrAUC([0.1, 0.4, 0.35, 0.8], []); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], []); -- { serverError BAD_ARGUMENTS } select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1], [1, 1, 0, 1]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +select arrayPrAUC(cast(['false', 'true'] as Array(Enum8('false' = -1, 'true' = 1))), [1, 0]); -- { serverError BAD_ARGUMENTS } +select arrayPrAUC(['a', 'b', 'c', 'd'], [1, 0, 1, 1]); -- { serverError BAD_ARGUMENTS } +select arrayPrAUC([0.1, 0.4, NULL, 0.8], [0, 0, 1, 1]); -- { serverError BAD_ARGUMENTS } +select arrayPrAUC([0.1, 0.4, 0.35, 0.8], [0, NULL, 1, 1]); -- { serverError BAD_ARGUMENTS } From 4d3b3c35884370d2196bef48fbb596f87d7d28d0 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Thu, 5 Dec 2024 11:23:24 +0800 Subject: [PATCH 372/502] add order by in sql test --- .../03278_enum_string_functions.reference | 34 +++++++++---------- .../03278_enum_string_functions.sql | 26 +++++++------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/tests/queries/0_stateless/03278_enum_string_functions.reference b/tests/queries/0_stateless/03278_enum_string_functions.reference index 1c7679c6e63..90f741e10fc 100644 --- a/tests/queries/0_stateless/03278_enum_string_functions.reference +++ b/tests/queries/0_stateless/03278_enum_string_functions.reference @@ -32,42 +32,42 @@ CREATE TABLE jsons ENGINE = Memory; INSERT INTO jsons VALUES ('{"a":1}'); INSERT INTO jsons VALUES ('a'); -SELECT simpleJSONHas(json, 'foo') FROM jsons; +SELECT simpleJSONHas(json, 'foo') as res FROM jsons order by res; 0 0 -SELECT simpleJSONHas(json, 'a') FROM jsons; +SELECT simpleJSONHas(json, 'a') as res FROM jsons order by res; +0 1 +SELECT simpleJSONExtractUInt(json, 'a') as res FROM jsons order by res; 0 -SELECT simpleJSONExtractUInt(json, 'a') FROM jsons; 1 -0 -SELECT simpleJSONExtractUInt(json, 'not exsits') FROM jsons; +SELECT simpleJSONExtractUInt(json, 'not exsits') as res FROM jsons order by res; 0 0 -SELECT simpleJSONExtractInt(json, 'a') FROM jsons; +SELECT simpleJSONExtractInt(json, 'a') as res FROM jsons order by res; +0 1 -0 -SELECT simpleJSONExtractInt(json, 'not exsits') FROM jsons; +SELECT simpleJSONExtractInt(json, 'not exsits') as res FROM jsons order by res; 0 0 -SELECT simpleJSONExtractFloat(json, 'a') FROM jsons; +SELECT simpleJSONExtractFloat(json, 'a') as res FROM jsons order by res; +0 1 -0 -SELECT simpleJSONExtractFloat(json, 'not exsits') FROM jsons; +SELECT simpleJSONExtractFloat(json, 'not exsits') as res FROM jsons order by res; 0 0 -SELECT simpleJSONExtractBool(json, 'a') FROM jsons; +SELECT simpleJSONExtractBool(json, 'a') as res FROM jsons order by res; 0 0 -SELECT simpleJSONExtractBool(json, 'not exsits') FROM jsons; +SELECT simpleJSONExtractBool(json, 'not exsits') as res FROM jsons order by res; 0 0 -SELECT positionUTF8(json, 'a') FROM jsons; +SELECT positionUTF8(json, 'a') as res FROM jsons order by res; +1 3 +SELECT positionCaseInsensitiveUTF8(json, 'A') as res FROM jsons order by res; 1 -SELECT positionCaseInsensitiveUTF8(json, 'A') FROM jsons; 3 +SELECT positionCaseInsensitive(json, 'A') as res FROM jsons order by res; 1 -SELECT positionCaseInsensitive(json, 'A') FROM jsons; 3 -1 diff --git a/tests/queries/0_stateless/03278_enum_string_functions.sql b/tests/queries/0_stateless/03278_enum_string_functions.sql index eda46704d75..9ab8487d86a 100644 --- a/tests/queries/0_stateless/03278_enum_string_functions.sql +++ b/tests/queries/0_stateless/03278_enum_string_functions.sql @@ -23,16 +23,16 @@ CREATE TABLE jsons ENGINE = Memory; INSERT INTO jsons VALUES ('{"a":1}'); INSERT INTO jsons VALUES ('a'); -SELECT simpleJSONHas(json, 'foo') FROM jsons; -SELECT simpleJSONHas(json, 'a') FROM jsons; -SELECT simpleJSONExtractUInt(json, 'a') FROM jsons; -SELECT simpleJSONExtractUInt(json, 'not exsits') FROM jsons; -SELECT simpleJSONExtractInt(json, 'a') FROM jsons; -SELECT simpleJSONExtractInt(json, 'not exsits') FROM jsons; -SELECT simpleJSONExtractFloat(json, 'a') FROM jsons; -SELECT simpleJSONExtractFloat(json, 'not exsits') FROM jsons; -SELECT simpleJSONExtractBool(json, 'a') FROM jsons; -SELECT simpleJSONExtractBool(json, 'not exsits') FROM jsons; -SELECT positionUTF8(json, 'a') FROM jsons; -SELECT positionCaseInsensitiveUTF8(json, 'A') FROM jsons; -SELECT positionCaseInsensitive(json, 'A') FROM jsons; +SELECT simpleJSONHas(json, 'foo') as res FROM jsons order by res; +SELECT simpleJSONHas(json, 'a') as res FROM jsons order by res; +SELECT simpleJSONExtractUInt(json, 'a') as res FROM jsons order by res; +SELECT simpleJSONExtractUInt(json, 'not exsits') as res FROM jsons order by res; +SELECT simpleJSONExtractInt(json, 'a') as res FROM jsons order by res; +SELECT simpleJSONExtractInt(json, 'not exsits') as res FROM jsons order by res; +SELECT simpleJSONExtractFloat(json, 'a') as res FROM jsons order by res; +SELECT simpleJSONExtractFloat(json, 'not exsits') as res FROM jsons order by res; +SELECT simpleJSONExtractBool(json, 'a') as res FROM jsons order by res; +SELECT simpleJSONExtractBool(json, 'not exsits') as res FROM jsons order by res; +SELECT positionUTF8(json, 'a') as res FROM jsons order by res; +SELECT positionCaseInsensitiveUTF8(json, 'A') as res FROM jsons order by res; +SELECT positionCaseInsensitive(json, 'A') as res FROM jsons order by res; From e2cf96535abd4177b8d52df799270dc2e665ff8d Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Thu, 5 Dec 2024 10:58:00 +0100 Subject: [PATCH 373/502] Update rust_vendor submodule again --- contrib/rust_vendor | 2 +- rust/workspace/Cargo.lock | 585 +++++++++++++++++++++++---------- rust/workspace/prql/Cargo.toml | 2 +- rust/workspace/prql/src/lib.rs | 1 + rust/workspace/skim/Cargo.toml | 5 +- rust/workspace/skim/src/lib.rs | 6 +- 6 files changed, 412 insertions(+), 189 deletions(-) diff --git a/contrib/rust_vendor b/contrib/rust_vendor index a73510df047..043d386c6ac 160000 --- a/contrib/rust_vendor +++ b/contrib/rust_vendor @@ -1 +1 @@ -Subproject commit a73510df0475fad6897f9c422c15d82f10abc6f0 +Subproject commit 043d386c6ac6818db25a9ddd6fbc6b59b95e64b2 diff --git a/rust/workspace/Cargo.lock b/rust/workspace/Cargo.lock index d309b26f88d..850922fefc8 100644 --- a/rust/workspace/Cargo.lock +++ b/rust/workspace/Cargo.lock @@ -15,27 +15,13 @@ dependencies = [ name = "_ch_rust_skim_rust" version = "0.1.0" dependencies = [ + "clap", "cxx", "cxx-build", "skim", - "term", + "term 1.0.0", ] -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" - [[package]] name = "ahash" version = "0.8.11" @@ -114,7 +100,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -124,17 +110,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" -dependencies = [ - "backtrace", -] +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "ariadne" @@ -142,7 +125,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44055e597c674aef7cb903b2b9f6e4cba1277ed0d2d61dae7cd52d7ffa81f8e2" dependencies = [ - "unicode-width", + "unicode-width 0.1.14", "yansi", ] @@ -152,27 +135,23 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" -[[package]] -name = "backtrace" -version = "0.3.74" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets", -] - [[package]] name = "beef" version = "0.5.2" @@ -191,12 +170,29 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "bstr" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a68f1f47cdf0ec8ee4b941b2eee2a80cb796db73118c0dd09ac63fbe405be22" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "cc" version = "1.2.2" @@ -212,6 +208,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.38" @@ -238,22 +240,36 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.21" +version = "4.5.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +checksum = "69371e34337c4c984bbe322360c2547210bf632eb2814bbe78a6e87a2935bd2b" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] name = "clap_builder" -version = "4.5.21" +version = "4.5.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +checksum = "6e24c1b4099818523236a8ca881d2b45db98dadfb4625cf6608c12069fcbbde1" dependencies = [ + "anstream", "anstyle", "clap_lex", - "strsim 0.11.1", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -269,7 +285,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" dependencies = [ "termcolor", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -386,7 +402,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.90", + "syn", ] [[package]] @@ -399,7 +415,7 @@ dependencies = [ "codespan-reporting", "proc-macro2", "quote", - "syn 2.0.90", + "syn", ] [[package]] @@ -417,14 +433,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.90", + "syn", ] [[package]] name = "darling" -version = "0.14.4" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ "darling_core", "darling_macro", @@ -432,27 +448,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.4" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", - "syn 1.0.109", + "strsim", + "syn", ] [[package]] name = "darling_macro" -version = "0.14.4" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 1.0.109", + "syn", ] [[package]] @@ -476,33 +492,33 @@ dependencies = [ [[package]] name = "derive_builder" -version = "0.11.2" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" dependencies = [ "derive_builder_macro", ] [[package]] name = "derive_builder_core" -version = "0.11.2" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" dependencies = [ "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "derive_builder_macro" -version = "0.11.2" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 1.0.109", + "syn", ] [[package]] @@ -526,6 +542,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dyn-clone" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" + [[package]] name = "either" version = "1.13.0" @@ -541,7 +563,30 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.90", + "syn", +] + +[[package]] +name = "env_filter" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", ] [[package]] @@ -550,6 +595,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "fnv" version = "1.0.7" @@ -582,12 +637,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - [[package]] name = "hashbrown" version = "0.14.5" @@ -610,6 +659,30 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "iana-time-zone" version = "0.1.61" @@ -657,9 +730,9 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] @@ -711,6 +784,12 @@ dependencies = [ "cc", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "log" version = "0.4.22" @@ -723,30 +802,12 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - [[package]] name = "minimal-lexical" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" -dependencies = [ - "adler2", -] - [[package]] name = "nix" version = "0.24.3" @@ -760,16 +821,14 @@ dependencies = [ [[package]] name = "nix" -version = "0.25.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "autocfg", - "bitflags 1.3.2", + "bitflags 2.6.0", "cfg-if", + "cfg_aliases", "libc", - "memoffset", - "pin-utils", ] [[package]] @@ -798,12 +857,12 @@ dependencies = [ ] [[package]] -name = "object" -version = "0.36.5" +name = "num_threads" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ - "memchr", + "libc", ] [[package]] @@ -812,18 +871,21 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "powerfmt" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.92" @@ -835,56 +897,45 @@ dependencies = [ [[package]] name = "prqlc" -version = "0.11.3" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beb05b6b71ce096fa56d73006ab1c42a8d11bf190d193fa511a134f7730ec43" +checksum = "f4e46dd8e5e577feadd15c1f30c1e703cb9b34ee06ebf167526fc54aa22cfefb" dependencies = [ "anstream", - "anyhow", "ariadne", "chrono", "csv", "enum-as-inner", "itertools", "log", - "once_cell", - "prqlc-ast", "prqlc-parser", "regex", + "schemars", "semver", "serde", "serde_json", - "serde_yaml", "sqlformat", "sqlparser", "strum", "strum_macros", -] - -[[package]] -name = "prqlc-ast" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c98923b046bc48046e3846b14a5fde5a059f681c7c367bd0ab96ebd3ecc33a71" -dependencies = [ - "anyhow", - "enum-as-inner", - "semver", - "serde", - "strum", + "vergen-gitcl", ] [[package]] name = "prqlc-parser" -version = "0.11.3" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "855ad9aba599ef608efc88a30ebd731155997d9bbe780639eb175de060b6cddc" +checksum = "0ea2bceb578101432556cf8798f252657ef48f7f502d450dac083d663c40c6db" dependencies = [ "chumsky", + "enum-as-inner", "itertools", - "prqlc-ast", + "log", + "schemars", "semver", + "serde", "stacker", + "strum", ] [[package]] @@ -905,6 +956,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "rayon" version = "1.10.0" @@ -936,6 +1017,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ref-cast" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf0a6f84d5f1d581da8b41b47ec8600871962f2a528115b542b362d4b744931" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "regex" version = "1.11.1" @@ -966,10 +1067,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] -name = "rustc-demangle" -version = "0.1.24" +name = "rustix" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] [[package]] name = "rustversion" @@ -983,6 +1091,31 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "schemars" +version = "1.0.0-alpha.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ef2a6523400a2228db974a8ddc9e1d3deaa04f51bddd7832ef8d7e531bafcc" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.0.0-alpha.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6d4e1945a3c9e58edaa708449b026519f7f4197185e1b5dbc689615c1ad724d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + [[package]] name = "scratch" version = "1.0.7" @@ -1015,7 +1148,18 @@ checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1031,16 +1175,12 @@ dependencies = [ ] [[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" +name = "shell-quote" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +checksum = "ae4c63bdcc11eea49b562941b914d5ac30d42cad982e3f6e846a513ee6a3ce7e" dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", + "bstr", ] [[package]] @@ -1051,27 +1191,35 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "skim" -version = "0.10.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d28de0a6cb2cdd83a076f1de9d965b973ae08b244df1aa70b432946dda0f32" +checksum = "713b5a46e3df69e892e024779d2c5418e886d52454b150569962a04c0ed88c61" dependencies = [ + "atty", "beef", "bitflags 1.3.2", "chrono", + "clap", "crossbeam", "defer-drop", "derive_builder", + "env_logger", "fuzzy-matcher", + "indexmap", "lazy_static", "log", - "nix 0.25.1", + "nix 0.29.0", + "rand", "rayon", "regex", + "shell-quote", + "shlex", "time", "timer", "tuikit", - "unicode-width", + "unicode-width 0.2.0", "vte", + "which", ] [[package]] @@ -1086,9 +1234,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.43.1" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f95c4bae5aba7cd30bd506f7140026ade63cff5afd778af8854026f9606bf5d4" +checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" dependencies = [ "log", "serde", @@ -1104,15 +1252,9 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys", + "windows-sys 0.59.0", ] -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "strsim" version = "0.11.1" @@ -1138,18 +1280,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.90", -] - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "syn", ] [[package]] @@ -1174,6 +1305,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "term" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4df4175de05129f31b80458c6df371a15e7fc3fd367272e6bf938e5c351c7ea0" +dependencies = [ + "home", + "windows-sys 0.52.0", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -1200,7 +1341,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn", ] [[package]] @@ -1220,10 +1361,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", + "itoa", + "libc", "num-conv", + "num_threads", "powerfmt", "serde", "time-core", + "time-macros", ] [[package]] @@ -1232,6 +1377,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "timer" version = "0.2.0" @@ -1250,8 +1405,8 @@ dependencies = [ "lazy_static", "log", "nix 0.24.3", - "term", - "unicode-width", + "term 0.7.0", + "unicode-width 0.1.14", ] [[package]] @@ -1266,24 +1421,62 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode_categories" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vergen" +version = "9.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f25fc8f8f05df455c7941e87f093ad22522a9ff33d7a027774815acf6f0639" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", + "time", + "vergen-lib", +] + +[[package]] +name = "vergen-gitcl" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0227006d09f98ab00ea69e9a5e055e676a813cfbed4232986176c86a6080b997" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", + "time", + "vergen", + "vergen-lib", +] + +[[package]] +name = "vergen-lib" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0c767e6751c09fc85cde58722cf2f1007e80e4c8d5a4321fc90d83dc54ca147" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", +] + [[package]] name = "version_check" version = "0.9.5" @@ -1292,9 +1485,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vte" -version = "0.11.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +checksum = "40eb22ae96f050e0c0d6f7ce43feeae26c348fc4dea56928ca81537cfaa6188b" dependencies = [ "arrayvec", "utf8parse", @@ -1339,7 +1532,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.90", + "syn", "wasm-bindgen-shared", ] @@ -1361,7 +1554,7 @@ checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1372,6 +1565,18 @@ version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" +[[package]] +name = "which" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cad3279ade7346b96e38731a641d7343dd6a53d55083dd54eadfa5a1b38c6b" +dependencies = [ + "either", + "home", + "rustix", + "winsafe", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1394,7 +1599,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -1412,6 +1617,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -1485,6 +1699,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winsafe" +version = "0.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" + [[package]] name = "yansi" version = "1.0.1" @@ -1497,6 +1717,7 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] @@ -1508,5 +1729,5 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn", ] diff --git a/rust/workspace/prql/Cargo.toml b/rust/workspace/prql/Cargo.toml index 38d3a58e8c0..26e70fd399e 100644 --- a/rust/workspace/prql/Cargo.toml +++ b/rust/workspace/prql/Cargo.toml @@ -5,7 +5,7 @@ version = "0.1.0" [dependencies] anstream = {version = "0.6.12"} -prqlc = {version = "0.11.3", default-features = false} +prqlc = {version = "0.13.2", default-features = false} serde_json = "1.0" [lib] diff --git a/rust/workspace/prql/src/lib.rs b/rust/workspace/prql/src/lib.rs index 2e5b2061fcb..920e2e96b69 100644 --- a/rust/workspace/prql/src/lib.rs +++ b/rust/workspace/prql/src/lib.rs @@ -36,6 +36,7 @@ pub unsafe extern "C" fn prql_to_sql_impl( target: Target::Sql(Some(Dialect::ClickHouse)), signature_comment: false, color: false, + display: prqlc::DisplayOptions::Plain, }; if let Ok(sql_str) = prqlc::compile(&query_str, &opts) { diff --git a/rust/workspace/skim/Cargo.toml b/rust/workspace/skim/Cargo.toml index e077fe7a8b9..e153e9058ae 100644 --- a/rust/workspace/skim/Cargo.toml +++ b/rust/workspace/skim/Cargo.toml @@ -6,9 +6,10 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -skim = { version = "0.10.2", default-features = false } +skim = { version = "0.15.5", default-features = false, features = ["cli"] } cxx = "1.0.83" -term = "0.7.0" +term = "1.0.0" +clap = "4.5.22" [build-dependencies] cxx-build = "1.0.83" diff --git a/rust/workspace/skim/src/lib.rs b/rust/workspace/skim/src/lib.rs index d03059fae5c..cb18564053d 100644 --- a/rust/workspace/skim/src/lib.rs +++ b/rust/workspace/skim/src/lib.rs @@ -44,15 +44,15 @@ fn skim_impl(prefix: &CxxString, words: &CxxVector) -> Result Date: Thu, 5 Dec 2024 10:26:03 +0000 Subject: [PATCH 374/502] Missing include --- src/Storages/MergeTree/MergeTreeDataPartWide.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 06cb3c865a0..9da9c36bd19 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include From 25399837285b3d729e5fc63faf5a367ae40ce430 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 5 Dec 2024 12:09:36 +0100 Subject: [PATCH 375/502] Fix segfault in CascadeWriteBuffer::getResultBuffers() --- src/IO/CascadeWriteBuffer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/IO/CascadeWriteBuffer.cpp b/src/IO/CascadeWriteBuffer.cpp index 6cd2c540643..d024e40f893 100644 --- a/src/IO/CascadeWriteBuffer.cpp +++ b/src/IO/CascadeWriteBuffer.cpp @@ -56,6 +56,9 @@ void CascadeWriteBuffer::nextImpl() CascadeWriteBuffer::WriteBufferPtrs CascadeWriteBuffer::getResultBuffers() { + if (!curr_buffer) + return {}; + /// Sync position with underlying buffer before invalidating curr_buffer->position() = position(); From 90d655dd96b7b8bea494692b2dfc7223481a3030 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 5 Dec 2024 12:32:43 +0100 Subject: [PATCH 376/502] Limit the docker memory to {HOST_MEM-2G} --- tests/integration/runner | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/runner b/tests/integration/runner index 9406e9a0286..64d483298b9 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -15,7 +15,7 @@ import sys from integration_test_images import get_docker_env -def random_str(length=6): +def random_str(length: int = 6) -> str: alphabet = string.ascii_lowercase + string.digits return "".join(random.SystemRandom().choice(alphabet) for _ in range(length)) @@ -33,6 +33,15 @@ UTILS_DIR_IN_REPO = "utils" DIND_INTEGRATION_TESTS_IMAGE_NAME = "clickhouse/integration-tests-runner" +def physical_memory() -> int: + try: + # for linux + return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") + except ValueError: + # for MacOS + return int(subprocess.check_output(["sysctl", "-n", "hw.memsize"]).strip()) + + def check_args_and_update_paths(args): if args.clickhouse_root: if not os.path.isabs(args.clickhouse_root): @@ -469,6 +478,7 @@ if __name__ == "__main__": cmd_base = ( f"docker run {net} {tty} --rm --name {CONTAINER_NAME} " "--privileged --dns-search='.' " # since recent dns search leaks from host + f"--memory={physical_memory() - 2 * 1024**3} " f"--volume={args.odbc_bridge_binary}:/clickhouse-odbc-bridge " f"--volume={args.binary}:/clickhouse " f"--volume={args.library_bridge_binary}:/clickhouse-library-bridge " From aaf4d860fe3fbecbfc6a8df3696e7e9b40d46cb0 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Thu, 5 Dec 2024 12:36:14 +0100 Subject: [PATCH 377/502] Poke CI From 9489137e23365ed522c93d4212438ca98d77a685 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 5 Dec 2024 12:39:52 +0100 Subject: [PATCH 378/502] Fix typing warnings --- tests/integration/runner | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/runner b/tests/integration/runner index 64d483298b9..bb4b7d7e2cb 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -11,6 +11,7 @@ import signal import string import subprocess import sys +from typing import Any from integration_test_images import get_docker_env @@ -42,7 +43,7 @@ def physical_memory() -> int: return int(subprocess.check_output(["sysctl", "-n", "hw.memsize"]).strip()) -def check_args_and_update_paths(args): +def check_args_and_update_paths(args: argparse.Namespace) -> None: if args.clickhouse_root: if not os.path.isabs(args.clickhouse_root): CLICKHOUSE_ROOT = os.path.abspath(args.clickhouse_root) @@ -145,7 +146,7 @@ def check_args_and_update_paths(args): ) -def check_iptables_legacy(): +def check_iptables_legacy() -> None: iptables_path = shutil.which("iptables") ip6tables_path = shutil.which("ip6tables") @@ -192,7 +193,7 @@ def check_iptables_legacy(): sys.exit(1) -def docker_kill_handler_handler(signum, frame): +def docker_kill_handler_handler(signum: Any, frame: Any) -> None: _, _ = signum, frame subprocess.check_call( f"docker ps --all --quiet --filter name={CONTAINER_NAME}", From ba517d6762f47b6b909ea381d18694c41fffdb6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 5 Dec 2024 12:47:02 +0100 Subject: [PATCH 379/502] Revert "More insistent compression in `StorageMemory`" --- docs/en/engines/table-engines/special/memory.md | 2 -- src/Columns/ColumnArray.cpp | 6 +++--- src/Columns/ColumnArray.h | 2 +- src/Columns/ColumnCompressed.cpp | 5 ++--- src/Columns/ColumnCompressed.h | 2 +- src/Columns/ColumnDecimal.cpp | 4 ++-- src/Columns/ColumnDecimal.h | 2 +- src/Columns/ColumnDynamic.cpp | 4 ++-- src/Columns/ColumnDynamic.h | 2 +- src/Columns/ColumnFixedString.cpp | 4 ++-- src/Columns/ColumnFixedString.h | 2 +- src/Columns/ColumnMap.cpp | 4 ++-- src/Columns/ColumnMap.h | 2 +- src/Columns/ColumnNullable.cpp | 6 +++--- src/Columns/ColumnNullable.h | 2 +- src/Columns/ColumnObject.cpp | 8 ++++---- src/Columns/ColumnObject.h | 2 +- src/Columns/ColumnSparse.cpp | 6 +++--- src/Columns/ColumnSparse.h | 2 +- src/Columns/ColumnString.cpp | 6 +++--- src/Columns/ColumnString.h | 2 +- src/Columns/ColumnTuple.cpp | 4 ++-- src/Columns/ColumnTuple.h | 2 +- src/Columns/ColumnVariant.cpp | 8 ++++---- src/Columns/ColumnVariant.h | 2 +- src/Columns/ColumnVector.cpp | 4 ++-- src/Columns/ColumnVector.h | 2 +- src/Columns/IColumn.h | 3 +-- src/Core/Block.cpp | 2 +- src/Interpreters/Cache/QueryCache.cpp | 2 +- src/Storages/StorageMemory.cpp | 7 ++++--- 31 files changed, 54 insertions(+), 57 deletions(-) diff --git a/docs/en/engines/table-engines/special/memory.md b/docs/en/engines/table-engines/special/memory.md index 3eb3e617ff9..f28157ebde2 100644 --- a/docs/en/engines/table-engines/special/memory.md +++ b/docs/en/engines/table-engines/special/memory.md @@ -36,8 +36,6 @@ Upper and lower bounds can be specified to limit Memory engine table size, effec - Requires `max_rows_to_keep` - `max_rows_to_keep` — Maximum rows to keep within memory table where oldest rows are deleted on each insertion (i.e circular buffer). Max rows can exceed the stated limit if the oldest batch of rows to remove falls under the `min_rows_to_keep` limit when adding a large block. - Default value: `0` -- `compress` - Whether to compress data in memory. - - Default value: `false` ## Usage {#usage} diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 013821db2c9..3f88ca93a97 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -1024,10 +1024,10 @@ void ColumnArray::updatePermutationWithCollation(const Collator & collator, Perm DefaultPartialSort()); } -ColumnPtr ColumnArray::compress(bool force_compression) const +ColumnPtr ColumnArray::compress() const { - ColumnPtr data_compressed = data->compress(force_compression); - ColumnPtr offsets_compressed = offsets->compress(force_compression); + ColumnPtr data_compressed = data->compress(); + ColumnPtr offsets_compressed = offsets->compress(); size_t byte_size = data_compressed->byteSize() + offsets_compressed->byteSize(); diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index dee6ae931f2..a66f9041213 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -159,7 +159,7 @@ public: /// For example, `getDataInRange(0, size())` is the same as `getDataPtr()->clone()`. MutableColumnPtr getDataInRange(size_t start, size_t length) const; - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnCompressed.cpp b/src/Columns/ColumnCompressed.cpp index adb2a5f391d..3bdc514d6d8 100644 --- a/src/Columns/ColumnCompressed.cpp +++ b/src/Columns/ColumnCompressed.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes } -std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, size_t data_size, bool force_compression) +std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, size_t data_size, bool always_compress) { size_t max_dest_size = LZ4_COMPRESSBOUND(data_size); @@ -35,8 +35,7 @@ std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, si throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress column"); /// If compression is inefficient. - const size_t threshold = force_compression ? 1 : 2; - if (static_cast(compressed_size) * threshold > data_size) + if (!always_compress && static_cast(compressed_size) * 2 > data_size) return {}; /// Shrink to fit. diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h index b030e762acd..c4270e8216b 100644 --- a/src/Columns/ColumnCompressed.h +++ b/src/Columns/ColumnCompressed.h @@ -72,7 +72,7 @@ public: /// If data is not worth to be compressed and not 'always_compress' - returns nullptr. /// Note: shared_ptr is to allow to be captured by std::function. - static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool force_compression); + static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool always_compress); static void decompressBuffer( const void * compressed_data, void * decompressed_data, size_t compressed_size, size_t decompressed_size); diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index c286c54198a..73366150e7d 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -478,7 +478,7 @@ ColumnPtr ColumnDecimal::replicate(const IColumn::Offsets & offsets) const } template -ColumnPtr ColumnDecimal::compress(bool force_compression) const +ColumnPtr ColumnDecimal::compress() const { const size_t data_size = data.size(); const size_t source_size = data_size * sizeof(T); @@ -487,7 +487,7 @@ ColumnPtr ColumnDecimal::compress(bool force_compression) const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, force_compression); + auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, false); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 3e5c189b731..690549e4a56 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -140,7 +140,7 @@ public: return false; } - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; void insertValue(const T value) { data.push_back(value); } Container & getData() { return data; } diff --git a/src/Columns/ColumnDynamic.cpp b/src/Columns/ColumnDynamic.cpp index 2d05701c57b..a4c932eafdd 100644 --- a/src/Columns/ColumnDynamic.cpp +++ b/src/Columns/ColumnDynamic.cpp @@ -991,9 +991,9 @@ void ColumnDynamic::updatePermutation(IColumn::PermutationSortDirection directio updatePermutationImpl(limit, res, equal_ranges, ComparatorDescendingStable(*this, nan_direction_hint), comparator_equal, DefaultSort(), DefaultPartialSort()); } -ColumnPtr ColumnDynamic::compress(bool force_compression) const +ColumnPtr ColumnDynamic::compress() const { - ColumnPtr variant_compressed = variant_column_ptr->compress(force_compression); + ColumnPtr variant_compressed = variant_column_ptr->compress(); size_t byte_size = variant_compressed->byteSize(); return ColumnCompressed::create(size(), byte_size, [my_variant_compressed = std::move(variant_compressed), my_variant_info = variant_info, my_max_dynamic_types = max_dynamic_types, my_global_max_dynamic_types = global_max_dynamic_types, my_statistics = statistics]() mutable diff --git a/src/Columns/ColumnDynamic.h b/src/Columns/ColumnDynamic.h index 093aaaf2793..bdbad99519f 100644 --- a/src/Columns/ColumnDynamic.h +++ b/src/Columns/ColumnDynamic.h @@ -335,7 +335,7 @@ public: return false; } - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; double getRatioOfDefaultRows(double sample_ratio) const override { diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index f076f904768..04e894ee5ab 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -419,7 +419,7 @@ void ColumnFixedString::getExtremes(Field & min, Field & max) const get(max_idx, max); } -ColumnPtr ColumnFixedString::compress(bool force_compression) const +ColumnPtr ColumnFixedString::compress() const { size_t source_size = chars.size(); @@ -427,7 +427,7 @@ ColumnPtr ColumnFixedString::compress(bool force_compression) const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(chars.data(), source_size, force_compression); + auto compressed = ColumnCompressed::compressBuffer(chars.data(), source_size, false); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index f55fb60a976..8cf0a6a57da 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -175,7 +175,7 @@ public: ColumnPtr replicate(const Offsets & offsets) const override; - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; void reserve(size_t size) override { diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp index fb9c8c9fbaf..a5511dfeeb4 100644 --- a/src/Columns/ColumnMap.cpp +++ b/src/Columns/ColumnMap.cpp @@ -352,9 +352,9 @@ bool ColumnMap::dynamicStructureEquals(const IColumn & rhs) const return false; } -ColumnPtr ColumnMap::compress(bool force_compression) const +ColumnPtr ColumnMap::compress() const { - auto compressed = nested->compress(force_compression); + auto compressed = nested->compress(); const auto byte_size = compressed->byteSize(); /// The order of evaluation of function arguments is unspecified /// and could cause interacting with object in moved-from state diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h index 31404a3e152..8dfa5bb5845 100644 --- a/src/Columns/ColumnMap.h +++ b/src/Columns/ColumnMap.h @@ -120,7 +120,7 @@ public: const ColumnTuple & getNestedData() const { return assert_cast(getNestedColumn().getData()); } ColumnTuple & getNestedData() { return assert_cast(getNestedColumn().getData()); } - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; bool hasDynamicStructure() const override { return nested->hasDynamicStructure(); } bool dynamicStructureEquals(const IColumn & rhs) const override; diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 640550fcf9a..6e8bd3fc70c 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -773,10 +773,10 @@ void ColumnNullable::protect() getNullMapColumn().protect(); } -ColumnPtr ColumnNullable::compress(bool force_compression) const +ColumnPtr ColumnNullable::compress() const { - ColumnPtr nested_compressed = nested_column->compress(force_compression); - ColumnPtr null_map_compressed = null_map->compress(force_compression); + ColumnPtr nested_compressed = nested_column->compress(); + ColumnPtr null_map_compressed = null_map->compress(); size_t byte_size = nested_column->byteSize() + null_map->byteSize(); diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 3a0be008cc2..32ce66c5965 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -141,7 +141,7 @@ public: // Special function for nullable minmax index void getExtremesNullLast(Field & min, Field & max) const; - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 17a90cc9b50..064145c5d4f 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -1224,14 +1224,14 @@ bool ColumnObject::structureEquals(const IColumn & rhs) const return true; } -ColumnPtr ColumnObject::compress(bool force_compression) const +ColumnPtr ColumnObject::compress() const { std::unordered_map compressed_typed_paths; compressed_typed_paths.reserve(typed_paths.size()); size_t byte_size = 0; for (const auto & [path, column] : typed_paths) { - auto compressed_column = column->compress(force_compression); + auto compressed_column = column->compress(); byte_size += compressed_column->byteSize(); compressed_typed_paths[path] = std::move(compressed_column); } @@ -1240,12 +1240,12 @@ ColumnPtr ColumnObject::compress(bool force_compression) const compressed_dynamic_paths.reserve(dynamic_paths_ptrs.size()); for (const auto & [path, column] : dynamic_paths_ptrs) { - auto compressed_column = column->compress(force_compression); + auto compressed_column = column->compress(); byte_size += compressed_column->byteSize(); compressed_dynamic_paths[path] = std::move(compressed_column); } - auto compressed_shared_data = shared_data->compress(force_compression); + auto compressed_shared_data = shared_data->compress(); byte_size += compressed_shared_data->byteSize(); auto decompress = diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index 3160b66cd20..7b8a381d571 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -171,7 +171,7 @@ public: bool structureEquals(const IColumn & rhs) const override; - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; void finalize() override; bool isFinalized() const override; diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp index b7d82ed8a09..a0e47e65fc6 100644 --- a/src/Columns/ColumnSparse.cpp +++ b/src/Columns/ColumnSparse.cpp @@ -774,10 +774,10 @@ UInt64 ColumnSparse::getNumberOfDefaultRows() const return _size - offsets->size(); } -ColumnPtr ColumnSparse::compress(bool force_compression) const +ColumnPtr ColumnSparse::compress() const { - auto values_compressed = values->compress(force_compression); - auto offsets_compressed = offsets->compress(force_compression); + auto values_compressed = values->compress(); + auto offsets_compressed = offsets->compress(); size_t byte_size = values_compressed->byteSize() + offsets_compressed->byteSize(); diff --git a/src/Columns/ColumnSparse.h b/src/Columns/ColumnSparse.h index f95752cd546..619dce63c1e 100644 --- a/src/Columns/ColumnSparse.h +++ b/src/Columns/ColumnSparse.h @@ -147,7 +147,7 @@ public: double getRatioOfDefaultRows(double sample_ratio) const override; UInt64 getNumberOfDefaultRows() const override; - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 0ed4f5f432d..269c20397b4 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -627,7 +627,7 @@ void ColumnString::getExtremes(Field & min, Field & max) const get(max_idx, max); } -ColumnPtr ColumnString::compress(bool force_compression) const +ColumnPtr ColumnString::compress() const { const size_t source_chars_size = chars.size(); const size_t source_offsets_elements = offsets.size(); @@ -637,13 +637,13 @@ ColumnPtr ColumnString::compress(bool force_compression) const if (source_chars_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto chars_compressed = ColumnCompressed::compressBuffer(chars.data(), source_chars_size, force_compression); + auto chars_compressed = ColumnCompressed::compressBuffer(chars.data(), source_chars_size, false); /// Return original column if not compressible. if (!chars_compressed) return ColumnCompressed::wrap(this->getPtr()); - auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, /*force_compression=*/true); + auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, true); const size_t chars_compressed_size = chars_compressed->size(); const size_t offsets_compressed_size = offsets_compressed->size(); diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index b2e340be61b..c2371412437 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -284,7 +284,7 @@ public: ColumnPtr replicate(const Offsets & replicate_offsets) const override; - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; void reserve(size_t n) override; size_t capacity() const override; diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 9bb377f56ae..28e5f03cc3c 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -796,7 +796,7 @@ void ColumnTuple::takeDynamicStructureFromSourceColumns(const Columns & source_c } -ColumnPtr ColumnTuple::compress(bool force_compression) const +ColumnPtr ColumnTuple::compress() const { if (columns.empty()) { @@ -812,7 +812,7 @@ ColumnPtr ColumnTuple::compress(bool force_compression) const compressed.reserve(columns.size()); for (const auto & column : columns) { - auto compressed_column = column->compress(force_compression); + auto compressed_column = column->compress(); byte_size += compressed_column->byteSize(); compressed.emplace_back(std::move(compressed_column)); } diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index b8b3697b84d..d5eee911edc 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -125,7 +125,7 @@ public: void forEachSubcolumnRecursively(RecursiveMutableColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; bool isCollationSupported() const override; - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; void finalize() override; bool isFinalized() const override; diff --git a/src/Columns/ColumnVariant.cpp b/src/Columns/ColumnVariant.cpp index 38d3bac3c10..2fa59b8e33c 100644 --- a/src/Columns/ColumnVariant.cpp +++ b/src/Columns/ColumnVariant.cpp @@ -1426,16 +1426,16 @@ bool ColumnVariant::dynamicStructureEquals(const IColumn & rhs) const return true; } -ColumnPtr ColumnVariant::compress(bool force_compression) const +ColumnPtr ColumnVariant::compress() const { - ColumnPtr local_discriminators_compressed = local_discriminators->compress(force_compression); - ColumnPtr offsets_compressed = offsets->compress(force_compression); + ColumnPtr local_discriminators_compressed = local_discriminators->compress(); + ColumnPtr offsets_compressed = offsets->compress(); size_t byte_size = local_discriminators_compressed->byteSize() + offsets_compressed->byteSize(); Columns compressed; compressed.reserve(variants.size()); for (const auto & variant : variants) { - auto compressed_variant = variant->compress(force_compression); + auto compressed_variant = variant->compress(); byte_size += compressed_variant->byteSize(); compressed.emplace_back(std::move(compressed_variant)); } diff --git a/src/Columns/ColumnVariant.h b/src/Columns/ColumnVariant.h index c7e37517004..a68a961169c 100644 --- a/src/Columns/ColumnVariant.h +++ b/src/Columns/ColumnVariant.h @@ -254,7 +254,7 @@ public: void forEachSubcolumn(MutableColumnCallback callback) override; void forEachSubcolumnRecursively(RecursiveMutableColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; double getRatioOfDefaultRows(double sample_ratio) const override; UInt64 getNumberOfDefaultRows() const override; void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override; diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 62f6c23c4f8..3c7727f37c4 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -951,7 +951,7 @@ void ColumnVector::getExtremes(Field & min, Field & max) const } template -ColumnPtr ColumnVector::compress(bool force_compression) const +ColumnPtr ColumnVector::compress() const { const size_t data_size = data.size(); const size_t source_size = data_size * sizeof(T); @@ -960,7 +960,7 @@ ColumnPtr ColumnVector::compress(bool force_compression) const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, force_compression); + auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, false); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 22b064ae053..1387cca1ece 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -286,7 +286,7 @@ public: ColumnPtr createWithOffsets(const IColumn::Offsets & offsets, const ColumnConst & column_with_default_value, size_t total_rows, size_t shift) const override; - ColumnPtr compress(bool force_compression) const override; + ColumnPtr compress() const override; /// Replace elements that match the filter with zeroes. If inverted replaces not matched elements. void applyZeroMap(const IColumn::Filter & filt, bool inverted = false); diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index e2099ac34b9..9d1b42d2bc1 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -601,8 +601,7 @@ public: /// Compress column in memory to some representation that allows to decompress it back. /// Return itself if compression is not applicable for this column type. - /// The flag `force_compression` indicates that compression should be performed even if it's not efficient (if only compression factor < 1). - [[nodiscard]] virtual Ptr compress([[maybe_unused]] bool force_compression) const + [[nodiscard]] virtual Ptr compress() const { /// No compression by default. return getPtr(); diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 0efb4596dcd..02176a6b77a 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -608,7 +608,7 @@ Block Block::compress() const size_t num_columns = data.size(); Columns new_columns(num_columns); for (size_t i = 0; i < num_columns; ++i) - new_columns[i] = data[i].column->compress(/*force_compression=*/false); + new_columns[i] = data[i].column->compress(); return cloneWithColumns(new_columns); } diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp index de3d720fc35..7dbee567c5b 100644 --- a/src/Interpreters/Cache/QueryCache.cpp +++ b/src/Interpreters/Cache/QueryCache.cpp @@ -469,7 +469,7 @@ void QueryCache::Writer::finalizeWrite() Columns compressed_columns; for (const auto & column : columns) { - auto compressed_column = column->compress(/*force_compression=*/false); + auto compressed_column = column->compress(); compressed_columns.push_back(compressed_column); } Chunk compressed_chunk(compressed_columns, chunk.getNumRows()); diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 55574e7b1db..d798e1b4fb5 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -91,7 +91,8 @@ public: { Block compressed_block; for (const auto & elem : block) - compressed_block.insert({elem.column->compress(/*force_compression=*/true), elem.type, elem.name}); + compressed_block.insert({ elem.column->compress(), elem.type, elem.name }); + new_blocks.push_back(std::move(compressed_block)); } else @@ -258,7 +259,7 @@ void StorageMemory::mutate(const MutationCommands & commands, ContextPtr context { if ((*memory_settings)[MemorySetting::compress]) for (auto & elem : block) - elem.column = elem.column->compress(/*force_compression=*/true); + elem.column = elem.column->compress(); out.push_back(block); } @@ -573,7 +574,7 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat { Block compressed_block; for (const auto & elem : block) - compressed_block.insert({elem.column->compress(/*force_compression=*/true), elem.type, elem.name}); + compressed_block.insert({ elem.column->compress(), elem.type, elem.name }); new_blocks.push_back(std::move(compressed_block)); } From 5b1d454f2ebf17fe821316bee72b4effac3ad82d Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 5 Dec 2024 13:14:33 +0100 Subject: [PATCH 380/502] Make docker container memory a CLI argument --- tests/integration/runner | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/integration/runner b/tests/integration/runner index bb4b7d7e2cb..0adaf90200d 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -286,6 +286,14 @@ if __name__ == "__main__": help="Set network driver for runnner container (defaults to `host`)", ) + parser.add_argument( + "--memory", + # Default is host memory - 2GiB + default=physical_memory() - 2 * 1024**3, + help="Set the memory limit for the docker container, " + "accepts the same argument as `docker run --memory`", + ) + parser.add_argument( "--docker-image-version", default="latest", @@ -479,7 +487,7 @@ if __name__ == "__main__": cmd_base = ( f"docker run {net} {tty} --rm --name {CONTAINER_NAME} " "--privileged --dns-search='.' " # since recent dns search leaks from host - f"--memory={physical_memory() - 2 * 1024**3} " + f"--memory={args.memory} " f"--volume={args.odbc_bridge_binary}:/clickhouse-odbc-bridge " f"--volume={args.binary}:/clickhouse " f"--volume={args.library_bridge_binary}:/clickhouse-library-bridge " From 726054d161a90ee94a3daa8a034a3398cf5d3cd6 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 5 Dec 2024 13:15:36 +0100 Subject: [PATCH 381/502] Print the argparse help with defaults --- tests/integration/runner | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/runner b/tests/integration/runner index 0adaf90200d..134c32d0441 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -222,7 +222,10 @@ if __name__ == "__main__": format="%(asctime)s [ %(process)d ] %(levelname)s : %(message)s (%(filename)s:%(lineno)s, %(funcName)s)", ) - parser = argparse.ArgumentParser(description="ClickHouse integration tests runner") + parser = argparse.ArgumentParser( + description="ClickHouse integration tests runner", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) parser.add_argument( "--binary", From c23182846a0c811d7b97cf11ca54051ca185185a Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:15:44 +0100 Subject: [PATCH 382/502] Remove templates from GroupConcatData --- .../AggregateFunctionGroupConcat.cpp | 11 ++++------- src/AggregateFunctions/AggregateFunctionGroupConcat.h | 3 +-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp index 81dd7541e3a..b1ff062f270 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp @@ -38,20 +38,17 @@ void GroupConcatDataBase::insert(const IColumn * column, const SerializationPtr insertChar(string.data(), string.size(), arena); } -template -UInt64 GroupConcatData::getSize(size_t i) const +UInt64 GroupConcatData::getSize(size_t i) const { return offsets[i * 2 + 1] - offsets[i * 2]; } -template -UInt64 GroupConcatData::getString(size_t i) const +UInt64 GroupConcatData::getString(size_t i) const { return offsets[i * 2]; } -template -void GroupConcatData::insert(const IColumn * column, const SerializationPtr & serialization, size_t row_num, Arena * arena) +void GroupConcatData::insert(const IColumn * column, const SerializationPtr & serialization, size_t row_num, Arena * arena) { WriteBufferFromOwnString buff; serialization->serializeText(*column, row_num, buff, {}); @@ -68,7 +65,7 @@ void GroupConcatData::insert(const IColumn * column, const Serializat template GroupConcatImpl::GroupConcatImpl( const DataTypePtr & data_type_, const Array & parameters_, UInt64 limit_, const String & delimiter_) - : IAggregateFunctionDataHelper, GroupConcatImpl>( + : IAggregateFunctionDataHelper>( {data_type_}, parameters_, std::make_shared()) , limit(limit_) , delimiter(delimiter_) diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.h b/src/AggregateFunctions/AggregateFunctionGroupConcat.h index 585dec8e90f..3652b9d8d4b 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupConcat.h +++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.h @@ -28,7 +28,6 @@ struct GroupConcatDataBase void insert(const IColumn * column, const SerializationPtr & serialization, size_t row_num, Arena * arena); }; -template struct GroupConcatData : public GroupConcatDataBase { using Offset = UInt64; @@ -45,7 +44,7 @@ struct GroupConcatData : public GroupConcatDataBase }; template -class GroupConcatImpl : public IAggregateFunctionDataHelper, GroupConcatImpl> +class GroupConcatImpl : public IAggregateFunctionDataHelper> { static constexpr auto name = "groupConcat"; From df04a39b97f8f5c3ff3144f6aa55f20aceb04d9e Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 5 Dec 2024 13:16:48 +0000 Subject: [PATCH 383/502] Fix private --- src/Storages/MergeTree/MergeTreeReaderCompact.cpp | 7 +++++-- .../MergeTree/MergeTreeReaderCompactSingleBuffer.cpp | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index a4fa3a30d23..3c76ebe5d5e 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -188,7 +188,7 @@ void MergeTreeReaderCompact::readData( auto serialization = getSerializationInPart({name_in_storage, type_in_storage}); ColumnPtr temp_column = type_in_storage->createColumn(*serialization); - serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); + serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name_in_storage], nullptr); auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); /// TODO: Avoid extra copying. @@ -231,6 +231,9 @@ void MergeTreeReaderCompact::readPrefix( { try { + if (deserialize_binary_bulk_state_map.contains(name_and_type.getNameInStorage())) + return; + ISerialization::DeserializeBinaryBulkSettings deserialize_settings; if (name_level_for_offsets.has_value()) @@ -253,7 +256,7 @@ void MergeTreeReaderCompact::readPrefix( deserialize_settings.getter = buffer_getter; deserialize_settings.object_and_dynamic_read_statistics = true; - serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name_and_type.name], nullptr); + serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name_and_type.getNameInStorage()], nullptr); } catch (Exception & e) { diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index 0c3e67d5078..d72dd3f2269 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -25,6 +25,7 @@ try while (read_rows < max_rows_to_read) { size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark); + deserialize_binary_bulk_state_map.clear(); /// Use cache to avoid reading the column with the same name twice. /// It may happen if there are empty array Nested in the part. From 41c16b71d699642656c5b2d26ce0ef64fd840c1c Mon Sep 17 00:00:00 2001 From: Pavel Kruglov <48961922+Avogar@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:19:57 +0100 Subject: [PATCH 384/502] Fix build --- src/TableFunctions/ITableFunctionFileLike.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index b7956a74ddc..64c34311af2 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -41,7 +41,7 @@ public: static size_t getMaxNumberOfArguments() { return max_number_of_arguments; } - static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &, bool with_structure) + static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr & context, bool with_structure) { if (args.empty() || args.size() > getMaxNumberOfArguments()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function, got {}", getMaxNumberOfArguments(), args.size()); From cf02976024e8a4cfcfba8ac5a7debe457d8604bc Mon Sep 17 00:00:00 2001 From: Emmanuel Dias Date: Thu, 5 Dec 2024 10:20:34 -0300 Subject: [PATCH 385/502] remove performance test --- tests/performance/array_pr_auc.xml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 tests/performance/array_pr_auc.xml diff --git a/tests/performance/array_pr_auc.xml b/tests/performance/array_pr_auc.xml deleted file mode 100644 index cd5155da992..00000000000 --- a/tests/performance/array_pr_auc.xml +++ /dev/null @@ -1,3 +0,0 @@ - - SELECT avg(ifNotFinite(arrayPrAUC(arrayMap(x -> rand(x) / 0x100000000, range(2 + rand() % 100)), arrayMap(x -> rand(x) % 2, range(2 + rand() % 100))), 0)) FROM numbers(100000) - From 1de568d72c83c2e2f2f16bbccf281c53160d2d8e Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 5 Dec 2024 13:34:55 +0000 Subject: [PATCH 386/502] Fix --- src/Functions/empty.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Functions/empty.cpp b/src/Functions/empty.cpp index f258cd8a369..95804994312 100644 --- a/src/Functions/empty.cpp +++ b/src/Functions/empty.cpp @@ -29,11 +29,11 @@ struct NameNotEmpty }; /// Implements the empty function for JSON type. -template +template class ExecutableFunctionJSONEmpty : public IExecutableFunction { public: - std::string getName() const override { return negative ? NameNotEmpty::name : NameEmpty::name; } + std::string getName() const override { return Name::name; } private: bool useDefaultImplementationForConstants() const override { return true; } @@ -87,13 +87,13 @@ private: } }; -template +template class FunctionEmptyJSON final : public IFunctionBase { public: FunctionEmptyJSON(const DataTypes & argument_types_, const DataTypePtr & return_type_) : argument_types(argument_types_), return_type(return_type_) {} - String getName() const override { return negative ? NameNotEmpty::name : NameEmpty::name; } + String getName() const override { return Name::name; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } @@ -102,7 +102,7 @@ public: ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName &) const override { - return std::make_unique>(); + return std::make_unique>(); } private: @@ -110,11 +110,11 @@ private: DataTypePtr return_type; }; -template +template class FunctionEmptyOverloadResolver final : public IFunctionOverloadResolver { public: - static constexpr auto name = negative ? NameNotEmpty::name : NameEmpty::name; + static constexpr auto name = Name::name; static FunctionOverloadResolverPtr create(ContextPtr) { @@ -132,9 +132,9 @@ public: argument_types.push_back(arg.type); if (argument_types.size() == 1 && isObject(argument_types[0])) - return std::make_shared>(argument_types, return_type); + return std::make_shared>(argument_types, return_type); - return std::make_shared(std::make_shared, NameEmpty, UInt8, false>>(), argument_types, return_type); + return std::make_shared(std::make_shared, Name, UInt8, false>>(), argument_types, return_type); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -161,8 +161,8 @@ public: REGISTER_FUNCTION(Empty) { - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } From 133ba836e856c8379c446a2a1930fd6bda92b450 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov <48961922+Avogar@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:38:51 +0100 Subject: [PATCH 387/502] Update 03280_dynamic_if_null.reference --- tests/queries/0_stateless/03280_dynamic_if_null.reference | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/queries/0_stateless/03280_dynamic_if_null.reference b/tests/queries/0_stateless/03280_dynamic_if_null.reference index 48f569502c5..64ca8cdbd88 100644 --- a/tests/queries/0_stateless/03280_dynamic_if_null.reference +++ b/tests/queries/0_stateless/03280_dynamic_if_null.reference @@ -3,8 +3,3 @@ 2 42 4 -0 -42 -2 -42 -4 From db3be9e6ec2a5015c588bdba00aca28fe38cf4ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 5 Dec 2024 15:03:11 +0100 Subject: [PATCH 388/502] Missing includes and notes --- src/AggregateFunctions/AggregateFunctionAvg.h | 2 +- src/Client/ClientBase.h | 1 + src/Common/examples/sip_hash_perf.cpp | 1 + src/Coordination/KeeperSnapshotManager.cpp | 1 + src/Core/MySQL/MySQLGtid.cpp | 5 ++++- src/Core/MySQL/MySQLReplication.cpp | 1 + src/Storages/MergeTree/MergeTreeMarksLoader.h | 1 + tests/queries/0_stateless/01668_avg_weighted_ubsan.sql | 4 ++++ 8 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionAvg.h b/src/AggregateFunctions/AggregateFunctionAvg.h index d16824b71dc..b89a2209166 100644 --- a/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/src/AggregateFunctions/AggregateFunctionAvg.h @@ -348,7 +348,7 @@ public: void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const ValuesWithType & arguments) const override { - if constexpr(canBeNativeType() && canBeNativeType()) + if constexpr (canBeNativeType() && canBeNativeType()) compileAddImpl(builder, aggregate_data_ptr, arguments); } diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 5fa7006b313..60bddb03ff1 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include diff --git a/src/Common/examples/sip_hash_perf.cpp b/src/Common/examples/sip_hash_perf.cpp index b7a8e97182b..64a8551e1a6 100644 --- a/src/Common/examples/sip_hash_perf.cpp +++ b/src/Common/examples/sip_hash_perf.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp index 495c6c174ce..0be1160f6c7 100644 --- a/src/Coordination/KeeperSnapshotManager.cpp +++ b/src/Coordination/KeeperSnapshotManager.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Core/MySQL/MySQLGtid.cpp b/src/Core/MySQL/MySQLGtid.cpp index 1210a6c3700..985e954ed2f 100644 --- a/src/Core/MySQL/MySQLGtid.cpp +++ b/src/Core/MySQL/MySQLGtid.cpp @@ -1,8 +1,11 @@ #include "MySQLGtid.h" -#include + +#include #include #include +#include + namespace DB { diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 3e603d023c4..7217e749430 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.h b/src/Storages/MergeTree/MergeTreeMarksLoader.h index de202ef76ba..3cd860cfa74 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -4,6 +4,7 @@ #include #include +#include namespace DB { diff --git a/tests/queries/0_stateless/01668_avg_weighted_ubsan.sql b/tests/queries/0_stateless/01668_avg_weighted_ubsan.sql index 24e7dc0cb90..b4dd6a6bdab 100644 --- a/tests/queries/0_stateless/01668_avg_weighted_ubsan.sql +++ b/tests/queries/0_stateless/01668_avg_weighted_ubsan.sql @@ -1 +1,5 @@ +-- https://github.com/ClickHouse/ClickHouse/pull/19475 +-- The result of this test is not important, but just so you know, it's wrong as it might overflow depending on which +-- underlying type is used. The expected result should be 10: +-- https://www.wolframalpha.com/input?i=%281023+*+1000000000.0+%2B+10+*+-9223372036854775808.0%29+%2F+%281000000000.0+%2B+-9223372036854775808.0%29 SELECT round(avgWeighted(x, y)) FROM (SELECT 1023 AS x, 1000000000 AS y UNION ALL SELECT 10 AS x, -9223372036854775808 AS y); From 70b09a83f9c375c1424c4b555c06006218e62cd5 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov <48961922+Avogar@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:31:07 +0100 Subject: [PATCH 389/502] Update example in JSON docs --- docs/en/sql-reference/data-types/newjson.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/sql-reference/data-types/newjson.md b/docs/en/sql-reference/data-types/newjson.md index bd65742d06f..87c34de63db 100644 --- a/docs/en/sql-reference/data-types/newjson.md +++ b/docs/en/sql-reference/data-types/newjson.md @@ -73,6 +73,7 @@ SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::JSON AS json Using CAST from `Tuple`: ```sql +SET enable_named_columns_in_function_tuple = 1; SELECT (tuple(42 AS b) AS a, [1, 2, 3] AS c, 'Hello, World!' AS d)::JSON AS json; ``` From e7177a5192ef9c8c3518d380ba1603312017bf3f Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 5 Dec 2024 14:51:14 +0000 Subject: [PATCH 390/502] Do not swap tables in join in case filter without index applied --- src/Common/ProfileEvents.cpp | 4 ++ .../QueryPlan/Optimizations/optimizeJoin.cpp | 37 +++++++++++++++---- .../Transforms/JoiningTransform.cpp | 17 +++++++++ 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 007c3ab4419..59a98b1f8e5 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -210,6 +210,10 @@ M(ExternalJoinCompressedBytes, "Number of compressed bytes written for JOIN in external memory.", ValueType::Bytes) \ M(ExternalJoinUncompressedBytes, "Amount of data (uncompressed, before compression) written for JOIN in external memory.", ValueType::Bytes) \ \ + M(JoinBuildTableRowCount, "Total number of rows in the build table for a JOIN operation.", ValueType::Number) \ + M(JoinProbeTableRowCount, "Total number of rows in the probe table for a JOIN operation.", ValueType::Number) \ + M(JoinResultRowCount, "Total number of rows in the result of a JOIN operation.", ValueType::Number) \ + \ M(SlowRead, "Number of reads from a file that were slow. This indicate system overload. Thresholds are controlled by read_backoff_* settings.", ValueType::Number) \ M(ReadBackoff, "Number of times the number of query processing threads was lowered due to slow reads.", ValueType::Number) \ \ diff --git a/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp b/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp index ca858559886..718e1905162 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp @@ -21,16 +21,35 @@ namespace DB::QueryPlanOptimizations { -static std::optional estimateReadRowsCount(QueryPlan::Node & node) +static std::optional estimateReadRowsCount(QueryPlan::Node & node, bool has_filter = false) { IQueryPlanStep * step = node.step.get(); if (const auto * reading = typeid_cast(step)) { - if (auto analyzed_result = reading->getAnalyzedResult()) - return analyzed_result->selected_rows; - if (auto analyzed_result = reading->selectRangesToRead()) - return analyzed_result->selected_rows; - return {}; + ReadFromMergeTree::AnalysisResultPtr analyzed_result = nullptr; + analyzed_result = analyzed_result ? analyzed_result : reading->getAnalyzedResult(); + analyzed_result = analyzed_result ? analyzed_result : reading->selectRangesToRead(); + if (!analyzed_result) + return {}; + + bool is_filtered_by_index = false; + for (const auto & idx_stat : analyzed_result->index_stats) + { + is_filtered_by_index = is_filtered_by_index + || (idx_stat.type == ReadFromMergeTree::IndexType::PrimaryKey && !idx_stat.used_keys.empty()) + || idx_stat.type == ReadFromMergeTree::IndexType::Skip + || idx_stat.type == ReadFromMergeTree::IndexType::MinMax; + if (is_filtered_by_index) + break; + } + has_filter = has_filter || reading->getPrewhereInfo(); + + /// If any conditions are pushed down to storage but not used in the index, + /// we cannot precisely estimate the row count + if (has_filter && !is_filtered_by_index) + return {}; + + return analyzed_result->selected_rows; } if (const auto * reading = typeid_cast(step)) @@ -39,8 +58,10 @@ static std::optional estimateReadRowsCount(QueryPlan::Node & node) if (node.children.size() != 1) return {}; - if (typeid_cast(step) || typeid_cast(step)) - return estimateReadRowsCount(*node.children.front()); + if (typeid_cast(step)) + return estimateReadRowsCount(*node.children.front(), has_filter); + if (typeid_cast(step)) + return estimateReadRowsCount(*node.children.front(), true); return {}; } diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp index 7772518cc1c..473de823bf6 100644 --- a/src/Processors/Transforms/JoiningTransform.cpp +++ b/src/Processors/Transforms/JoiningTransform.cpp @@ -4,6 +4,15 @@ #include + + +namespace ProfileEvents +{ + extern const Event JoinBuildTableRowCount; + extern const Event JoinProbeTableRowCount; + extern const Event JoinResultRowCount; +} + namespace DB { @@ -157,6 +166,7 @@ void JoiningTransform::work() if (block.rows()) { + ProfileEvents::increment(ProfileEvents::JoinResultRowCount, block.rows()); output_chunks.emplace_back(block.getColumns(), block.rows()); has_output = true; } @@ -200,7 +210,10 @@ void JoiningTransform::transform(Chunk & chunk) for (const auto & block : res) { if (block.rows()) + { + ProfileEvents::increment(ProfileEvents::JoinResultRowCount, block.rows()); output_chunks.emplace_back(block.getColumns(), block.rows()); + } } } @@ -211,6 +224,7 @@ Blocks JoiningTransform::readExecute(Chunk & chunk) auto join_block = [&]() { + ProfileEvents::increment(ProfileEvents::JoinProbeTableRowCount, block.rows()); if (join->isScatteredJoin()) { join->joinBlock(block, remaining_blocks, res); @@ -334,7 +348,10 @@ void FillingRightJoinSideTransform::work() if (for_totals) join->setTotals(block); else + { + ProfileEvents::increment(ProfileEvents::JoinBuildTableRowCount, block.rows()); stop_reading = !join->addBlockToJoin(block); + } if (input.isFinished()) join->tryRerangeRightTableData(); From 9bea1382939fda3c17211a7594329af0e7097e31 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 5 Dec 2024 14:53:45 +0000 Subject: [PATCH 391/502] add 03279_join_choose_build_table --- .../03279_join_choose_build_table.reference | 5 + .../03279_join_choose_build_table.sql | 97 +++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 tests/queries/0_stateless/03279_join_choose_build_table.reference create mode 100644 tests/queries/0_stateless/03279_join_choose_build_table.sql diff --git a/tests/queries/0_stateless/03279_join_choose_build_table.reference b/tests/queries/0_stateless/03279_join_choose_build_table.reference new file mode 100644 index 00000000000..2e4f278c022 --- /dev/null +++ b/tests/queries/0_stateless/03279_join_choose_build_table.reference @@ -0,0 +1,5 @@ +ok ok ok auto +ok ok ok auto +ok ok ok auto +ok ok ok auto + diff --git a/tests/queries/0_stateless/03279_join_choose_build_table.sql b/tests/queries/0_stateless/03279_join_choose_build_table.sql new file mode 100644 index 00000000000..6051342a2c7 --- /dev/null +++ b/tests/queries/0_stateless/03279_join_choose_build_table.sql @@ -0,0 +1,97 @@ + +DROP TABLE IF EXISTS products; +DROP TABLE IF EXISTS sales; + +SET allow_experimental_analyzer = 1; + +CREATE TABLE sales ( + id Int32, + date Date, + amount Decimal(10, 2), + product_id Int32 +) ENGINE = MergeTree ORDER BY id; + +INSERT INTO sales SELECT number, '2024-05-05' + INTERVAL intDiv(number, 1000) DAY , (number + 1) % 100, number % 100_000 FROM numbers(1_000_000); + +CREATE TABLE products (id Int32, name String) ENGINE = MergeTree ORDER BY id; +INSERT INTO products SELECT number, 'product ' || toString(number) FROM numbers(100_000); + +SET query_plan_join_swap_table = 'auto'; + +SELECT * FROM products, sales +WHERE sales.product_id = products.id AND date = '2024-05-07' +SETTINGS log_comment = '03279_join_choose_build_table_no_idx' FORMAT Null; + +SELECT * FROM sales, products +WHERE sales.product_id = products.id AND date = '2024-05-07' +SETTINGS log_comment = '03279_join_choose_build_table_no_idx' FORMAT Null; + +SET mutations_sync = 2; +ALTER TABLE sales ADD INDEX date_idx date TYPE minmax GRANULARITY 1; +ALTER TABLE sales MATERIALIZE INDEX date_idx; + +SELECT * FROM products, sales +WHERE sales.product_id = products.id AND date = '2024-05-07' +SETTINGS log_comment = '03279_join_choose_build_table_idx' FORMAT Null; + +SELECT * FROM sales, products +WHERE sales.product_id = products.id AND date = '2024-05-07' +SETTINGS log_comment = '03279_join_choose_build_table_idx' FORMAT Null; + +SYSTEM FLUSH LOGS; + +-- condtitions are pushed down, but no filter by index applied +-- build table is as it's written in query + +SELECT + if(ProfileEvents['JoinBuildTableRowCount'] BETWEEN 100 AND 2000, 'ok', 'fail: ' || toString(ProfileEvents['JoinBuildTableRowCount'])), + if(ProfileEvents['JoinProbeTableRowCount'] BETWEEN 90_000 AND 110_000, 'ok', 'fail: ' || toString(ProfileEvents['JoinProbeTableRowCount'])), + if(ProfileEvents['JoinResultRowCount'] == 1000, 'ok', 'fail: ' || toString(ProfileEvents['JoinResultRowCount'])), + Settings['query_plan_join_swap_table'], +FROM system.query_log +WHERE type = 'QueryFinish' AND event_date = today() AND query_kind = 'Select' +AND query like '%products, sales%' +AND log_comment = '03279_join_choose_build_table_no_idx' +ORDER BY event_time DESC +LIMIT 1; + +SELECT + if(ProfileEvents['JoinBuildTableRowCount'] BETWEEN 90_000 AND 110_000, 'ok', 'fail: ' || toString(ProfileEvents['JoinBuildTableRowCount'])), + if(ProfileEvents['JoinProbeTableRowCount'] BETWEEN 100 AND 2000, 'ok', 'fail: ' || toString(ProfileEvents['JoinProbeTableRowCount'])), + if(ProfileEvents['JoinResultRowCount'] == 1000, 'ok', 'fail: ' || toString(ProfileEvents['JoinResultRowCount'])), + Settings['query_plan_join_swap_table'], +FROM system.query_log +WHERE type = 'QueryFinish' AND event_date = today() AND query_kind = 'Select' +AND query like '%sales, products%' +AND log_comment = '03279_join_choose_build_table_no_idx' +ORDER BY event_time DESC +LIMIT 1; + +-- after adding index, optimizer can choose best table order + +SELECT + if(ProfileEvents['JoinBuildTableRowCount'] BETWEEN 100 AND 2000, 'ok', 'fail: ' || toString(ProfileEvents['JoinBuildTableRowCount'])), + if(ProfileEvents['JoinProbeTableRowCount'] BETWEEN 90_000 AND 110_000, 'ok', 'fail: ' || toString(ProfileEvents['JoinProbeTableRowCount'])), + if(ProfileEvents['JoinResultRowCount'] == 1000, 'ok', 'fail: ' || toString(ProfileEvents['JoinResultRowCount'])), + Settings['query_plan_join_swap_table'], +FROM system.query_log +WHERE type = 'QueryFinish' AND event_date = today() AND query_kind = 'Select' +AND query like '%products, sales%' +AND log_comment = '03279_join_choose_build_table_idx' +ORDER BY event_time DESC +LIMIT 1; + +SELECT + if(ProfileEvents['JoinBuildTableRowCount'] BETWEEN 100 AND 2000, 'ok', 'fail: ' || toString(ProfileEvents['JoinBuildTableRowCount'])), + if(ProfileEvents['JoinProbeTableRowCount'] BETWEEN 90_000 AND 110_000, 'ok', 'fail: ' || toString(ProfileEvents['JoinProbeTableRowCount'])), + if(ProfileEvents['JoinResultRowCount'] == 1000, 'ok', 'fail: ' || toString(ProfileEvents['JoinResultRowCount'])), + Settings['query_plan_join_swap_table'], +FROM system.query_log +WHERE type = 'QueryFinish' AND event_date = today() AND query_kind = 'Select' +AND query like '%sales, products%' +AND log_comment = '03279_join_choose_build_table_idx' +ORDER BY event_time DESC +LIMIT 1; + +DROP TABLE IF EXISTS products; +DROP TABLE IF EXISTS sales; From 96bb7818054462ab8eb2fda743fb14f14c19d97a Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 5 Dec 2024 12:44:51 +0000 Subject: [PATCH 392/502] devirtualize two calls and fix UB --- src/AggregateFunctions/IAggregateFunction.h | 36 +++++++++++++++++++ .../SerializationAggregateFunction.cpp | 24 ++----------- src/IO/ReadHelpers.h | 10 +++--- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 66330465a39..86e72dd09b0 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -14,6 +14,7 @@ #include #include +#include #include "config.h" #include @@ -176,11 +177,15 @@ public: /// Serializes state (to transmit it over the network, for example). virtual void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version = std::nullopt) const = 0; /// NOLINT + /// Devirtualize serialize call. virtual void serializeBatch(const PaddedPODArray & data, size_t start, size_t size, WriteBuffer & buf, std::optional version = std::nullopt) const = 0; /// NOLINT /// Deserializes state. This function is called only for empty (just created) states. virtual void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional version = std::nullopt, Arena * arena = nullptr) const = 0; /// NOLINT + /// Devirtualize create and deserialize calls. Used in deserialization of ColumnAggregateFunction. + virtual void createAndDeserializeBatch(PaddedPODArray & data, AggregateDataPtr __restrict place, size_t total_size_of_state, size_t limit, ReadBuffer & buf, std::optional version, Arena * arena) const = 0; + /// Returns true if a function requires Arena to handle own states (see add(), merge(), deserialize()). virtual bool allocatesMemoryInArena() const = 0; @@ -479,6 +484,37 @@ public: static_cast(this)->serialize(data[i], buf, version); } + void createAndDeserializeBatch( + PaddedPODArray & data, + AggregateDataPtr __restrict place, + size_t total_size_of_state, + size_t limit, + ReadBuffer & buf, + std::optional version, + Arena * arena) const override + { + for (size_t i = 0; i < limit; ++i) + { + if (buf.eof()) + break; + + static_cast(this)->create(place); + + try + { + static_cast(this)->deserialize(place, buf, version, arena); + } + catch (...) + { + static_cast(this)->destroy(place); + throw; + } + + data.push_back(place); + place += total_size_of_state; + } + } + void addBatchSparse( size_t row_begin, size_t row_end, diff --git a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp index 905de3f9e78..f63094ef13f 100644 --- a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp +++ b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp @@ -81,29 +81,9 @@ void SerializationAggregateFunction::deserializeBinaryBulk(IColumn & column, Rea /// Adjust the size of state to make all states aligned in vector. size_t total_size_of_state = (size_of_state + align_of_state - 1) / align_of_state * align_of_state; - char * memory = arena.alignedAlloc(total_size_of_state * limit, align_of_state); - char * place = memory; + char * place = arena.alignedAlloc(total_size_of_state * limit, align_of_state); - for (size_t i = 0; i < limit; ++i) - { - if (istr.eof()) - break; - - function->create(place); - - try - { - function->deserialize(place, istr, version, &arena); - } - catch (...) - { - function->destroy(place); - throw; - } - - vec.push_back(place); - place += total_size_of_state; - } + function->createAndDeserializeBatch(vec, place, total_size_of_state, limit, istr, version, &arena); } static String serializeToString(const AggregateFunctionPtr & function, const IColumn & column, size_t row_num, size_t version) diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 7e32f163671..4d54a1f0618 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -110,16 +110,18 @@ inline void readChar(char & x, ReadBuffer & buf) template inline void readPODBinary(T & x, ReadBuffer & buf) { + static constexpr size_t size = sizeof(T); /// NOLINT + /// If the whole value fits in buffer do not call readStrict and copy with /// __builtin_memcpy since it is faster than generic memcpy for small copies. - if (buf.position() + sizeof(T) <= buf.buffer().end()) [[likely]] + if (buf.position() && buf.position() + size <= buf.buffer().end()) [[likely]] { - __builtin_memcpy(reinterpret_cast(&x), buf.position(), sizeof(T)); - buf.position() += sizeof(T); + __builtin_memcpy(reinterpret_cast(&x), buf.position(), size); + buf.position() += size; } else { - buf.readStrict(reinterpret_cast(&x), sizeof(T)); /// NOLINT + buf.readStrict(reinterpret_cast(&x), size); } } From 6f220c37f9649d4fa495325a0fb76f5304d62100 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov <48961922+Avogar@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:44:47 +0100 Subject: [PATCH 393/502] Apply suggestions --- docs/en/sql-reference/data-types/newjson.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/data-types/newjson.md b/docs/en/sql-reference/data-types/newjson.md index 87c34de63db..4ce22b32adc 100644 --- a/docs/en/sql-reference/data-types/newjson.md +++ b/docs/en/sql-reference/data-types/newjson.md @@ -98,8 +98,9 @@ SELECT map('a', map('b', 42), 'c', [1,2,3], 'd', 'Hello, World!')::JSON AS json; Using CAST from deprecated `Object('json')`: ```sql - SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::Object('json')::JSON AS json; - ``` +SET allow_experimental_object_type = 1; +SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::Object('json')::JSON AS json; +``` ```text ┌─json───────────────────────────────────────────┐ From 74b6f0a6b46a5ba00a07a4db7d80a68b873cabaa Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Thu, 5 Dec 2024 17:04:05 +0100 Subject: [PATCH 394/502] Poke CI From d14b360ceb71f1c21a821c4fba653fc5140cc378 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 5 Dec 2024 16:38:05 +0000 Subject: [PATCH 395/502] fix update of all columns in presence of _block_number column --- src/Interpreters/MutationsInterpreter.cpp | 20 ++++++++++---- src/Interpreters/MutationsInterpreter.h | 5 ++++ .../03275_block_number_update.reference | 4 +++ .../0_stateless/03275_block_number_update.sql | 27 +++++++++++++++++++ 4 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/03275_block_number_update.reference create mode 100644 tests/queries/0_stateless/03275_block_number_update.sql diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index d95e2392ad6..575d3197879 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -72,7 +72,6 @@ namespace ErrorCodes extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int CANNOT_UPDATE_COLUMN; extern const int UNEXPECTED_EXPRESSION; - extern const int THERE_IS_NO_COLUMN; extern const int ILLEGAL_STATISTICS; } @@ -602,10 +601,6 @@ void MutationsInterpreter::prepare(bool dry_run) if (available_columns_set.emplace(name).second) available_columns.push_back(name); } - else if (!available_columns_set.contains(name)) - { - throw Exception(ErrorCodes::THERE_IS_NO_COLUMN, "Column {} is updated but not requested to read", name); - } updated_columns.insert(name); } @@ -1106,6 +1101,21 @@ void MutationsInterpreter::prepareMutationStages(std::vector & prepared_s } } + auto storage_columns = metadata_snapshot->getColumns().getNamesOfPhysical(); + + /// Add persistent virtual columns if the whole part is rewritten, + /// because we should preserve them in parts after mutation. + if (prepared_stages.back().isAffectingAllColumns(storage_columns)) + { + for (const auto & column_name : available_columns) + { + if (column_name == RowExistsColumn::name && has_filters && !deleted_mask_updated) + continue; + + prepared_stages.back().output_columns.insert(column_name); + } + } + /// Now, calculate `expressions_chain` for each stage except the first. /// Do it backwards to propagate information about columns required as input for a stage to the previous stage. for (int64_t i = prepared_stages.size() - 1; i >= 0; --i) diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h index 3f07d57362d..a5a7f28f40d 100644 --- a/src/Interpreters/MutationsInterpreter.h +++ b/src/Interpreters/MutationsInterpreter.h @@ -170,7 +170,12 @@ private: Source source; StorageMetadataPtr metadata_snapshot; MutationCommands commands; + + /// List of columns in table or in data part that can be updated by mutation. + /// If mutation affects all columns (e.g. DELETE), all of this columns + /// must be returned by pipeline created in MutationsInterpreter. Names available_columns; + ContextPtr context; Settings settings; SelectQueryOptions select_limits; diff --git a/tests/queries/0_stateless/03275_block_number_update.reference b/tests/queries/0_stateless/03275_block_number_update.reference new file mode 100644 index 00000000000..e7d6081b647 --- /dev/null +++ b/tests/queries/0_stateless/03275_block_number_update.reference @@ -0,0 +1,4 @@ +2 +3 +2 +3 diff --git a/tests/queries/0_stateless/03275_block_number_update.sql b/tests/queries/0_stateless/03275_block_number_update.sql new file mode 100644 index 00000000000..ca8160b48f6 --- /dev/null +++ b/tests/queries/0_stateless/03275_block_number_update.sql @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS t_block_number_mut; + +SET mutations_sync = 2; + +CREATE TABLE t_block_number_mut (n int) ENGINE = MergeTree ORDER BY tuple() SETTINGS enable_block_number_column = 1, min_bytes_for_wide_part = 0; + +INSERT INTO t_block_number_mut VALUES (1) (2); + +OPTIMIZE TABLE t_block_number_mut FINAL; + +ALTER TABLE t_block_number_mut UPDATE n = n + 1 WHERE 1; + +SELECT * FROM t_block_number_mut; + +DROP TABLE IF EXISTS t_block_number_mut; + +CREATE TABLE t_block_number_mut (n int) ENGINE = MergeTree ORDER BY tuple() SETTINGS enable_block_number_column = 1, min_bytes_for_wide_part = '1G'; + +INSERT INTO t_block_number_mut VALUES (1) (2); + +OPTIMIZE TABLE t_block_number_mut FINAL; + +ALTER TABLE t_block_number_mut UPDATE n = n + 1 WHERE 1; + +SELECT * FROM t_block_number_mut; + +DROP TABLE IF EXISTS t_block_number_mut; From f75cd514ab0f4dc096fbd966825fa14b0b8dfcfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 5 Dec 2024 18:53:33 +0100 Subject: [PATCH 396/502] Another missing include --- src/IO/examples/parse_date_time_best_effort.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/IO/examples/parse_date_time_best_effort.cpp b/src/IO/examples/parse_date_time_best_effort.cpp index 2c237b97089..a525d219ed4 100644 --- a/src/IO/examples/parse_date_time_best_effort.cpp +++ b/src/IO/examples/parse_date_time_best_effort.cpp @@ -1,3 +1,4 @@ +#include #include #include From 273830bacf8e8ebb6561d816e9484e2b7cd69229 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Thu, 5 Dec 2024 19:02:29 +0100 Subject: [PATCH 397/502] setup_minio: Do not add aws credentials twice --- ci/jobs/scripts/functional_tests/setup_minio.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci/jobs/scripts/functional_tests/setup_minio.sh b/ci/jobs/scripts/functional_tests/setup_minio.sh index 88839c39674..0eb87b127a0 100755 --- a/ci/jobs/scripts/functional_tests/setup_minio.sh +++ b/ci/jobs/scripts/functional_tests/setup_minio.sh @@ -120,6 +120,12 @@ setup_aws_credentials() { local minio_root_user=${MINIO_ROOT_USER:-clickhouse} local minio_root_password=${MINIO_ROOT_PASSWORD:-clickhouse} mkdir -p ~/.aws + if [[ -f ~/.aws/credentials ]]; then + if grep -q "^\[default\]" ~/.aws/credentials; then + echo "The credentials file contains a [default] section." + return + fi + fi cat <> ~/.aws/credentials [default] aws_access_key_id=${minio_root_user} From 13dbd31608f9abca889ec6ab7376a0773f56b112 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 5 Dec 2024 19:05:52 +0100 Subject: [PATCH 398/502] Fix backup/restore of empty file --- .../DiskObjectStorageTransaction.cpp | 5 +++- src/Disks/ObjectStorages/IMetadataStorage.h | 2 ++ .../MetadataStorageFromPlainObjectStorage.h | 4 +-- ...aStorageFromPlainRewritableObjectStorage.h | 25 +++++++++++++++++++ .../configs/query_log.xml | 8 ++++++ .../test_backup_restore_s3/test.py | 24 ++++++++++++++++++ 6 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 tests/integration/test_backup_restore_s3/configs/query_log.xml diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 5314ed42cca..83c1bb641b4 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -911,8 +911,11 @@ void DiskObjectStorageTransaction::chmod(const String & path, mode_t mode) void DiskObjectStorageTransaction::createFile(const std::string & path) { operations_to_execute.emplace_back( - std::make_unique(object_storage, metadata_storage, [path](MetadataTransactionPtr tx) + std::make_unique(object_storage, metadata_storage, [path, this](MetadataTransactionPtr tx) { + if (object_storage.isPlain() && !object_storage.isWriteOnce()) + tx->createEmptyFile(path); + tx->createEmptyMetadataFile(path); })); } diff --git a/src/Disks/ObjectStorages/IMetadataStorage.h b/src/Disks/ObjectStorages/IMetadataStorage.h index 145e81aa447..27d5ca4f983 100644 --- a/src/Disks/ObjectStorages/IMetadataStorage.h +++ b/src/Disks/ObjectStorages/IMetadataStorage.h @@ -135,6 +135,8 @@ public: /// Create empty file in metadata storage virtual void createEmptyMetadataFile(const std::string & path) = 0; + virtual void createEmptyFile(const std::string & /* path */) {} + /// Create metadata file on paths with content (blob_name, size_in_bytes) virtual void createMetadataFile(const std::string & path, ObjectStorageKey key, uint64_t size_in_bytes) = 0; diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h index db7390af5fd..771a522ff5d 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h +++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h @@ -98,9 +98,9 @@ protected: ObjectMetadataEntryPtr getObjectMetadataEntryWithCache(const std::string & path) const; }; -class MetadataStorageFromPlainObjectStorageTransaction final : public IMetadataTransaction, private MetadataOperationsHolder +class MetadataStorageFromPlainObjectStorageTransaction : public IMetadataTransaction, private MetadataOperationsHolder { -private: +protected: MetadataStorageFromPlainObjectStorage & metadata_storage; ObjectStoragePtr object_storage; diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h index 983e379d292..dea5f367c4f 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h +++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h @@ -4,11 +4,31 @@ #include #include +#include namespace DB { +class MetadataStorageFromPlainRewritableObjectStorageTransaction final : public MetadataStorageFromPlainObjectStorageTransaction +{ +public: + explicit MetadataStorageFromPlainRewritableObjectStorageTransaction( + MetadataStorageFromPlainObjectStorage & metadata_storage_, ObjectStoragePtr object_storage_) + : DB::MetadataStorageFromPlainObjectStorageTransaction(metadata_storage_, object_storage_) + { + } + + void createEmptyFile(const std::string & path) override + { + const auto key = object_storage->generateObjectKeyForPath(path, std::nullopt); + StoredObject object(key.serialize(), "", /* file_size */0); + auto buf = object_storage->writeObject(object, WriteMode::Rewrite); + buf->finalize(); + } +}; + + class MetadataStorageFromPlainRewritableObjectStorage final : public MetadataStorageFromPlainObjectStorage { private: @@ -22,6 +42,11 @@ public: MetadataStorageType getType() const override { return MetadataStorageType::PlainRewritable; } + MetadataTransactionPtr createTransaction() override + { + return std::make_shared(*this, object_storage); + } + bool existsFile(const std::string & path) const override; bool existsDirectory(const std::string & path) const override; diff --git a/tests/integration/test_backup_restore_s3/configs/query_log.xml b/tests/integration/test_backup_restore_s3/configs/query_log.xml new file mode 100644 index 00000000000..2bdcf25800f --- /dev/null +++ b/tests/integration/test_backup_restore_s3/configs/query_log.xml @@ -0,0 +1,8 @@ + + + + system +
query_log
+ Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + INTERVAL 30 day SETTINGS storage_policy='policy_s3_plain_rewritable', ttl_only_drop_parts=1, min_rows_for_wide_part=0, min_bytes_for_wide_part=0 + + diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index f58190522a3..cb2c45b8cb4 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -18,6 +18,7 @@ node = cluster.add_instance( "configs/s3_settings.xml", "configs/blob_log.xml", "configs/remote_servers.xml", + "configs/query_log.xml", ], user_configs=[ "configs/zookeeper_retries.xml", @@ -776,3 +777,26 @@ def test_backup_to_s3_different_credentials(): check_backup_restore(False) check_backup_restore(True) + + +def test_backup_restore_system_tables_with_plain_rewritable_disk(): + instance = cluster.instances["node"] + backup_name = new_backup_name() + backup_destination = ( + f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')" + ) + + instance.query("SYSTEM FLUSH LOGS") + + backup_query_id = uuid.uuid4().hex + instance.query( + f"BACKUP TABLE system.query_log TO {backup_destination}", + query_id=backup_query_id, + ) + restore_query_id = uuid.uuid4().hex + instance.query( + f""" + RESTORE TABLE system.query_log AS data_restored FROM {backup_destination}; + """, + query_id=restore_query_id, + ) From d20af6ca9a02859ff53490301886b26cbade83cc Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 5 Dec 2024 15:45:19 +0000 Subject: [PATCH 399/502] fix stylecheck --- src/Processors/Transforms/JoiningTransform.cpp | 2 -- .../queries/0_stateless/03279_join_choose_build_table.sql | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp index 473de823bf6..869b8395cdd 100644 --- a/src/Processors/Transforms/JoiningTransform.cpp +++ b/src/Processors/Transforms/JoiningTransform.cpp @@ -4,8 +4,6 @@ #include - - namespace ProfileEvents { extern const Event JoinBuildTableRowCount; diff --git a/tests/queries/0_stateless/03279_join_choose_build_table.sql b/tests/queries/0_stateless/03279_join_choose_build_table.sql index 6051342a2c7..e1d8010158d 100644 --- a/tests/queries/0_stateless/03279_join_choose_build_table.sql +++ b/tests/queries/0_stateless/03279_join_choose_build_table.sql @@ -49,7 +49,7 @@ SELECT if(ProfileEvents['JoinResultRowCount'] == 1000, 'ok', 'fail: ' || toString(ProfileEvents['JoinResultRowCount'])), Settings['query_plan_join_swap_table'], FROM system.query_log -WHERE type = 'QueryFinish' AND event_date = today() AND query_kind = 'Select' +WHERE type = 'QueryFinish' AND event_date >= yesterday() AND query_kind = 'Select' AND current_database = currentDatabase() AND query like '%products, sales%' AND log_comment = '03279_join_choose_build_table_no_idx' ORDER BY event_time DESC @@ -61,7 +61,7 @@ SELECT if(ProfileEvents['JoinResultRowCount'] == 1000, 'ok', 'fail: ' || toString(ProfileEvents['JoinResultRowCount'])), Settings['query_plan_join_swap_table'], FROM system.query_log -WHERE type = 'QueryFinish' AND event_date = today() AND query_kind = 'Select' +WHERE type = 'QueryFinish' AND event_date >= yesterday() AND query_kind = 'Select' AND current_database = currentDatabase() AND query like '%sales, products%' AND log_comment = '03279_join_choose_build_table_no_idx' ORDER BY event_time DESC @@ -75,7 +75,7 @@ SELECT if(ProfileEvents['JoinResultRowCount'] == 1000, 'ok', 'fail: ' || toString(ProfileEvents['JoinResultRowCount'])), Settings['query_plan_join_swap_table'], FROM system.query_log -WHERE type = 'QueryFinish' AND event_date = today() AND query_kind = 'Select' +WHERE type = 'QueryFinish' AND event_date >= yesterday() AND query_kind = 'Select' AND current_database = currentDatabase() AND query like '%products, sales%' AND log_comment = '03279_join_choose_build_table_idx' ORDER BY event_time DESC @@ -87,7 +87,7 @@ SELECT if(ProfileEvents['JoinResultRowCount'] == 1000, 'ok', 'fail: ' || toString(ProfileEvents['JoinResultRowCount'])), Settings['query_plan_join_swap_table'], FROM system.query_log -WHERE type = 'QueryFinish' AND event_date = today() AND query_kind = 'Select' +WHERE type = 'QueryFinish' AND event_date >= yesterday() AND query_kind = 'Select' AND current_database = currentDatabase() AND query like '%sales, products%' AND log_comment = '03279_join_choose_build_table_idx' ORDER BY event_time DESC From 8694ce545646fe9d0250293fc654d1030e9c6c9e Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Thu, 5 Dec 2024 23:37:29 +0000 Subject: [PATCH 400/502] try fix test_backup_restore_s3 --- tests/integration/test_backup_restore_s3/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index cb2c45b8cb4..caa0ef05c72 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -794,9 +794,11 @@ def test_backup_restore_system_tables_with_plain_rewritable_disk(): query_id=backup_query_id, ) restore_query_id = uuid.uuid4().hex + instance.query("DROP TABLE IF EXISTS data_restored SYNC") instance.query( f""" RESTORE TABLE system.query_log AS data_restored FROM {backup_destination}; """, query_id=restore_query_id, ) + instance.query("DROP TABLE data_restored SYNC") From 92a2a8046a7e0ee0684560bfeeeb6c9753245a54 Mon Sep 17 00:00:00 2001 From: Tanya Bragin Date: Thu, 5 Dec 2024 17:12:53 -0800 Subject: [PATCH 401/502] Update README.md - Update meetups --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b22826e13f..8eb264073c3 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ Keep an eye out for upcoming meetups and events around the world. Somewhere else Upcoming meetups -* [Amsterdam Meetup](https://www.meetup.com/clickhouse-netherlands-user-group/events/303638814) - December 3 * [Stockholm Meetup](https://www.meetup.com/clickhouse-stockholm-user-group/events/304382411) - December 9 * [New York Meetup](https://www.meetup.com/clickhouse-new-york-user-group/events/304268174) - December 9 * [Kuala Lampur Meetup](https://www.meetup.com/clickhouse-malaysia-meetup-group/events/304576472/) - December 11 @@ -52,6 +51,7 @@ Upcoming meetups Recently completed meetups +* [Amsterdam Meetup](https://www.meetup.com/clickhouse-netherlands-user-group/events/303638814) - December 3 * [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/303096434) - November 26 * [Ghent Meetup](https://www.meetup.com/clickhouse-belgium-user-group/events/303049405/) - November 19 * [Barcelona Meetup](https://www.meetup.com/clickhouse-spain-user-group/events/303096876/) - November 12 From 0a9e2fa2c4c7ddf9c146688249ee437138eb271b Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 6 Dec 2024 08:59:58 +0000 Subject: [PATCH 402/502] fix test --- .../queries/0_stateless/03279_join_choose_build_table.reference | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/03279_join_choose_build_table.reference b/tests/queries/0_stateless/03279_join_choose_build_table.reference index 2e4f278c022..1e7b7d8bde0 100644 --- a/tests/queries/0_stateless/03279_join_choose_build_table.reference +++ b/tests/queries/0_stateless/03279_join_choose_build_table.reference @@ -2,4 +2,3 @@ ok ok ok auto ok ok ok auto ok ok ok auto ok ok ok auto - From d3fbaa7cdbf4a2fa7355cda8e948893d48211ecf Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 6 Dec 2024 11:26:57 +0000 Subject: [PATCH 403/502] 03279_join_choose_build_table not flaky --- tests/queries/0_stateless/03279_join_choose_build_table.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03279_join_choose_build_table.sql b/tests/queries/0_stateless/03279_join_choose_build_table.sql index e1d8010158d..03733e1c9b7 100644 --- a/tests/queries/0_stateless/03279_join_choose_build_table.sql +++ b/tests/queries/0_stateless/03279_join_choose_build_table.sql @@ -9,7 +9,8 @@ CREATE TABLE sales ( date Date, amount Decimal(10, 2), product_id Int32 -) ENGINE = MergeTree ORDER BY id; +) ENGINE = MergeTree ORDER BY id +SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'; INSERT INTO sales SELECT number, '2024-05-05' + INTERVAL intDiv(number, 1000) DAY , (number + 1) % 100, number % 100_000 FROM numbers(1_000_000); From 3034a97631a3bab25c7a01d88b50a7e28344d5ee Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:52:03 +0100 Subject: [PATCH 404/502] Revert "Revert "Revert "make d-tor Finalizer more obvious""" --- src/Storages/MergeTree/MergeTreeSink.cpp | 3 --- .../MergeTree/MergedBlockOutputStream.cpp | 17 ++++++++++++----- .../MergeTree/MergedBlockOutputStream.h | 2 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 2 -- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 582a722d678..d65d1f3212f 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -45,8 +44,6 @@ MergeTreeSink::~MergeTreeSink() if (!delayed_chunk) return; - chassert(isCancelled() || std::uncaught_exceptions()); - for (auto & partition : delayed_chunk->partitions) { partition.temp_part.cancel(); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 87b3d238e3b..f8863a8a6d6 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -119,7 +119,7 @@ struct MergedBlockOutputStream::Finalizer::Impl } void finish(); - void cancel() noexcept; + void cancel(); }; void MergedBlockOutputStream::Finalizer::finish() @@ -130,7 +130,7 @@ void MergedBlockOutputStream::Finalizer::finish() to_finish->finish(); } -void MergedBlockOutputStream::Finalizer::cancel() noexcept +void MergedBlockOutputStream::Finalizer::cancel() { std::unique_ptr to_cancel = std::move(impl); impl.reset(); @@ -167,7 +167,7 @@ void MergedBlockOutputStream::Finalizer::Impl::finish() part->getDataPartStorage().removeFile(file_name); } -void MergedBlockOutputStream::Finalizer::Impl::cancel() noexcept +void MergedBlockOutputStream::Finalizer::Impl::cancel() { writer.cancel(); @@ -183,8 +183,15 @@ MergedBlockOutputStream::Finalizer::Finalizer(std::unique_ptr impl_) : imp MergedBlockOutputStream::Finalizer::~Finalizer() { - if (impl) - cancel(); + try + { + if (impl) + finish(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } } diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.h b/src/Storages/MergeTree/MergedBlockOutputStream.h index 2f6205427d1..b6fc13cbe42 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -55,7 +55,7 @@ public: ~Finalizer(); void finish(); - void cancel() noexcept; + void cancel(); }; /// Finalize writing part and fill inner structures diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 507b4bfd214..19a69eb46be 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -179,8 +179,6 @@ ReplicatedMergeTreeSinkImpl::~ReplicatedMergeTreeSinkImpl() if (!delayed_chunk) return; - chassert(isCancelled() || std::uncaught_exceptions()); - for (auto & partition : delayed_chunk->partitions) { partition.temp_part.cancel(); From 044fdba25167153a5d97e56ae094a0d9c6f4c94e Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 6 Dec 2024 12:58:49 +0100 Subject: [PATCH 405/502] Better --- .../DiskObjectStorageTransaction.cpp | 47 +++++++++++++++++-- ...aStorageFromPlainRewritableObjectStorage.h | 24 ---------- 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 83c1bb641b4..4633432aa0a 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -10,6 +10,7 @@ #include #include +#include #include namespace DB @@ -593,6 +594,39 @@ struct TruncateFileObjectStorageOperation final : public IDiskObjectStorageOpera } }; +struct CreateEmptyFileObjectStorageOperation final : public IDiskObjectStorageOperation +{ + StoredObject object; + + CreateEmptyFileObjectStorageOperation( + IObjectStorage & object_storage_, + IMetadataStorage & metadata_storage_, + const std::string & path_) + : IDiskObjectStorageOperation(object_storage_, metadata_storage_) + { + const auto key = object_storage.generateObjectKeyForPath(path_, std::nullopt); + object = StoredObject(key.serialize(), path_, /* file_size */0); + } + + std::string getInfoForLog() const override + { + return fmt::format("CreateEmptyFileObjectStorageOperation (remote path: {}, local path: {})", object.remote_path, object.local_path); + } + + void execute(MetadataTransactionPtr /* tx */) override + { + auto buf = object_storage.writeObject(object, WriteMode::Rewrite); + buf->finalize(); + } + + void undo() override + { + object_storage.removeObjectIfExists(object); + } + + void finalize() override {} +}; + } void DiskObjectStorageTransaction::createDirectory(const std::string & path) @@ -910,12 +944,15 @@ void DiskObjectStorageTransaction::chmod(const String & path, mode_t mode) void DiskObjectStorageTransaction::createFile(const std::string & path) { - operations_to_execute.emplace_back( - std::make_unique(object_storage, metadata_storage, [path, this](MetadataTransactionPtr tx) - { - if (object_storage.isPlain() && !object_storage.isWriteOnce()) - tx->createEmptyFile(path); + if (object_storage.isPlain() && !object_storage.isWriteOnce()) + { + operations_to_execute.emplace_back( + std::make_unique(object_storage, metadata_storage, path)); + } + operations_to_execute.emplace_back( + std::make_unique(object_storage, metadata_storage, [path](MetadataTransactionPtr tx) + { tx->createEmptyMetadataFile(path); })); } diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h index dea5f367c4f..c2d514c1105 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h +++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h @@ -10,25 +10,6 @@ namespace DB { -class MetadataStorageFromPlainRewritableObjectStorageTransaction final : public MetadataStorageFromPlainObjectStorageTransaction -{ -public: - explicit MetadataStorageFromPlainRewritableObjectStorageTransaction( - MetadataStorageFromPlainObjectStorage & metadata_storage_, ObjectStoragePtr object_storage_) - : DB::MetadataStorageFromPlainObjectStorageTransaction(metadata_storage_, object_storage_) - { - } - - void createEmptyFile(const std::string & path) override - { - const auto key = object_storage->generateObjectKeyForPath(path, std::nullopt); - StoredObject object(key.serialize(), "", /* file_size */0); - auto buf = object_storage->writeObject(object, WriteMode::Rewrite); - buf->finalize(); - } -}; - - class MetadataStorageFromPlainRewritableObjectStorage final : public MetadataStorageFromPlainObjectStorage { private: @@ -42,11 +23,6 @@ public: MetadataStorageType getType() const override { return MetadataStorageType::PlainRewritable; } - MetadataTransactionPtr createTransaction() override - { - return std::make_shared(*this, object_storage); - } - bool existsFile(const std::string & path) const override; bool existsDirectory(const std::string & path) const override; From f2c328aa7a9c0ac7cc7a13f5d99201d46dc79361 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 6 Dec 2024 13:11:00 +0100 Subject: [PATCH 406/502] Revert "Merge pull request #72836 from ClickHouse/revert-72770-more_insistent_compress_in_memory_eng" This reverts commit f752c0b89d1a84fc92bfbb7fc510a092b5aa4c84, reversing changes made to 65d895a6dbd5f07383b871430f3cee5ce1b984a3. --- docs/en/engines/table-engines/special/memory.md | 2 ++ src/Columns/ColumnArray.cpp | 6 +++--- src/Columns/ColumnArray.h | 2 +- src/Columns/ColumnCompressed.cpp | 5 +++-- src/Columns/ColumnCompressed.h | 2 +- src/Columns/ColumnDecimal.cpp | 4 ++-- src/Columns/ColumnDecimal.h | 2 +- src/Columns/ColumnDynamic.cpp | 4 ++-- src/Columns/ColumnDynamic.h | 2 +- src/Columns/ColumnFixedString.cpp | 4 ++-- src/Columns/ColumnFixedString.h | 2 +- src/Columns/ColumnMap.cpp | 4 ++-- src/Columns/ColumnMap.h | 2 +- src/Columns/ColumnNullable.cpp | 6 +++--- src/Columns/ColumnNullable.h | 2 +- src/Columns/ColumnObject.cpp | 8 ++++---- src/Columns/ColumnObject.h | 2 +- src/Columns/ColumnSparse.cpp | 6 +++--- src/Columns/ColumnSparse.h | 2 +- src/Columns/ColumnString.cpp | 6 +++--- src/Columns/ColumnString.h | 2 +- src/Columns/ColumnTuple.cpp | 4 ++-- src/Columns/ColumnTuple.h | 2 +- src/Columns/ColumnVariant.cpp | 8 ++++---- src/Columns/ColumnVariant.h | 2 +- src/Columns/ColumnVector.cpp | 4 ++-- src/Columns/ColumnVector.h | 2 +- src/Columns/IColumn.h | 3 ++- src/Core/Block.cpp | 2 +- src/Interpreters/Cache/QueryCache.cpp | 2 +- src/Storages/StorageMemory.cpp | 7 +++---- 31 files changed, 57 insertions(+), 54 deletions(-) diff --git a/docs/en/engines/table-engines/special/memory.md b/docs/en/engines/table-engines/special/memory.md index f28157ebde2..3eb3e617ff9 100644 --- a/docs/en/engines/table-engines/special/memory.md +++ b/docs/en/engines/table-engines/special/memory.md @@ -36,6 +36,8 @@ Upper and lower bounds can be specified to limit Memory engine table size, effec - Requires `max_rows_to_keep` - `max_rows_to_keep` — Maximum rows to keep within memory table where oldest rows are deleted on each insertion (i.e circular buffer). Max rows can exceed the stated limit if the oldest batch of rows to remove falls under the `min_rows_to_keep` limit when adding a large block. - Default value: `0` +- `compress` - Whether to compress data in memory. + - Default value: `false` ## Usage {#usage} diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 3f88ca93a97..013821db2c9 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -1024,10 +1024,10 @@ void ColumnArray::updatePermutationWithCollation(const Collator & collator, Perm DefaultPartialSort()); } -ColumnPtr ColumnArray::compress() const +ColumnPtr ColumnArray::compress(bool force_compression) const { - ColumnPtr data_compressed = data->compress(); - ColumnPtr offsets_compressed = offsets->compress(); + ColumnPtr data_compressed = data->compress(force_compression); + ColumnPtr offsets_compressed = offsets->compress(force_compression); size_t byte_size = data_compressed->byteSize() + offsets_compressed->byteSize(); diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index a66f9041213..dee6ae931f2 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -159,7 +159,7 @@ public: /// For example, `getDataInRange(0, size())` is the same as `getDataPtr()->clone()`. MutableColumnPtr getDataInRange(size_t start, size_t length) const; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnCompressed.cpp b/src/Columns/ColumnCompressed.cpp index 3bdc514d6d8..adb2a5f391d 100644 --- a/src/Columns/ColumnCompressed.cpp +++ b/src/Columns/ColumnCompressed.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes } -std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, size_t data_size, bool always_compress) +std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, size_t data_size, bool force_compression) { size_t max_dest_size = LZ4_COMPRESSBOUND(data_size); @@ -35,7 +35,8 @@ std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, si throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress column"); /// If compression is inefficient. - if (!always_compress && static_cast(compressed_size) * 2 > data_size) + const size_t threshold = force_compression ? 1 : 2; + if (static_cast(compressed_size) * threshold > data_size) return {}; /// Shrink to fit. diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h index c4270e8216b..b030e762acd 100644 --- a/src/Columns/ColumnCompressed.h +++ b/src/Columns/ColumnCompressed.h @@ -72,7 +72,7 @@ public: /// If data is not worth to be compressed and not 'always_compress' - returns nullptr. /// Note: shared_ptr is to allow to be captured by std::function. - static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool always_compress); + static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool force_compression); static void decompressBuffer( const void * compressed_data, void * decompressed_data, size_t compressed_size, size_t decompressed_size); diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 73366150e7d..c286c54198a 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -478,7 +478,7 @@ ColumnPtr ColumnDecimal::replicate(const IColumn::Offsets & offsets) const } template -ColumnPtr ColumnDecimal::compress() const +ColumnPtr ColumnDecimal::compress(bool force_compression) const { const size_t data_size = data.size(); const size_t source_size = data_size * sizeof(T); @@ -487,7 +487,7 @@ ColumnPtr ColumnDecimal::compress() const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, false); + auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, force_compression); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 690549e4a56..3e5c189b731 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -140,7 +140,7 @@ public: return false; } - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void insertValue(const T value) { data.push_back(value); } Container & getData() { return data; } diff --git a/src/Columns/ColumnDynamic.cpp b/src/Columns/ColumnDynamic.cpp index a4c932eafdd..2d05701c57b 100644 --- a/src/Columns/ColumnDynamic.cpp +++ b/src/Columns/ColumnDynamic.cpp @@ -991,9 +991,9 @@ void ColumnDynamic::updatePermutation(IColumn::PermutationSortDirection directio updatePermutationImpl(limit, res, equal_ranges, ComparatorDescendingStable(*this, nan_direction_hint), comparator_equal, DefaultSort(), DefaultPartialSort()); } -ColumnPtr ColumnDynamic::compress() const +ColumnPtr ColumnDynamic::compress(bool force_compression) const { - ColumnPtr variant_compressed = variant_column_ptr->compress(); + ColumnPtr variant_compressed = variant_column_ptr->compress(force_compression); size_t byte_size = variant_compressed->byteSize(); return ColumnCompressed::create(size(), byte_size, [my_variant_compressed = std::move(variant_compressed), my_variant_info = variant_info, my_max_dynamic_types = max_dynamic_types, my_global_max_dynamic_types = global_max_dynamic_types, my_statistics = statistics]() mutable diff --git a/src/Columns/ColumnDynamic.h b/src/Columns/ColumnDynamic.h index bdbad99519f..093aaaf2793 100644 --- a/src/Columns/ColumnDynamic.h +++ b/src/Columns/ColumnDynamic.h @@ -335,7 +335,7 @@ public: return false; } - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; double getRatioOfDefaultRows(double sample_ratio) const override { diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 04e894ee5ab..f076f904768 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -419,7 +419,7 @@ void ColumnFixedString::getExtremes(Field & min, Field & max) const get(max_idx, max); } -ColumnPtr ColumnFixedString::compress() const +ColumnPtr ColumnFixedString::compress(bool force_compression) const { size_t source_size = chars.size(); @@ -427,7 +427,7 @@ ColumnPtr ColumnFixedString::compress() const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(chars.data(), source_size, false); + auto compressed = ColumnCompressed::compressBuffer(chars.data(), source_size, force_compression); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index 8cf0a6a57da..f55fb60a976 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -175,7 +175,7 @@ public: ColumnPtr replicate(const Offsets & offsets) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void reserve(size_t size) override { diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp index a5511dfeeb4..fb9c8c9fbaf 100644 --- a/src/Columns/ColumnMap.cpp +++ b/src/Columns/ColumnMap.cpp @@ -352,9 +352,9 @@ bool ColumnMap::dynamicStructureEquals(const IColumn & rhs) const return false; } -ColumnPtr ColumnMap::compress() const +ColumnPtr ColumnMap::compress(bool force_compression) const { - auto compressed = nested->compress(); + auto compressed = nested->compress(force_compression); const auto byte_size = compressed->byteSize(); /// The order of evaluation of function arguments is unspecified /// and could cause interacting with object in moved-from state diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h index 8dfa5bb5845..31404a3e152 100644 --- a/src/Columns/ColumnMap.h +++ b/src/Columns/ColumnMap.h @@ -120,7 +120,7 @@ public: const ColumnTuple & getNestedData() const { return assert_cast(getNestedColumn().getData()); } ColumnTuple & getNestedData() { return assert_cast(getNestedColumn().getData()); } - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; bool hasDynamicStructure() const override { return nested->hasDynamicStructure(); } bool dynamicStructureEquals(const IColumn & rhs) const override; diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 6e8bd3fc70c..640550fcf9a 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -773,10 +773,10 @@ void ColumnNullable::protect() getNullMapColumn().protect(); } -ColumnPtr ColumnNullable::compress() const +ColumnPtr ColumnNullable::compress(bool force_compression) const { - ColumnPtr nested_compressed = nested_column->compress(); - ColumnPtr null_map_compressed = null_map->compress(); + ColumnPtr nested_compressed = nested_column->compress(force_compression); + ColumnPtr null_map_compressed = null_map->compress(force_compression); size_t byte_size = nested_column->byteSize() + null_map->byteSize(); diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 32ce66c5965..3a0be008cc2 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -141,7 +141,7 @@ public: // Special function for nullable minmax index void getExtremesNullLast(Field & min, Field & max) const; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index dae7b4c36b9..f110e8b442b 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -1225,14 +1225,14 @@ bool ColumnObject::structureEquals(const IColumn & rhs) const return true; } -ColumnPtr ColumnObject::compress() const +ColumnPtr ColumnObject::compress(bool force_compression) const { std::unordered_map compressed_typed_paths; compressed_typed_paths.reserve(typed_paths.size()); size_t byte_size = 0; for (const auto & [path, column] : typed_paths) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(force_compression); byte_size += compressed_column->byteSize(); compressed_typed_paths[path] = std::move(compressed_column); } @@ -1241,12 +1241,12 @@ ColumnPtr ColumnObject::compress() const compressed_dynamic_paths.reserve(dynamic_paths_ptrs.size()); for (const auto & [path, column] : dynamic_paths_ptrs) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(force_compression); byte_size += compressed_column->byteSize(); compressed_dynamic_paths[path] = std::move(compressed_column); } - auto compressed_shared_data = shared_data->compress(); + auto compressed_shared_data = shared_data->compress(force_compression); byte_size += compressed_shared_data->byteSize(); auto decompress = diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index 7b8a381d571..3160b66cd20 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -171,7 +171,7 @@ public: bool structureEquals(const IColumn & rhs) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void finalize() override; bool isFinalized() const override; diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp index a0e47e65fc6..b7d82ed8a09 100644 --- a/src/Columns/ColumnSparse.cpp +++ b/src/Columns/ColumnSparse.cpp @@ -774,10 +774,10 @@ UInt64 ColumnSparse::getNumberOfDefaultRows() const return _size - offsets->size(); } -ColumnPtr ColumnSparse::compress() const +ColumnPtr ColumnSparse::compress(bool force_compression) const { - auto values_compressed = values->compress(); - auto offsets_compressed = offsets->compress(); + auto values_compressed = values->compress(force_compression); + auto offsets_compressed = offsets->compress(force_compression); size_t byte_size = values_compressed->byteSize() + offsets_compressed->byteSize(); diff --git a/src/Columns/ColumnSparse.h b/src/Columns/ColumnSparse.h index 619dce63c1e..f95752cd546 100644 --- a/src/Columns/ColumnSparse.h +++ b/src/Columns/ColumnSparse.h @@ -147,7 +147,7 @@ public: double getRatioOfDefaultRows(double sample_ratio) const override; UInt64 getNumberOfDefaultRows() const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 9569e9ec252..4bdc253bfc4 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -628,7 +628,7 @@ void ColumnString::getExtremes(Field & min, Field & max) const get(max_idx, max); } -ColumnPtr ColumnString::compress() const +ColumnPtr ColumnString::compress(bool force_compression) const { const size_t source_chars_size = chars.size(); const size_t source_offsets_elements = offsets.size(); @@ -638,13 +638,13 @@ ColumnPtr ColumnString::compress() const if (source_chars_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto chars_compressed = ColumnCompressed::compressBuffer(chars.data(), source_chars_size, false); + auto chars_compressed = ColumnCompressed::compressBuffer(chars.data(), source_chars_size, force_compression); /// Return original column if not compressible. if (!chars_compressed) return ColumnCompressed::wrap(this->getPtr()); - auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, true); + auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, /*force_compression=*/true); const size_t chars_compressed_size = chars_compressed->size(); const size_t offsets_compressed_size = offsets_compressed->size(); diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 062315219b5..4bf24217383 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -272,7 +272,7 @@ public: ColumnPtr replicate(const Offsets & replicate_offsets) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void reserve(size_t n) override; size_t capacity() const override; diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 28e5f03cc3c..9bb377f56ae 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -796,7 +796,7 @@ void ColumnTuple::takeDynamicStructureFromSourceColumns(const Columns & source_c } -ColumnPtr ColumnTuple::compress() const +ColumnPtr ColumnTuple::compress(bool force_compression) const { if (columns.empty()) { @@ -812,7 +812,7 @@ ColumnPtr ColumnTuple::compress() const compressed.reserve(columns.size()); for (const auto & column : columns) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(force_compression); byte_size += compressed_column->byteSize(); compressed.emplace_back(std::move(compressed_column)); } diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index d5eee911edc..b8b3697b84d 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -125,7 +125,7 @@ public: void forEachSubcolumnRecursively(RecursiveMutableColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; bool isCollationSupported() const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void finalize() override; bool isFinalized() const override; diff --git a/src/Columns/ColumnVariant.cpp b/src/Columns/ColumnVariant.cpp index 2fa59b8e33c..38d3bac3c10 100644 --- a/src/Columns/ColumnVariant.cpp +++ b/src/Columns/ColumnVariant.cpp @@ -1426,16 +1426,16 @@ bool ColumnVariant::dynamicStructureEquals(const IColumn & rhs) const return true; } -ColumnPtr ColumnVariant::compress() const +ColumnPtr ColumnVariant::compress(bool force_compression) const { - ColumnPtr local_discriminators_compressed = local_discriminators->compress(); - ColumnPtr offsets_compressed = offsets->compress(); + ColumnPtr local_discriminators_compressed = local_discriminators->compress(force_compression); + ColumnPtr offsets_compressed = offsets->compress(force_compression); size_t byte_size = local_discriminators_compressed->byteSize() + offsets_compressed->byteSize(); Columns compressed; compressed.reserve(variants.size()); for (const auto & variant : variants) { - auto compressed_variant = variant->compress(); + auto compressed_variant = variant->compress(force_compression); byte_size += compressed_variant->byteSize(); compressed.emplace_back(std::move(compressed_variant)); } diff --git a/src/Columns/ColumnVariant.h b/src/Columns/ColumnVariant.h index a68a961169c..c7e37517004 100644 --- a/src/Columns/ColumnVariant.h +++ b/src/Columns/ColumnVariant.h @@ -254,7 +254,7 @@ public: void forEachSubcolumn(MutableColumnCallback callback) override; void forEachSubcolumnRecursively(RecursiveMutableColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; double getRatioOfDefaultRows(double sample_ratio) const override; UInt64 getNumberOfDefaultRows() const override; void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override; diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 3c7727f37c4..62f6c23c4f8 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -951,7 +951,7 @@ void ColumnVector::getExtremes(Field & min, Field & max) const } template -ColumnPtr ColumnVector::compress() const +ColumnPtr ColumnVector::compress(bool force_compression) const { const size_t data_size = data.size(); const size_t source_size = data_size * sizeof(T); @@ -960,7 +960,7 @@ ColumnPtr ColumnVector::compress() const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, false); + auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, force_compression); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 5247bfdf972..e5ece863a3b 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -287,7 +287,7 @@ public: ColumnPtr createWithOffsets(const IColumn::Offsets & offsets, const ColumnConst & column_with_default_value, size_t total_rows, size_t shift) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; /// Replace elements that match the filter with zeroes. If inverted replaces not matched elements. void applyZeroMap(const IColumn::Filter & filt, bool inverted = false); diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 9d1b42d2bc1..e2099ac34b9 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -601,7 +601,8 @@ public: /// Compress column in memory to some representation that allows to decompress it back. /// Return itself if compression is not applicable for this column type. - [[nodiscard]] virtual Ptr compress() const + /// The flag `force_compression` indicates that compression should be performed even if it's not efficient (if only compression factor < 1). + [[nodiscard]] virtual Ptr compress([[maybe_unused]] bool force_compression) const { /// No compression by default. return getPtr(); diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 02176a6b77a..0efb4596dcd 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -608,7 +608,7 @@ Block Block::compress() const size_t num_columns = data.size(); Columns new_columns(num_columns); for (size_t i = 0; i < num_columns; ++i) - new_columns[i] = data[i].column->compress(); + new_columns[i] = data[i].column->compress(/*force_compression=*/false); return cloneWithColumns(new_columns); } diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp index 7dbee567c5b..de3d720fc35 100644 --- a/src/Interpreters/Cache/QueryCache.cpp +++ b/src/Interpreters/Cache/QueryCache.cpp @@ -469,7 +469,7 @@ void QueryCache::Writer::finalizeWrite() Columns compressed_columns; for (const auto & column : columns) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(/*force_compression=*/false); compressed_columns.push_back(compressed_column); } Chunk compressed_chunk(compressed_columns, chunk.getNumRows()); diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index b56d6d98680..fcc24a06881 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -91,8 +91,7 @@ public: { Block compressed_block; for (const auto & elem : block) - compressed_block.insert({ elem.column->compress(), elem.type, elem.name }); - + compressed_block.insert({elem.column->compress(/*force_compression=*/true), elem.type, elem.name}); new_blocks.push_back(std::move(compressed_block)); } else @@ -259,7 +258,7 @@ void StorageMemory::mutate(const MutationCommands & commands, ContextPtr context { if ((*memory_settings)[MemorySetting::compress]) for (auto & elem : block) - elem.column = elem.column->compress(); + elem.column = elem.column->compress(/*force_compression=*/true); out.push_back(block); } @@ -574,7 +573,7 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat { Block compressed_block; for (const auto & elem : block) - compressed_block.insert({ elem.column->compress(), elem.type, elem.name }); + compressed_block.insert({elem.column->compress(/*force_compression=*/true), elem.type, elem.name}); new_blocks.push_back(std::move(compressed_block)); } From d1a35d5b285443aef1d981df450083ac4010ee11 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 6 Dec 2024 13:17:58 +0100 Subject: [PATCH 407/502] Properly cancel inserts in DistributedAsyncInsertDirectoryQueue --- .../DistributedAsyncInsertDirectoryQueue.cpp | 12 ++++++------ .../DistributedAsyncInsertDirectoryQueue.h | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp index 377919366b5..993f9e229e5 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -169,7 +169,7 @@ void DistributedAsyncInsertDirectoryQueue::flushAllData(const SettingsChanges & std::lock_guard lock{mutex}; if (!hasPendingFiles()) return; - processFiles(settings_changes); + processFiles(settings_changes, /*force=*/true); } void DistributedAsyncInsertDirectoryQueue::shutdownAndDropAllData() @@ -380,18 +380,18 @@ void DistributedAsyncInsertDirectoryQueue::initializeFilesFromDisk() status.broken_bytes_count = broken_bytes_count; } } -void DistributedAsyncInsertDirectoryQueue::processFiles(const SettingsChanges & settings_changes) +void DistributedAsyncInsertDirectoryQueue::processFiles(const SettingsChanges & settings_changes, bool force) try { if (should_batch_inserts) - processFilesWithBatching(settings_changes); + processFilesWithBatching(settings_changes, force); else { /// Process unprocessed file. if (!current_file.empty()) processFile(current_file, settings_changes); - while (!pending_files.isFinished() && pending_files.tryPop(current_file)) + while ((force || !monitor_blocker.isCancelled()) && !pending_files.isFinished() && pending_files.tryPop(current_file)) processFile(current_file, settings_changes); } @@ -539,7 +539,7 @@ DistributedAsyncInsertDirectoryQueue::Status DistributedAsyncInsertDirectoryQueu return current_status; } -void DistributedAsyncInsertDirectoryQueue::processFilesWithBatching(const SettingsChanges & settings_changes) +void DistributedAsyncInsertDirectoryQueue::processFilesWithBatching(const SettingsChanges & settings_changes, bool force) { /// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch. if (fs::exists(current_batch_file_path)) @@ -569,7 +569,7 @@ void DistributedAsyncInsertDirectoryQueue::processFilesWithBatching(const Settin try { - while (pending_files.tryPop(file_path)) + while ((force || !monitor_blocker.isCancelled()) && !pending_files.isFinished() && pending_files.tryPop(file_path)) { if (!fs::exists(file_path)) { diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h index 972e6b3143a..6fcbaf9f117 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h @@ -100,9 +100,9 @@ private: void addFile(const std::string & file_path); void initializeFilesFromDisk(); - void processFiles(const SettingsChanges & settings_changes = {}); + void processFiles(const SettingsChanges & settings_changes = {}, bool force = false); void processFile(std::string & file_path, const SettingsChanges & settings_changes); - void processFilesWithBatching(const SettingsChanges & settings_changes); + void processFilesWithBatching(const SettingsChanges & settings_changes, bool force); void markAsBroken(const std::string & file_path); void markAsSend(const std::string & file_path); From 6c6e1118c5601e3f54232908ba24baf6424995ed Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 6 Dec 2024 13:43:30 +0100 Subject: [PATCH 408/502] Better --- .../DistributedAsyncInsertDirectoryQueue.cpp | 10 +++++----- .../Distributed/DistributedAsyncInsertDirectoryQueue.h | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp index 993f9e229e5..480efd11ba1 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -169,7 +169,7 @@ void DistributedAsyncInsertDirectoryQueue::flushAllData(const SettingsChanges & std::lock_guard lock{mutex}; if (!hasPendingFiles()) return; - processFiles(settings_changes, /*force=*/true); + processFiles(/*force=*/true, settings_changes); } void DistributedAsyncInsertDirectoryQueue::shutdownAndDropAllData() @@ -212,7 +212,7 @@ void DistributedAsyncInsertDirectoryQueue::run() { try { - processFiles(); + processFiles(/*force=*/false); /// No errors while processing existing files. /// Let's see maybe there are more files to process. do_sleep = false; @@ -380,11 +380,11 @@ void DistributedAsyncInsertDirectoryQueue::initializeFilesFromDisk() status.broken_bytes_count = broken_bytes_count; } } -void DistributedAsyncInsertDirectoryQueue::processFiles(const SettingsChanges & settings_changes, bool force) +void DistributedAsyncInsertDirectoryQueue::processFiles(bool force, const SettingsChanges & settings_changes) try { if (should_batch_inserts) - processFilesWithBatching(settings_changes, force); + processFilesWithBatching(force, settings_changes); else { /// Process unprocessed file. @@ -539,7 +539,7 @@ DistributedAsyncInsertDirectoryQueue::Status DistributedAsyncInsertDirectoryQueu return current_status; } -void DistributedAsyncInsertDirectoryQueue::processFilesWithBatching(const SettingsChanges & settings_changes, bool force) +void DistributedAsyncInsertDirectoryQueue::processFilesWithBatching(bool force, const SettingsChanges & settings_changes) { /// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch. if (fs::exists(current_batch_file_path)) diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h index 6fcbaf9f117..023edb01f5a 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h @@ -100,9 +100,10 @@ private: void addFile(const std::string & file_path); void initializeFilesFromDisk(); - void processFiles(const SettingsChanges & settings_changes = {}, bool force = false); + /// Set `force = true` if processing of files must be finished fully despite cancellation flag being set + void processFiles(bool force, const SettingsChanges & settings_changes = {}); void processFile(std::string & file_path, const SettingsChanges & settings_changes); - void processFilesWithBatching(const SettingsChanges & settings_changes, bool force); + void processFilesWithBatching(bool force, const SettingsChanges & settings_changes); void markAsBroken(const std::string & file_path); void markAsSend(const std::string & file_path); From 1917ed1d07af2fc221160c4af2036ed8cccf156c Mon Sep 17 00:00:00 2001 From: Dale Mcdiarmid Date: Fri, 6 Dec 2024 13:04:44 +0000 Subject: [PATCH 409/502] ALL not supported in Cloud --- docs/en/sql-reference/statements/grant.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 6decaf19d5b..5062c6729d9 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -605,6 +605,10 @@ Allows using a specified table engine when creating a table. Applies to [table e Grants all the privileges on regulated entity to a user account or a role. +:::note +The privilege `ALL` is not supported in ClickHouse Cloud, where the `default` user has limited permissions. Users can grant the maximium permissions to a user by granting the `default_role`. See [here](/docs/en/cloud/security/cloud-access-management#initial-settings) for further details. +::: + ### NONE Doesn’t grant any privileges. From 52fc14eb029d55af5134423b5a41cc4e31d84a60 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Dec 2024 13:19:55 +0000 Subject: [PATCH 410/502] Docs: Follow-up to #71441 --- .../functions/string-replace-functions.md | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index f8d46cacbdc..58bdab224fd 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -253,7 +253,11 @@ SELECT format('{} {}', 'Hello', 'World') ## translate -Replaces characters in the string `s` using a one-to-one character mapping defined by `from` and `to` strings. `from` and `to` must be constant ASCII strings. Non-ASCII characters in the original string are not modified. If the number of characters in `from` list is larger than the `to` list, non overlapping characters will be deleted from the input string. +Replaces characters in the string `s` using a one-to-one character mapping defined by `from` and `to` strings. +`from` and `to` must be constant ASCII strings. +If `from` and `to` have equal sizes, each occurrence of the 1st character of `first` in `s` is replaced by the 1st character of `to`, the 2nd character of `first` in `s` is replaced by the 2nd character of `to`, etc. +If `from` contains more characters than `to`, all occurrences of the characters at the end of `from` that have no corresponding character in `to` are deleted from `s`. +Non-ASCII characters in `s` are not modified by the function. **Syntax** @@ -275,6 +279,20 @@ Result: └───────────────┘ ``` +`from` and `to` arguments have different lengths: + +``` sql +SELECT translate('clickhouse', 'clickhouse', 'CLICK') AS res +``` + +Result: + +``` text +┌─res───┐ +│ CLICK │ +└───────┘ +``` + ## translateUTF8 Like [translate](#translate) but assumes `s`, `from` and `to` are UTF-8 encoded strings. From 05871aa2e8132ed969a1117ed396e0f5e59f29c6 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 6 Dec 2024 13:34:07 +0000 Subject: [PATCH 411/502] fix parsing of incorrect data into sparse columns --- src/Core/Block.cpp | 8 +++++ src/Core/Block.h | 3 ++ src/Processors/Formats/IRowInputFormat.cpp | 6 +--- .../RowInputFormatWithDiagnosticInfo.cpp | 2 +- ...3279_insert_sparse_parsing_error.reference | 6 ++++ .../03279_insert_sparse_parsing_error.sh | 31 +++++++++++++++++++ 6 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/03279_insert_sparse_parsing_error.reference create mode 100755 tests/queries/0_stateless/03279_insert_sparse_parsing_error.sh diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 02176a6b77a..7d58c25a372 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -449,6 +449,14 @@ MutableColumns Block::cloneEmptyColumns() const return columns; } +MutableColumns Block::cloneEmptyColumns(const Serializations & serializations) const +{ + size_t num_columns = data.size(); + MutableColumns columns(num_columns); + for (size_t i = 0; i < num_columns; ++i) + columns[i] = data[i].type->createColumn(*serializations[i]); + return columns; +} Columns Block::getColumns() const { diff --git a/src/Core/Block.h b/src/Core/Block.h index 841fb3fb663..a045e2acdf0 100644 --- a/src/Core/Block.h +++ b/src/Core/Block.h @@ -141,6 +141,9 @@ public: /** Get empty columns with the same types as in block. */ MutableColumns cloneEmptyColumns() const; + /** Get empty columns with the same types as in block and given serializations. */ + MutableColumns cloneEmptyColumns(const Serializations & serializations) const; + /** Get columns from block for mutation. Columns in block will be nullptr. */ MutableColumns mutateColumns(); diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp index ea316f97521..928be5d7c39 100644 --- a/src/Processors/Formats/IRowInputFormat.cpp +++ b/src/Processors/Formats/IRowInputFormat.cpp @@ -104,12 +104,8 @@ Chunk IRowInputFormat::read() } const Block & header = getPort().getHeader(); - size_t num_columns = header.columns(); - MutableColumns columns(num_columns); - - for (size_t i = 0; i < num_columns; ++i) - columns[i] = header.getByPosition(i).type->createColumn(*serializations[i]); + MutableColumns columns = header.cloneEmptyColumns(serializations); ColumnCheckpoints checkpoints(columns.size()); for (size_t column_idx = 0; column_idx < columns.size(); ++column_idx) diff --git a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp index c1dd77aecaf..12e9b650acd 100644 --- a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp +++ b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp @@ -44,7 +44,7 @@ std::pair RowInputFormatWithDiagnosticInfo::getDiagnosticAndRawD "Buffer has gone, cannot extract information about what has been parsed."); const auto & header = getPort().getHeader(); - MutableColumns columns = header.cloneEmptyColumns(); + MutableColumns columns = header.cloneEmptyColumns(serializations); /// It is possible to display detailed diagnostics only if the last and next to last rows are still in the read buffer. size_t bytes_read_at_start_of_buffer = in->count() - in->offset(); diff --git a/tests/queries/0_stateless/03279_insert_sparse_parsing_error.reference b/tests/queries/0_stateless/03279_insert_sparse_parsing_error.reference new file mode 100644 index 00000000000..07d1b660d36 --- /dev/null +++ b/tests/queries/0_stateless/03279_insert_sparse_parsing_error.reference @@ -0,0 +1,6 @@ +Code: 27 +1 0 0 +2 0 0 +a Default +b Sparse +c Sparse diff --git a/tests/queries/0_stateless/03279_insert_sparse_parsing_error.sh b/tests/queries/0_stateless/03279_insert_sparse_parsing_error.sh new file mode 100755 index 00000000000..1d46c084c69 --- /dev/null +++ b/tests/queries/0_stateless/03279_insert_sparse_parsing_error.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query " + DROP TABLE IF EXISTS t_insert_sparse_columns; + + CREATE TABLE t_insert_sparse_columns (a UInt64, b UInt64, c UInt64) + ENGINE = MergeTree ORDER BY a + SETTINGS ratio_of_defaults_for_sparse_serialization = 0.5, enable_block_number_column = 0, enable_block_offset_column = 0; + + SYSTEM STOP MERGES t_insert_sparse_columns; +" + +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_insert_sparse_columns+FORMAT+CSV" --data-binary @- <&1 | grep -o "Code: 27" +3, 0 +4, 0 +EOF + +$CLICKHOUSE_CLIENT --query " + SELECT * FROM t_insert_sparse_columns; + SELECT column, serialization_kind FROM system.parts_columns WHERE database = currentDatabase() AND table = 't_insert_sparse_columns' AND active ORDER BY column, serialization_kind; + DROP TABLE IF EXISTS t_insert_sparse_columns; +" From e6de632c190f62daddf966b6fb5bf2c992fd22b0 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov <48961922+Avogar@users.noreply.github.com> Date: Fri, 6 Dec 2024 14:35:20 +0100 Subject: [PATCH 412/502] Fix flaky 03271_ghdata_object_to_json_alter test --- tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh b/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh index fc217780aa3..5b838fed0e6 100755 --- a/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh +++ b/tests/queries/0_stateless/03271_ghdata_object_to_json_alter.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: no-fasttest, long +# Tags: no-fasttest, long, no-s3-storage +# ^ no-s3-storage: too slow CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 7c7b2314765033493f326df4dafc462c25bcdd24 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 6 Dec 2024 13:58:35 +0000 Subject: [PATCH 413/502] Update test --- tests/performance/compact_part_subcolumns.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/performance/compact_part_subcolumns.xml b/tests/performance/compact_part_subcolumns.xml index 234b33ab8a7..d0544458589 100644 --- a/tests/performance/compact_part_subcolumns.xml +++ b/tests/performance/compact_part_subcolumns.xml @@ -3,9 +3,6 @@ 1 - - CREATE TABLE t_json (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=100000000000 INSERT INTO t_json SELECT toJSONString(map(number % 10, repeat('a', number % 100))) FROM numbers(10000000) SELECT data.k0, data.k1, data.k2, data.k3, data.k4, data.k5, data.k6, data.k7, data.k8, data.k9 FROM t_json FORMAT Null From f9b8a293ef94409cd6c561b5ae8df308a8697139 Mon Sep 17 00:00:00 2001 From: Dale Mcdiarmid Date: Fri, 6 Dec 2024 13:58:53 +0000 Subject: [PATCH 414/502] request headers in s3 --- .../table-engines/integrations/s3queue.md | 2 +- docs/en/sql-reference/table-functions/s3.md | 22 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/s3queue.md b/docs/en/engines/table-engines/integrations/s3queue.md index 89a70420069..41ce40cbac2 100644 --- a/docs/en/engines/table-engines/integrations/s3queue.md +++ b/docs/en/engines/table-engines/integrations/s3queue.md @@ -12,7 +12,7 @@ This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ec ``` sql CREATE TABLE s3_queue_engine_table (name String, value UInt32) - ENGINE = S3Queue(path, [NOSIGN, | aws_access_key_id, aws_secret_access_key,] format, [compression]) + ENGINE = S3Queue(path, [NOSIGN, | aws_access_key_id, aws_secret_access_key,] format, [compression], [headers]) [SETTINGS] [mode = '',] [after_processing = 'keep',] diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index ea7820c1aec..4bde1cb75ed 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -16,7 +16,7 @@ When using the `s3 table function` with [`INSERT INTO...SELECT`](../../sql-refer **Syntax** ``` sql -s3(url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,structure] [,compression_method]) +s3(url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,structure] [,compression_method],[,headers]) s3(named_collection[, option=value [,..]]) ``` @@ -44,6 +44,7 @@ For GCS, substitute your HMAC key and HMAC secret where you see `access_key_id` - `format` — The [format](../../interfaces/formats.md#formats) of the file. - `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. - `compression_method` — Parameter is optional. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. By default, it will autodetect compression method by file extension. +- `headers` - Parameter is optional. Allows headers to be passed in the S3 request. Pass in the format `headers(key=value)` e.g. `headers('x-amz-request-payer' = 'requester')`. Arguments can also be passed using [named collections](/docs/en/operations/named-collections.md). In this case `url`, `access_key_id`, `secret_access_key`, `format`, `structure`, `compression_method` work in the same way, and some extra parameters are supported: @@ -313,6 +314,25 @@ SET use_hive_partitioning = 1; SELECT * from s3('s3://data/path/date=*/country=*/code=*/*.parquet') where _date > '2020-01-01' and _country = 'Netherlands' and _code = 42; ``` +## Accessing requester-pays buckets + +To access a requester-pays bucket, a header `x-amz-request-payer = requester` must be passed in any requests. This is achieved by passing the parameter `headers('x-amz-request-payer' = 'requester')` to the s3 function. For example: + +```sql +SELECT + count() AS num_rows, + uniqExact(_file) AS num_files +FROM s3('https://coiled-datasets-rp.s3.us-east-1.amazonaws.com/1trc/measurements-100*.parquet', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', headers('x-amz-request-payer' = 'requester')) + +┌───num_rows─┬─num_files─┐ +│ 1110000000 │ 111 │ +└────────────┴───────────┘ + +1 row in set. Elapsed: 3.089 sec. Processed 1.09 billion rows, 0.00 B (353.55 million rows/s., 0.00 B/s.) +Peak memory usage: 192.27 KiB. +``` + + ## Storage Settings {#storage-settings} - [s3_truncate_on_insert](/docs/en/operations/settings/settings.md#s3_truncate_on_insert) - allows to truncate file before insert into it. Disabled by default. From ebc908faa0f36578156a11dcea5543e4abaf5cd6 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Fri, 6 Dec 2024 14:59:02 +0100 Subject: [PATCH 415/502] Push CI From 4cc1767a5dabfe4fa2a80a5c46ec952fa88f8df6 Mon Sep 17 00:00:00 2001 From: Dale Mcdiarmid Date: Fri, 6 Dec 2024 14:02:26 +0000 Subject: [PATCH 416/502] fix spelling --- docs/en/sql-reference/statements/grant.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 5062c6729d9..9b4edb02c12 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -606,7 +606,7 @@ Allows using a specified table engine when creating a table. Applies to [table e Grants all the privileges on regulated entity to a user account or a role. :::note -The privilege `ALL` is not supported in ClickHouse Cloud, where the `default` user has limited permissions. Users can grant the maximium permissions to a user by granting the `default_role`. See [here](/docs/en/cloud/security/cloud-access-management#initial-settings) for further details. +The privilege `ALL` is not supported in ClickHouse Cloud, where the `default` user has limited permissions. Users can grant the maximum permissions to a user by granting the `default_role`. See [here](/docs/en/cloud/security/cloud-access-management#initial-settings) for further details. ::: ### NONE From 82da99cfd317f4087696548f5173736eb6eba60f Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 6 Dec 2024 15:34:39 +0100 Subject: [PATCH 417/502] fix --- src/Columns/ColumnCompressed.h | 2 +- src/Columns/ColumnString.cpp | 43 ++++++++++---- src/Columns/ColumnString.h | 2 + src/Columns/tests/gtest_column_string.cpp | 71 +++++++++++++++++++++++ 4 files changed, 107 insertions(+), 11 deletions(-) create mode 100644 src/Columns/tests/gtest_column_string.cpp diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h index b030e762acd..7d7970cce8a 100644 --- a/src/Columns/ColumnCompressed.h +++ b/src/Columns/ColumnCompressed.h @@ -70,7 +70,7 @@ public: /// Helper methods for compression. - /// If data is not worth to be compressed and not 'always_compress' - returns nullptr. + /// If data is not worth to be compressed and not `force_compression` - returns nullptr. /// Note: shared_ptr is to allow to be captured by std::function. static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool force_compression); diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 4bdc253bfc4..3c73a005673 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -635,26 +635,39 @@ ColumnPtr ColumnString::compress(bool force_compression) const const size_t source_offsets_size = source_offsets_elements * sizeof(Offset); /// Don't compress small blocks. - if (source_chars_size < 4096) /// A wild guess. + if (source_chars_size < min_size_to_compress) + { return ColumnCompressed::wrap(this->getPtr()); + } auto chars_compressed = ColumnCompressed::compressBuffer(chars.data(), source_chars_size, force_compression); /// Return original column if not compressible. if (!chars_compressed) + { return ColumnCompressed::wrap(this->getPtr()); + } auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, /*force_compression=*/true); + const bool offsets_were_compressed = !!offsets_compressed; + + /// Offsets are not compressible. Use the source data. + if (!offsets_compressed) + { + offsets_compressed = std::make_shared>(source_offsets_size); + memcpy(offsets_compressed->data(), offsets.data(), source_offsets_size); + } const size_t chars_compressed_size = chars_compressed->size(); const size_t offsets_compressed_size = offsets_compressed->size(); - return ColumnCompressed::create(source_offsets_elements, chars_compressed_size + offsets_compressed_size, - [ - my_chars_compressed = std::move(chars_compressed), - my_offsets_compressed = std::move(offsets_compressed), - source_chars_size, - source_offsets_elements - ] + return ColumnCompressed::create( + source_offsets_elements, + chars_compressed_size + offsets_compressed_size, + [my_chars_compressed = std::move(chars_compressed), + my_offsets_compressed = std::move(offsets_compressed), + source_chars_size, + source_offsets_elements, + offsets_were_compressed] { auto res = ColumnString::create(); @@ -664,8 +677,18 @@ ColumnPtr ColumnString::compress(bool force_compression) const ColumnCompressed::decompressBuffer( my_chars_compressed->data(), res->getChars().data(), my_chars_compressed->size(), source_chars_size); - ColumnCompressed::decompressBuffer( - my_offsets_compressed->data(), res->getOffsets().data(), my_offsets_compressed->size(), source_offsets_elements * sizeof(Offset)); + if (offsets_were_compressed) + { + ColumnCompressed::decompressBuffer( + my_offsets_compressed->data(), + res->getOffsets().data(), + my_offsets_compressed->size(), + source_offsets_elements * sizeof(Offset)); + } + else + { + memcpy(res->getOffsets().data(), my_offsets_compressed->data(), my_offsets_compressed->size()); + } return res; }); diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 4bf24217383..245164ca31b 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -29,6 +29,8 @@ public: using Char = UInt8; using Chars = PaddedPODArray; + static constexpr size_t min_size_to_compress = 4096; + private: friend class COWHelper, ColumnString>; diff --git a/src/Columns/tests/gtest_column_string.cpp b/src/Columns/tests/gtest_column_string.cpp new file mode 100644 index 00000000000..13a29616802 --- /dev/null +++ b/src/Columns/tests/gtest_column_string.cpp @@ -0,0 +1,71 @@ +#include + +#include + +#include +#include + +using namespace DB; + +static pcg64 rng(randomSeed()); + +constexpr size_t bytes_per_string = sizeof(size_t) + 1; +/// Column should have enough bytes to be compressed +constexpr size_t column_size = ColumnString::min_size_to_compress / bytes_per_string + 42; + +TEST(ColumnString, Incompressible) +{ + auto col = ColumnString::create(); + auto & chars = col->getChars(); + auto & offsets = col->getOffsets(); + chars.resize(column_size * bytes_per_string); + for (size_t i = 0; i < column_size; ++i) + { + auto value = rng(); + memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); + chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + offsets.push_back((i + 1) * bytes_per_string); + } + + auto compressed = col->compress(true); + auto decompressed = compressed->decompress(); + ASSERT_EQ(decompressed.get(), col.get()); +} + +TEST(ColumnString, CompressibleCharsAndIncompressibleOffsets) +{ + auto col = ColumnString::create(); + auto & chars = col->getChars(); + auto & offsets = col->getOffsets(); + chars.resize(column_size * bytes_per_string); + for (size_t i = 0; i < column_size; ++i) + { + static const size_t value = 42; + memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); + chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + } + offsets.push_back(chars.size()); + + auto compressed = col->compress(true); + auto decompressed = compressed->decompress(); + ASSERT_NE(decompressed.get(), col.get()); +} + +TEST(ColumnString, CompressibleCharsAndCompressibleOffsets) +{ + auto col = ColumnString::create(); + auto & chars = col->getChars(); + auto & offsets = col->getOffsets(); + chars.resize(column_size * bytes_per_string); + for (size_t i = 0; i < column_size; ++i) + { + static const size_t value = 42; + memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); + chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + offsets.push_back((i + 1) * bytes_per_string); + } + + auto compressed = col->compress(true); + auto decompressed = compressed->decompress(); + ASSERT_NE(decompressed.get(), col.get()); +} From cc3ebd893168228facd66bc393b75d36c6de0259 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Fri, 6 Dec 2024 15:35:04 +0100 Subject: [PATCH 418/502] Poke CI From bbc73f6577bedbb102d88f8638c8b81e7a28ee6c Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Fri, 6 Dec 2024 15:41:06 +0100 Subject: [PATCH 419/502] Update rust_vendor submodule again --- contrib/rust_vendor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/rust_vendor b/contrib/rust_vendor index 043d386c6ac..b25b16b0b10 160000 --- a/contrib/rust_vendor +++ b/contrib/rust_vendor @@ -1 +1 @@ -Subproject commit 043d386c6ac6818db25a9ddd6fbc6b59b95e64b2 +Subproject commit b25b16b0b10a1cbb33eb0922f77aeedb72119792 From 0745342f6e9df4370a673101ca7ac227c3d61d78 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Dec 2024 15:19:41 +0000 Subject: [PATCH 420/502] Docs: Fix data type links --- .../functions/string-search-functions.md | 82 +++++++++---------- src/Functions/FunctionsStringSearch.h | 15 ++-- 2 files changed, 46 insertions(+), 51 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index c8435d02d65..0570d4fac5b 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -36,8 +36,8 @@ Alias: **Arguments** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md) or [Enum](../data-types/string.md). +- `needle` — Substring to be searched. [String](../data-types/string.md). - `start_pos` – Position (1-based) in `haystack` at which the search starts. [UInt](../data-types/int-uint.md). Optional. **Returned value** @@ -203,7 +203,7 @@ multiSearchAllPositions(haystack, [needle1, needle2, ..., needleN]) **Arguments** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md). - `needle` — Substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -238,7 +238,7 @@ multiSearchAllPositionsCaseInsensitive(haystack, [needle1, needle2, ..., needleN **Parameters** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md). - `needle` — Substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -272,7 +272,7 @@ multiSearchAllPositionsUTF8(haystack, [needle1, needle2, ..., needleN]) **Parameters** -- `haystack` — UTF-8 encoded string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — UTF-8 encoded string in which the search is performed. [String](../data-types/string.md). - `needle` — UTF-8 encoded substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -308,7 +308,7 @@ multiSearchAllPositionsCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., nee **Parameters** -- `haystack` — UTF-8 encoded string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — UTF-8 encoded string in which the search is performed. [String](../data-types/string.md). - `needle` — UTF-8 encoded substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -346,7 +346,7 @@ multiSearchFirstPosition(haystack, [needle1, needle2, ..., needleN]) **Parameters** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md). - `needle` — Substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -380,7 +380,7 @@ multiSearchFirstPositionCaseInsensitive(haystack, [needle1, needle2, ..., needle **Parameters** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md). - `needle` — Array of substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -414,7 +414,7 @@ multiSearchFirstPositionUTF8(haystack, [needle1, needle2, ..., needleN]) **Parameters** -- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — UTF-8 string in which the search is performed. [String](../data-types/string.md). - `needle` — Array of UTF-8 substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -450,7 +450,7 @@ multiSearchFirstPositionCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., ne **Parameters** -- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — UTF-8 string in which the search is performed. [String](../data-types/string.md). - `needle` — Array of UTF-8 substrings to be searched. [Array](../data-types/array.md) **Returned value** @@ -487,7 +487,7 @@ multiSearchFirstIndex(haystack, [needle1, needle2, ..., needleN]) ``` **Parameters** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md). - `needle` — Substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -520,7 +520,7 @@ multiSearchFirstIndexCaseInsensitive(haystack, [needle1, needle2, ..., needleN]) **Parameters** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md). - `needle` — Substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -553,7 +553,7 @@ multiSearchFirstIndexUTF8(haystack, [needle1, needle2, ..., needleN]) **Parameters** -- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — UTF-8 string in which the search is performed. [String](../data-types/string.md). - `needle` — Array of UTF-8 substrings to be searched. [Array](../data-types/array.md) **Returned value** @@ -588,7 +588,7 @@ multiSearchFirstIndexCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needl **Parameters** -- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — UTF-8 string in which the search is performed. [String](../data-types/string.md). - `needle` — Array of UTF-8 substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -625,7 +625,7 @@ multiSearchAny(haystack, [needle1, needle2, ..., needleN]) **Parameters** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md). - `needle` — Substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -659,7 +659,7 @@ multiSearchAnyCaseInsensitive(haystack, [needle1, needle2, ..., needleN]) **Parameters** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md). - `needle` — Substrings to be searched. [Array](../data-types/array.md) **Returned value** @@ -693,7 +693,7 @@ multiSearchAnyUTF8(haystack, [needle1, needle2, ..., needleN]) **Parameters** -- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — UTF-8 string in which the search is performed. [String](../data-types/string.md). - `needle` — UTF-8 substrings to be searched. [Array](../data-types/array.md). **Returned value** @@ -729,7 +729,7 @@ multiSearchAnyCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN]) **Parameters** -- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — UTF-8 string in which the search is performed. [String](../data-types/string.md). - `needle` — UTF-8 substrings to be searched. [Array](../data-types/array.md) **Returned value** @@ -1414,8 +1414,8 @@ countSubstrings(haystack, needle[, start_pos]) **Arguments** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md) or [Enum](../data-types/enum.md). +- `needle` — Substring to be searched. [String](../data-types/string.md). - `start_pos` – Position (1-based) in `haystack` at which the search starts. [UInt](../data-types/int-uint.md). Optional. **Returned value** @@ -1461,8 +1461,8 @@ countSubstringsCaseInsensitive(haystack, needle[, start_pos]) **Arguments** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md) or [Enum](../data-types/enum.md). +- `needle` — Substring to be searched. [String](../data-types/string.md). - `start_pos` – Position (1-based) in `haystack` at which the search starts. [UInt](../data-types/int-uint.md). Optional. **Returned value** @@ -1513,8 +1513,8 @@ countSubstringsCaseInsensitiveUTF8(haystack, needle[, start_pos]) **Arguments** -- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — UTF-8 string in which the search is performed. [String](../data-types/string.md) or [Enum](../data-types/enum.md). +- `needle` — Substring to be searched. [String](../data-types/string.md). - `start_pos` – Position (1-based) in `haystack` at which the search starts. [UInt](../data-types/int-uint.md). Optional. **Returned value** @@ -1565,7 +1565,7 @@ countMatches(haystack, pattern) **Arguments** -- `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — The string to search in. [String](../data-types/string.md). - `pattern` — The regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). [String](../data-types/string.md). **Returned value** @@ -1610,7 +1610,7 @@ countMatchesCaseInsensitive(haystack, pattern) **Arguments** -- `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — The string to search in. [String](../data-types/string.md). - `pattern` — The regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). [String](../data-types/string.md). **Returned value** @@ -1647,8 +1647,8 @@ Alias: `REGEXP_EXTRACT(haystack, pattern[, index])`. **Arguments** -- `haystack` — String, in which regexp pattern will to be matched. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `pattern` — String, regexp expression, must be constant. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String, in which regexp pattern will to be matched. [String](../data-types/string.md). +- `pattern` — String, regexp expression, must be constant. [String](../data-types/string.md). - `index` – An integer number greater or equal 0 with default 1. It represents which regex group to extract. [UInt or Int](../data-types/int-uint.md). Optional. **Returned value** @@ -1687,8 +1687,8 @@ hasSubsequence(haystack, needle) **Arguments** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Subsequence to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md). +- `needle` — Subsequence to be searched. [String](../data-types/string.md). **Returned value** @@ -1722,8 +1722,8 @@ hasSubsequenceCaseInsensitive(haystack, needle) **Arguments** -- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Subsequence to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. [String](../data-types/string.md). +- `needle` — Subsequence to be searched. [String](../data-types/string.md). **Returned value** @@ -1757,8 +1757,8 @@ hasSubsequenceUTF8(haystack, needle) **Arguments** -- `haystack` — String in which the search is performed. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Subsequence to be searched. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. UTF-8 encoded [String](../data-types/string.md). +- `needle` — Subsequence to be searched. UTF-8 encoded [String](../data-types/string.md). **Returned value** @@ -1792,8 +1792,8 @@ hasSubsequenceCaseInsensitiveUTF8(haystack, needle) **Arguments** -- `haystack` — String in which the search is performed. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — Subsequence to be searched. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String in which the search is performed. UTF-8 encoded [String](../data-types/string.md). +- `needle` — Subsequence to be searched. UTF-8 encoded [String](../data-types/string.md). **Returned value** @@ -1827,7 +1827,7 @@ hasToken(haystack, token) **Parameters** -- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack`: String in which the search is performed. [String](../data-types/string.md) or [Enum](../data-types/enum.md). - `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). **Returned value** @@ -1862,12 +1862,12 @@ hasTokenOrNull(haystack, token) **Parameters** -- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack`: String in which the search is performed. [String](../data-types/string.md) or [Enum](../data-types/enum.md). - `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). **Returned value** -- 1, if the token is present in the haystack, 0 if it is not present, and null if the token is ill formed. +- 1, if the token is present in the haystack, 0 if it is not present, and null if the token is ill formed. **Implementation details** @@ -1899,7 +1899,7 @@ hasTokenCaseInsensitive(haystack, token) **Parameters** -- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack`: String in which the search is performed. [String](../data-types/string.md) or [Enum](../data-types/enum.md). - `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). **Returned value** @@ -1934,7 +1934,7 @@ hasTokenCaseInsensitiveOrNull(haystack, token) **Parameters** -- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack`: String in which the search is performed. [String](../data-types/string.md) or [Enum](../data-types/enum.md). - `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). **Returned value** diff --git a/src/Functions/FunctionsStringSearch.h b/src/Functions/FunctionsStringSearch.h index e3f19119539..fc0eb1d0258 100644 --- a/src/Functions/FunctionsStringSearch.h +++ b/src/Functions/FunctionsStringSearch.h @@ -7,10 +7,10 @@ #include #include #include +#include #include #include #include -#include #include #include #include @@ -179,8 +179,10 @@ public: { const auto * col = argument.column.get(); const auto * type = argument.type.get(); + auto res = ColumnString::create(); res->reserve(col->size()); + if constexpr (std::is_same_v) { const auto * enum_col = typeid_cast(col); @@ -212,17 +214,10 @@ public: ColumnPtr column_haystack = haystack_argument.column; const ColumnPtr & column_needle = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[1].column : arguments[0].column; - bool is_enum8 = isEnum8(haystack_argument.type); - bool is_enum16 = isEnum16(haystack_argument.type); - - if (is_enum8) - { + if (isEnum8(haystack_argument.type)) column_haystack = genStringColumnFromEnumColumn(haystack_argument); - } - if (is_enum16) - { + if (isEnum16(haystack_argument.type)) column_haystack = genStringColumnFromEnumColumn(haystack_argument); - } ColumnPtr column_start_pos = nullptr; if (arguments.size() >= 3) From 5a9062d1200c9f9c3bae5c9601f4ab4805be59b8 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 6 Dec 2024 16:21:07 +0100 Subject: [PATCH 421/502] better --- src/Columns/tests/gtest_column_string.cpp | 37 +++++++++++++++++------ 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/Columns/tests/gtest_column_string.cpp b/src/Columns/tests/gtest_column_string.cpp index 13a29616802..4a0de2b5515 100644 --- a/src/Columns/tests/gtest_column_string.cpp +++ b/src/Columns/tests/gtest_column_string.cpp @@ -9,7 +9,7 @@ using namespace DB; static pcg64 rng(randomSeed()); -constexpr size_t bytes_per_string = sizeof(size_t) + 1; +constexpr size_t bytes_per_string = sizeof(uint64_t) + 1; /// Column should have enough bytes to be compressed constexpr size_t column_size = ColumnString::min_size_to_compress / bytes_per_string + 42; @@ -21,15 +21,20 @@ TEST(ColumnString, Incompressible) chars.resize(column_size * bytes_per_string); for (size_t i = 0; i < column_size; ++i) { - auto value = rng(); - memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); - chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + const uint64_t value = rng(); + memcpy(&chars[i * bytes_per_string], &value, sizeof(uint64_t)); + chars[i * bytes_per_string + sizeof(uint64_t)] = '\0'; offsets.push_back((i + 1) * bytes_per_string); } auto compressed = col->compress(true); auto decompressed = compressed->decompress(); + // When column is incompressible, we return the original column wrapped in CompressedColumn ASSERT_EQ(decompressed.get(), col.get()); + ASSERT_EQ(compressed->size(), col->size()); + ASSERT_EQ(compressed->allocatedBytes(), col->allocatedBytes()); + ASSERT_EQ(decompressed->size(), col->size()); + ASSERT_EQ(decompressed->allocatedBytes(), col->allocatedBytes()); } TEST(ColumnString, CompressibleCharsAndIncompressibleOffsets) @@ -40,15 +45,21 @@ TEST(ColumnString, CompressibleCharsAndIncompressibleOffsets) chars.resize(column_size * bytes_per_string); for (size_t i = 0; i < column_size; ++i) { - static const size_t value = 42; - memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); - chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + static const uint64_t value = 42; + memcpy(&chars[i * bytes_per_string], &value, sizeof(uint64_t)); + chars[i * bytes_per_string + sizeof(uint64_t)] = '\0'; } offsets.push_back(chars.size()); auto compressed = col->compress(true); auto decompressed = compressed->decompress(); + // For actually compressed column only compressed `chars` and `offsets` arrays are stored. + // Upon decompression, a new column is created. ASSERT_NE(decompressed.get(), col.get()); + ASSERT_EQ(compressed->size(), col->size()); + ASSERT_LE(compressed->allocatedBytes(), col->allocatedBytes()); + ASSERT_EQ(decompressed->size(), col->size()); + ASSERT_LE(decompressed->allocatedBytes(), col->allocatedBytes()); } TEST(ColumnString, CompressibleCharsAndCompressibleOffsets) @@ -59,13 +70,19 @@ TEST(ColumnString, CompressibleCharsAndCompressibleOffsets) chars.resize(column_size * bytes_per_string); for (size_t i = 0; i < column_size; ++i) { - static const size_t value = 42; - memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); - chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + static const uint64_t value = 42; + memcpy(&chars[i * bytes_per_string], &value, sizeof(uint64_t)); + chars[i * bytes_per_string + sizeof(uint64_t)] = '\0'; offsets.push_back((i + 1) * bytes_per_string); } auto compressed = col->compress(true); auto decompressed = compressed->decompress(); + // For actually compressed column only compressed `chars` and `offsets` arrays are stored. + // Upon decompression, a new column is created. ASSERT_NE(decompressed.get(), col.get()); + ASSERT_EQ(compressed->size(), col->size()); + ASSERT_LE(compressed->allocatedBytes(), col->allocatedBytes()); + ASSERT_EQ(decompressed->size(), col->size()); + ASSERT_LE(decompressed->allocatedBytes(), col->allocatedBytes()); } From bd577939b13c436fe29d04e2a6c6324fbe749235 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 6 Dec 2024 16:36:02 +0100 Subject: [PATCH 422/502] Fix --- tests/integration/test_storage_s3_queue/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index e12f23b994c..820f02b1dbd 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -2328,7 +2328,7 @@ def test_list_and_delete_race(started_cluster): keeper_path = f"/clickhouse/test_{table_name}" files_path = f"{table_name}_data" files_to_generate = 1000 - row_num = 10 + row_num = 1 for instance in [node, node_2]: create_table( @@ -2350,7 +2350,7 @@ def test_list_and_delete_race(started_cluster): }, ) - threads = 6 + threads = 10 total_rows = row_num * files_to_generate * (threads + 1) busy_pool = Pool(threads) From ba57068fa83b1e4d9b947a1745bbce1d9928bc34 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Fri, 6 Dec 2024 16:53:27 +0100 Subject: [PATCH 423/502] Push CI From 9b28d404e2ea1184f1c939e8bc58aaa146dcace6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 6 Dec 2024 17:31:42 +0100 Subject: [PATCH 424/502] Review fixes --- .../ObjectStorageQueueMetadata.cpp | 25 ++++++------ .../ObjectStorageQueueMetadataFactory.cpp | 39 +++++++++++++++---- .../ObjectStorageQueueMetadataFactory.h | 8 +++- .../integration/test_storage_s3_queue/test.py | 4 +- 4 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp index 6f536d78ac0..d9cda0f614c 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp @@ -494,8 +494,7 @@ void ObjectStorageQueueMetadata::registerIfNot(const StorageID & storage_id) std::string registry_str; auto zk_client = getZooKeeper(); - bool node_exists = zk_client->tryGet(registry_path, registry_str, &stat); - if (node_exists) + if (zk_client->tryGet(registry_path, registry_str, &stat)) { Strings registered; splitInto<','>(registered, registry_str); @@ -512,22 +511,18 @@ void ObjectStorageQueueMetadata::registerIfNot(const StorageID & storage_id) return; } } - } - LOG_TRACE(log, "Adding {} to registry", self.table_id); - - if (node_exists) - { auto new_registry_str = registry_str + "," + self.serialize(); code = zk_client->trySet(registry_path, new_registry_str, stat.version); } else - { code = zk_client->tryCreate(registry_path, self.serialize(), zkutil::CreateMode::Persistent); - } if (code == Coordination::Error::ZOK) + { + LOG_TRACE(log, "Added {} to registry", self.table_id); return; + } if (code == Coordination::Error::ZBADVERSION || code == Coordination::Error::ZSESSIONEXPIRED) @@ -543,7 +538,7 @@ size_t ObjectStorageQueueMetadata::unregister(const StorageID & storage_id) const auto registry_path = zookeeper_path / "registry"; const auto self = Info::create(storage_id); - Coordination::Error code; + Coordination::Error code = Coordination::Error::ZOK; for (size_t i = 0; i < 1000; ++i) { Coordination::Stat stat; @@ -588,13 +583,17 @@ size_t ObjectStorageQueueMetadata::unregister(const StorageID & storage_id) if (code == Coordination::Error::ZOK) return count; - if (code == Coordination::Error::ZBADVERSION - || code == Coordination::Error::ZSESSIONEXPIRED) + if (Coordination::isHardwareError(code) + || code == Coordination::Error::ZBADVERSION) continue; throw zkutil::KeeperException(code); } - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot unregister in keeper. Last error: {}", code); + + if (Coordination::isHardwareError(code)) + throw zkutil::KeeperException(code); + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot unregister in keeper. Last error: {}", code); } void ObjectStorageQueueMetadata::cleanupThreadFunc() diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp index f9902e556f0..6f7946f15a4 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp @@ -28,13 +28,14 @@ ObjectStorageQueueMetadataFactory::FilesMetadataPtr ObjectStorageQueueMetadataFa else { auto & metadata_from_table = metadata->getTableMetadata(); - auto & metadata_from_keeper = it->second->getTableMetadata(); + auto & metadata_from_keeper = it->second.metadata->getTableMetadata(); metadata_from_table.checkEquals(metadata_from_keeper); } - it->second->registerIfNot(storage_id); - return it->second; + it->second.metadata->registerIfNot(storage_id); + it->second.ref_count += 1; + return it->second.metadata; } void ObjectStorageQueueMetadataFactory::remove(const std::string & zookeeper_path, const StorageID & storage_id) @@ -45,8 +46,29 @@ void ObjectStorageQueueMetadataFactory::remove(const std::string & zookeeper_pat if (it == metadata_by_path.end()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Metadata with zookeeper path {} does not exist", zookeeper_path); - const size_t registry_size = it->second->unregister(storage_id); - LOG_TRACE(log, "Remaining registry size: {}", registry_size); + it->second.ref_count -= 1; + + size_t registry_size; + try + { + registry_size = it->second.metadata->unregister(storage_id); + LOG_TRACE(log, "Remaining registry size: {}", registry_size); + } + catch (const zkutil::KeeperException & e) + { + if (!Coordination::isHardwareError(e.code)) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + /// Any non-zero value would do. + registry_size = 1; + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + /// Any non-zero value would do. + registry_size = 1; + } if (registry_size == 0) { @@ -59,16 +81,17 @@ void ObjectStorageQueueMetadataFactory::remove(const std::string & zookeeper_pat { tryLogCurrentException(log); } - - metadata_by_path.erase(it); } + + if (!it->second.ref_count) + metadata_by_path.erase(it); } std::unordered_map ObjectStorageQueueMetadataFactory::getAll() { std::unordered_map result; for (const auto & [zk_path, metadata] : metadata_by_path) - result.emplace(zk_path, metadata); + result.emplace(zk_path, metadata.metadata); return result; } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h index 0b35d42a9a1..18da1d2f6f4 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h @@ -22,7 +22,13 @@ public: std::unordered_map getAll(); private: - using MetadataByPath = std::unordered_map>; + struct MetadataWithRefCount + { + explicit MetadataWithRefCount(std::shared_ptr metadata_) : metadata(metadata_) {} + std::shared_ptr metadata; + size_t ref_count = 0; + }; + using MetadataByPath = std::unordered_map; MetadataByPath metadata_by_path; std::mutex mutex; diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 10938ee32d6..1f6ef1f9120 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -2318,10 +2318,10 @@ def test_registry(started_cluster): node2.query(f"DROP DATABASE IF EXISTS {db_name}") node1.query( - f"CREATE DATABASE {db_name} ENGINE=Replicated('/clickhouse/databases/replicateddb', 'shard1', 'node1')" + f"CREATE DATABASE {db_name} ENGINE=Replicated('/clickhouse/databases/replicateddb2', 'shard1', 'node1')" ) node2.query( - f"CREATE DATABASE {db_name} ENGINE=Replicated('/clickhouse/databases/replicateddb', 'shard1', 'node2')" + f"CREATE DATABASE {db_name} ENGINE=Replicated('/clickhouse/databases/replicateddb2', 'shard1', 'node2')" ) create_table( From 71849444fc165975775c898715c667316e492c4f Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 6 Dec 2024 18:11:12 +0000 Subject: [PATCH 425/502] Fix tests --- src/Storages/MergeTree/IMergeTreeReader.h | 4 ++- .../MergeTree/MergeTreeReaderCompact.cpp | 27 +++++++++++-------- .../MergeTreeReaderCompactSingleBuffer.cpp | 1 + 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeReader.h b/src/Storages/MergeTree/IMergeTreeReader.h index c68617d3995..a80467e630e 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.h +++ b/src/Storages/MergeTree/IMergeTreeReader.h @@ -81,8 +81,10 @@ protected: /// avg_value_size_hints are used to reduce the number of reallocations when creating columns of variable size. ValueSizeMap avg_value_size_hints; - /// Stores states for IDataType::deserializeBinaryBulk + /// Stores states for IDataType::deserializeBinaryBulk for regular columns. DeserializeBinaryBulkStateMap deserialize_binary_bulk_state_map; + /// The same as above, but for subcolumns. + DeserializeBinaryBulkStateMap deserialize_binary_bulk_state_map_for_subcolumns; /// Actual column names and types of columns in part, /// which may differ from table metadata. diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 3c76ebe5d5e..685b37a6bde 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -188,7 +188,7 @@ void MergeTreeReaderCompact::readData( auto serialization = getSerializationInPart({name_in_storage, type_in_storage}); ColumnPtr temp_column = type_in_storage->createColumn(*serialization); - serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name_in_storage], nullptr); + serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map_for_subcolumns[name_in_storage], nullptr); auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); /// TODO: Avoid extra copying. @@ -231,9 +231,6 @@ void MergeTreeReaderCompact::readPrefix( { try { - if (deserialize_binary_bulk_state_map.contains(name_and_type.getNameInStorage())) - return; - ISerialization::DeserializeBinaryBulkSettings deserialize_settings; if (name_level_for_offsets.has_value()) @@ -248,15 +245,23 @@ void MergeTreeReaderCompact::readPrefix( serialization_for_prefix->deserializeBinaryBulkStatePrefix(deserialize_settings, state_for_prefix, nullptr); } - SerializationPtr serialization; - if (name_and_type.isSubcolumn()) - serialization = getSerializationInPart({name_and_type.getNameInStorage(), name_and_type.getTypeInStorage()}); - else - serialization = getSerializationInPart(name_and_type); - deserialize_settings.getter = buffer_getter; deserialize_settings.object_and_dynamic_read_statistics = true; - serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name_and_type.getNameInStorage()], nullptr); + + if (name_and_type.isSubcolumn()) + { + /// For subcolumns of the same column we need to deserialize prefix only once. + if (deserialize_binary_bulk_state_map_for_subcolumns.contains(name_and_type.getNameInStorage())) + return; + + auto serialization = getSerializationInPart({name_and_type.getNameInStorage(), name_and_type.getTypeInStorage()}); + serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map_for_subcolumns[name_and_type.getNameInStorage()], nullptr); + } + else + { + auto serialization = getSerializationInPart(name_and_type); + serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name_and_type.getNameInStorage()], nullptr); + } } catch (Exception & e) { diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index d72dd3f2269..4b59540b993 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -26,6 +26,7 @@ try { size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark); deserialize_binary_bulk_state_map.clear(); + deserialize_binary_bulk_state_map_for_subcolumns.clear(); /// Use cache to avoid reading the column with the same name twice. /// It may happen if there are empty array Nested in the part. From d5383d4d1b9f1dbd120875cce30b4113b84f1390 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 6 Dec 2024 18:31:56 +0000 Subject: [PATCH 426/502] Fix build --- src/TableFunctions/ITableFunctionFileLike.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index 64c34311af2..321bdcf0d3b 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -4,6 +4,9 @@ #include "Core/Names.h" #include "Parsers/IAST_fwd.h" +#include +#include + namespace DB { From 702a610d2bf52fa564f85747420cf1e7e57604bd Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 9 Dec 2024 10:55:01 +0100 Subject: [PATCH 427/502] Fixing build. --- src/Parsers/ASTOrderByElement.cpp | 6 +++--- src/Parsers/ASTOrderByElement.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Parsers/ASTOrderByElement.cpp b/src/Parsers/ASTOrderByElement.cpp index 0f3fabb7e0c..61a4d496675 100644 --- a/src/Parsers/ASTOrderByElement.cpp +++ b/src/Parsers/ASTOrderByElement.cpp @@ -68,12 +68,12 @@ void ASTStorageOrderByElement::updateTreeHashImpl(SipHash & hash_state, bool ign IAST::updateTreeHashImpl(hash_state, ignore_aliases); } -void ASTStorageOrderByElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +void ASTStorageOrderByElement::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - children.front()->formatImpl(settings, state, frame); + children.front()->formatImpl(ostr, settings, state, frame); if (direction == -1) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " DESC" << (settings.hilite ? hilite_none : ""); + ostr << (settings.hilite ? hilite_keyword : "") << " DESC" << (settings.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/ASTOrderByElement.h b/src/Parsers/ASTOrderByElement.h index ef0584ce704..ed4591ce3ce 100644 --- a/src/Parsers/ASTOrderByElement.h +++ b/src/Parsers/ASTOrderByElement.h @@ -101,7 +101,7 @@ public: void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; } From 663d26e475d019da091f9f1493ce57648195afcf Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 9 Dec 2024 11:19:26 +0100 Subject: [PATCH 428/502] Update SettingsChangesHistory --- src/Core/SettingsChangesHistory.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 97353d03c34..aa91ce4ec5d 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -623,6 +623,7 @@ static std::initializer_list Date: Mon, 9 Dec 2024 11:52:40 +0100 Subject: [PATCH 429/502] Fix test --- tests/integration/test_storage_s3_queue/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index df62a0959b7..d7f498f9972 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -2420,10 +2420,10 @@ def test_registry(started_cluster): node2.query(f"DROP DATABASE IF EXISTS {db_name}") node1.query( - f"CREATE DATABASE {db_name} ENGINE=Replicated('/clickhouse/databases/replicateddb', 'shard1', 'node1')" + f"CREATE DATABASE {db_name} ENGINE=Replicated('/clickhouse/databases/replicateddb2', 'shard1', 'node1')" ) node2.query( - f"CREATE DATABASE {db_name} ENGINE=Replicated('/clickhouse/databases/replicateddb', 'shard1', 'node2')" + f"CREATE DATABASE {db_name} ENGINE=Replicated('/clickhouse/databases/replicateddb2', 'shard1', 'node2')" ) create_table( From de0fdba3467a18d2f83baec5c541992a1f5f067c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 9 Dec 2024 12:00:41 +0100 Subject: [PATCH 430/502] Update next community call --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8eb264073c3..431e177d7ce 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ curl https://clickhouse.com/ | sh Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know. -* [v24.10 Community Call](https://clickhouse.com/company/events/v24-10-community-release-call) - October 31 +* [v24.12 Community Call](https://clickhouse.com/company/events/v24-12-community-release-call) - December 19 ## Upcoming Events From 9256d370c8370477d9dbe7683c0e2cf84f3a6b82 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 9 Dec 2024 12:48:12 +0100 Subject: [PATCH 431/502] Resolve conflicts --- src/Core/Settings.cpp | 7 ------- src/Databases/Iceberg/DatabaseIceberg.cpp | 3 ++- src/Databases/Iceberg/DatabaseIcebergStorageType.h | 1 + src/Databases/Iceberg/RestCatalog.cpp | 10 +++++----- .../ObjectStorage/DataLakes/IcebergMetadata.cpp | 4 ---- src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h | 7 ++----- src/Storages/ObjectStorage/StorageObjectStorage.cpp | 10 ++++++---- 7 files changed, 16 insertions(+), 26 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index e4b3a28c60e..284a5572bec 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -5560,13 +5560,6 @@ Only available in ClickHouse Cloud. Exclude new data parts from SELECT queries u )", 0) \ DECLARE(Int64, prefer_warmed_unmerged_parts_seconds, 0, R"( Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm. -)", 0) \ - DECLARE(Bool, iceberg_engine_ignore_schema_evolution, false, R"( -Allow to ignore schema evolution in Iceberg table engine and read all data using schema specified by the user on table creation or latest schema parsed from metadata on table creation. - -:::note -Enabling this setting can lead to incorrect result as in case of evolved schema all data files will be read using the same schema. -::: )", 0) \ DECLARE(Bool, allow_experimental_database_iceberg, false, R"( Allow experimental database engine Iceberg diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 62af8288933..7a49d11b19e 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -229,10 +229,11 @@ StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_ storage_type = table_metadata.getStorageType(); const auto configuration = getConfiguration(storage_type); + auto storage_settings = std::make_unique(); /// with_table_structure = false: because there will be /// no table structure in table definition AST. - StorageObjectStorage::Configuration::initialize(*configuration, args, context_, /* with_table_structure */false); + StorageObjectStorage::Configuration::initialize(*configuration, args, context_, /* with_table_structure */false, std::move(storage_settings)); return std::make_shared( configuration, diff --git a/src/Databases/Iceberg/DatabaseIcebergStorageType.h b/src/Databases/Iceberg/DatabaseIcebergStorageType.h index cc3c8f8cb1d..9b8dc3a0633 100644 --- a/src/Databases/Iceberg/DatabaseIcebergStorageType.h +++ b/src/Databases/Iceberg/DatabaseIcebergStorageType.h @@ -1,3 +1,4 @@ +#pragma once #include namespace DB diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index 460fce78696..ef947e05f7a 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -577,11 +577,11 @@ bool RestCatalog::getTableMetadataImpl( if (result.requiresSchema()) { - const auto & settings = getContext()->getSettingsRef(); - int format_version = metadata_object->getValue("format-version"); - auto schema = DB::IcebergMetadata::parseTableSchema( - metadata_object, log, format_version, settings[DB::Setting::iceberg_engine_ignore_schema_evolution]).first; - result.setSchema(schema); + // int format_version = metadata_object->getValue("format-version"); + auto schema_processor = DB::IcebergSchemaProcessor(); + auto id = DB::IcebergMetadata::parseTableSchema(metadata_object, schema_processor, log); + auto schema = schema_processor.getClickhouseTableSchemaById(id); + result.setSchema(*schema); } if (result.requiresCredentials() && object->has("config")) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 293969d1c7e..8550717d4c7 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -52,9 +52,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -Int32 parseTableSchema( - const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor, LoggerPtr metadata_logger); - IcebergMetadata::IcebergMetadata( ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, @@ -125,7 +122,6 @@ bool operator!=(const Poco::JSON::Object & first, const Poco::JSON::Object & sec { return !(first == second); } -} DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index ac55d90badc..d4e70030012 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -207,11 +207,8 @@ public: bool supportsExternalMetadataChange() const override { return true; } - static std::pair parseTableSchema( - const Poco::JSON::Object::Ptr & metadata_object, - LoggerPtr metadata_logger, - int format_version, - bool ignore_schema_evolution); + static Int32 parseTableSchema( + const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor, LoggerPtr metadata_logger); private: mutable std::unordered_map schema_id_by_data_file; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index ba2f9095887..2091e6185b4 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -104,10 +104,12 @@ StorageObjectStorage::StorageObjectStorage( try { if (!lazy_init) - if (configuration->hasExternalDynamicMetadata()) - configuration->updateAndGetCurrentSchema(object_storage, context); - else - configuration->update(object_storage, context); + { + if (configuration->hasExternalDynamicMetadata()) + configuration->updateAndGetCurrentSchema(object_storage, context); + else + configuration->update(object_storage, context); + } } catch (...) { From 1475ccca26f40926964d9c4032e33b66d8bee98f Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:13:56 +0100 Subject: [PATCH 432/502] Update MergeTreeData.cpp --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 197620b1310..fac290abe9b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6054,7 +6054,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartRestoredFromBackup(cons if (!retryable || (try_no + 1 == loading_parts_max_tries)) { if (Exception * e = exception_cast(error)) - e->addMessage("while restoring part {} of table {}", part->name, getStorageID()); + e->addMessage("while restoring part {} of table {}", part_name, getStorageID()); std::rethrow_exception(error); } From bed4eaaafab943d2b030e142f234f086aaa03626 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 9 Dec 2024 14:04:48 +0100 Subject: [PATCH 433/502] Fix style check --- src/Databases/Iceberg/RestCatalog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index ef947e05f7a..39108b1648e 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -20,12 +20,12 @@ #include #include + namespace DB::ErrorCodes { extern const int ICEBERG_CATALOG_ERROR; extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; - extern const int NOT_IMPLEMENTED; } namespace DB::Setting From e17973130d6e52f8bcffcfacaf0da8691dc32f76 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 9 Dec 2024 14:44:00 +0100 Subject: [PATCH 434/502] impl --- .../0_stateless/01290_max_execution_speed_distributed.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql index d0dc554f425..4ab7c0672e6 100644 --- a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql +++ b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql @@ -21,6 +21,7 @@ FROM system.query_log WHERE current_database = currentDatabase() AND event_date >= yesterday() AND + event_time >= now() - INTERVAL '5 MINUTES' AND -- time limit for tests not marked `long` is 3 minutes, 5 should be more than enough query LIKE '%special query for 01290_max_execution_speed_distributed%' AND query NOT LIKE '%system.query_log%' AND type = 2; From 646d44e30ddcfc7d4193854c765a93b2853aa887 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 9 Dec 2024 15:14:58 +0100 Subject: [PATCH 435/502] fix comment --- src/Columns/ColumnCompressed.h | 4 +++- src/Columns/ColumnString.cpp | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h index 7d7970cce8a..48f3717291b 100644 --- a/src/Columns/ColumnCompressed.h +++ b/src/Columns/ColumnCompressed.h @@ -70,7 +70,9 @@ public: /// Helper methods for compression. - /// If data is not worth to be compressed and not `force_compression` - returns nullptr. + /// If data is not worth to be compressed - returns nullptr. + /// By default it requires that compressed data is at least 50% smaller than original. + /// With `force_compression` set to true, it requires compressed data to be not larger than the source data. /// Note: shared_ptr is to allow to be captured by std::function. static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool force_compression); diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 3c73a005673..4785f0ce28d 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -648,7 +648,7 @@ ColumnPtr ColumnString::compress(bool force_compression) const return ColumnCompressed::wrap(this->getPtr()); } - auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, /*force_compression=*/true); + auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, force_compression); const bool offsets_were_compressed = !!offsets_compressed; /// Offsets are not compressible. Use the source data. From b3419881ae2b6737c30e6d2424a3be76f9d6863d Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 9 Dec 2024 15:30:23 +0100 Subject: [PATCH 436/502] better --- .../0_stateless/01290_max_execution_speed_distributed.sql | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql index 4ab7c0672e6..8962f15e101 100644 --- a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql +++ b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql @@ -1,15 +1,12 @@ -- Tags: distributed -SET max_execution_speed = 1000000; -SET timeout_before_checking_execution_speed = 0; -SET max_block_size = 100; - SET log_queries=1; CREATE TEMPORARY TABLE times (t DateTime); INSERT INTO times SELECT now(); -SELECT count('special query for 01290_max_execution_speed_distributed') FROM remote('127.0.0.{2,3}', numbers(1000000)); +SELECT count('special query for 01290_max_execution_speed_distributed') FROM remote('127.0.0.{2,3}', numbers(1000000)) +SETTINGS max_execution_speed = 1000000, timeout_before_checking_execution_speed = 0, max_block_size = 100; INSERT INTO times SELECT now(); SELECT max(t) - min(t) >= 1 FROM times; From 6da4d19fe0fdbbe574f94638cfb1c3180b19d488 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 9 Dec 2024 15:26:40 +0000 Subject: [PATCH 437/502] Fix tests --- src/Storages/StorageURL.cpp | 7 ++++--- src/Storages/StorageURL.h | 2 +- tests/queries/0_stateless/02968_url_args.reference | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 9b7b7227ad2..5ac37cbd418 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -1488,7 +1488,7 @@ FormatSettings StorageURL::getFormatSettingsFromArgs(const StorageFactory::Argum } size_t StorageURL::evalArgsAndCollectHeaders( - ASTs & url_function_args, HTTPHeaderEntries & header_entries, const ContextPtr & context) + ASTs & url_function_args, HTTPHeaderEntries & header_entries, const ContextPtr & context, bool evaluate_arguments) { ASTs::iterator headers_it = url_function_args.end(); @@ -1542,7 +1542,8 @@ size_t StorageURL::evalArgsAndCollectHeaders( if (headers_ast_function && headers_ast_function->name == "equals") continue; - (*arg_it) = evaluateConstantExpressionOrIdentifierAsLiteral((*arg_it), context); + if (evaluate_arguments) + (*arg_it) = evaluateConstantExpressionOrIdentifierAsLiteral((*arg_it), context); } if (headers_it == url_function_args.end()) @@ -1584,7 +1585,7 @@ StorageURL::Configuration StorageURL::getConfiguration(ASTs & args, const Contex if (auto named_collection = tryGetNamedCollectionWithOverrides(args, local_context)) { StorageURL::processNamedCollectionResult(configuration, *named_collection); - evalArgsAndCollectHeaders(args, configuration.headers, local_context); + evalArgsAndCollectHeaders(args, configuration.headers, local_context, false); } else { diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 7df5b90653d..b71f0cd68f8 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -322,7 +322,7 @@ public: /// Does evaluateConstantExpressionOrIdentifierAsLiteral() on all arguments. /// If `headers(...)` argument is present, parses it and moves it to the end of the array. /// Returns number of arguments excluding `headers(...)`. - static size_t evalArgsAndCollectHeaders(ASTs & url_function_args, HTTPHeaderEntries & header_entries, const ContextPtr & context); + static size_t evalArgsAndCollectHeaders(ASTs & url_function_args, HTTPHeaderEntries & header_entries, const ContextPtr & context, bool evaluate_arguments = true); static void processNamedCollectionResult(Configuration & configuration, const NamedCollection & collection); }; diff --git a/tests/queries/0_stateless/02968_url_args.reference b/tests/queries/0_stateless/02968_url_args.reference index e7e9e2c0d94..3c51a5edf81 100644 --- a/tests/queries/0_stateless/02968_url_args.reference +++ b/tests/queries/0_stateless/02968_url_args.reference @@ -1,7 +1,7 @@ CREATE TABLE default.a\n(\n `x` Int64\n)\nENGINE = URL(\'https://example.com/\', \'CSV\', headers(\'foo\' = \'[HIDDEN]\', \'a\' = \'[HIDDEN]\')) CREATE TABLE default.b\n(\n `x` Int64\n)\nENGINE = URL(\'https://example.com/\', \'CSV\', headers()) CREATE TABLE default.c\n(\n `x` Int64\n)\nENGINE = S3(\'https://example.s3.amazonaws.com/a.csv\', \'NOSIGN\', \'CSV\', headers(\'foo\' = \'[HIDDEN]\')) -CREATE TABLE default.d\n(\n `x` Int64\n)\nENGINE = S3(\'https://example.s3.amazonaws.com/a.csv\', \'NOSIGN\', headers(\'foo\' = \'[HIDDEN]\')) +CREATE TABLE default.d\n(\n `x` Int64\n)\nENGINE = S3(\'https://example.s3.amazonaws.com/a.csv\', \'NOSIGN\', headers(\'foo\' = \'bar\'), \'CSV\') CREATE VIEW default.e\n(\n `x` Int64\n)\nAS SELECT count()\nFROM url(\'https://example.com/\', CSV, headers(\'foo\' = \'[HIDDEN]\', \'a\' = \'[HIDDEN]\')) CREATE VIEW default.f\n(\n `x` Int64\n)\nAS SELECT count()\nFROM url(\'https://example.com/\', CSV, headers()) CREATE VIEW default.g\n(\n `x` Int64\n)\nAS SELECT count()\nFROM s3(\'https://example.s3.amazonaws.com/a.csv\', CSV, headers(\'foo\' = \'[HIDDEN]\')) From e8f9f981c2940c4ac45fa58f40a9bfe69bccabb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 9 Dec 2024 16:32:16 +0100 Subject: [PATCH 438/502] Use absl::Mutex instead of std::shared_mutex on non-Linux OSs --- src/Common/AsynchronousMetrics.h | 5 ---- src/Common/ProfilingScopedRWLock.h | 2 ++ src/Common/SharedMutex.cpp | 4 +-- src/Common/SharedMutex.h | 30 ++++++++++++++++++--- src/Common/StackTrace.cpp | 3 ++- src/Interpreters/DatabaseCatalog.h | 1 + src/Interpreters/Set.cpp | 1 + src/Processors/Executors/ExecutingGraph.cpp | 5 +++- src/Server/InterserverIOHTTPHandler.cpp | 2 ++ 9 files changed, 41 insertions(+), 12 deletions(-) diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index e8d4b7980f9..b60f0f25e74 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -112,12 +112,7 @@ private: /// Protects saved values. mutable SharedMutex values_mutex; /// Values store the result of the last update prepared for reading. -#ifdef OS_LINUX AsynchronousMetricValues values TSA_GUARDED_BY(values_mutex); -#else - /// When SharedMutex == std::shared_mutex it may not be annotated with the 'capability'. - AsynchronousMetricValues values; -#endif #if defined(OS_LINUX) || defined(OS_FREEBSD) MemoryStatisticsOS memory_stat TSA_GUARDED_BY(data_mutex); diff --git a/src/Common/ProfilingScopedRWLock.h b/src/Common/ProfilingScopedRWLock.h index 50e52e66e2d..d2ac050f10d 100644 --- a/src/Common/ProfilingScopedRWLock.h +++ b/src/Common/ProfilingScopedRWLock.h @@ -4,6 +4,8 @@ #include #include +#include + namespace DB { diff --git a/src/Common/SharedMutex.cpp b/src/Common/SharedMutex.cpp index 2d63f8542b0..f3d1cf95aeb 100644 --- a/src/Common/SharedMutex.cpp +++ b/src/Common/SharedMutex.cpp @@ -1,8 +1,8 @@ +#ifdef OS_LINUX /// Because of futex + #include #include -#ifdef OS_LINUX /// Because of futex - #include #include diff --git a/src/Common/SharedMutex.h b/src/Common/SharedMutex.h index d2947645eca..90e80ceedc3 100644 --- a/src/Common/SharedMutex.h +++ b/src/Common/SharedMutex.h @@ -1,11 +1,11 @@ #pragma once -#include +#include #ifdef OS_LINUX /// Because of futex #include -#include + #include namespace DB @@ -46,11 +46,35 @@ private: #else +#include + namespace DB { -using SharedMutex = std::shared_mutex; +class TSA_CAPABILITY("SharedMutex") SharedMutex : public absl::Mutex +{ + using absl::Mutex::Mutex; +public: + SharedMutex(const SharedMutex &) = delete; + SharedMutex & operator=(const SharedMutex &) = delete; + SharedMutex(SharedMutex &&) = delete; + SharedMutex & operator=(SharedMutex &&) = delete; + + // Exclusive ownership + void lock() TSA_ACQUIRE() { WriterLock(); } + + bool try_lock() TSA_TRY_ACQUIRE(true) { return WriterTryLock(); } + + void unlock() TSA_RELEASE() { WriterUnlock(); } + + // Shared ownership + void lock_shared() TSA_ACQUIRE_SHARED() { ReaderLock(); } + + bool try_lock_shared() TSA_TRY_ACQUIRE_SHARED(true) { return ReaderTryLock(); } + + void unlock_shared() TSA_RELEASE_SHARED() { ReaderUnlock(); } +}; } #endif diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 78f5c9fee01..bccae285d29 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -19,9 +19,10 @@ #include #include #include +#include #include -#include #include +#include #include diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 308d1b33e8b..55c4836cfd2 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index 60328123aa0..4d47a33a2bf 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -1,4 +1,5 @@ #include +#include #include diff --git a/src/Processors/Executors/ExecutingGraph.cpp b/src/Processors/Executors/ExecutingGraph.cpp index 8cf73c49f59..046a0895b50 100644 --- a/src/Processors/Executors/ExecutingGraph.cpp +++ b/src/Processors/Executors/ExecutingGraph.cpp @@ -1,7 +1,10 @@ #include -#include #include +#include +#include + + namespace DB { diff --git a/src/Server/InterserverIOHTTPHandler.cpp b/src/Server/InterserverIOHTTPHandler.cpp index e125738a18e..c5ff737d67e 100644 --- a/src/Server/InterserverIOHTTPHandler.cpp +++ b/src/Server/InterserverIOHTTPHandler.cpp @@ -15,6 +15,8 @@ #include #include +#include + namespace DB { From 77e52b5527897e1bc0d848f572168f6dbc13573e Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 9 Dec 2024 16:38:35 +0100 Subject: [PATCH 439/502] fix again --- .../01290_max_execution_speed_distributed.reference | 2 +- .../0_stateless/01290_max_execution_speed_distributed.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01290_max_execution_speed_distributed.reference b/tests/queries/0_stateless/01290_max_execution_speed_distributed.reference index ad0e80d8e69..4646cfab789 100644 --- a/tests/queries/0_stateless/01290_max_execution_speed_distributed.reference +++ b/tests/queries/0_stateless/01290_max_execution_speed_distributed.reference @@ -1,3 +1,3 @@ -2000000 +200000 1 1 diff --git a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql index 8962f15e101..2994d6d3ff3 100644 --- a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql +++ b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql @@ -5,8 +5,8 @@ SET log_queries=1; CREATE TEMPORARY TABLE times (t DateTime); INSERT INTO times SELECT now(); -SELECT count('special query for 01290_max_execution_speed_distributed') FROM remote('127.0.0.{2,3}', numbers(1000000)) -SETTINGS max_execution_speed = 1000000, timeout_before_checking_execution_speed = 0, max_block_size = 100; +SELECT count('special query for 01290_max_execution_speed_distributed') FROM remote('127.0.0.{2,3}', numbers(100000)) +SETTINGS max_execution_speed = 100000, timeout_before_checking_execution_speed = 0, max_block_size = 100; INSERT INTO times SELECT now(); SELECT max(t) - min(t) >= 1 FROM times; From 831487fb6bbbdd32528d5281becf795a67deb410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 9 Dec 2024 16:51:34 +0100 Subject: [PATCH 440/502] Build --- src/Common/OvercommitTracker.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index dbc645390b0..f98d5fe20a3 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -1,11 +1,12 @@ #include "OvercommitTracker.h" -#include -#include #include #include #include +#include +#include + namespace CurrentMetrics { extern const Metric ThreadsInOvercommitTracker; From c9c8c9d26708da0b0648aa4aa94bc1277228e021 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 30 Jul 2024 16:35:29 +0200 Subject: [PATCH 441/502] Tests: add RMV tests --- .../sql-reference/statements/create/view.md | 11 +- tests/integration/helpers/client.py | 25 +- tests/integration/helpers/cluster.py | 20 +- .../configs/config.xml | 2 + .../configs/remote_servers.xml | 27 + .../configs/settings.xml | 8 + .../schedule_model.py | 143 + .../test_refreshable_mat_view/test.py | 2530 +++++++++++++++++ .../test_schedule_model.py | 90 + .../__init__.py | 0 .../configs/settings.xml | 16 + 11 files changed, 2860 insertions(+), 12 deletions(-) create mode 100644 tests/integration/test_refreshable_mat_view/configs/config.xml create mode 100644 tests/integration/test_refreshable_mat_view/configs/remote_servers.xml create mode 100644 tests/integration/test_refreshable_mat_view/configs/settings.xml create mode 100644 tests/integration/test_refreshable_mat_view/schedule_model.py create mode 100644 tests/integration/test_refreshable_mat_view/test.py create mode 100644 tests/integration/test_refreshable_mat_view/test_schedule_model.py create mode 100644 tests/integration/test_refreshable_mat_view_db_replicated/__init__.py create mode 100644 tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index 141538d502c..a6f9f394871 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -157,13 +157,14 @@ For your convenience, the old documentation is located [here](https://pastila.nl ## Refreshable Materialized View {#refreshable-materialized-view} ```sql -CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name +CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] REFRESH EVERY|AFTER interval [OFFSET interval] -RANDOMIZE FOR interval -DEPENDS ON [db.]name [, [db.]name [, ...]] -SETTINGS name = value [, name = value [, ...]] +[RANDOMIZE FOR interval] +[DEPENDS ON [db.]name [, [db.]name [, ...]]] +[SETTINGS name = value [, name = value [, ...]]] [APPEND] -[TO[db.]name] [(columns)] [ENGINE = engine] [EMPTY] +[TO[db.]name] [(columns)] [ENGINE = engine] +[EMPTY] AS SELECT ... [COMMENT 'comment'] ``` diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index 1677a1536c5..634bf1fc701 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -4,6 +4,8 @@ import subprocess as sp import tempfile from threading import Timer +import numpy as np + DEFAULT_QUERY_TIMEOUT = 600 @@ -59,6 +61,7 @@ class Client: host=None, ignore_error=False, query_id=None, + parse=False, ): return self.get_query_request( sql, @@ -71,6 +74,7 @@ class Client: host=host, ignore_error=ignore_error, query_id=query_id, + parse=parse, ).get_answer() def get_query_request( @@ -85,6 +89,7 @@ class Client: host=None, ignore_error=False, query_id=None, + parse=False, ): command = self.command[:] @@ -107,8 +112,10 @@ class Client: command += ["--host", host] if query_id is not None: command += ["--query_id", query_id] + if parse: + command += ["--format=TabSeparatedWithNames"] - return CommandRequest(command, stdin, timeout, ignore_error) + return CommandRequest(command, stdin, timeout, ignore_error, parse) @stacktraces_on_timeout_decorator def query_and_get_error( @@ -169,7 +176,9 @@ class QueryRuntimeException(Exception): class CommandRequest: - def __init__(self, command, stdin=None, timeout=None, ignore_error=False): + def __init__( + self, command, stdin=None, timeout=None, ignore_error=False, parse=False + ): # Write data to tmp file to avoid PIPEs and execution blocking stdin_file = tempfile.TemporaryFile(mode="w+") stdin_file.write(stdin) @@ -177,7 +186,7 @@ class CommandRequest: self.stdout_file = tempfile.TemporaryFile() self.stderr_file = tempfile.TemporaryFile() self.ignore_error = ignore_error - + self.parse = parse # print " ".join(command) # we suppress stderror on client becase sometimes thread sanitizer @@ -243,6 +252,16 @@ class CommandRequest: stderr, ) + if self.parse: + import pandas as pd + from io import StringIO + + return ( + pd.read_csv(StringIO(stdout), sep="\t") + .replace(r"\N", None) + .replace(np.nan, None) + ) + return stdout def get_error(self): diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index d47bd5a1c01..31c10b7d053 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -25,6 +25,7 @@ from typing import Any, List, Sequence, Tuple, Union import requests import urllib3 +import shlex try: # Please, add modules that required for specific tests only here. @@ -49,6 +50,7 @@ except Exception as e: logging.warning(f"Cannot import some modules, some tests may not work: {e}") import docker +from docker.models.containers import Container from dict2xml import dict2xml from kazoo.client import KazooClient from kazoo.exceptions import KazooException @@ -525,7 +527,7 @@ class ClickHouseCluster: self.base_jdbc_bridge_cmd = [] self.base_redis_cmd = [] self.pre_zookeeper_commands = [] - self.instances = {} + self.instances: dict[str, ClickHouseInstance] = {} self.with_zookeeper = False self.with_zookeeper_secure = False self.with_mysql_client = False @@ -765,7 +767,7 @@ class ClickHouseCluster: self.prometheus_remote_read_handler_port = 9092 self.prometheus_remote_read_handler_path = "/read" - self.docker_client = None + self.docker_client: docker.DockerClient = None self.is_up = False self.env = os.environ.copy() logging.debug(f"CLUSTER INIT base_config_dir:{self.base_config_dir}") @@ -956,7 +958,7 @@ class ClickHouseCluster: except: pass - def get_docker_handle(self, docker_id): + def get_docker_handle(self, docker_id) -> Container: exception = None for i in range(20): try: @@ -3353,6 +3355,12 @@ class ClickHouseCluster: logging.info("Starting zookeeper node: %s", n) subprocess_check_call(self.base_zookeeper_cmd + ["start", n]) + def query_all_nodes(self, sql, *args, **kwargs): + return { + name: instance.query(sql, ignore_error=True, *args, **kwargs) + for name, instance in self.instances.items() + } + DOCKER_COMPOSE_TEMPLATE = """--- services: @@ -3628,6 +3636,7 @@ class ClickHouseInstance: host=None, ignore_error=False, query_id=None, + parse=False, ): sql_for_log = "" if len(sql) > 1000: @@ -3646,6 +3655,7 @@ class ClickHouseInstance: ignore_error=ignore_error, query_id=query_id, host=host, + parse=parse, ) def query_with_retry( @@ -3662,6 +3672,7 @@ class ClickHouseInstance: retry_count=20, sleep_time=0.5, check_callback=lambda x: True, + parse=False, ): # logging.debug(f"Executing query {sql} on {self.name}") result = None @@ -3678,6 +3689,7 @@ class ClickHouseInstance: database=database, host=host, ignore_error=ignore_error, + parse=parse, ) if check_callback(result): return result @@ -4410,7 +4422,7 @@ class ClickHouseInstance: else: self.wait_start(time_left) - def get_docker_handle(self): + def get_docker_handle(self) -> Container: return self.cluster.get_docker_handle(self.docker_id) def stop(self): diff --git a/tests/integration/test_refreshable_mat_view/configs/config.xml b/tests/integration/test_refreshable_mat_view/configs/config.xml new file mode 100644 index 00000000000..3cbf717bb67 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/configs/config.xml @@ -0,0 +1,2 @@ + + diff --git a/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml new file mode 100644 index 00000000000..6504141033a --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml @@ -0,0 +1,27 @@ + + + + + + node1_1 + 9000 + + + node1_2 + 9000 + + + + + + node2_1 + 9000 + + + node2_2 + 9000 + + + + + diff --git a/tests/integration/test_refreshable_mat_view/configs/settings.xml b/tests/integration/test_refreshable_mat_view/configs/settings.xml new file mode 100644 index 00000000000..d15acfaa303 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/configs/settings.xml @@ -0,0 +1,8 @@ + + + + 1 + Etc/UTC + + + diff --git a/tests/integration/test_refreshable_mat_view/schedule_model.py b/tests/integration/test_refreshable_mat_view/schedule_model.py new file mode 100644 index 00000000000..857a1285076 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/schedule_model.py @@ -0,0 +1,143 @@ +from datetime import datetime, timedelta + +from dateutil.relativedelta import relativedelta + +""" +It is the model to test the scheduling of refreshable mat view +""" + + +def relative_offset(unit, value): + if unit == "SECONDS": + return relativedelta(seconds=value) + elif unit == "MINUTE": + return relativedelta(minutes=value) + elif unit == "HOUR": + return relativedelta(hours=value) + elif unit == "DAY": + return relativedelta(days=value) + elif unit == "WEEK": + return relativedelta(days=7 * value) + elif unit == "MONTH": + return relativedelta(months=value) + elif unit == "YEAR": + return relativedelta(years=value) + + raise Exception("Cant parse unit: {}".format(unit)) + + +def group_and_sort(parts, reverse=False): + order = ["YEAR", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECONDS"] + grouped_parts = [] + + for i in range(0, len(parts), 2): + grouped_parts.append((parts[i], parts[i + 1])) + + sorted_parts = sorted( + grouped_parts, key=lambda x: order.index(x[1]), reverse=reverse + ) + return sorted_parts + + +def get_next_refresh_time(schedule, current_time: datetime): + parts = schedule.split() + + randomize_offset = timedelta() + + if "RANDOMIZE" in parts: + randomize_index = parts.index("RANDOMIZE") + randomize_parts = parts[randomize_index + 2 :] + + for i in range(0, len(randomize_parts), 2): + value = int(randomize_parts[i]) + randomize_offset += relative_offset(randomize_parts[i + 1], value) + + parts = parts[:randomize_index] + + offset = timedelta() + if "OFFSET" in parts: + offset_index = parts.index("OFFSET") + for i in range(offset_index + 1, len(parts), 2): + value = int(parts[i]) + offset += relative_offset(parts[i + 1], value) + + parts = parts[:offset_index] + + if parts[0] == "EVERY": + parts = group_and_sort(parts[1:]) + for part in parts: + value = int(part[0]) + unit = part[1] + + if unit == "SECONDS": + current_time = current_time.replace(microsecond=0) + relativedelta( + seconds=value + ) + elif unit == "MINUTE": + current_time = current_time.replace( + second=0, microsecond=0 + ) + relativedelta(minutes=value) + elif unit == "HOUR": + current_time = current_time.replace( + minute=0, second=0, microsecond=0 + ) + relativedelta(hours=value) + elif unit == "DAY": + current_time = current_time.replace( + hour=0, minute=0, second=0, microsecond=0 + ) + relativedelta(days=value) + elif unit == "WEEK": + current_time = current_time.replace( + hour=0, minute=0, second=0, microsecond=0 + ) + relativedelta(weekday=0, weeks=value) + elif unit == "MONTH": + current_time = current_time.replace( + day=1, hour=0, minute=0, second=0, microsecond=0 + ) + relativedelta(months=value) + elif unit == "YEAR": + current_time = current_time.replace( + month=1, day=1, hour=0, minute=0, second=0, microsecond=0 + ) + relativedelta(years=value) + + current_time += offset + + if randomize_offset: + half_offset = (current_time + randomize_offset - current_time) / 2 + return ( + current_time - half_offset, + current_time + half_offset, + ) + + return current_time + + elif parts[0] == "AFTER": + parts = group_and_sort(parts[1:], reverse=True) + interval = relativedelta() + for part in parts: + value = int(part[0]) + unit = part[1] + + if unit == "SECONDS": + interval += relativedelta(seconds=value) + elif unit == "MINUTE": + interval += relativedelta(minutes=value) + elif unit == "HOUR": + interval += relativedelta(hours=value) + elif unit == "DAY": + interval += relativedelta(days=value) + elif unit == "WEEK": + interval += relativedelta(weeks=value) + elif unit == "MONTH": + interval += relativedelta(months=value) + elif unit == "YEAR": + interval += relativedelta(years=value) + + current_time += interval + if randomize_offset: + half_offset = (current_time + randomize_offset - current_time) / 2 + return ( + current_time - half_offset, + current_time + half_offset, + ) + + return current_time + raise ValueError("Invalid refresh schedule") diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py new file mode 100644 index 00000000000..48c29072134 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -0,0 +1,2530 @@ +import datetime +import os +import random +import shutil +import time +import re +from typing import Optional + +import numpy as np +import pytest +import threading + +from dateutil.relativedelta import relativedelta +from jinja2 import Template, Environment +from datetime import datetime, timedelta + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry, assert_logs_contain +from helpers.network import PartitionManager +from test_refreshable_mat_view.schedule_model import get_next_refresh_time + +test_recover_staled_replica_run = 1 + +cluster = ClickHouseCluster(__file__) + +node1_1 = cluster.add_instance( + "node1_1", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + macros={"shard": 1, "replica": 1}, +) +node1_2 = cluster.add_instance( + "node1_2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + macros={"shard": 1, "replica": 2}, +) +node2_1 = cluster.add_instance( + "node2_1", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + macros={"shard": 2, "replica": 1}, +) + +node2_2 = cluster.add_instance( + "node2_2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + macros={"shard": 2, "replica": 2}, +) + +# all_nodes = [ +# main_node, +# dummy_node, +# competing_node, +# ] + +uuid_regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") + + +def assert_create_query(nodes, table_name, expected): + replace_uuid = lambda x: re.sub(uuid_regex, "uuid", x) + query = "show create table {}".format(table_name) + for node in nodes: + assert_eq_with_retry(node, query, expected, get_result=replace_uuid) + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(scope="module", autouse=True) +def setup_tables(started_cluster): + print(node1_1.query("SELECT version()")) + + node1_1.query(f"CREATE DATABASE test_db ON CLUSTER test_cluster ENGINE = Atomic") + + node1_1.query( + f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + + node1_1.query( + f"CREATE TABLE src2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + + node1_1.query( + f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + + node1_1.query( + f"CREATE TABLE tgt2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + + node1_1.query( + f"CREATE MATERIALIZED VIEW dummy_rmv ON CLUSTER test_cluster " + f"REFRESH EVERY 10 HOUR engine Memory AS select number as x from numbers(10)" + ) + + +""" +#!/usr/bin/env bash +# Tags: atomic-database + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# reset --log_comment +CLICKHOUSE_LOG_COMMENT= +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Set session timezone to UTC to make all DateTime formatting and parsing use UTC, because refresh +# scheduling is done in UTC. +CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" +CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --session_timezone Etc/UTC"`" + +$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" + + +# Basic refreshing. +$CLICKHOUSE_CLIENT -nq " + create materialized view a + refresh after 2 second + engine Memory + empty + as select number as x from numbers(2) union all select rand64() as x; + select '<1: created view>', exception, view from refreshes; + show create a;" +# Wait for any refresh. (xargs trims the string and turns \t and \n into spaces) +while [ "`$CLICKHOUSE_CLIENT -nq "select last_success_time is null from refreshes -- $LINENO" | xargs`" != '0' ] +do + sleep 0.1 +done +start_time="`$CLICKHOUSE_CLIENT -nq "select reinterpret(now64(), 'Int64')"`" +# Check table contents. +$CLICKHOUSE_CLIENT -nq "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" +# Wait for table contents to change. +res1="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`" +while : +do + res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" + [ "$res2" == "$res1" ] || break + sleep 0.1 +done +# Wait for another change. +while : +do + res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" + [ "$res3" == "$res2" ] || break + sleep 0.1 +done +# Check that the two changes were at least 1 second apart, in particular that we're not refreshing +# like crazy. This is potentially flaky, but we need at least one test that uses non-mocked timer +# to make sure the clock+timer code works at all. If it turns out flaky, increase refresh period above. +$CLICKHOUSE_CLIENT -nq " + select '<3: time difference at least>', min2(reinterpret(now64(), 'Int64') - $start_time, 1000); + select '<4: next refresh in>', next_refresh_time-last_success_time from refreshes;" + +# Create a source table from which views will read. +$CLICKHOUSE_CLIENT -nq " + create table src (x Int8) engine Memory as select 1;" + +# Switch to fake clock, change refresh schedule, change query. +$CLICKHOUSE_CLIENT -nq " + system test view a set fake time '2050-01-01 00:00:01'; + system wait view a; + system refresh view a; + system wait view a; + select '<4.1: fake clock>', status, last_success_time, next_refresh_time from refreshes; + alter table a modify refresh every 2 year; + alter table a modify query select x*2 as x from src; + system wait view a; + select '<4.5: altered>', status, last_success_time, next_refresh_time from refreshes; + show create a;" +# Advance time to trigger the refresh. +$CLICKHOUSE_CLIENT -nq " + select '<5: no refresh>', count() from a; + system test view a set fake time '2052-02-03 04:05:06';" +while [ "`$CLICKHOUSE_CLIENT -nq "select last_success_time, status from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06 Scheduled' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<6: refreshed>', * from a; + select '<7: refreshed>', status, last_success_time, next_refresh_time from refreshes;" + +# Create a dependent view, refresh it once. +$CLICKHOUSE_CLIENT -nq " + create materialized view b refresh every 2 year depends on a (y Int32) engine MergeTree order by y empty as select x*10 as y from a; + show create b; + system test view b set fake time '2052-11-11 11:11:11'; + system refresh view b; + system wait view b; + select '<7.5: created dependent>', last_success_time from refreshes where view = 'b';" +# Next refresh shouldn't start until the dependency refreshes. +$CLICKHOUSE_CLIENT -nq " + select '<8: refreshed>', * from b; + select '<9: refreshed>', view, status, next_refresh_time from refreshes; + system test view b set fake time '2054-01-24 23:22:21';" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] +do + sleep 0.1 +done + +# Drop the source table, check that refresh fails and doesn't leave a temp table behind. +$CLICKHOUSE_CLIENT -nq " + select '<9.2: dropping>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase(); + drop table src; + system refresh view a;" +$CLICKHOUSE_CLIENT -nq "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -nq " + select '<9.4: dropped>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase();" + +# Create the source table again, check that refresh succeeds (in particular that tables are looked +# up by name rather than uuid). +$CLICKHOUSE_CLIENT -nq " + select '<10: creating>', view, status, next_refresh_time from refreshes; + create table src (x Int16) engine Memory as select 2; + system test view a set fake time '2054-01-01 00:00:01';" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] +do + sleep 0.1 +done +# Both tables should've refreshed. +$CLICKHOUSE_CLIENT -nq " + select '<11: chain-refreshed a>', * from a; + select '<12: chain-refreshed b>', * from b; + select '<13: chain-refreshed>', view, status, last_success_time, last_refresh_time, next_refresh_time, exception == '' from refreshes;" + +$CLICKHOUSE_CLIENT -nq " + system test view b set fake time '2061-01-01 00:00:00'; + truncate src; + insert into src values (3); + system test view a set fake time '2060-02-02 02:02:02';" +while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<15: chain-refreshed a>', * from a; + select '<16: chain-refreshed b>', * from b; + select '<17: chain-refreshed>', view, status, next_refresh_time from refreshes;" + +# Get to WaitingForDependencies state and remove the depencency. +$CLICKHOUSE_CLIENT -nq " + system test view b set fake time '2062-03-03 03:03:03'" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + alter table b modify refresh every 2 year" +while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<18: removed dependency>', view, status, last_success_time, last_refresh_time, next_refresh_time from refreshes where view = 'b'; + show create b;" + +# Select from a table that doesn't exist, get an exception. +$CLICKHOUSE_CLIENT -nq " + drop table a; + drop table b; + create materialized view c refresh every 1 second (x Int64) engine Memory empty as select * from src; + drop table src;" +while [ "`$CLICKHOUSE_CLIENT -nq "select exception == '' from refreshes -- $LINENO" | xargs`" != '0' ] +do + sleep 0.1 +done +# Check exception, create src, expect successful refresh. +$CLICKHOUSE_CLIENT -nq " + select '<19: exception>', exception ilike '%UNKNOWN_TABLE%' from refreshes; + create table src (x Int64) engine Memory as select 1; + system refresh view c; + system wait view c;" +# Rename table. +$CLICKHOUSE_CLIENT -nq " + select '<20: unexception>', * from c; + rename table c to d; + select '<21: rename>', * from d; + select '<22: rename>', view, status, last_success_time is null from refreshes;" + +# Do various things during a refresh. +# First make a nonempty view. +$CLICKHOUSE_CLIENT -nq " + drop table d; + truncate src; + insert into src values (1) + create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x as select x + sleepEachRow(1) as x from src settings max_block_size = 1; + system wait view e;" +# Stop refreshes. +$CLICKHOUSE_CLIENT -nq " + select '<23: simple refresh>', * from e; + system stop view e;" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] +do + sleep 0.1 +done +# Make refreshes slow, wait for a slow refresh to start. (We stopped refreshes first to make sure +# we wait for a slow refresh, not a previous fast one.) +$CLICKHOUSE_CLIENT -nq " + insert into src select * from numbers(1000) settings max_block_size=1; + system start view e;" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] +do + sleep 0.1 +done +# Rename. +$CLICKHOUSE_CLIENT -nq " + rename table e to f; + select '<24: rename during refresh>', * from f; + select '<25: rename during refresh>', view, status from refreshes; + alter table f modify refresh after 10 year settings refresh_retries = 0;" +sleep 2 # make it likely that at least one row was processed +# Cancel. +$CLICKHOUSE_CLIENT -nq " + system cancel view f;" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Scheduled' ] +do + sleep 0.1 +done +# Check that another refresh doesn't immediately start after the cancelled one. +sleep 1 +$CLICKHOUSE_CLIENT -nq " + select '<27: cancelled>', view, status, exception from refreshes; + system refresh view f;" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] +do + sleep 0.1 +done +# Drop. +$CLICKHOUSE_CLIENT -nq " + drop table f; + select '<28: drop during refresh>', view, status from refreshes; + select '<28: drop during refresh>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase()" + +# Try OFFSET and RANDOMIZE FOR. +$CLICKHOUSE_CLIENT -nq " + create materialized view g refresh every 1 week offset 3 day 4 hour randomize for 4 day 1 hour (x Int64) engine Memory empty as select 42 as x; + show create g; + system test view g set fake time '2050-02-03 15:30:13';" +while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + with '2050-02-10 04:00:00'::DateTime as expected + select '<29: randomize>', abs(next_refresh_time::Int64 - expected::Int64) <= 3600*(24*4+1), next_refresh_time != expected from refreshes;" + +# Send data 'TO' an existing table. +$CLICKHOUSE_CLIENT -nq " + drop table g; + create table dest (x Int64) engine MergeTree order by x; + truncate src; + insert into src values (1); + create materialized view h refresh every 1 second to dest as select x*10 as x from src; + show create h; + system wait view h; + select '<30: to existing table>', * from dest; + insert into src values (2);" +while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest -- $LINENO" | xargs`" != '2' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<31: to existing table>', * from dest; + drop table dest; + drop table h;" + +# Retries. +$CLICKHOUSE_CLIENT -nq " + create materialized view h2 refresh after 1 year settings refresh_retries = 10 (x Int64) engine Memory as select x*10 + throwIf(x % 2 == 0) as x from src;" +$CLICKHOUSE_CLIENT -nq "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -nq " + select '<31.5: will retry>', exception != '', retry > 0 from refreshes; + create table src2 empty as src; + insert into src2 values (1) + exchange tables src and src2; + drop table src2;" +while [ "`$CLICKHOUSE_CLIENT -nq "select status, retry from refreshes -- $LINENO" | xargs`" != 'Scheduled 0' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<31.6: did retry>', x from h2; + drop table h2" + +# EMPTY +$CLICKHOUSE_CLIENT -nq " + create materialized view i refresh after 1 year engine Memory empty as select number as x from numbers(2); + system wait view i; + create materialized view j refresh after 1 year engine Memory as select number as x from numbers(2);" +while [ "`$CLICKHOUSE_CLIENT -nq "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<32: empty>', view, status, last_success_time is null, retry from refreshes order by view; + drop table i; + drop table j;" + +# APPEND +$CLICKHOUSE_CLIENT -nq " + create materialized view k refresh every 10 year append (x Int64) engine Memory empty as select x*10 as x from src; + select '<33: append>', * from k; + system refresh view k; + system wait view k; + select '<34: append>', * from k; + truncate table src; + insert into src values (2), (3); + system refresh view k; + system wait view k; + select '<35: append>', * from k order by x;" +# ALTER to non-APPEND +$CLICKHOUSE_CLIENT -nq " + alter table k modify refresh every 10 year;" 2>/dev/null || echo "ALTER from APPEND to non-APPEND failed, as expected" +$CLICKHOUSE_CLIENT -nq " + drop table k; + truncate table src;" + +# APPEND + TO + regular materialized view reading from it. +$CLICKHOUSE_CLIENT -nq " + create table mid (x Int64) engine MergeTree order by x; + create materialized view l refresh every 10 year append to mid empty as select x*10 as x from src; + create materialized view m (x Int64) engine Memory as select x*10 as x from mid; + insert into src values (1); + system refresh view l; + system wait view l; + select '<37: append chain>', * from m; + insert into src values (2); + system refresh view l; + system wait view l; + select '<38: append chain>', * from m order by x; + drop table l; + drop table m; + drop table mid;" + +# Failing to create inner table. +$CLICKHOUSE_CLIENT -nq " + create materialized view n refresh every 1 second (x Int64) engine MergeTree as select 1 as x from numbers(2);" 2>/dev/null || echo "creating MergeTree without ORDER BY failed, as expected" +$CLICKHOUSE_CLIENT -nq " + create materialized view n refresh every 1 second (x Int64) engine MergeTree order by x as select 1 as x from numbers(2); + drop table n;" + +$CLICKHOUSE_CLIENT -nq " + drop table refreshes;" + +Information about Refreshable Materialized Views. Contains all refreshable materialized views, regardless of whether there's a refresh in progress or not. + +Columns: + +database (String) — The name of the database the table is in. +view (String) — Table name. +status (String) — Current state of the refresh. +last_success_time (Nullable(DateTime)) — Time when the latest successful refresh started. NULL if no successful refreshes happened since server startup or table creation. +last_success_duration_ms (Nullable(UInt64)) — How long the latest refresh took. +last_refresh_time (Nullable(DateTime)) — Time when the latest refresh attempt finished (if known) or started (if unknown or still running). NULL if no refresh attempts happened since server startup or table creation. +last_refresh_replica (String) — If coordination is enabled, name of the replica that made the current (if running) or previous (if not running) refresh attempt. +next_refresh_time (Nullable(DateTime)) — Time at which the next refresh is scheduled to start, if status = Scheduled. +exception (String) — Error message from previous attempt if it failed. +retry (UInt64) — How many failed attempts there were so far, for the current refresh. +progress (Float64) — Progress of the current refresh, between 0 and 1. Not available if status is RunningOnAnotherReplica. +read_rows (UInt64) — Number of rows read by the current refresh so far. Not available if status is RunningOnAnotherReplica. +total_rows (UInt64) — Estimated total number of rows that need to be read by the current refresh. Not available if status is RunningOnAnotherReplica. + +Available refresh settings: + * `refresh_retries` - How many times to retry if refresh query fails with an exception. If all retries fail, skip to the next scheduled refresh time. 0 means no retries, -1 means infinite retries. Default: 0. + * `refresh_retry_initial_backoff_ms` - Delay before the first retry, if `refresh_retries` is not zero. Each subsequent retry doubles the delay, up to `refresh_retry_max_backoff_ms`. Default: 100 ms. + * `refresh_retry_max_backoff_ms` - Limit on the exponential growth of delay between refresh attempts. Default: 60000 ms (1 minute). + + + +CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name +REFRESH EVERY|AFTER interval [OFFSET interval] +RANDOMIZE FOR interval +DEPENDS ON [db.]name [, [db.]name [, ...]] +SETTINGS name = value [, name = value [, ...]] +[APPEND] +[TO[db.]name] [(columns)] [ENGINE = engine] [EMPTY] +AS SELECT ... +where interval is a sequence of simple intervals: + +number SECOND|MINUTE|HOUR|DAY|WEEK|MONTH|YEAR + +Example refresh schedules: +```sql +REFRESH EVERY 1 DAY -- every day, at midnight (UTC) +REFRESH EVERY 1 MONTH -- on 1st day of every month, at midnight +REFRESH EVERY 1 MONTH OFFSET 5 DAY 2 HOUR -- on 6th day of every month, at 2:00 am +REFRESH EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE -- every other Saturday, at 3:10 pm +REFRESH EVERY 30 MINUTE -- at 00:00, 00:30, 01:00, 01:30, etc +REFRESH AFTER 30 MINUTE -- 30 minutes after the previous refresh completes, no alignment with time of day +-- REFRESH AFTER 1 HOUR OFFSET 1 MINUTE -- syntax errror, OFFSET is not allowed with AFTER +``` + +`RANDOMIZE FOR` randomly adjusts the time of each refresh, e.g.: +```sql +REFRESH EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR -- every day at random time between 01:30 and 02:30 + + +SYSTEM REFRESH VIEW + +Wait for the currently running refresh to complete. If the refresh fails, throws an exception. If no refresh is running, completes immediately, throwing an exception if previous refresh failed. + +### SYSTEM STOP VIEW, SYSTEM STOP VIEWS + +### SYSTEM WAIT VIEW + +Waits for the running refresh to complete. If no refresh is running, returns immediately. If the latest refresh attempt failed, reports an error. + +Can be used right after creating a new refreshable materialized view (without EMPTY keyword) to wait for the initial refresh to complete. + +```sql +SYSTEM WAIT VIEW [db.]name + +### TESTS +1. ON CLUSTER? +2. Append mode +3. Drop node and wait for restore +4. Simple functional testing: all values in refresh result correct (two and more rmv) +5. Combinations of time +6. Two (and more) rmv from single to single [APPEND] +7. ALTER rmv ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] +8. RMV without tgt table (automatic table) (check APPEND) + +8 DROP rmv +9 CREATE - DROP - CREATE - ALTER +10 Backups? +11. Long queries over refresh time (check settings) +12. RMV settings +13. incorrect intervals (interval 1 sec, offset 1 minute) +14. ALTER on cluster + +15. write to distributed with / without APPEND +16. `default_replica_path` empty on database replicated +17. Not existent TO table (ON CLUSTER) +18. Not existent FROM table (ON CLUSTER) +19. Not existent BOTH tables (ON CLUSTER) +20. retry failed +21. overflow with wait test + +22. ALTER of SELECT? + +23. OFFSET must be less than the period. 'EVERY 1 MONTH OFFSET 5 WEEK' + + + +""" + + +def sql_template( + string: str, + globals: Optional[dict] = None, + filters: Optional[dict] = None, + tests: Optional[dict] = None, +) -> Template: + def uppercase(value: str): + return value.upper() + + def lowercase(value: str): + return value.lower() + + def format_settings(items: dict): + return ", ".join([f"{k}={v}" for k, v in items.items()]) + + # Create a custom environment and add the functions + env = Environment( + trim_blocks=False, lstrip_blocks=True, keep_trailing_newline=False + ) + env.globals["uppercase"] = uppercase + env.globals["lowercase"] = lowercase + env.filters["format_settings"] = format_settings + + if filters: + env.filters.update(filters) + if globals: + env.globals.update(globals) + if tests: + env.tests.update(tests) + + return env.from_string(string) + + +def assert_same_values(lst: list): + if not isinstance(lst, list): + lst = list(lst) + assert all(x == lst[0] for x in lst) + + +CREATE_RMV_TEMPLATE = sql_template( + """CREATE MATERIALIZED VIEW +{% if if_not_exists %}IF NOT EXISTS{% endif %} +{% if db %}{{db}}.{% endif %}{{ table_name }} +{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} +REFRESH {{ refresh_interval }} +{% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} +{% if settings %}SETTINGS {{ settings|format_settings }}{% endif %} +{% if with_append %}APPEND{% endif %} +{% if to_clause %}TO {{ to_clause }}{% endif %} +{% if table_clause %}{{ table_clause }}{% endif %} +{% if empty %}EMPTY{% endif %} +AS {{ select_query }}""" +) + +# ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] + +ALTER_RMV_TEMPLATE = sql_template( + """ALTER TABLE +{% if db %}{{db}}.{% endif %}{{ table_name }} +{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} +MODIFY REFRESH {{ refresh_interval }} +{% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} +{% if settings %}SETTINGS {{ settings|format_settings }}{% endif %}""" +) + + +@pytest.mark.parametrize("with_append", [True, False]) +@pytest.mark.parametrize("create_target_table", [True, False]) +@pytest.mark.parametrize("if_not_exists", [True, False]) +@pytest.mark.parametrize("on_cluster", [True, False]) +@pytest.mark.parametrize( + "depends_on", [None, ["dummy_rmv"], ["default.dummy_rmv", "src1"]] +) +@pytest.mark.parametrize("randomize", [True, False]) +@pytest.mark.parametrize("empty", [True, False]) +@pytest.mark.parametrize("database_name", [None, "default", "test_db"]) +def test_correct_states( + request, + started_cluster, + with_append, + create_target_table, + if_not_exists, + on_cluster, + depends_on, + randomize, + empty, + database_name, +): + """ + Check correctness of functional states of RMV after CREATE, DROP, ALTER, trigger of RMV, ... + Check ref + """ + + def teardown(): + node1_1.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + + request.addfinalizer(teardown) + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + if_not_exists=if_not_exists, + db_name=database_name, + refresh_interval="EVERY 1 HOUR" + " 30 MINUTE" if randomize else None, + depends_on=depends_on, + to_clause="tgt1", + select_query="SELECT * FROM src1", + with_append=with_append, + on_cluster="test_cluster" if on_cluster else None, + # settings={'setting1':'value1', 'setting2': 'value2'}, + ) + print(create_sql) + node1_1.query(create_sql) + refreshes = node1_1.query( + "SELECT hostname(), * FROM clusterAllReplicas('test_cluster', system.view_refreshes)", + parse=True, + ) + + # Check same RMV is created on cluster + def compare_create_all_nodes(): + show_create_all_nodes = cluster.query_all_nodes("SHOW CREATE test_rmv") + + if not on_cluster: + del show_create_all_nodes["node1_1"] + + assert_same_values(show_create_all_nodes.values()) + + compare_create_all_nodes() + + show_create = node1_1.query("SHOW CREATE test_rmv") + + alter_sql = ALTER_RMV_TEMPLATE.render( + table_name="test_rmv", + if_not_exists=if_not_exists, + db_name=database_name, + refresh_interval="EVERY 1 HOUR" + " 30 MINUTE" if randomize else None, + depends_on=depends_on, + select_query="SELECT * FROM src1", + on_cluster="test_cluster" if on_cluster else None, + # settings={'setting1':'value1', 'setting2': 'value2'}, + ) + + node1_1.query(alter_sql) + show_create_after = node1_1.query("SHOW CREATE test_rmv") + + compare_create_all_nodes() + assert show_create == show_create_after + # breakpoint() + + pass + + +def compare_dates( + date1: str | datetime, + date2: str | datetime, + inaccuracy=timedelta(minutes=10), + format_str="%Y-%m-%d %H:%M:%S", +) -> bool: + """ + Compares two dates with an inaccuracy of 2 minutes. + """ + if isinstance(date1, str): + date1 = datetime.strptime(date1, format_str) + if isinstance(date2, str): + date2 = datetime.strptime(date2, format_str) + + return abs(date1 - date2) <= inaccuracy + + +def date_in_interval( + date1: str | datetime, + date2: str | datetime, + inaccuracy=timedelta(minutes=2), + format_str="%Y-%m-%d %H:%M:%S", +): + pass + + +def get_rmv_info(node, table): + return node.query_with_retry( + f"SELECT * FROM system.view_refreshes WHERE view='{table}'", + check_callback=lambda r: r.iloc[0]["status"] == "Scheduled", + parse=True, + ).to_dict("records")[0] + + +@pytest.mark.parametrize("with_append", [True, False]) +@pytest.mark.parametrize("create_target_table", [True, False]) +# @pytest.mark.parametrize("if_not_exists", [True, False]) +@pytest.mark.parametrize("on_cluster", [True, False]) +@pytest.mark.parametrize("depends_on", [None, ["dummy_rmv"]]) +@pytest.mark.parametrize("randomize", [True, False]) +@pytest.mark.parametrize("empty", [True, False]) +@pytest.mark.parametrize("database_name", [None, "default", "test_db"]) +def test_check_data( + request, + started_cluster, + with_append, + create_target_table, + # if_not_exists, + on_cluster, + depends_on, + randomize, + empty, + database_name, +): + def teardown(): + node1_1.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + node1_1.query("DROP TABLE IF EXISTS tgt_new ON CLUSTER test_cluster") + node1_1.query("TRUNCATE TABLE tgt1 ON CLUSTER test_cluster") + + request.addfinalizer(teardown) + + # CREATE RMV can use already existed table + # or create new one + if create_target_table: + to_clause = "tgt1" + tgt = "tgt1" + else: + to_clause = "tgt_new Engine = Memory" + tgt = "tgt_new" + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + # if_not_exists=if_not_exists, + db_name=database_name, + refresh_interval="EVERY 1 HOUR", + randomize_interval="30 MINUTE" if randomize else None, + depends_on=depends_on, + to_clause=to_clause, + select_query="SELECT now() as a, number as b FROM numbers(2)", + with_append=with_append, + on_cluster="test_cluster" if on_cluster else None, + empty=empty + # settings={'setting1':'value1', 'setting2': 'value2'}, + ) + print(create_sql) + node1_1.query(create_sql) + + # now = node1_1.query("SELECT now()", parse=True)[0][0] + now = datetime.utcnow() + + rmv = get_rmv_info(node1_1, "test_rmv") + print(rmv) + + """ + 1. check table view_refreshes + 2. check inserted data if without EMPTY + 3. set time, wait for refresh + 4. check data inserted + 5. alter table + """ + + assert rmv["exception"] is None + assert rmv["status"] == "Scheduled" + assert rmv["progress"] is None + + if not empty: + # Insert + assert compare_dates( + rmv["last_refresh_time"], now, format_str="%Y-%m-%d %H:%M:%S" + ) + assert rmv["last_success_duration_ms"] > 0 + inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) + print(inserted_data) + assert len(inserted_data) == 2 + + if empty: + assert rmv["last_refresh_time"] is None + assert rmv["last_success_duration_ms"] is None + + assert rmv["retry"] == 0 + assert rmv["read_rows"] == 0 + assert rmv["read_bytes"] == 0 + assert rmv["total_rows"] == 0 + assert rmv["total_bytes"] == 0 + assert rmv["written_rows"] == 0 + assert rmv["written_bytes"] == 0 + assert rmv["result_rows"] == 0 + assert rmv["result_bytes"] == 0 + + # Rewind to the next trigger and wait for it + # node1_1.query("SYSTEM TEST VIEW test_rmv set fake time '2050-01-01 00:00:01';") + # system test view a set fake time '2050-01-01 00:00:01'; + + now = datetime.utcnow() + + inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) + if with_append: + assert len(inserted_data) == 4 + else: + assert len(inserted_data) == 2 + + # alter time + + breakpoint() + + +""" + +(Pdb) pp interval +'EVERY 1 WEEK' +(Pdb) pp next_refresh_time +datetime.datetime(2024, 5, 6, 0, 0) +(Pdb) pp predicted_next_refresh_time +datetime.datetime(2024, 5, 6, 0, 0) + + +(Pdb) pp interval +'EVERY 2 WEEK' +(Pdb) pp next_refresh_time +datetime.datetime(2024, 5, 6, 0, 0) +(Pdb) pp predicted_next_refresh_time +datetime.datetime(2024, 5, 13, 0, 0) + + +(Pdb) pp interval +'EVERY 2 WEEK OFFSET 1 DAY' +(Pdb) pp next_refresh_time +datetime.datetime(2024, 5, 7, 0, 0) +(Pdb) pp predicted_next_refresh_time +datetime.datetime(2024, 5, 14, 0, 0) + + +(Pdb) pp interval +'EVERY 2 WEEK OFFSET 2 DAY' +(Pdb) pp next_refresh_time +datetime.datetime(2024, 5, 8, 0, 0) +(Pdb) pp predicted_next_refresh_time +datetime.datetime(2024, 5, 15, 0, 0) + + +'EVERY 1 WEEK OFFSET 2 DAY' +(Pdb) pp next_refresh_time +datetime.datetime(2024, 5, 1, 0, 0) +(Pdb) pp predicted_next_refresh_time +datetime.datetime(2024, 5, 8, 0, 0) + +""" + + +def parse_ch_datetime(date_str): + return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") + + +INTERVALS = [ + # Same units + # "EVERY 1 YEAR 2 YEAR", + # "AFTER 1 YEAR 2 YEAR" + "EVERY 1 MINUTE", + "EVERY 1 HOUR", + "EVERY 1 DAY", + "EVERY 1 MONTH", + "EVERY 1 YEAR", + "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECONDS", + # smth wrong with + # "EVERY 1 WEEK", + # "EVERY 2 WEEK", + # "EVERY 3 WEEK", + # "EVERY 4 WEEK", + # "EVERY 5 WEEK", + # "EVERY 6 WEEK", + # "EVERY 7 WEEK", + # "EVERY 8 WEEK", + # "EVERY 9 WEEK", + # "EVERY 10 WEEK", + # "EVERY 11 WEEK", + # "EVERY 12 WEEK", + # "EVERY 13 WEEK", + # "EVERY 14 WEEK", + # "EVERY 15 WEEK", + # "EVERY 20 WEEK", + # "EVERY 21 WEEK", + # "EVERY 31 WEEK", + # "EVERY 32 WEEK", + # "EVERY 33 WEEK", + # "EVERY 50 WEEK", + # "EVERY 51 WEEK", + # "EVERY 52 WEEK", + # "EVERY 59 WEEK", + # + "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", + "EVERY 1 MONTH OFFSET 1 WEEK", + "EVERY 1 MONTH OFFSET 2 WEEK", + # + "AFTER 30 SECONDS", + "AFTER 30 MINUTE", + "AFTER 2 HOUR", + "AFTER 2 DAY", + "AFTER 2 WEEK", + "AFTER 2 MONTH", + "AFTER 2 YEAR", + "AFTER 2 MONTH", + "AFTER 2 MONTH 3 DAY", + "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", + "AFTER 1 YEAR RANDOMIZE FOR 11 MONTH", + # Randomize bigger than interval + "AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", + "EVERY 1 MINUTE RANDOMIZE FOR 3 YEAR", + "EVERY 1 MINUTE OFFSET 1 SECONDS RANDOMIZE FOR 3 YEAR", + # + "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", + "EVERY 1 YEAR OFFSET 11 MONTH RANDOMIZE FOR 1 YEAR", + "EVERY 1 YEAR OFFSET 11 MONTH RANDOMIZE FOR 1 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", + # + "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS RANDOMIZE FOR 2 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", + # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS OFFSET 11 MONTH RANDOMIZE FOR 2 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", + # + # Two different units in EVERY + # "EVERY 1 YEAR", + # "EVERY 1 YEAR 0 MONTH", + # "EVERY 1 YEAR 1 MONTH", + # "EVERY 1 YEAR 2 MONTH", + # "EVERY 1 YEAR 3 MONTH", + # "EVERY 1 YEAR 4 MONTH", + # "EVERY 1 YEAR 8 MONTH", + # "EVERY 1 YEAR 9 MONTH", + # "EVERY 1 YEAR 10 MONTH", + # "EVERY 1 YEAR 11 MONTH", + # "EVERY 1 YEAR 12 MONTH", + # "EVERY 1 YEAR 13 MONTH", + # "EVERY 1 DAY", + # "EVERY 1 DAY 0 HOUR", + # "EVERY 1 DAY 1 HOUR", + # "EVERY 1 DAY 2 HOUR", + # "EVERY 1 DAY 28 HOUR", + # "EVERY 1 DAY 29 HOUR", + # "EVERY 1 DAY 30 HOUR", + # "EVERY 1 DAY 31 HOUR", + # "EVERY 1 DAY 32 HOUR", + # + "AFTER 1 MONTH", + "AFTER 1 MONTH 0 DAY", + "AFTER 1 MONTH 1 DAY", + "AFTER 1 MONTH 3 DAY", + "AFTER 1 MONTH 50 DAY", + "AFTER 1 YEAR 10 MONTH", + "AFTER 1 YEAR 10 MONTH 3 DAY", + "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR", + "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS", + # Interval shouldn't contain both calendar units and clock units (e.g. months and days) + # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS", +] + + +@pytest.mark.parametrize( + "interval", + INTERVALS, +) +@pytest.mark.parametrize( + "append", + [ + # True, + False + ], +) +@pytest.mark.parametrize( + "empty", + [ + True, + # False + ], +) +def test_schedule( + request, + started_cluster, + interval, + append, + empty, +): + """ + - Create RMV + - Check table view_refreshes + - Check inserted data if without EMPTY + - Set time, wait for refresh + - Check data is inserted/appended + - Alter table + - Check everything again + - DROP target table + """ + + if "WEEK" in interval: + pytest.skip() + + def teardown(): + node1_1.query("DROP TABLE IF EXISTS test_rmv_schedule") + # node1_1.query("DROP TABLE IF EXISTS tgt_new ON CLUSTER test_cluster") + # node1_1.query("TRUNCATE TABLE tgt1 ON CLUSTER test_cluster") + + request.addfinalizer(teardown) + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv_schedule", + refresh_interval=interval, + table_clause="ENGINE = Memory", + select_query="SELECT now() as a, number as b FROM numbers(2)", + with_append=append, + empty=empty, + ) + print(create_sql) + node1_1.query(create_sql) + + now = datetime.utcnow() + + rmv = get_rmv_info(node1_1, "test_rmv_schedule") + print(rmv) + + next_refresh_time = parse_ch_datetime(rmv["next_refresh_time"]) + predicted_next_refresh_time = get_next_refresh_time(interval, now) + + print("----") + print("current_time:", now) + print("Interval:", interval) + print("next_refresh_time", next_refresh_time) + print("predicted", predicted_next_refresh_time) + print("----") + + # RANDOMIZE means the next refresh time will be randomly chosen + # within a range of RANDOMIZE interval + if "RANDOMIZE" in interval: + assert ( + predicted_next_refresh_time[0] + <= next_refresh_time + <= predicted_next_refresh_time[1] + ) + else: + assert compare_dates(next_refresh_time, predicted_next_refresh_time) + + assert next_refresh_time > now + + append_expect_rows = 0 + + def check_data(): + inserted_data = node1_1.query_with_retry( + f"SELECT * FROM test_rmv_schedule", + parse=True, + check_callback=lambda x: len(x) > 0, + retry_count=200, + ) + + # Check data if not EMPTY + if append: + # Append adds rows + global append_expect_rows + append_expect_rows += 2 + assert len(inserted_data) == append_expect_rows + if not append: + # Rewrite without append + assert len(inserted_data) == 2 + + inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) + if empty: + assert len(inserted_data) == 0 + else: + assert rmv["last_success_duration_ms"] > 0 + check_data() + + # Trigger next refresh + node1_1.query( + f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time}'" + ) + + if "RANDOMIZE" not in interval: + check_data() + + # rmv = get_rmv_info(node1_1, "test_rmv_schedule") + # next_refresh_time = parse_ch_datetime(rmv["next_refresh_time"]) + + # # Alter RMV to random interval and test schedule is changed + # # TODO: remove week filter after fix + # interval_alter = random.choice(list(filter(lambda x: "WEEK" not in x, INTERVALS))) + # alter_sql = ALTER_RMV_TEMPLATE.render( + # table_name="test_rmv_schedule", + # refresh_interval=interval_alter, + # # select_query="SELECT * FROM src1", + # ) + # print(alter_sql) + # node1_1.query(alter_sql) + # now_alter = datetime.utcnow() + # + # rmv = get_rmv_info(node1_1, "test_rmv_schedule") + # next_refresh_time_alter = parse_ch_datetime(rmv["next_refresh_time"]) + # predicted_next_refresh_time = get_next_refresh_time(interval_alter, now_alter) + # + # if "RANDOMIZE" in interval_alter: + # assert ( + # predicted_next_refresh_time[0] + # <= next_refresh_time_alter + # <= predicted_next_refresh_time[1] + # ) + # else: + # assert compare_dates(next_refresh_time_alter, predicted_next_refresh_time) + # + # assert next_refresh_time_alter > now_alter + # + # # Trigger next refresh + # node1_1.query( + # f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time_alter}'" + # ) + # check_data() + + # breakpoint() + + +# def test_create_replicated_table(started_cluster): +# main_node.query( +# "CREATE DATABASE create_replicated_table ENGINE = Replicated('/test/create_replicated_table', 'shard1', 'replica' || '1');" +# ) +# dummy_node.query( +# "CREATE DATABASE create_replicated_table ENGINE = Replicated('/test/create_replicated_table', 'shard1', 'replica2');" +# ) +# assert ( +# "Explicit zookeeper_path and replica_name are specified" +# in main_node.query_and_get_error( +# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " +# "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" +# ) +# ) +# +# assert ( +# "Explicit zookeeper_path and replica_name are specified" +# in main_node.query_and_get_error( +# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " +# "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" +# ) +# ) +# +# assert ( +# "This syntax for *MergeTree engine is deprecated" +# in main_node.query_and_get_error( +# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " +# "ENGINE=ReplicatedMergeTree('/test/tmp/{shard}', '{replica}', d, k, 8192);" +# ) +# ) +# +# main_node.query( +# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);" +# ) +# +# expected = ( +# "CREATE TABLE create_replicated_table.replicated_table\\n(\\n `d` Date,\\n `k` UInt64,\\n `i32` Int32\\n)\\n" +# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\n" +# "PARTITION BY toYYYYMM(d)\\nORDER BY k\\nSETTINGS index_granularity = 8192" +# ) +# assert_create_query( +# [main_node, dummy_node], "create_replicated_table.replicated_table", expected +# ) +# # assert without replacing uuid +# assert main_node.query( +# "show create create_replicated_table.replicated_table" +# ) == dummy_node.query("show create create_replicated_table.replicated_table") +# main_node.query("DROP DATABASE create_replicated_table SYNC") +# dummy_node.query("DROP DATABASE create_replicated_table SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_simple_alter_table(started_cluster, engine): +# database = f"test_simple_alter_table_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# # test_simple_alter_table +# name = f"{database}.alter_test" +# main_node.query( +# "CREATE TABLE {} " +# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " +# "ENGINE = {} PARTITION BY StartDate ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);".format( +# name, engine +# ) +# ) +# main_node.query("ALTER TABLE {} ADD COLUMN Added0 UInt32;".format(name)) +# main_node.query("ALTER TABLE {} ADD COLUMN Added2 UInt32;".format(name)) +# main_node.query( +# "ALTER TABLE {} ADD COLUMN Added1 UInt32 AFTER Added0;".format(name) +# ) +# main_node.query( +# "ALTER TABLE {} ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;".format( +# name +# ) +# ) +# main_node.query( +# "ALTER TABLE {} ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;".format( +# name +# ) +# ) +# main_node.query( +# "ALTER TABLE {} ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;".format( +# name +# ) +# ) +# +# full_engine = ( +# engine +# if not "Replicated" in engine +# else engine + "(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')" +# ) +# expected = ( +# "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" +# " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n" +# " `AddedNested1.A` Array(UInt32),\\n `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n" +# " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64)\\n)\\n" +# "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" +# "SETTINGS index_granularity = 8192".format(name, full_engine) +# ) +# +# assert_create_query([main_node, dummy_node], name, expected) +# +# # test_create_replica_after_delay +# competing_node.query( +# f"CREATE DATABASE IF NOT EXISTS {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica3');" +# ) +# +# main_node.query("ALTER TABLE {} ADD COLUMN Added3 UInt32;".format(name)) +# main_node.query("ALTER TABLE {} DROP COLUMN AddedNested1;".format(name)) +# main_node.query("ALTER TABLE {} RENAME COLUMN Added1 TO AddedNested1;".format(name)) +# +# full_engine = ( +# engine +# if not "Replicated" in engine +# else engine + "(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')" +# ) +# expected = ( +# "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" +# " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `AddedNested1` UInt32,\\n `Added2` UInt32,\\n" +# " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64),\\n `Added3` UInt32\\n)\\n" +# "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" +# "SETTINGS index_granularity = 8192".format(name, full_engine) +# ) +# +# assert_create_query([main_node, dummy_node, competing_node], name, expected) +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# competing_node.query(f"DROP DATABASE {database} SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_delete_from_table(started_cluster, engine): +# database = f"delete_from_table_{engine}" +# +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard2', 'replica1');" +# ) +# +# name = f"{database}.delete_test" +# main_node.query( +# "CREATE TABLE {} " +# "(id UInt64, value String) " +# "ENGINE = {} PARTITION BY id%2 ORDER BY (id);".format(name, engine) +# ) +# main_node.query("INSERT INTO TABLE {} VALUES(1, 'aaaa');".format(name)) +# main_node.query("INSERT INTO TABLE {} VALUES(2, 'aaaa');".format(name)) +# dummy_node.query("INSERT INTO TABLE {} VALUES(1, 'bbbb');".format(name)) +# dummy_node.query("INSERT INTO TABLE {} VALUES(2, 'bbbb');".format(name)) +# +# main_node.query("DELETE FROM {} WHERE id=2;".format(name)) +# +# expected = "1\taaaa\n1\tbbbb" +# +# table_for_select = name +# if not "Replicated" in engine: +# table_for_select = f"cluster('{database}', {name})" +# for node in [main_node, dummy_node]: +# assert_eq_with_retry( +# node, +# "SELECT * FROM {} ORDER BY id, value;".format(table_for_select), +# expected, +# ) +# +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# +# +# def get_table_uuid(database, name): +# return main_node.query( +# f"SELECT uuid FROM system.tables WHERE database = '{database}' and name = '{name}'" +# ).strip() +# +# +# @pytest.fixture(scope="module", name="attachable_part") +# def fixture_attachable_part(started_cluster): +# main_node.query(f"CREATE DATABASE testdb_attach_atomic ENGINE = Atomic") +# main_node.query( +# f"CREATE TABLE testdb_attach_atomic.test (CounterID UInt32) ENGINE = MergeTree ORDER BY (CounterID)" +# ) +# main_node.query(f"INSERT INTO testdb_attach_atomic.test VALUES (123)") +# main_node.query( +# f"ALTER TABLE testdb_attach_atomic.test FREEZE WITH NAME 'test_attach'" +# ) +# table_uuid = get_table_uuid("testdb_attach_atomic", "test") +# return os.path.join( +# main_node.path, +# f"database/shadow/test_attach/store/{table_uuid[:3]}/{table_uuid}/all_1_1_0", +# ) +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_alter_attach(started_cluster, attachable_part, engine): +# database = f"alter_attach_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# +# main_node.query( +# f"CREATE TABLE {database}.alter_attach_test (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" +# ) +# table_uuid = get_table_uuid(database, "alter_attach_test") +# # Provide and attach a part to the main node +# shutil.copytree( +# attachable_part, +# os.path.join( +# main_node.path, +# f"database/store/{table_uuid[:3]}/{table_uuid}/detached/all_1_1_0", +# ), +# ) +# main_node.query(f"ALTER TABLE {database}.alter_attach_test ATTACH PART 'all_1_1_0'") +# # On the main node, data is attached +# assert ( +# main_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") +# == "123\n" +# ) +# # On the other node, data is replicated only if using a Replicated table engine +# if engine == "ReplicatedMergeTree": +# assert ( +# dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") +# == "123\n" +# ) +# else: +# assert ( +# dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") +# == "" +# ) +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_alter_drop_part(started_cluster, engine): +# database = f"alter_drop_part_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# +# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" +# main_node.query( +# f"CREATE TABLE {database}.alter_drop_part (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" +# ) +# main_node.query(f"INSERT INTO {database}.alter_drop_part VALUES (123)") +# if engine == "MergeTree": +# dummy_node.query(f"INSERT INTO {database}.alter_drop_part VALUES (456)") +# else: +# main_node.query(f"SYSTEM SYNC REPLICA {database}.alter_drop_part PULL") +# main_node.query(f"ALTER TABLE {database}.alter_drop_part DROP PART '{part_name}'") +# assert main_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") == "" +# if engine == "ReplicatedMergeTree": +# # The DROP operation is still replicated at the table engine level +# assert ( +# dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") == "" +# ) +# else: +# assert ( +# dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") +# == "456\n" +# ) +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_alter_detach_part(started_cluster, engine): +# database = f"alter_detach_part_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# +# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" +# main_node.query( +# f"CREATE TABLE {database}.alter_detach (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" +# ) +# main_node.query(f"INSERT INTO {database}.alter_detach VALUES (123)") +# if engine == "MergeTree": +# dummy_node.query(f"INSERT INTO {database}.alter_detach VALUES (456)") +# main_node.query(f"ALTER TABLE {database}.alter_detach DETACH PART '{part_name}'") +# detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='{database}' AND table='alter_detach'" +# assert main_node.query(detached_parts_query) == f"{part_name}\n" +# if engine == "ReplicatedMergeTree": +# # The detach operation is still replicated at the table engine level +# assert dummy_node.query(detached_parts_query) == f"{part_name}\n" +# else: +# assert dummy_node.query(detached_parts_query) == "" +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_alter_drop_detached_part(started_cluster, engine): +# database = f"alter_drop_detached_part_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# +# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" +# main_node.query( +# f"CREATE TABLE {database}.alter_drop_detached (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" +# ) +# main_node.query(f"INSERT INTO {database}.alter_drop_detached VALUES (123)") +# main_node.query( +# f"ALTER TABLE {database}.alter_drop_detached DETACH PART '{part_name}'" +# ) +# if engine == "MergeTree": +# dummy_node.query(f"INSERT INTO {database}.alter_drop_detached VALUES (456)") +# dummy_node.query( +# f"ALTER TABLE {database}.alter_drop_detached DETACH PART '{part_name}'" +# ) +# main_node.query( +# f"ALTER TABLE {database}.alter_drop_detached DROP DETACHED PART '{part_name}'" +# ) +# detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='{database}' AND table='alter_drop_detached'" +# assert main_node.query(detached_parts_query) == "" +# assert dummy_node.query(detached_parts_query) == f"{part_name}\n" +# +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_alter_drop_partition(started_cluster, engine): +# database = f"alter_drop_partition_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# snapshotting_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard2', 'replica1');" +# ) +# +# main_node.query( +# f"CREATE TABLE {database}.alter_drop (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" +# ) +# main_node.query(f"INSERT INTO {database}.alter_drop VALUES (123)") +# if engine == "MergeTree": +# dummy_node.query(f"INSERT INTO {database}.alter_drop VALUES (456)") +# snapshotting_node.query(f"INSERT INTO {database}.alter_drop VALUES (789)") +# main_node.query( +# f"ALTER TABLE {database}.alter_drop ON CLUSTER {database} DROP PARTITION ID 'all'", +# settings={"replication_alter_partitions_sync": 2}, +# ) +# assert ( +# main_node.query( +# f"SELECT CounterID FROM clusterAllReplicas('{database}', {database}.alter_drop)" +# ) +# == "" +# ) +# assert dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop") == "" +# main_node.query(f"DROP DATABASE {database}") +# dummy_node.query(f"DROP DATABASE {database}") +# snapshotting_node.query(f"DROP DATABASE {database}") +# +# +# def test_alter_fetch(started_cluster): +# main_node.query( +# "CREATE DATABASE alter_fetch ENGINE = Replicated('/test/alter_fetch', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE alter_fetch ENGINE = Replicated('/test/alter_fetch', 'shard1', 'replica2');" +# ) +# +# main_node.query( +# "CREATE TABLE alter_fetch.fetch_source (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" +# ) +# main_node.query( +# "CREATE TABLE alter_fetch.fetch_target (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" +# ) +# main_node.query("INSERT INTO alter_fetch.fetch_source VALUES (123)") +# table_uuid = get_table_uuid("alter_fetch", "fetch_source") +# main_node.query( +# f"ALTER TABLE alter_fetch.fetch_target FETCH PART 'all_0_0_0' FROM '/clickhouse/tables/{table_uuid}/{{shard}}' " +# ) +# detached_parts_query = "SELECT name FROM system.detached_parts WHERE database='alter_fetch' AND table='fetch_target'" +# assert main_node.query(detached_parts_query) == "all_0_0_0\n" +# assert dummy_node.query(detached_parts_query) == "" +# +# main_node.query("DROP DATABASE alter_fetch SYNC") +# dummy_node.query("DROP DATABASE alter_fetch SYNC") +# +# +# def test_alters_from_different_replicas(started_cluster): +# main_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica2');" +# ) +# +# # test_alters_from_different_replicas +# competing_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica3');" +# ) +# +# main_node.query( +# "CREATE TABLE alters_from_different_replicas.concurrent_test " +# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " +# "ENGINE = MergeTree PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);" +# ) +# +# main_node.query( +# "CREATE TABLE alters_from_different_replicas.dist AS alters_from_different_replicas.concurrent_test ENGINE = Distributed(alters_from_different_replicas, alters_from_different_replicas, concurrent_test, CounterID)" +# ) +# +# dummy_node.stop_clickhouse(kill=True) +# +# settings = {"distributed_ddl_task_timeout": 5} +# assert "is not finished on 1 of 3 hosts" in competing_node.query_and_get_error( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added0 UInt32;", +# settings=settings, +# ) +# settings = { +# "distributed_ddl_task_timeout": 5, +# "distributed_ddl_output_mode": "null_status_on_timeout", +# } +# assert "shard1\treplica2\tQUEUED\t" in main_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added2 UInt32;", +# settings=settings, +# ) +# settings = { +# "distributed_ddl_task_timeout": 5, +# "distributed_ddl_output_mode": "never_throw", +# } +# assert "shard1\treplica2\tQUEUED\t" in competing_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added1 UInt32 AFTER Added0;", +# settings=settings, +# ) +# dummy_node.start_clickhouse() +# main_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;" +# ) +# competing_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;" +# ) +# main_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;" +# ) +# +# expected = ( +# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32,\\n" +# " `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n `AddedNested1.A` Array(UInt32),\\n" +# " `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n `AddedNested2.A` Array(UInt32),\\n" +# " `AddedNested2.B` Array(UInt64)\\n)\\n" +# "ENGINE = MergeTree\\nPARTITION BY toYYYYMM(StartDate)\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\nSETTINGS index_granularity = 8192" +# ) +# +# assert_create_query( +# [main_node, competing_node], +# "alters_from_different_replicas.concurrent_test", +# expected, +# ) +# +# # test_create_replica_after_delay +# main_node.query("DROP TABLE alters_from_different_replicas.concurrent_test SYNC") +# main_node.query( +# "CREATE TABLE alters_from_different_replicas.concurrent_test " +# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " +# "ENGINE = ReplicatedMergeTree ORDER BY CounterID;" +# ) +# +# expected = ( +# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" +# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" +# ) +# +# assert_create_query( +# [main_node, competing_node], +# "alters_from_different_replicas.concurrent_test", +# expected, +# ) +# +# main_node.query( +# "INSERT INTO alters_from_different_replicas.dist (CounterID, StartDate, UserID) SELECT number, addDays(toDate('2020-02-02'), number), intHash32(number) FROM numbers(10)" +# ) +# +# # test_replica_restart +# main_node.restart_clickhouse() +# +# expected = ( +# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" +# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" +# ) +# +# # test_snapshot_and_snapshot_recover +# snapshotting_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard2', 'replica1');" +# ) +# snapshot_recovering_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard2', 'replica2');" +# ) +# assert_create_query( +# all_nodes, "alters_from_different_replicas.concurrent_test", expected +# ) +# +# main_node.query("SYSTEM FLUSH DISTRIBUTED alters_from_different_replicas.dist") +# main_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test UPDATE StartDate = addYears(StartDate, 1) WHERE 1" +# ) +# res = main_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test DELETE WHERE UserID % 2" +# ) +# assert ( +# "shard1\treplica1\tOK" in res +# and "shard1\treplica2\tOK" in res +# and "shard1\treplica3\tOK" in res +# ) +# assert "shard2\treplica1\tOK" in res and "shard2\treplica2\tOK" in res +# +# expected = ( +# "1\t1\tmain_node\n" +# "1\t2\tdummy_node\n" +# "1\t3\tcompeting_node\n" +# "2\t1\tsnapshotting_node\n" +# "2\t2\tsnapshot_recovering_node\n" +# ) +# assert ( +# main_node.query( +# "SELECT shard_num, replica_num, host_name FROM system.clusters WHERE cluster='alters_from_different_replicas'" +# ) +# == expected +# ) +# +# # test_drop_and_create_replica +# main_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# main_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica1');" +# ) +# +# expected = ( +# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" +# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" +# ) +# +# assert_create_query( +# [main_node, competing_node], +# "alters_from_different_replicas.concurrent_test", +# expected, +# ) +# assert_create_query( +# all_nodes, "alters_from_different_replicas.concurrent_test", expected +# ) +# +# for node in all_nodes: +# node.query("SYSTEM SYNC REPLICA alters_from_different_replicas.concurrent_test") +# +# expected = ( +# "0\t2021-02-02\t4249604106\n" +# "1\t2021-02-03\t1343103100\n" +# "4\t2021-02-06\t3902320246\n" +# "7\t2021-02-09\t3844986530\n" +# "9\t2021-02-11\t1241149650\n" +# ) +# +# assert_eq_with_retry( +# dummy_node, +# "SELECT CounterID, StartDate, UserID FROM alters_from_different_replicas.dist ORDER BY CounterID", +# expected, +# ) +# main_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# dummy_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# competing_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# snapshotting_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# snapshot_recovering_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# +# +# def create_some_tables(db): +# settings = { +# "distributed_ddl_task_timeout": 0, +# "allow_experimental_object_type": 1, +# "allow_suspicious_codecs": 1, +# } +# main_node.query(f"CREATE TABLE {db}.t1 (n int) ENGINE=Memory", settings=settings) +# dummy_node.query( +# f"CREATE TABLE {db}.t2 (s String) ENGINE=Memory", settings=settings +# ) +# main_node.query( +# f"CREATE TABLE {db}.mt1 (n int) ENGINE=MergeTree order by n", +# settings=settings, +# ) +# dummy_node.query( +# f"CREATE TABLE {db}.mt2 (n int) ENGINE=MergeTree order by n", +# settings=settings, +# ) +# main_node.query( +# f"CREATE TABLE {db}.rmt1 (n int) ENGINE=ReplicatedMergeTree order by n", +# settings=settings, +# ) +# dummy_node.query( +# f"CREATE TABLE {db}.rmt2 (n int CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12))) ENGINE=ReplicatedMergeTree order by n", +# settings=settings, +# ) +# main_node.query( +# f"CREATE TABLE {db}.rmt3 (n int, json Object('json') materialized '') ENGINE=ReplicatedMergeTree order by n", +# settings=settings, +# ) +# dummy_node.query( +# f"CREATE TABLE {db}.rmt5 (n int) ENGINE=ReplicatedMergeTree order by n", +# settings=settings, +# ) +# main_node.query( +# f"CREATE MATERIALIZED VIEW {db}.mv1 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt1", +# settings=settings, +# ) +# dummy_node.query( +# f"CREATE MATERIALIZED VIEW {db}.mv2 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt2", +# settings=settings, +# ) +# main_node.query( +# f"CREATE DICTIONARY {db}.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " +# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " +# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" +# ) +# dummy_node.query( +# f"CREATE DICTIONARY {db}.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " +# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt2' PASSWORD '' DB 'recover')) " +# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" +# ) +# +# +# # These tables are used to check that DatabaseReplicated correctly renames all the tables in case when it restores from the lost state +# def create_table_for_exchanges(db): +# settings = {"distributed_ddl_task_timeout": 0} +# for table in ["a1", "a2", "a3", "a4", "a5", "a6"]: +# main_node.query( +# f"CREATE TABLE {db}.{table} (s String) ENGINE=ReplicatedMergeTree order by s", +# settings=settings, +# ) +# +# +# def test_recover_staled_replica(started_cluster): +# main_node.query( +# "CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica1');" +# ) +# started_cluster.get_kazoo_client("zoo1").set( +# "/clickhouse/databases/recover/logs_to_keep", b"10" +# ) +# dummy_node.query( +# "CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica2');" +# ) +# +# settings = {"distributed_ddl_task_timeout": 0} +# create_some_tables("recover") +# create_table_for_exchanges("recover") +# +# for table in ["t1", "t2", "mt1", "mt2", "rmt1", "rmt2", "rmt3", "rmt5"]: +# main_node.query(f"INSERT INTO recover.{table} VALUES (42)") +# for table in ["t1", "t2", "mt1", "mt2"]: +# dummy_node.query(f"INSERT INTO recover.{table} VALUES (42)") +# +# for i, table in enumerate(["a1", "a2", "a3", "a4", "a5", "a6"]): +# main_node.query(f"INSERT INTO recover.{table} VALUES ('{str(i + 1) * 10}')") +# +# for table in ["rmt1", "rmt2", "rmt3", "rmt5"]: +# main_node.query(f"SYSTEM SYNC REPLICA recover.{table}") +# for table in ["a1", "a2", "a3", "a4", "a5", "a6"]: +# main_node.query(f"SYSTEM SYNC REPLICA recover.{table}") +# +# with PartitionManager() as pm: +# pm.drop_instance_zk_connections(dummy_node) +# dummy_node.query_and_get_error("RENAME TABLE recover.t1 TO recover.m1") +# +# main_node.query_with_retry( +# "RENAME TABLE recover.t1 TO recover.m1", settings=settings +# ) +# main_node.query_with_retry( +# "ALTER TABLE recover.mt1 ADD COLUMN m int", settings=settings +# ) +# main_node.query_with_retry( +# "ALTER TABLE recover.rmt1 ADD COLUMN m int", settings=settings +# ) +# main_node.query_with_retry( +# "RENAME TABLE recover.rmt3 TO recover.rmt4", settings=settings +# ) +# main_node.query_with_retry("DROP TABLE recover.rmt5", settings=settings) +# main_node.query_with_retry("DROP DICTIONARY recover.d2", settings=settings) +# main_node.query_with_retry( +# "CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " +# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " +# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());", +# settings=settings, +# ) +# +# inner_table = ( +# ".inner_id." +# + dummy_node.query_with_retry( +# "SELECT uuid FROM system.tables WHERE database='recover' AND name='mv1'" +# ).strip() +# ) +# main_node.query_with_retry( +# f"ALTER TABLE recover.`{inner_table}` MODIFY COLUMN n int DEFAULT 42", +# settings=settings, +# ) +# main_node.query_with_retry( +# "ALTER TABLE recover.mv1 MODIFY QUERY SELECT m as n FROM recover.rmt1", +# settings=settings, +# ) +# main_node.query_with_retry( +# "RENAME TABLE recover.mv2 TO recover.mv3", +# settings=settings, +# ) +# +# main_node.query_with_retry( +# "CREATE TABLE recover.tmp AS recover.m1", settings=settings +# ) +# main_node.query_with_retry("DROP TABLE recover.tmp", settings=settings) +# main_node.query_with_retry( +# "CREATE TABLE recover.tmp AS recover.m1", settings=settings +# ) +# main_node.query_with_retry("DROP TABLE recover.tmp", settings=settings) +# main_node.query_with_retry( +# "CREATE TABLE recover.tmp AS recover.m1", settings=settings +# ) +# +# main_node.query("EXCHANGE TABLES recover.a1 AND recover.a2", settings=settings) +# main_node.query("EXCHANGE TABLES recover.a3 AND recover.a4", settings=settings) +# main_node.query("EXCHANGE TABLES recover.a5 AND recover.a4", settings=settings) +# main_node.query("EXCHANGE TABLES recover.a6 AND recover.a3", settings=settings) +# main_node.query("RENAME TABLE recover.a6 TO recover.a7", settings=settings) +# main_node.query("RENAME TABLE recover.a1 TO recover.a8", settings=settings) +# +# assert ( +# main_node.query( +# "SELECT name FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' ORDER BY name" +# ) +# == "a2\na3\na4\na5\na7\na8\nd1\nd2\nm1\nmt1\nmt2\nmv1\nmv3\nrmt1\nrmt2\nrmt4\nt2\ntmp\n" +# ) +# query = ( +# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' " +# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" +# ) +# expected = main_node.query(query) +# assert_eq_with_retry(dummy_node, query, expected) +# assert ( +# main_node.query( +# "SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'" +# ) +# == "2\n" +# ) +# assert ( +# dummy_node.query( +# "SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'" +# ) +# == "2\n" +# ) +# +# # Check that Database Replicated renamed all the tables correctly +# for i, table in enumerate(["a2", "a8", "a5", "a7", "a4", "a3"]): +# assert ( +# dummy_node.query(f"SELECT * FROM recover.{table}") == f"{str(i + 1) * 10}\n" +# ) +# +# for table in [ +# "m1", +# "t2", +# "mt1", +# "mt2", +# "rmt1", +# "rmt2", +# "rmt4", +# "d1", +# "d2", +# "mv1", +# "mv3", +# ]: +# assert main_node.query(f"SELECT (*,).1 FROM recover.{table}") == "42\n" +# for table in ["t2", "rmt1", "rmt2", "rmt4", "d1", "d2", "mt2", "mv1", "mv3"]: +# assert ( +# dummy_node.query(f"SELECT '{table}', (*,).1 FROM recover.{table}") +# == f"{table}\t42\n" +# ) +# for table in ["m1", "mt1"]: +# assert dummy_node.query(f"SELECT count() FROM recover.{table}") == "0\n" +# global test_recover_staled_replica_run +# assert ( +# dummy_node.query( +# "SELECT count() FROM system.tables WHERE database='recover_broken_tables'" +# ) +# == f"{test_recover_staled_replica_run}\n" +# ) +# assert ( +# dummy_node.query( +# "SELECT count() FROM system.tables WHERE database='recover_broken_replicated_tables'" +# ) +# == f"{test_recover_staled_replica_run}\n" +# ) +# test_recover_staled_replica_run += 1 +# +# print(dummy_node.query("SHOW DATABASES")) +# print(dummy_node.query("SHOW TABLES FROM recover_broken_tables")) +# print(dummy_node.query("SHOW TABLES FROM recover_broken_replicated_tables")) +# +# table = dummy_node.query( +# "SHOW TABLES FROM recover_broken_tables LIKE 'mt1_41_%' LIMIT 1" +# ).strip() +# assert ( +# dummy_node.query(f"SELECT (*,).1 FROM recover_broken_tables.{table}") == "42\n" +# ) +# table = dummy_node.query( +# "SHOW TABLES FROM recover_broken_replicated_tables LIKE 'rmt5_41_%' LIMIT 1" +# ).strip() +# assert ( +# dummy_node.query(f"SELECT (*,).1 FROM recover_broken_replicated_tables.{table}") +# == "42\n" +# ) +# +# expected = "Cleaned 6 outdated objects: dropped 1 dictionaries and 3 tables, moved 2 tables" +# assert_logs_contain(dummy_node, expected) +# +# dummy_node.query("DROP TABLE recover.tmp") +# assert_eq_with_retry( +# main_node, +# "SELECT count() FROM system.tables WHERE database='recover' AND name='tmp'", +# "0\n", +# ) +# main_node.query("DROP DATABASE recover SYNC") +# dummy_node.query("DROP DATABASE recover SYNC") +# +# +# def test_recover_staled_replica_many_mvs(started_cluster): +# main_node.query("DROP DATABASE IF EXISTS recover_mvs") +# dummy_node.query("DROP DATABASE IF EXISTS recover_mvs") +# +# main_node.query_with_retry( +# "CREATE DATABASE IF NOT EXISTS recover_mvs ENGINE = Replicated('/clickhouse/databases/recover_mvs', 'shard1', 'replica1');" +# ) +# started_cluster.get_kazoo_client("zoo1").set( +# "/clickhouse/databases/recover_mvs/logs_to_keep", b"10" +# ) +# dummy_node.query_with_retry( +# "CREATE DATABASE IF NOT EXISTS recover_mvs ENGINE = Replicated('/clickhouse/databases/recover_mvs', 'shard1', 'replica2');" +# ) +# +# settings = {"distributed_ddl_task_timeout": 0} +# +# with PartitionManager() as pm: +# pm.drop_instance_zk_connections(dummy_node) +# dummy_node.query_and_get_error("RENAME TABLE recover_mvs.t1 TO recover_mvs.m1") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query( +# f"CREATE TABLE recover_mvs.rmt{identifier} (n int) ENGINE=ReplicatedMergeTree ORDER BY n", +# settings=settings, +# ) +# +# print("Created tables") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query( +# f"CREATE TABLE recover_mvs.mv_inner{identifier} (n int) ENGINE=ReplicatedMergeTree ORDER BY n", +# settings=settings, +# ) +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE MATERIALIZED VIEW recover_mvs.mv{identifier} +# TO recover_mvs.mv_inner{identifier} +# AS SELECT * FROM recover_mvs.rmt{identifier}""", +# settings=settings, +# ) +# +# print("Created MVs") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE VIEW recover_mvs.view_from_mv{identifier} +# AS SELECT * FROM recover_mvs.mv{identifier}""", +# settings=settings, +# ) +# +# print("Created Views on top of MVs") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE MATERIALIZED VIEW recover_mvs.cascade_mv{identifier} +# ENGINE=MergeTree() ORDER BY tuple() +# POPULATE AS SELECT * FROM recover_mvs.mv_inner{identifier};""", +# settings=settings, +# ) +# +# print("Created cascade MVs") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE VIEW recover_mvs.view_from_cascade_mv{identifier} +# AS SELECT * FROM recover_mvs.cascade_mv{identifier}""", +# settings=settings, +# ) +# +# print("Created Views on top of cascade MVs") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE MATERIALIZED VIEW recover_mvs.double_cascade_mv{identifier} +# ENGINE=MergeTree() ORDER BY tuple() +# POPULATE AS SELECT * FROM recover_mvs.`.inner_id.{get_table_uuid("recover_mvs", f"cascade_mv{identifier}")}`""", +# settings=settings, +# ) +# +# print("Created double cascade MVs") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE VIEW recover_mvs.view_from_double_cascade_mv{identifier} +# AS SELECT * FROM recover_mvs.double_cascade_mv{identifier}""", +# settings=settings, +# ) +# +# print("Created Views on top of double cascade MVs") +# +# # This weird table name is actually makes sence because it starts with letter `a` and may break some internal sorting +# main_node.query_with_retry( +# """ +# CREATE VIEW recover_mvs.anime +# AS +# SELECT n +# FROM +# ( +# SELECT * +# FROM +# ( +# SELECT * +# FROM +# ( +# SELECT * +# FROM recover_mvs.mv_inner1 AS q1 +# INNER JOIN recover_mvs.mv_inner2 AS q2 ON q1.n = q2.n +# ) AS new_table_1 +# INNER JOIN recover_mvs.mv_inner3 AS q3 ON new_table_1.n = q3.n +# ) AS new_table_2 +# INNER JOIN recover_mvs.mv_inner4 AS q4 ON new_table_2.n = q4.n +# ) +# """, +# settings=settings, +# ) +# +# print("Created final boss") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE DICTIONARY recover_mvs.`11111d{identifier}` (n UInt64) +# PRIMARY KEY n +# SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'double_cascade_mv{identifier}' DB 'recover_mvs')) +# LAYOUT(FLAT()) LIFETIME(1)""", +# settings=settings, +# ) +# +# print("Created dictionaries") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE VIEW recover_mvs.`00000vd{identifier}` +# AS SELECT * FROM recover_mvs.`11111d{identifier}`""", +# settings=settings, +# ) +# +# print("Created Views on top of dictionaries") +# +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA recover_mvs") +# query = "SELECT name FROM system.tables WHERE database='recover_mvs' ORDER BY name" +# assert main_node.query(query) == dummy_node.query(query) +# +# main_node.query("DROP DATABASE IF EXISTS recover_mvs") +# dummy_node.query("DROP DATABASE IF EXISTS recover_mvs") +# +# +# def test_startup_without_zk(started_cluster): +# with PartitionManager() as pm: +# pm.drop_instance_zk_connections(main_node) +# err = main_node.query_and_get_error( +# "CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');" +# ) +# assert "ZooKeeper" in err or "Coordination::Exception" in err +# main_node.query( +# "CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');" +# ) +# main_node.query( +# "CREATE TABLE startup.rmt (n int) ENGINE=ReplicatedMergeTree order by n" +# ) +# +# main_node.query("INSERT INTO startup.rmt VALUES (42)") +# with PartitionManager() as pm: +# pm.drop_instance_zk_connections(main_node) +# main_node.restart_clickhouse(stop_start_wait_sec=60) +# assert main_node.query("SELECT (*,).1 FROM startup.rmt") == "42\n" +# +# # we need to wait until the table is not readonly +# main_node.query_with_retry("INSERT INTO startup.rmt VALUES(42)") +# +# main_node.query_with_retry("CREATE TABLE startup.m (n int) ENGINE=Memory") +# +# main_node.query("EXCHANGE TABLES startup.rmt AND startup.m") +# assert main_node.query("SELECT (*,).1 FROM startup.m") == "42\n" +# +# main_node.query("DROP DATABASE startup SYNC") +# +# +# def test_server_uuid(started_cluster): +# uuid1 = main_node.query("select serverUUID()") +# uuid2 = dummy_node.query("select serverUUID()") +# assert uuid1 != uuid2 +# main_node.restart_clickhouse() +# uuid1_after_restart = main_node.query("select serverUUID()") +# assert uuid1 == uuid1_after_restart +# +# +# def test_sync_replica(started_cluster): +# main_node.query( +# "CREATE DATABASE test_sync_database ENGINE = Replicated('/test/sync_replica', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE test_sync_database ENGINE = Replicated('/test/sync_replica', 'shard1', 'replica2');" +# ) +# +# number_of_tables = 1000 +# +# settings = {"distributed_ddl_task_timeout": 0} +# +# with PartitionManager() as pm: +# pm.drop_instance_zk_connections(dummy_node) +# +# for i in range(number_of_tables): +# main_node.query( +# "CREATE TABLE test_sync_database.table_{} (n int) ENGINE=MergeTree order by n".format( +# i +# ), +# settings=settings, +# ) +# +# # wait for host to reconnect +# dummy_node.query_with_retry("SELECT * FROM system.zookeeper WHERE path='/'") +# +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_sync_database") +# +# assert "2\n" == main_node.query( +# "SELECT sum(is_active) FROM system.clusters WHERE cluster='test_sync_database'" +# ) +# +# assert dummy_node.query( +# "SELECT count() FROM system.tables where database='test_sync_database'" +# ).strip() == str(number_of_tables) +# +# assert main_node.query( +# "SELECT count() FROM system.tables where database='test_sync_database'" +# ).strip() == str(number_of_tables) +# +# engine_settings = {"default_table_engine": "ReplicatedMergeTree"} +# dummy_node.query( +# "CREATE TABLE test_sync_database.table (n int, primary key n) partition by n", +# settings=engine_settings, +# ) +# main_node.query("INSERT INTO test_sync_database.table SELECT * FROM numbers(10)") +# dummy_node.query("TRUNCATE TABLE test_sync_database.table", settings=settings) +# dummy_node.query( +# "ALTER TABLE test_sync_database.table ADD COLUMN m int", settings=settings +# ) +# +# main_node.query( +# "SYSTEM SYNC DATABASE REPLICA ON CLUSTER test_sync_database test_sync_database" +# ) +# +# lp1 = main_node.query( +# "select value from system.zookeeper where path='/test/sync_replica/replicas/shard1|replica1' and name='log_ptr'" +# ) +# lp2 = main_node.query( +# "select value from system.zookeeper where path='/test/sync_replica/replicas/shard1|replica2' and name='log_ptr'" +# ) +# max_lp = main_node.query( +# "select value from system.zookeeper where path='/test/sync_replica/' and name='max_log_ptr'" +# ) +# assert lp1 == max_lp +# assert lp2 == max_lp +# +# main_node.query("DROP DATABASE test_sync_database SYNC") +# dummy_node.query("DROP DATABASE test_sync_database SYNC") +# +# +# def test_force_synchronous_settings(started_cluster): +# main_node.query( +# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard1', 'replica2');" +# ) +# snapshotting_node.query( +# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard2', 'replica1');" +# ) +# main_node.query( +# "CREATE TABLE test_force_synchronous_settings.t (n int) ENGINE=ReplicatedMergeTree('/test/same/path/{shard}', '{replica}') ORDER BY tuple()" +# ) +# main_node.query( +# "INSERT INTO test_force_synchronous_settings.t SELECT * FROM numbers(10)" +# ) +# snapshotting_node.query( +# "INSERT INTO test_force_synchronous_settings.t SELECT * FROM numbers(10)" +# ) +# snapshotting_node.query( +# "SYSTEM SYNC DATABASE REPLICA test_force_synchronous_settings" +# ) +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_force_synchronous_settings") +# +# snapshotting_node.query("SYSTEM STOP MERGES test_force_synchronous_settings.t") +# +# def start_merges_func(): +# time.sleep(5) +# snapshotting_node.query("SYSTEM START MERGES test_force_synchronous_settings.t") +# +# start_merges_thread = threading.Thread(target=start_merges_func) +# start_merges_thread.start() +# +# settings = { +# "mutations_sync": 2, +# "database_replicated_enforce_synchronous_settings": 1, +# } +# main_node.query( +# "ALTER TABLE test_force_synchronous_settings.t UPDATE n = n * 10 WHERE 1", +# settings=settings, +# ) +# assert "10\t450\n" == snapshotting_node.query( +# "SELECT count(), sum(n) FROM test_force_synchronous_settings.t" +# ) +# start_merges_thread.join() +# +# def select_func(): +# dummy_node.query( +# "SELECT sleepEachRow(1) FROM test_force_synchronous_settings.t SETTINGS function_sleep_max_microseconds_per_block = 0" +# ) +# +# select_thread = threading.Thread(target=select_func) +# select_thread.start() +# +# settings = {"database_replicated_enforce_synchronous_settings": 1} +# snapshotting_node.query( +# "DROP TABLE test_force_synchronous_settings.t SYNC", settings=settings +# ) +# main_node.query( +# "CREATE TABLE test_force_synchronous_settings.t (n String) ENGINE=ReplicatedMergeTree('/test/same/path/{shard}', '{replica}') ORDER BY tuple()" +# ) +# select_thread.join() +# +# +# def test_recover_digest_mismatch(started_cluster): +# main_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") +# dummy_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") +# +# main_node.query( +# "CREATE DATABASE recover_digest_mismatch ENGINE = Replicated('/clickhouse/databases/recover_digest_mismatch', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE recover_digest_mismatch ENGINE = Replicated('/clickhouse/databases/recover_digest_mismatch', 'shard1', 'replica2');" +# ) +# +# create_some_tables("recover_digest_mismatch") +# +# main_node.query("SYSTEM SYNC DATABASE REPLICA recover_digest_mismatch") +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA recover_digest_mismatch") +# +# ways_to_corrupt_metadata = [ +# "mv /var/lib/clickhouse/metadata/recover_digest_mismatch/t1.sql /var/lib/clickhouse/metadata/recover_digest_mismatch/m1.sql", +# "sed --follow-symlinks -i 's/Int32/String/' /var/lib/clickhouse/metadata/recover_digest_mismatch/mv1.sql", +# "rm -f /var/lib/clickhouse/metadata/recover_digest_mismatch/d1.sql", +# "rm -rf /var/lib/clickhouse/metadata/recover_digest_mismatch/", # Will trigger "Directory already exists" +# "rm -rf /var/lib/clickhouse/store", +# ] +# +# for command in ways_to_corrupt_metadata: +# print(f"Corrupting data using `{command}`") +# need_remove_is_active_node = "rm -rf" in command +# dummy_node.stop_clickhouse(kill=not need_remove_is_active_node) +# dummy_node.exec_in_container(["bash", "-c", command]) +# +# query = ( +# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover_digest_mismatch' AND name NOT LIKE '.inner_id.%' " +# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" +# ) +# expected = main_node.query(query) +# +# if need_remove_is_active_node: +# # NOTE Otherwise it fails to recreate ReplicatedMergeTree table due to "Replica already exists" +# main_node.query( +# "SYSTEM DROP REPLICA '2' FROM DATABASE recover_digest_mismatch" +# ) +# +# # There is a race condition between deleting active node and creating it on server startup +# # So we start a server only after we deleted all table replicas from the Keeper +# dummy_node.start_clickhouse() +# assert_eq_with_retry(dummy_node, query, expected) +# +# main_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") +# dummy_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") +# +# print("Everything Okay") +# +# +# def test_replicated_table_structure_alter(started_cluster): +# main_node.query("DROP DATABASE IF EXISTS table_structure") +# dummy_node.query("DROP DATABASE IF EXISTS table_structure") +# +# main_node.query( +# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica2');" +# ) +# competing_node.query( +# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica3');" +# ) +# +# competing_node.query("CREATE TABLE table_structure.mem (n int) ENGINE=Memory") +# dummy_node.query("DETACH DATABASE table_structure") +# +# settings = {"distributed_ddl_task_timeout": 0} +# main_node.query( +# "CREATE TABLE table_structure.rmt (n int, v UInt64) ENGINE=ReplicatedReplacingMergeTree(v) ORDER BY n", +# settings=settings, +# ) +# +# competing_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") +# competing_node.query("DETACH DATABASE table_structure") +# +# main_node.query( +# "ALTER TABLE table_structure.rmt ADD COLUMN m int", settings=settings +# ) +# main_node.query( +# "ALTER TABLE table_structure.rmt COMMENT COLUMN v 'version'", settings=settings +# ) +# main_node.query("INSERT INTO table_structure.rmt VALUES (1, 2, 3)") +# +# command = "rm -f /var/lib/clickhouse/metadata/table_structure/mem.sql" +# competing_node.exec_in_container(["bash", "-c", command]) +# competing_node.restart_clickhouse(kill=True) +# +# dummy_node.query("ATTACH DATABASE table_structure") +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") +# dummy_node.query("SYSTEM SYNC REPLICA table_structure.rmt") +# assert "1\t2\t3\n" == dummy_node.query("SELECT * FROM table_structure.rmt") +# +# competing_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") +# competing_node.query("SYSTEM SYNC REPLICA table_structure.rmt") +# # time.sleep(600) +# assert "mem" in competing_node.query("SHOW TABLES FROM table_structure") +# assert "1\t2\t3\n" == competing_node.query("SELECT * FROM table_structure.rmt") +# +# main_node.query("ALTER TABLE table_structure.rmt ADD COLUMN k int") +# main_node.query("INSERT INTO table_structure.rmt VALUES (1, 2, 3, 4)") +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") +# dummy_node.query("SYSTEM SYNC REPLICA table_structure.rmt") +# assert "1\t2\t3\t0\n1\t2\t3\t4\n" == dummy_node.query( +# "SELECT * FROM table_structure.rmt ORDER BY k" +# ) +# +# +# def test_modify_comment(started_cluster): +# main_node.query( +# "CREATE DATABASE modify_comment_db ENGINE = Replicated('/test/modify_comment', 'shard1', 'replica' || '1');" +# ) +# +# dummy_node.query( +# "CREATE DATABASE modify_comment_db ENGINE = Replicated('/test/modify_comment', 'shard1', 'replica' || '2');" +# ) +# +# main_node.query( +# "CREATE TABLE modify_comment_db.modify_comment_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);" +# ) +# +# def restart_verify_not_readonly(): +# main_node.restart_clickhouse() +# assert ( +# main_node.query( +# "SELECT is_readonly FROM system.replicas WHERE table = 'modify_comment_table'" +# ) +# == "0\n" +# ) +# dummy_node.restart_clickhouse() +# assert ( +# dummy_node.query( +# "SELECT is_readonly FROM system.replicas WHERE table = 'modify_comment_table'" +# ) +# == "0\n" +# ) +# +# main_node.query( +# "ALTER TABLE modify_comment_db.modify_comment_table COMMENT COLUMN d 'Some comment'" +# ) +# +# restart_verify_not_readonly() +# +# main_node.query( +# "ALTER TABLE modify_comment_db.modify_comment_table MODIFY COMMENT 'Some error comment'" +# ) +# +# restart_verify_not_readonly() +# +# main_node.query("DROP DATABASE modify_comment_db SYNC") +# dummy_node.query("DROP DATABASE modify_comment_db SYNC") +# +# +# def test_table_metadata_corruption(started_cluster): +# main_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") +# dummy_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") +# +# main_node.query( +# "CREATE DATABASE table_metadata_corruption ENGINE = Replicated('/clickhouse/databases/table_metadata_corruption', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE table_metadata_corruption ENGINE = Replicated('/clickhouse/databases/table_metadata_corruption', 'shard1', 'replica2');" +# ) +# +# create_some_tables("table_metadata_corruption") +# +# main_node.query("SYSTEM SYNC DATABASE REPLICA table_metadata_corruption") +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_metadata_corruption") +# +# # Server should handle this by throwing an exception during table loading, which should lead to server shutdown +# corrupt = "sed --follow-symlinks -i 's/ReplicatedMergeTree/CorruptedMergeTree/' /var/lib/clickhouse/metadata/table_metadata_corruption/rmt1.sql" +# +# print(f"Corrupting metadata using `{corrupt}`") +# dummy_node.stop_clickhouse(kill=True) +# dummy_node.exec_in_container(["bash", "-c", corrupt]) +# +# query = ( +# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='table_metadata_corruption' AND name NOT LIKE '.inner_id.%' " +# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" +# ) +# expected = main_node.query(query) +# +# # We expect clickhouse server to shutdown without LOGICAL_ERRORs or deadlocks +# dummy_node.start_clickhouse(expected_to_fail=True) +# assert not dummy_node.contains_in_log("LOGICAL_ERROR") +# +# fix_corrupt = "sed --follow-symlinks -i 's/CorruptedMergeTree/ReplicatedMergeTree/' /var/lib/clickhouse/metadata/table_metadata_corruption/rmt1.sql" +# print(f"Fix corrupted metadata using `{fix_corrupt}`") +# dummy_node.exec_in_container(["bash", "-c", fix_corrupt]) +# +# dummy_node.start_clickhouse() +# assert_eq_with_retry(dummy_node, query, expected) +# +# main_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") +# dummy_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") diff --git a/tests/integration/test_refreshable_mat_view/test_schedule_model.py b/tests/integration/test_refreshable_mat_view/test_schedule_model.py new file mode 100644 index 00000000000..ac15333cd11 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/test_schedule_model.py @@ -0,0 +1,90 @@ +from datetime import datetime + +from test_refreshable_mat_view.schedule_model import get_next_refresh_time + + +def test_refresh_schedules(): + time_ = datetime(2000, 1, 1, 1, 1, 1) + + assert get_next_refresh_time( + "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", time_ + ) == datetime(2000, 1, 1, 1, 1, 1) + + assert get_next_refresh_time("EVERY 1 SECOND", time_) == datetime( + 2000, 1, 1, 1, 1, 2 + ) + assert get_next_refresh_time("EVERY 1 MINUTE", time_) == datetime( + 2000, + 1, + 1, + 1, + 2, + ) + assert get_next_refresh_time("EVERY 1 HOUR", time_) == datetime( + 2000, + 1, + 1, + 2, + ) + assert get_next_refresh_time("EVERY 1 DAY", time_) == datetime(2000, 1, 2) + assert get_next_refresh_time("EVERY 1 WEEK", time_) == datetime(2000, 1, 10) + assert get_next_refresh_time("EVERY 2 WEEK", time_) == datetime(2000, 1, 17) + assert get_next_refresh_time("EVERY 1 MONTH", time_) == datetime(2000, 2, 1) + assert get_next_refresh_time("EVERY 1 YEAR", time_) == datetime(2001, 1, 1) + + assert get_next_refresh_time("EVERY 3 YEAR 4 MONTH 10 DAY", time_) == datetime( + 2003, 5, 11 + ) + + # OFFSET + assert get_next_refresh_time( + "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", time_ + ) == datetime(2000, 2, 6, 2, 30, 15) + assert get_next_refresh_time( + "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", time_ + ) == datetime(2001, 3, 6, 2, 30, 15) + + assert get_next_refresh_time( + "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", time_ + ) == datetime(2000, 1, 22, 15, 10) + + # AFTER + assert get_next_refresh_time("AFTER 30 SECOND", time_) == datetime( + 2000, 1, 1, 1, 1, 31 + ) + assert get_next_refresh_time("AFTER 30 MINUTE", time_) == datetime( + 2000, 1, 1, 1, 31, 1 + ) + assert get_next_refresh_time("AFTER 2 HOUR", time_) == datetime(2000, 1, 1, 3, 1, 1) + assert get_next_refresh_time("AFTER 2 DAY", time_) == datetime(2000, 1, 3, 1, 1, 1) + assert get_next_refresh_time("AFTER 2 WEEK", time_) == datetime( + 2000, 1, 15, 1, 1, 1 + ) + assert get_next_refresh_time("AFTER 2 MONTH", time_) == datetime( + 2000, 3, 1, 1, 1, 1 + ) + assert get_next_refresh_time("AFTER 2 YEAR", time_) == datetime(2002, 1, 1, 1, 1, 1) + + assert get_next_refresh_time("AFTER 2 YEAR 1 MONTH", time_) == datetime( + 2002, 2, 1, 1, 1, 1 + ) + + assert get_next_refresh_time("AFTER 1 MONTH 2 YEAR", time_) == datetime( + 2002, 2, 1, 1, 1, 1 + ) + + # RANDOMIZE + next_refresh = get_next_refresh_time( + "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", time_ + ) + + assert next_refresh == (datetime(2000, 1, 2, 2, 0), datetime(2000, 1, 2, 3, 0)) + + next_refresh = get_next_refresh_time( + "EVERY 2 MONTH 3 DAY 5 HOUR OFFSET 3 HOUR 20 SECOND RANDOMIZE FOR 3 DAY 1 HOUR", + time_, + ) + assert next_refresh == ( + datetime(2000, 3, 4, 11, 0, 20), + datetime(2000, 3, 7, 12, 0, 20), + ) diff --git a/tests/integration/test_refreshable_mat_view_db_replicated/__init__.py b/tests/integration/test_refreshable_mat_view_db_replicated/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml b/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml new file mode 100644 index 00000000000..7c0e60a044e --- /dev/null +++ b/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml @@ -0,0 +1,16 @@ + + + + 1 + 1 + 1 + 0 + 0 + + + + + default + + + From 92a9199c7635e89d02b607a3ac69d9e18bb9b870 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Thu, 1 Aug 2024 11:57:36 +0200 Subject: [PATCH 442/502] Tests: add RMV tests --- .../configs/config.xml | 1 + .../configs/remote_servers.xml | 11 - .../schedule_model.py | 66 +- .../test_refreshable_mat_view/test.py | 1721 ++--------------- .../test_schedule_model.py | 133 +- 5 files changed, 312 insertions(+), 1620 deletions(-) diff --git a/tests/integration/test_refreshable_mat_view/configs/config.xml b/tests/integration/test_refreshable_mat_view/configs/config.xml index 3cbf717bb67..fdaad39175f 100644 --- a/tests/integration/test_refreshable_mat_view/configs/config.xml +++ b/tests/integration/test_refreshable_mat_view/configs/config.xml @@ -1,2 +1,3 @@ + Etc/UTC diff --git a/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml index 6504141033a..cb35cec266d 100644 --- a/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml +++ b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml @@ -10,17 +10,6 @@ node1_2 9000 - - - - - node2_1 - 9000 - - - node2_2 - 9000 - diff --git a/tests/integration/test_refreshable_mat_view/schedule_model.py b/tests/integration/test_refreshable_mat_view/schedule_model.py index 857a1285076..5195df3dec6 100644 --- a/tests/integration/test_refreshable_mat_view/schedule_model.py +++ b/tests/integration/test_refreshable_mat_view/schedule_model.py @@ -1,6 +1,6 @@ from datetime import datetime, timedelta -from dateutil.relativedelta import relativedelta +import dateutil.relativedelta as rd """ It is the model to test the scheduling of refreshable mat view @@ -8,26 +8,29 @@ It is the model to test the scheduling of refreshable mat view def relative_offset(unit, value): - if unit == "SECONDS": - return relativedelta(seconds=value) + if unit == "SECOND": + return rd.relativedelta(seconds=value) elif unit == "MINUTE": - return relativedelta(minutes=value) + return rd.relativedelta(minutes=value) elif unit == "HOUR": - return relativedelta(hours=value) + return rd.relativedelta(hours=value) elif unit == "DAY": - return relativedelta(days=value) + return rd.relativedelta(days=value) + # elif unit == "WEEK": + # return rd.relativedelta(days=7 * value) elif unit == "WEEK": - return relativedelta(days=7 * value) - elif unit == "MONTH": - return relativedelta(months=value) - elif unit == "YEAR": - return relativedelta(years=value) + return rd.relativedelta(weeks=7 * value) - raise Exception("Cant parse unit: {}".format(unit)) + elif unit == "MONTH": + return rd.relativedelta(months=value) + elif unit == "YEAR": + return rd.relativedelta(years=value) + + raise Exception("Can't parse unit: {}".format(unit)) def group_and_sort(parts, reverse=False): - order = ["YEAR", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECONDS"] + order = ["YEAR", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECOND"] grouped_parts = [] for i in range(0, len(parts), 2): @@ -41,7 +44,6 @@ def group_and_sort(parts, reverse=False): def get_next_refresh_time(schedule, current_time: datetime): parts = schedule.split() - randomize_offset = timedelta() if "RANDOMIZE" in parts: @@ -69,37 +71,36 @@ def get_next_refresh_time(schedule, current_time: datetime): value = int(part[0]) unit = part[1] - if unit == "SECONDS": - current_time = current_time.replace(microsecond=0) + relativedelta( + if unit == "SECOND": + current_time = current_time.replace(microsecond=0) + rd.relativedelta( seconds=value ) elif unit == "MINUTE": current_time = current_time.replace( second=0, microsecond=0 - ) + relativedelta(minutes=value) + ) + rd.relativedelta(minutes=value) elif unit == "HOUR": current_time = current_time.replace( minute=0, second=0, microsecond=0 - ) + relativedelta(hours=value) + ) + rd.relativedelta(hours=value) elif unit == "DAY": current_time = current_time.replace( hour=0, minute=0, second=0, microsecond=0 - ) + relativedelta(days=value) + ) + rd.relativedelta(days=value) elif unit == "WEEK": current_time = current_time.replace( hour=0, minute=0, second=0, microsecond=0 - ) + relativedelta(weekday=0, weeks=value) + ) + rd.relativedelta(weekday=0, weeks=value) elif unit == "MONTH": current_time = current_time.replace( day=1, hour=0, minute=0, second=0, microsecond=0 - ) + relativedelta(months=value) + ) + rd.relativedelta(months=value) elif unit == "YEAR": current_time = current_time.replace( month=1, day=1, hour=0, minute=0, second=0, microsecond=0 - ) + relativedelta(years=value) + ) + rd.relativedelta(years=value) current_time += offset - if randomize_offset: half_offset = (current_time + randomize_offset - current_time) / 2 return ( @@ -111,31 +112,32 @@ def get_next_refresh_time(schedule, current_time: datetime): elif parts[0] == "AFTER": parts = group_and_sort(parts[1:], reverse=True) - interval = relativedelta() + interval = rd.relativedelta() for part in parts: value = int(part[0]) unit = part[1] - if unit == "SECONDS": - interval += relativedelta(seconds=value) + if unit == "SECOND": + interval += rd.relativedelta(seconds=value) elif unit == "MINUTE": - interval += relativedelta(minutes=value) + interval += rd.relativedelta(minutes=value) elif unit == "HOUR": - interval += relativedelta(hours=value) + interval += rd.relativedelta(hours=value) elif unit == "DAY": - interval += relativedelta(days=value) + interval += rd.relativedelta(days=value) elif unit == "WEEK": - interval += relativedelta(weeks=value) + interval += rd.relativedelta(weeks=value) elif unit == "MONTH": - interval += relativedelta(months=value) + interval += rd.relativedelta(months=value) elif unit == "YEAR": - interval += relativedelta(years=value) + interval += rd.relativedelta(years=value) current_time += interval if randomize_offset: half_offset = (current_time + randomize_offset - current_time) / 2 return ( current_time - half_offset, + # current_time, current_time + half_offset, ) diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py index 48c29072134..95d15fe369d 100644 --- a/tests/integration/test_refreshable_mat_view/test.py +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -19,8 +19,6 @@ from helpers.test_tools import assert_eq_with_retry, assert_logs_contain from helpers.network import PartitionManager from test_refreshable_mat_view.schedule_model import get_next_refresh_time -test_recover_staled_replica_run = 1 - cluster = ClickHouseCluster(__file__) node1_1 = cluster.add_instance( @@ -39,36 +37,13 @@ node1_2 = cluster.add_instance( stay_alive=True, macros={"shard": 1, "replica": 2}, ) -node2_1 = cluster.add_instance( - "node2_1", - main_configs=["configs/remote_servers.xml"], - user_configs=["configs/settings.xml"], - with_zookeeper=True, - stay_alive=True, - macros={"shard": 2, "replica": 1}, -) - -node2_2 = cluster.add_instance( - "node2_2", - main_configs=["configs/remote_servers.xml"], - user_configs=["configs/settings.xml"], - with_zookeeper=True, - stay_alive=True, - macros={"shard": 2, "replica": 2}, -) - -# all_nodes = [ -# main_node, -# dummy_node, -# competing_node, -# ] uuid_regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") def assert_create_query(nodes, table_name, expected): replace_uuid = lambda x: re.sub(uuid_regex, "uuid", x) - query = "show create table {}".format(table_name) + query = "SHOW CREATE TABLE {}".format(table_name) for node in nodes: assert_eq_with_retry(node, query, expected, get_result=replace_uuid) @@ -466,7 +441,7 @@ database (String) — The name of the database the table is in. view (String) — Table name. status (String) — Current state of the refresh. last_success_time (Nullable(DateTime)) — Time when the latest successful refresh started. NULL if no successful refreshes happened since server startup or table creation. -last_success_duration_ms (Nullable(UInt64)) — How long the latest refresh took. +last_success_time (Nullable(UInt64)) — How long the latest refresh took. last_refresh_time (Nullable(DateTime)) — Time when the latest refresh attempt finished (if known) or started (if unknown or still running). NULL if no refresh attempts happened since server startup or table creation. last_refresh_replica (String) — If coordination is enabled, name of the replica that made the current (if running) or previous (if not running) refresh attempt. next_refresh_time (Nullable(DateTime)) — Time at which the next refresh is scheduled to start, if status = Scheduled. @@ -623,6 +598,7 @@ ALTER_RMV_TEMPLATE = sql_template( {% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} MODIFY REFRESH {{ refresh_interval }} {% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} +{% if with_append %}APPEND{% endif %} {% if settings %}SETTINGS {{ settings|format_settings }}{% endif %}""" ) @@ -634,7 +610,6 @@ MODIFY REFRESH {{ refresh_interval }} @pytest.mark.parametrize( "depends_on", [None, ["dummy_rmv"], ["default.dummy_rmv", "src1"]] ) -@pytest.mark.parametrize("randomize", [True, False]) @pytest.mark.parametrize("empty", [True, False]) @pytest.mark.parametrize("database_name", [None, "default", "test_db"]) def test_correct_states( @@ -645,7 +620,6 @@ def test_correct_states( if_not_exists, on_cluster, depends_on, - randomize, empty, database_name, ): @@ -663,7 +637,7 @@ def test_correct_states( table_name="test_rmv", if_not_exists=if_not_exists, db_name=database_name, - refresh_interval="EVERY 1 HOUR" + " 30 MINUTE" if randomize else None, + refresh_interval="EVERY 1 HOUR", depends_on=depends_on, to_clause="tgt1", select_query="SELECT * FROM src1", @@ -691,36 +665,39 @@ def test_correct_states( show_create = node1_1.query("SHOW CREATE test_rmv") + # Alter of RMV replaces all non-specified alter_sql = ALTER_RMV_TEMPLATE.render( table_name="test_rmv", if_not_exists=if_not_exists, db_name=database_name, - refresh_interval="EVERY 1 HOUR" + " 30 MINUTE" if randomize else None, + refresh_interval="EVERY 1 HOUR", depends_on=depends_on, select_query="SELECT * FROM src1", + with_append=with_append, on_cluster="test_cluster" if on_cluster else None, # settings={'setting1':'value1', 'setting2': 'value2'}, ) node1_1.query(alter_sql) - show_create_after = node1_1.query("SHOW CREATE test_rmv") + show_create_after_alter = node1_1.query("SHOW CREATE test_rmv") compare_create_all_nodes() - assert show_create == show_create_after + assert show_create == show_create_after_alter # breakpoint() - pass - def compare_dates( date1: str | datetime, - date2: str | datetime, - inaccuracy=timedelta(minutes=10), + date2: str | datetime | tuple[datetime], + inaccuracy=timedelta(hours=1), format_str="%Y-%m-%d %H:%M:%S", ) -> bool: """ Compares two dates with an inaccuracy of 2 minutes. """ + if isinstance(date2, tuple): + return date2[0] <= date1 <= date2[1] + if isinstance(date1, str): date1 = datetime.strptime(date1, format_str) if isinstance(date2, str): @@ -739,12 +716,17 @@ def date_in_interval( def get_rmv_info(node, table): - return node.query_with_retry( + rmv_info = node.query_with_retry( f"SELECT * FROM system.view_refreshes WHERE view='{table}'", check_callback=lambda r: r.iloc[0]["status"] == "Scheduled", parse=True, ).to_dict("records")[0] + rmv_info["next_refresh_time"] = parse_ch_datetime(rmv_info["next_refresh_time"]) + + return rmv_info + + @pytest.mark.parametrize("with_append", [True, False]) @pytest.mark.parametrize("create_target_table", [True, False]) @@ -822,14 +804,14 @@ def test_check_data( assert compare_dates( rmv["last_refresh_time"], now, format_str="%Y-%m-%d %H:%M:%S" ) - assert rmv["last_success_duration_ms"] > 0 + assert rmv["last_success_time"] > 0 inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) print(inserted_data) assert len(inserted_data) == 2 if empty: assert rmv["last_refresh_time"] is None - assert rmv["last_success_duration_ms"] is None + assert rmv["last_success_time"] is None assert rmv["retry"] == 0 assert rmv["read_rows"] == 0 @@ -842,11 +824,11 @@ def test_check_data( assert rmv["result_bytes"] == 0 # Rewind to the next trigger and wait for it - # node1_1.query("SYSTEM TEST VIEW test_rmv set fake time '2050-01-01 00:00:01';") - # system test view a set fake time '2050-01-01 00:00:01'; + node1_1.query( + f"SYSTEM TEST VIEW test_rmv set fake time '{rmv['next_refresh_time']}';" + ) now = datetime.utcnow() - inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) if with_append: assert len(inserted_data) == 4 @@ -855,8 +837,6 @@ def test_check_data( # alter time - breakpoint() - """ @@ -905,69 +885,29 @@ def parse_ch_datetime(date_str): return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") -INTERVALS = [ +INTERVALS_EVERY = [ # Same units - # "EVERY 1 YEAR 2 YEAR", - # "AFTER 1 YEAR 2 YEAR" + "EVERY 1 YEAR 2 YEAR", + # "EVERY 1 MINUTE", "EVERY 1 HOUR", "EVERY 1 DAY", + "EVERY 1 WEEK", + "EVERY 2 WEEK", "EVERY 1 MONTH", "EVERY 1 YEAR", - "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECONDS", + "EVERY 1 WEEK RANDOMIZE FOR 1 HOUR", + "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", + "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", + "EVERY 1 MONTH OFFSET 1 DAY RANDOMIZE FOR 10 HOUR", # smth wrong with # "EVERY 1 WEEK", # "EVERY 2 WEEK", # "EVERY 3 WEEK", # "EVERY 4 WEEK", # "EVERY 5 WEEK", - # "EVERY 6 WEEK", - # "EVERY 7 WEEK", - # "EVERY 8 WEEK", - # "EVERY 9 WEEK", - # "EVERY 10 WEEK", - # "EVERY 11 WEEK", - # "EVERY 12 WEEK", - # "EVERY 13 WEEK", - # "EVERY 14 WEEK", - # "EVERY 15 WEEK", - # "EVERY 20 WEEK", - # "EVERY 21 WEEK", - # "EVERY 31 WEEK", - # "EVERY 32 WEEK", - # "EVERY 33 WEEK", - # "EVERY 50 WEEK", - # "EVERY 51 WEEK", - # "EVERY 52 WEEK", - # "EVERY 59 WEEK", - # - "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", - "EVERY 1 MONTH OFFSET 1 WEEK", - "EVERY 1 MONTH OFFSET 2 WEEK", - # - "AFTER 30 SECONDS", - "AFTER 30 MINUTE", - "AFTER 2 HOUR", - "AFTER 2 DAY", - "AFTER 2 WEEK", - "AFTER 2 MONTH", - "AFTER 2 YEAR", - "AFTER 2 MONTH", - "AFTER 2 MONTH 3 DAY", - "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", - "AFTER 1 YEAR RANDOMIZE FOR 11 MONTH", - # Randomize bigger than interval - "AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", - "EVERY 1 MINUTE RANDOMIZE FOR 3 YEAR", - "EVERY 1 MINUTE OFFSET 1 SECONDS RANDOMIZE FOR 3 YEAR", - # - "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", - "EVERY 1 YEAR OFFSET 11 MONTH RANDOMIZE FOR 1 YEAR", - "EVERY 1 YEAR OFFSET 11 MONTH RANDOMIZE FOR 1 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", - # - "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS RANDOMIZE FOR 2 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", - # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS OFFSET 11 MONTH RANDOMIZE FOR 2 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", - # + # "EVERY 1 MONTH OFFSET 1 WEEK", + # "EVERY 1 MONTH OFFSET 2 WEEK", # Two different units in EVERY # "EVERY 1 YEAR", # "EVERY 1 YEAR 0 MONTH", @@ -990,7 +930,26 @@ INTERVALS = [ # "EVERY 1 DAY 30 HOUR", # "EVERY 1 DAY 31 HOUR", # "EVERY 1 DAY 32 HOUR", - # + # Interval shouldn't contain both calendar units and clock units (e.g. months and days) + # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", +] + +INTERVALS_AFTER = [ + "AFTER 1 YEAR 2 YEAR", + "AFTER 30 SECOND", + "AFTER 30 MINUTE", + "AFTER 2 HOUR", + "AFTER 2 DAY", + "AFTER 2 WEEK", + "AFTER 2 MONTH", + "AFTER 2 YEAR", + "AFTER 2 MONTH 3 DAY", + "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", + "AFTER 1 YEAR RANDOMIZE FOR 11 MONTH", + # Randomize bigger than interval + "AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", + "EVERY 1 MINUTE RANDOMIZE FOR 3 YEAR", + "EVERY 1 MINUTE OFFSET 1 SECOND RANDOMIZE FOR 3 YEAR", "AFTER 1 MONTH", "AFTER 1 MONTH 0 DAY", "AFTER 1 MONTH 1 DAY", @@ -999,15 +958,131 @@ INTERVALS = [ "AFTER 1 YEAR 10 MONTH", "AFTER 1 YEAR 10 MONTH 3 DAY", "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR", - "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS", - # Interval shouldn't contain both calendar units and clock units (e.g. months and days) - # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS", + "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", ] @pytest.mark.parametrize( "interval", - INTERVALS, + INTERVALS_EVERY, +) +@pytest.mark.parametrize( + "append", + [ + # True, + False + ], +) +@pytest.mark.parametrize( + "empty", + [ + True, + # False + ], +) +def test_schedule_2( + request, + started_cluster, + interval, + append, + empty, +): + """ + - Create RMV + - Check table view_refreshes + - Check inserted data if without EMPTY + - Set time, wait for refresh + - Check data is inserted/appended + - Alter table + - Check everything again + - DROP target table + """ + + # if "AFTER" in interval: + # pytest.skip() + + def teardown(): + node1_1.query("DROP TABLE IF EXISTS test_rmv_schedule") + node1_1.query("DROP TABLE IF EXISTS tgt_new") + node1_1.query("TRUNCATE TABLE tgt1") + + request.addfinalizer(teardown) + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv_schedule", + refresh_interval=interval, + table_clause="ENGINE = Memory", + select_query="SELECT now() as a, number as b FROM numbers(2)", + with_append=append, + empty=empty, + ) + print(create_sql) + node1_1.query(create_sql) + + now = datetime.utcnow() + + rmv = get_rmv_info(node1_1, "test_rmv_schedule") + # this is for EVERY + predicted_next_refresh_time = get_next_refresh_time(interval, now) + # this is for AFTER + # diff = next_refresh_time - now.replace(microsecond=0) + + if 'WEEK' in + assert compare_dates(rmv["next_refresh_time"], predicted_next_refresh_time) + assert rmv["next_refresh_time"] > now + + def expect_rows(rows): + inserted_data = node1_1.query_with_retry( + f"SELECT * FROM test_rmv_schedule", + parse=True, + check_callback=lambda x: len(x) == rows, + retry_count=100, + ) + assert len(inserted_data) == rows + + # Check data if not EMPTY + append_expect_rows = 0 + if append: + # Append adds rows + append_expect_rows += 2 + expect_rows(append_expect_rows) + if not append: + # Rewrite without append + expect_rows(2) + + inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) + if empty: + # No data is inserted with empty + assert len(inserted_data) == 0 + else: + # Query is executed without empty + assert rmv["last_success_time"] > 0 + expect_rows(2) + + # Trigger refresh, update `next_refresh_time` and check interval again + node1_1.query( + f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{rmv['next_refresh_time']}'" + ) + predicted_next_refresh_time = get_next_refresh_time(interval, rmv['next_refresh_time']) + rmv = get_rmv_info(node1_1, "test_rmv_schedule") + + assert compare_dates(rmv["next_refresh_time"], predicted_next_refresh_time) + + # Check data if not EMPTY + if append: + # Append adds rows + append_expect_rows += 2 + expect_rows(append_expect_rows) + else: + # Rewrite + expect_rows(2) + + # breakpoint() + + +@pytest.mark.parametrize( + "interval", + INTERVALS_EVERY, ) @pytest.mark.parametrize( "append", @@ -1041,13 +1116,13 @@ def test_schedule( - DROP target table """ - if "WEEK" in interval: - pytest.skip() + # if "WEEK" in interval: + # pytest.skip() def teardown(): node1_1.query("DROP TABLE IF EXISTS test_rmv_schedule") - # node1_1.query("DROP TABLE IF EXISTS tgt_new ON CLUSTER test_cluster") - # node1_1.query("TRUNCATE TABLE tgt1 ON CLUSTER test_cluster") + node1_1.query("DROP TABLE IF EXISTS tgt_new") + node1_1.query("TRUNCATE TABLE tgt1") request.addfinalizer(teardown) @@ -1090,41 +1165,41 @@ def test_schedule( assert next_refresh_time > now - append_expect_rows = 0 - - def check_data(): - inserted_data = node1_1.query_with_retry( - f"SELECT * FROM test_rmv_schedule", - parse=True, - check_callback=lambda x: len(x) > 0, - retry_count=200, - ) - - # Check data if not EMPTY - if append: - # Append adds rows - global append_expect_rows - append_expect_rows += 2 - assert len(inserted_data) == append_expect_rows - if not append: - # Rewrite without append - assert len(inserted_data) == 2 - - inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) - if empty: - assert len(inserted_data) == 0 - else: - assert rmv["last_success_duration_ms"] > 0 - check_data() - - # Trigger next refresh - node1_1.query( - f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time}'" - ) - - if "RANDOMIZE" not in interval: - check_data() - + # append_expect_rows = 0 + # + # def check_data(): + # inserted_data = node1_1.query_with_retry( + # f"SELECT * FROM test_rmv_schedule", + # parse=True, + # check_callback=lambda x: len(x) > 0, + # retry_count=200, + # ) + # + # # Check data if not EMPTY + # if append: + # # Append adds rows + # global append_expect_rows + # append_expect_rows += 2 + # assert len(inserted_data) == append_expect_rows + # if not append: + # # Rewrite without append + # assert len(inserted_data) == 2 + # + # inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) + # if empty: + # assert len(inserted_data) == 0 + # else: + # assert rmv["last_success_time"] > 0 + # check_data() + # + # # Trigger next refresh + # node1_1.query( + # f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time}'" + # ) + # + # if "RANDOMIZE" not in interval: + # check_data() + # ----------------------------- # rmv = get_rmv_info(node1_1, "test_rmv_schedule") # next_refresh_time = parse_ch_datetime(rmv["next_refresh_time"]) @@ -1162,1369 +1237,3 @@ def test_schedule( # check_data() # breakpoint() - - -# def test_create_replicated_table(started_cluster): -# main_node.query( -# "CREATE DATABASE create_replicated_table ENGINE = Replicated('/test/create_replicated_table', 'shard1', 'replica' || '1');" -# ) -# dummy_node.query( -# "CREATE DATABASE create_replicated_table ENGINE = Replicated('/test/create_replicated_table', 'shard1', 'replica2');" -# ) -# assert ( -# "Explicit zookeeper_path and replica_name are specified" -# in main_node.query_and_get_error( -# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " -# "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" -# ) -# ) -# -# assert ( -# "Explicit zookeeper_path and replica_name are specified" -# in main_node.query_and_get_error( -# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " -# "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" -# ) -# ) -# -# assert ( -# "This syntax for *MergeTree engine is deprecated" -# in main_node.query_and_get_error( -# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " -# "ENGINE=ReplicatedMergeTree('/test/tmp/{shard}', '{replica}', d, k, 8192);" -# ) -# ) -# -# main_node.query( -# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);" -# ) -# -# expected = ( -# "CREATE TABLE create_replicated_table.replicated_table\\n(\\n `d` Date,\\n `k` UInt64,\\n `i32` Int32\\n)\\n" -# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\n" -# "PARTITION BY toYYYYMM(d)\\nORDER BY k\\nSETTINGS index_granularity = 8192" -# ) -# assert_create_query( -# [main_node, dummy_node], "create_replicated_table.replicated_table", expected -# ) -# # assert without replacing uuid -# assert main_node.query( -# "show create create_replicated_table.replicated_table" -# ) == dummy_node.query("show create create_replicated_table.replicated_table") -# main_node.query("DROP DATABASE create_replicated_table SYNC") -# dummy_node.query("DROP DATABASE create_replicated_table SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_simple_alter_table(started_cluster, engine): -# database = f"test_simple_alter_table_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# # test_simple_alter_table -# name = f"{database}.alter_test" -# main_node.query( -# "CREATE TABLE {} " -# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " -# "ENGINE = {} PARTITION BY StartDate ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);".format( -# name, engine -# ) -# ) -# main_node.query("ALTER TABLE {} ADD COLUMN Added0 UInt32;".format(name)) -# main_node.query("ALTER TABLE {} ADD COLUMN Added2 UInt32;".format(name)) -# main_node.query( -# "ALTER TABLE {} ADD COLUMN Added1 UInt32 AFTER Added0;".format(name) -# ) -# main_node.query( -# "ALTER TABLE {} ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;".format( -# name -# ) -# ) -# main_node.query( -# "ALTER TABLE {} ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;".format( -# name -# ) -# ) -# main_node.query( -# "ALTER TABLE {} ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;".format( -# name -# ) -# ) -# -# full_engine = ( -# engine -# if not "Replicated" in engine -# else engine + "(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')" -# ) -# expected = ( -# "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" -# " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n" -# " `AddedNested1.A` Array(UInt32),\\n `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n" -# " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64)\\n)\\n" -# "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" -# "SETTINGS index_granularity = 8192".format(name, full_engine) -# ) -# -# assert_create_query([main_node, dummy_node], name, expected) -# -# # test_create_replica_after_delay -# competing_node.query( -# f"CREATE DATABASE IF NOT EXISTS {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica3');" -# ) -# -# main_node.query("ALTER TABLE {} ADD COLUMN Added3 UInt32;".format(name)) -# main_node.query("ALTER TABLE {} DROP COLUMN AddedNested1;".format(name)) -# main_node.query("ALTER TABLE {} RENAME COLUMN Added1 TO AddedNested1;".format(name)) -# -# full_engine = ( -# engine -# if not "Replicated" in engine -# else engine + "(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')" -# ) -# expected = ( -# "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" -# " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `AddedNested1` UInt32,\\n `Added2` UInt32,\\n" -# " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64),\\n `Added3` UInt32\\n)\\n" -# "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" -# "SETTINGS index_granularity = 8192".format(name, full_engine) -# ) -# -# assert_create_query([main_node, dummy_node, competing_node], name, expected) -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# competing_node.query(f"DROP DATABASE {database} SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_delete_from_table(started_cluster, engine): -# database = f"delete_from_table_{engine}" -# -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard2', 'replica1');" -# ) -# -# name = f"{database}.delete_test" -# main_node.query( -# "CREATE TABLE {} " -# "(id UInt64, value String) " -# "ENGINE = {} PARTITION BY id%2 ORDER BY (id);".format(name, engine) -# ) -# main_node.query("INSERT INTO TABLE {} VALUES(1, 'aaaa');".format(name)) -# main_node.query("INSERT INTO TABLE {} VALUES(2, 'aaaa');".format(name)) -# dummy_node.query("INSERT INTO TABLE {} VALUES(1, 'bbbb');".format(name)) -# dummy_node.query("INSERT INTO TABLE {} VALUES(2, 'bbbb');".format(name)) -# -# main_node.query("DELETE FROM {} WHERE id=2;".format(name)) -# -# expected = "1\taaaa\n1\tbbbb" -# -# table_for_select = name -# if not "Replicated" in engine: -# table_for_select = f"cluster('{database}', {name})" -# for node in [main_node, dummy_node]: -# assert_eq_with_retry( -# node, -# "SELECT * FROM {} ORDER BY id, value;".format(table_for_select), -# expected, -# ) -# -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# -# -# def get_table_uuid(database, name): -# return main_node.query( -# f"SELECT uuid FROM system.tables WHERE database = '{database}' and name = '{name}'" -# ).strip() -# -# -# @pytest.fixture(scope="module", name="attachable_part") -# def fixture_attachable_part(started_cluster): -# main_node.query(f"CREATE DATABASE testdb_attach_atomic ENGINE = Atomic") -# main_node.query( -# f"CREATE TABLE testdb_attach_atomic.test (CounterID UInt32) ENGINE = MergeTree ORDER BY (CounterID)" -# ) -# main_node.query(f"INSERT INTO testdb_attach_atomic.test VALUES (123)") -# main_node.query( -# f"ALTER TABLE testdb_attach_atomic.test FREEZE WITH NAME 'test_attach'" -# ) -# table_uuid = get_table_uuid("testdb_attach_atomic", "test") -# return os.path.join( -# main_node.path, -# f"database/shadow/test_attach/store/{table_uuid[:3]}/{table_uuid}/all_1_1_0", -# ) -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_alter_attach(started_cluster, attachable_part, engine): -# database = f"alter_attach_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# -# main_node.query( -# f"CREATE TABLE {database}.alter_attach_test (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" -# ) -# table_uuid = get_table_uuid(database, "alter_attach_test") -# # Provide and attach a part to the main node -# shutil.copytree( -# attachable_part, -# os.path.join( -# main_node.path, -# f"database/store/{table_uuid[:3]}/{table_uuid}/detached/all_1_1_0", -# ), -# ) -# main_node.query(f"ALTER TABLE {database}.alter_attach_test ATTACH PART 'all_1_1_0'") -# # On the main node, data is attached -# assert ( -# main_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") -# == "123\n" -# ) -# # On the other node, data is replicated only if using a Replicated table engine -# if engine == "ReplicatedMergeTree": -# assert ( -# dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") -# == "123\n" -# ) -# else: -# assert ( -# dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") -# == "" -# ) -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_alter_drop_part(started_cluster, engine): -# database = f"alter_drop_part_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# -# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" -# main_node.query( -# f"CREATE TABLE {database}.alter_drop_part (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" -# ) -# main_node.query(f"INSERT INTO {database}.alter_drop_part VALUES (123)") -# if engine == "MergeTree": -# dummy_node.query(f"INSERT INTO {database}.alter_drop_part VALUES (456)") -# else: -# main_node.query(f"SYSTEM SYNC REPLICA {database}.alter_drop_part PULL") -# main_node.query(f"ALTER TABLE {database}.alter_drop_part DROP PART '{part_name}'") -# assert main_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") == "" -# if engine == "ReplicatedMergeTree": -# # The DROP operation is still replicated at the table engine level -# assert ( -# dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") == "" -# ) -# else: -# assert ( -# dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") -# == "456\n" -# ) -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_alter_detach_part(started_cluster, engine): -# database = f"alter_detach_part_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# -# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" -# main_node.query( -# f"CREATE TABLE {database}.alter_detach (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" -# ) -# main_node.query(f"INSERT INTO {database}.alter_detach VALUES (123)") -# if engine == "MergeTree": -# dummy_node.query(f"INSERT INTO {database}.alter_detach VALUES (456)") -# main_node.query(f"ALTER TABLE {database}.alter_detach DETACH PART '{part_name}'") -# detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='{database}' AND table='alter_detach'" -# assert main_node.query(detached_parts_query) == f"{part_name}\n" -# if engine == "ReplicatedMergeTree": -# # The detach operation is still replicated at the table engine level -# assert dummy_node.query(detached_parts_query) == f"{part_name}\n" -# else: -# assert dummy_node.query(detached_parts_query) == "" -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_alter_drop_detached_part(started_cluster, engine): -# database = f"alter_drop_detached_part_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# -# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" -# main_node.query( -# f"CREATE TABLE {database}.alter_drop_detached (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" -# ) -# main_node.query(f"INSERT INTO {database}.alter_drop_detached VALUES (123)") -# main_node.query( -# f"ALTER TABLE {database}.alter_drop_detached DETACH PART '{part_name}'" -# ) -# if engine == "MergeTree": -# dummy_node.query(f"INSERT INTO {database}.alter_drop_detached VALUES (456)") -# dummy_node.query( -# f"ALTER TABLE {database}.alter_drop_detached DETACH PART '{part_name}'" -# ) -# main_node.query( -# f"ALTER TABLE {database}.alter_drop_detached DROP DETACHED PART '{part_name}'" -# ) -# detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='{database}' AND table='alter_drop_detached'" -# assert main_node.query(detached_parts_query) == "" -# assert dummy_node.query(detached_parts_query) == f"{part_name}\n" -# -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_alter_drop_partition(started_cluster, engine): -# database = f"alter_drop_partition_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# snapshotting_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard2', 'replica1');" -# ) -# -# main_node.query( -# f"CREATE TABLE {database}.alter_drop (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" -# ) -# main_node.query(f"INSERT INTO {database}.alter_drop VALUES (123)") -# if engine == "MergeTree": -# dummy_node.query(f"INSERT INTO {database}.alter_drop VALUES (456)") -# snapshotting_node.query(f"INSERT INTO {database}.alter_drop VALUES (789)") -# main_node.query( -# f"ALTER TABLE {database}.alter_drop ON CLUSTER {database} DROP PARTITION ID 'all'", -# settings={"replication_alter_partitions_sync": 2}, -# ) -# assert ( -# main_node.query( -# f"SELECT CounterID FROM clusterAllReplicas('{database}', {database}.alter_drop)" -# ) -# == "" -# ) -# assert dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop") == "" -# main_node.query(f"DROP DATABASE {database}") -# dummy_node.query(f"DROP DATABASE {database}") -# snapshotting_node.query(f"DROP DATABASE {database}") -# -# -# def test_alter_fetch(started_cluster): -# main_node.query( -# "CREATE DATABASE alter_fetch ENGINE = Replicated('/test/alter_fetch', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE alter_fetch ENGINE = Replicated('/test/alter_fetch', 'shard1', 'replica2');" -# ) -# -# main_node.query( -# "CREATE TABLE alter_fetch.fetch_source (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" -# ) -# main_node.query( -# "CREATE TABLE alter_fetch.fetch_target (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" -# ) -# main_node.query("INSERT INTO alter_fetch.fetch_source VALUES (123)") -# table_uuid = get_table_uuid("alter_fetch", "fetch_source") -# main_node.query( -# f"ALTER TABLE alter_fetch.fetch_target FETCH PART 'all_0_0_0' FROM '/clickhouse/tables/{table_uuid}/{{shard}}' " -# ) -# detached_parts_query = "SELECT name FROM system.detached_parts WHERE database='alter_fetch' AND table='fetch_target'" -# assert main_node.query(detached_parts_query) == "all_0_0_0\n" -# assert dummy_node.query(detached_parts_query) == "" -# -# main_node.query("DROP DATABASE alter_fetch SYNC") -# dummy_node.query("DROP DATABASE alter_fetch SYNC") -# -# -# def test_alters_from_different_replicas(started_cluster): -# main_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica2');" -# ) -# -# # test_alters_from_different_replicas -# competing_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica3');" -# ) -# -# main_node.query( -# "CREATE TABLE alters_from_different_replicas.concurrent_test " -# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " -# "ENGINE = MergeTree PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);" -# ) -# -# main_node.query( -# "CREATE TABLE alters_from_different_replicas.dist AS alters_from_different_replicas.concurrent_test ENGINE = Distributed(alters_from_different_replicas, alters_from_different_replicas, concurrent_test, CounterID)" -# ) -# -# dummy_node.stop_clickhouse(kill=True) -# -# settings = {"distributed_ddl_task_timeout": 5} -# assert "is not finished on 1 of 3 hosts" in competing_node.query_and_get_error( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added0 UInt32;", -# settings=settings, -# ) -# settings = { -# "distributed_ddl_task_timeout": 5, -# "distributed_ddl_output_mode": "null_status_on_timeout", -# } -# assert "shard1\treplica2\tQUEUED\t" in main_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added2 UInt32;", -# settings=settings, -# ) -# settings = { -# "distributed_ddl_task_timeout": 5, -# "distributed_ddl_output_mode": "never_throw", -# } -# assert "shard1\treplica2\tQUEUED\t" in competing_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added1 UInt32 AFTER Added0;", -# settings=settings, -# ) -# dummy_node.start_clickhouse() -# main_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;" -# ) -# competing_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;" -# ) -# main_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;" -# ) -# -# expected = ( -# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32,\\n" -# " `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n `AddedNested1.A` Array(UInt32),\\n" -# " `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n `AddedNested2.A` Array(UInt32),\\n" -# " `AddedNested2.B` Array(UInt64)\\n)\\n" -# "ENGINE = MergeTree\\nPARTITION BY toYYYYMM(StartDate)\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\nSETTINGS index_granularity = 8192" -# ) -# -# assert_create_query( -# [main_node, competing_node], -# "alters_from_different_replicas.concurrent_test", -# expected, -# ) -# -# # test_create_replica_after_delay -# main_node.query("DROP TABLE alters_from_different_replicas.concurrent_test SYNC") -# main_node.query( -# "CREATE TABLE alters_from_different_replicas.concurrent_test " -# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " -# "ENGINE = ReplicatedMergeTree ORDER BY CounterID;" -# ) -# -# expected = ( -# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" -# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" -# ) -# -# assert_create_query( -# [main_node, competing_node], -# "alters_from_different_replicas.concurrent_test", -# expected, -# ) -# -# main_node.query( -# "INSERT INTO alters_from_different_replicas.dist (CounterID, StartDate, UserID) SELECT number, addDays(toDate('2020-02-02'), number), intHash32(number) FROM numbers(10)" -# ) -# -# # test_replica_restart -# main_node.restart_clickhouse() -# -# expected = ( -# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" -# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" -# ) -# -# # test_snapshot_and_snapshot_recover -# snapshotting_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard2', 'replica1');" -# ) -# snapshot_recovering_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard2', 'replica2');" -# ) -# assert_create_query( -# all_nodes, "alters_from_different_replicas.concurrent_test", expected -# ) -# -# main_node.query("SYSTEM FLUSH DISTRIBUTED alters_from_different_replicas.dist") -# main_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test UPDATE StartDate = addYears(StartDate, 1) WHERE 1" -# ) -# res = main_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test DELETE WHERE UserID % 2" -# ) -# assert ( -# "shard1\treplica1\tOK" in res -# and "shard1\treplica2\tOK" in res -# and "shard1\treplica3\tOK" in res -# ) -# assert "shard2\treplica1\tOK" in res and "shard2\treplica2\tOK" in res -# -# expected = ( -# "1\t1\tmain_node\n" -# "1\t2\tdummy_node\n" -# "1\t3\tcompeting_node\n" -# "2\t1\tsnapshotting_node\n" -# "2\t2\tsnapshot_recovering_node\n" -# ) -# assert ( -# main_node.query( -# "SELECT shard_num, replica_num, host_name FROM system.clusters WHERE cluster='alters_from_different_replicas'" -# ) -# == expected -# ) -# -# # test_drop_and_create_replica -# main_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# main_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica1');" -# ) -# -# expected = ( -# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" -# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" -# ) -# -# assert_create_query( -# [main_node, competing_node], -# "alters_from_different_replicas.concurrent_test", -# expected, -# ) -# assert_create_query( -# all_nodes, "alters_from_different_replicas.concurrent_test", expected -# ) -# -# for node in all_nodes: -# node.query("SYSTEM SYNC REPLICA alters_from_different_replicas.concurrent_test") -# -# expected = ( -# "0\t2021-02-02\t4249604106\n" -# "1\t2021-02-03\t1343103100\n" -# "4\t2021-02-06\t3902320246\n" -# "7\t2021-02-09\t3844986530\n" -# "9\t2021-02-11\t1241149650\n" -# ) -# -# assert_eq_with_retry( -# dummy_node, -# "SELECT CounterID, StartDate, UserID FROM alters_from_different_replicas.dist ORDER BY CounterID", -# expected, -# ) -# main_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# dummy_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# competing_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# snapshotting_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# snapshot_recovering_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# -# -# def create_some_tables(db): -# settings = { -# "distributed_ddl_task_timeout": 0, -# "allow_experimental_object_type": 1, -# "allow_suspicious_codecs": 1, -# } -# main_node.query(f"CREATE TABLE {db}.t1 (n int) ENGINE=Memory", settings=settings) -# dummy_node.query( -# f"CREATE TABLE {db}.t2 (s String) ENGINE=Memory", settings=settings -# ) -# main_node.query( -# f"CREATE TABLE {db}.mt1 (n int) ENGINE=MergeTree order by n", -# settings=settings, -# ) -# dummy_node.query( -# f"CREATE TABLE {db}.mt2 (n int) ENGINE=MergeTree order by n", -# settings=settings, -# ) -# main_node.query( -# f"CREATE TABLE {db}.rmt1 (n int) ENGINE=ReplicatedMergeTree order by n", -# settings=settings, -# ) -# dummy_node.query( -# f"CREATE TABLE {db}.rmt2 (n int CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12))) ENGINE=ReplicatedMergeTree order by n", -# settings=settings, -# ) -# main_node.query( -# f"CREATE TABLE {db}.rmt3 (n int, json Object('json') materialized '') ENGINE=ReplicatedMergeTree order by n", -# settings=settings, -# ) -# dummy_node.query( -# f"CREATE TABLE {db}.rmt5 (n int) ENGINE=ReplicatedMergeTree order by n", -# settings=settings, -# ) -# main_node.query( -# f"CREATE MATERIALIZED VIEW {db}.mv1 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt1", -# settings=settings, -# ) -# dummy_node.query( -# f"CREATE MATERIALIZED VIEW {db}.mv2 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt2", -# settings=settings, -# ) -# main_node.query( -# f"CREATE DICTIONARY {db}.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " -# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " -# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" -# ) -# dummy_node.query( -# f"CREATE DICTIONARY {db}.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " -# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt2' PASSWORD '' DB 'recover')) " -# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" -# ) -# -# -# # These tables are used to check that DatabaseReplicated correctly renames all the tables in case when it restores from the lost state -# def create_table_for_exchanges(db): -# settings = {"distributed_ddl_task_timeout": 0} -# for table in ["a1", "a2", "a3", "a4", "a5", "a6"]: -# main_node.query( -# f"CREATE TABLE {db}.{table} (s String) ENGINE=ReplicatedMergeTree order by s", -# settings=settings, -# ) -# -# -# def test_recover_staled_replica(started_cluster): -# main_node.query( -# "CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica1');" -# ) -# started_cluster.get_kazoo_client("zoo1").set( -# "/clickhouse/databases/recover/logs_to_keep", b"10" -# ) -# dummy_node.query( -# "CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica2');" -# ) -# -# settings = {"distributed_ddl_task_timeout": 0} -# create_some_tables("recover") -# create_table_for_exchanges("recover") -# -# for table in ["t1", "t2", "mt1", "mt2", "rmt1", "rmt2", "rmt3", "rmt5"]: -# main_node.query(f"INSERT INTO recover.{table} VALUES (42)") -# for table in ["t1", "t2", "mt1", "mt2"]: -# dummy_node.query(f"INSERT INTO recover.{table} VALUES (42)") -# -# for i, table in enumerate(["a1", "a2", "a3", "a4", "a5", "a6"]): -# main_node.query(f"INSERT INTO recover.{table} VALUES ('{str(i + 1) * 10}')") -# -# for table in ["rmt1", "rmt2", "rmt3", "rmt5"]: -# main_node.query(f"SYSTEM SYNC REPLICA recover.{table}") -# for table in ["a1", "a2", "a3", "a4", "a5", "a6"]: -# main_node.query(f"SYSTEM SYNC REPLICA recover.{table}") -# -# with PartitionManager() as pm: -# pm.drop_instance_zk_connections(dummy_node) -# dummy_node.query_and_get_error("RENAME TABLE recover.t1 TO recover.m1") -# -# main_node.query_with_retry( -# "RENAME TABLE recover.t1 TO recover.m1", settings=settings -# ) -# main_node.query_with_retry( -# "ALTER TABLE recover.mt1 ADD COLUMN m int", settings=settings -# ) -# main_node.query_with_retry( -# "ALTER TABLE recover.rmt1 ADD COLUMN m int", settings=settings -# ) -# main_node.query_with_retry( -# "RENAME TABLE recover.rmt3 TO recover.rmt4", settings=settings -# ) -# main_node.query_with_retry("DROP TABLE recover.rmt5", settings=settings) -# main_node.query_with_retry("DROP DICTIONARY recover.d2", settings=settings) -# main_node.query_with_retry( -# "CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " -# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " -# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());", -# settings=settings, -# ) -# -# inner_table = ( -# ".inner_id." -# + dummy_node.query_with_retry( -# "SELECT uuid FROM system.tables WHERE database='recover' AND name='mv1'" -# ).strip() -# ) -# main_node.query_with_retry( -# f"ALTER TABLE recover.`{inner_table}` MODIFY COLUMN n int DEFAULT 42", -# settings=settings, -# ) -# main_node.query_with_retry( -# "ALTER TABLE recover.mv1 MODIFY QUERY SELECT m as n FROM recover.rmt1", -# settings=settings, -# ) -# main_node.query_with_retry( -# "RENAME TABLE recover.mv2 TO recover.mv3", -# settings=settings, -# ) -# -# main_node.query_with_retry( -# "CREATE TABLE recover.tmp AS recover.m1", settings=settings -# ) -# main_node.query_with_retry("DROP TABLE recover.tmp", settings=settings) -# main_node.query_with_retry( -# "CREATE TABLE recover.tmp AS recover.m1", settings=settings -# ) -# main_node.query_with_retry("DROP TABLE recover.tmp", settings=settings) -# main_node.query_with_retry( -# "CREATE TABLE recover.tmp AS recover.m1", settings=settings -# ) -# -# main_node.query("EXCHANGE TABLES recover.a1 AND recover.a2", settings=settings) -# main_node.query("EXCHANGE TABLES recover.a3 AND recover.a4", settings=settings) -# main_node.query("EXCHANGE TABLES recover.a5 AND recover.a4", settings=settings) -# main_node.query("EXCHANGE TABLES recover.a6 AND recover.a3", settings=settings) -# main_node.query("RENAME TABLE recover.a6 TO recover.a7", settings=settings) -# main_node.query("RENAME TABLE recover.a1 TO recover.a8", settings=settings) -# -# assert ( -# main_node.query( -# "SELECT name FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' ORDER BY name" -# ) -# == "a2\na3\na4\na5\na7\na8\nd1\nd2\nm1\nmt1\nmt2\nmv1\nmv3\nrmt1\nrmt2\nrmt4\nt2\ntmp\n" -# ) -# query = ( -# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' " -# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" -# ) -# expected = main_node.query(query) -# assert_eq_with_retry(dummy_node, query, expected) -# assert ( -# main_node.query( -# "SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'" -# ) -# == "2\n" -# ) -# assert ( -# dummy_node.query( -# "SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'" -# ) -# == "2\n" -# ) -# -# # Check that Database Replicated renamed all the tables correctly -# for i, table in enumerate(["a2", "a8", "a5", "a7", "a4", "a3"]): -# assert ( -# dummy_node.query(f"SELECT * FROM recover.{table}") == f"{str(i + 1) * 10}\n" -# ) -# -# for table in [ -# "m1", -# "t2", -# "mt1", -# "mt2", -# "rmt1", -# "rmt2", -# "rmt4", -# "d1", -# "d2", -# "mv1", -# "mv3", -# ]: -# assert main_node.query(f"SELECT (*,).1 FROM recover.{table}") == "42\n" -# for table in ["t2", "rmt1", "rmt2", "rmt4", "d1", "d2", "mt2", "mv1", "mv3"]: -# assert ( -# dummy_node.query(f"SELECT '{table}', (*,).1 FROM recover.{table}") -# == f"{table}\t42\n" -# ) -# for table in ["m1", "mt1"]: -# assert dummy_node.query(f"SELECT count() FROM recover.{table}") == "0\n" -# global test_recover_staled_replica_run -# assert ( -# dummy_node.query( -# "SELECT count() FROM system.tables WHERE database='recover_broken_tables'" -# ) -# == f"{test_recover_staled_replica_run}\n" -# ) -# assert ( -# dummy_node.query( -# "SELECT count() FROM system.tables WHERE database='recover_broken_replicated_tables'" -# ) -# == f"{test_recover_staled_replica_run}\n" -# ) -# test_recover_staled_replica_run += 1 -# -# print(dummy_node.query("SHOW DATABASES")) -# print(dummy_node.query("SHOW TABLES FROM recover_broken_tables")) -# print(dummy_node.query("SHOW TABLES FROM recover_broken_replicated_tables")) -# -# table = dummy_node.query( -# "SHOW TABLES FROM recover_broken_tables LIKE 'mt1_41_%' LIMIT 1" -# ).strip() -# assert ( -# dummy_node.query(f"SELECT (*,).1 FROM recover_broken_tables.{table}") == "42\n" -# ) -# table = dummy_node.query( -# "SHOW TABLES FROM recover_broken_replicated_tables LIKE 'rmt5_41_%' LIMIT 1" -# ).strip() -# assert ( -# dummy_node.query(f"SELECT (*,).1 FROM recover_broken_replicated_tables.{table}") -# == "42\n" -# ) -# -# expected = "Cleaned 6 outdated objects: dropped 1 dictionaries and 3 tables, moved 2 tables" -# assert_logs_contain(dummy_node, expected) -# -# dummy_node.query("DROP TABLE recover.tmp") -# assert_eq_with_retry( -# main_node, -# "SELECT count() FROM system.tables WHERE database='recover' AND name='tmp'", -# "0\n", -# ) -# main_node.query("DROP DATABASE recover SYNC") -# dummy_node.query("DROP DATABASE recover SYNC") -# -# -# def test_recover_staled_replica_many_mvs(started_cluster): -# main_node.query("DROP DATABASE IF EXISTS recover_mvs") -# dummy_node.query("DROP DATABASE IF EXISTS recover_mvs") -# -# main_node.query_with_retry( -# "CREATE DATABASE IF NOT EXISTS recover_mvs ENGINE = Replicated('/clickhouse/databases/recover_mvs', 'shard1', 'replica1');" -# ) -# started_cluster.get_kazoo_client("zoo1").set( -# "/clickhouse/databases/recover_mvs/logs_to_keep", b"10" -# ) -# dummy_node.query_with_retry( -# "CREATE DATABASE IF NOT EXISTS recover_mvs ENGINE = Replicated('/clickhouse/databases/recover_mvs', 'shard1', 'replica2');" -# ) -# -# settings = {"distributed_ddl_task_timeout": 0} -# -# with PartitionManager() as pm: -# pm.drop_instance_zk_connections(dummy_node) -# dummy_node.query_and_get_error("RENAME TABLE recover_mvs.t1 TO recover_mvs.m1") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query( -# f"CREATE TABLE recover_mvs.rmt{identifier} (n int) ENGINE=ReplicatedMergeTree ORDER BY n", -# settings=settings, -# ) -# -# print("Created tables") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query( -# f"CREATE TABLE recover_mvs.mv_inner{identifier} (n int) ENGINE=ReplicatedMergeTree ORDER BY n", -# settings=settings, -# ) -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE MATERIALIZED VIEW recover_mvs.mv{identifier} -# TO recover_mvs.mv_inner{identifier} -# AS SELECT * FROM recover_mvs.rmt{identifier}""", -# settings=settings, -# ) -# -# print("Created MVs") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE VIEW recover_mvs.view_from_mv{identifier} -# AS SELECT * FROM recover_mvs.mv{identifier}""", -# settings=settings, -# ) -# -# print("Created Views on top of MVs") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE MATERIALIZED VIEW recover_mvs.cascade_mv{identifier} -# ENGINE=MergeTree() ORDER BY tuple() -# POPULATE AS SELECT * FROM recover_mvs.mv_inner{identifier};""", -# settings=settings, -# ) -# -# print("Created cascade MVs") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE VIEW recover_mvs.view_from_cascade_mv{identifier} -# AS SELECT * FROM recover_mvs.cascade_mv{identifier}""", -# settings=settings, -# ) -# -# print("Created Views on top of cascade MVs") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE MATERIALIZED VIEW recover_mvs.double_cascade_mv{identifier} -# ENGINE=MergeTree() ORDER BY tuple() -# POPULATE AS SELECT * FROM recover_mvs.`.inner_id.{get_table_uuid("recover_mvs", f"cascade_mv{identifier}")}`""", -# settings=settings, -# ) -# -# print("Created double cascade MVs") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE VIEW recover_mvs.view_from_double_cascade_mv{identifier} -# AS SELECT * FROM recover_mvs.double_cascade_mv{identifier}""", -# settings=settings, -# ) -# -# print("Created Views on top of double cascade MVs") -# -# # This weird table name is actually makes sence because it starts with letter `a` and may break some internal sorting -# main_node.query_with_retry( -# """ -# CREATE VIEW recover_mvs.anime -# AS -# SELECT n -# FROM -# ( -# SELECT * -# FROM -# ( -# SELECT * -# FROM -# ( -# SELECT * -# FROM recover_mvs.mv_inner1 AS q1 -# INNER JOIN recover_mvs.mv_inner2 AS q2 ON q1.n = q2.n -# ) AS new_table_1 -# INNER JOIN recover_mvs.mv_inner3 AS q3 ON new_table_1.n = q3.n -# ) AS new_table_2 -# INNER JOIN recover_mvs.mv_inner4 AS q4 ON new_table_2.n = q4.n -# ) -# """, -# settings=settings, -# ) -# -# print("Created final boss") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE DICTIONARY recover_mvs.`11111d{identifier}` (n UInt64) -# PRIMARY KEY n -# SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'double_cascade_mv{identifier}' DB 'recover_mvs')) -# LAYOUT(FLAT()) LIFETIME(1)""", -# settings=settings, -# ) -# -# print("Created dictionaries") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE VIEW recover_mvs.`00000vd{identifier}` -# AS SELECT * FROM recover_mvs.`11111d{identifier}`""", -# settings=settings, -# ) -# -# print("Created Views on top of dictionaries") -# -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA recover_mvs") -# query = "SELECT name FROM system.tables WHERE database='recover_mvs' ORDER BY name" -# assert main_node.query(query) == dummy_node.query(query) -# -# main_node.query("DROP DATABASE IF EXISTS recover_mvs") -# dummy_node.query("DROP DATABASE IF EXISTS recover_mvs") -# -# -# def test_startup_without_zk(started_cluster): -# with PartitionManager() as pm: -# pm.drop_instance_zk_connections(main_node) -# err = main_node.query_and_get_error( -# "CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');" -# ) -# assert "ZooKeeper" in err or "Coordination::Exception" in err -# main_node.query( -# "CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');" -# ) -# main_node.query( -# "CREATE TABLE startup.rmt (n int) ENGINE=ReplicatedMergeTree order by n" -# ) -# -# main_node.query("INSERT INTO startup.rmt VALUES (42)") -# with PartitionManager() as pm: -# pm.drop_instance_zk_connections(main_node) -# main_node.restart_clickhouse(stop_start_wait_sec=60) -# assert main_node.query("SELECT (*,).1 FROM startup.rmt") == "42\n" -# -# # we need to wait until the table is not readonly -# main_node.query_with_retry("INSERT INTO startup.rmt VALUES(42)") -# -# main_node.query_with_retry("CREATE TABLE startup.m (n int) ENGINE=Memory") -# -# main_node.query("EXCHANGE TABLES startup.rmt AND startup.m") -# assert main_node.query("SELECT (*,).1 FROM startup.m") == "42\n" -# -# main_node.query("DROP DATABASE startup SYNC") -# -# -# def test_server_uuid(started_cluster): -# uuid1 = main_node.query("select serverUUID()") -# uuid2 = dummy_node.query("select serverUUID()") -# assert uuid1 != uuid2 -# main_node.restart_clickhouse() -# uuid1_after_restart = main_node.query("select serverUUID()") -# assert uuid1 == uuid1_after_restart -# -# -# def test_sync_replica(started_cluster): -# main_node.query( -# "CREATE DATABASE test_sync_database ENGINE = Replicated('/test/sync_replica', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE test_sync_database ENGINE = Replicated('/test/sync_replica', 'shard1', 'replica2');" -# ) -# -# number_of_tables = 1000 -# -# settings = {"distributed_ddl_task_timeout": 0} -# -# with PartitionManager() as pm: -# pm.drop_instance_zk_connections(dummy_node) -# -# for i in range(number_of_tables): -# main_node.query( -# "CREATE TABLE test_sync_database.table_{} (n int) ENGINE=MergeTree order by n".format( -# i -# ), -# settings=settings, -# ) -# -# # wait for host to reconnect -# dummy_node.query_with_retry("SELECT * FROM system.zookeeper WHERE path='/'") -# -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_sync_database") -# -# assert "2\n" == main_node.query( -# "SELECT sum(is_active) FROM system.clusters WHERE cluster='test_sync_database'" -# ) -# -# assert dummy_node.query( -# "SELECT count() FROM system.tables where database='test_sync_database'" -# ).strip() == str(number_of_tables) -# -# assert main_node.query( -# "SELECT count() FROM system.tables where database='test_sync_database'" -# ).strip() == str(number_of_tables) -# -# engine_settings = {"default_table_engine": "ReplicatedMergeTree"} -# dummy_node.query( -# "CREATE TABLE test_sync_database.table (n int, primary key n) partition by n", -# settings=engine_settings, -# ) -# main_node.query("INSERT INTO test_sync_database.table SELECT * FROM numbers(10)") -# dummy_node.query("TRUNCATE TABLE test_sync_database.table", settings=settings) -# dummy_node.query( -# "ALTER TABLE test_sync_database.table ADD COLUMN m int", settings=settings -# ) -# -# main_node.query( -# "SYSTEM SYNC DATABASE REPLICA ON CLUSTER test_sync_database test_sync_database" -# ) -# -# lp1 = main_node.query( -# "select value from system.zookeeper where path='/test/sync_replica/replicas/shard1|replica1' and name='log_ptr'" -# ) -# lp2 = main_node.query( -# "select value from system.zookeeper where path='/test/sync_replica/replicas/shard1|replica2' and name='log_ptr'" -# ) -# max_lp = main_node.query( -# "select value from system.zookeeper where path='/test/sync_replica/' and name='max_log_ptr'" -# ) -# assert lp1 == max_lp -# assert lp2 == max_lp -# -# main_node.query("DROP DATABASE test_sync_database SYNC") -# dummy_node.query("DROP DATABASE test_sync_database SYNC") -# -# -# def test_force_synchronous_settings(started_cluster): -# main_node.query( -# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard1', 'replica2');" -# ) -# snapshotting_node.query( -# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard2', 'replica1');" -# ) -# main_node.query( -# "CREATE TABLE test_force_synchronous_settings.t (n int) ENGINE=ReplicatedMergeTree('/test/same/path/{shard}', '{replica}') ORDER BY tuple()" -# ) -# main_node.query( -# "INSERT INTO test_force_synchronous_settings.t SELECT * FROM numbers(10)" -# ) -# snapshotting_node.query( -# "INSERT INTO test_force_synchronous_settings.t SELECT * FROM numbers(10)" -# ) -# snapshotting_node.query( -# "SYSTEM SYNC DATABASE REPLICA test_force_synchronous_settings" -# ) -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_force_synchronous_settings") -# -# snapshotting_node.query("SYSTEM STOP MERGES test_force_synchronous_settings.t") -# -# def start_merges_func(): -# time.sleep(5) -# snapshotting_node.query("SYSTEM START MERGES test_force_synchronous_settings.t") -# -# start_merges_thread = threading.Thread(target=start_merges_func) -# start_merges_thread.start() -# -# settings = { -# "mutations_sync": 2, -# "database_replicated_enforce_synchronous_settings": 1, -# } -# main_node.query( -# "ALTER TABLE test_force_synchronous_settings.t UPDATE n = n * 10 WHERE 1", -# settings=settings, -# ) -# assert "10\t450\n" == snapshotting_node.query( -# "SELECT count(), sum(n) FROM test_force_synchronous_settings.t" -# ) -# start_merges_thread.join() -# -# def select_func(): -# dummy_node.query( -# "SELECT sleepEachRow(1) FROM test_force_synchronous_settings.t SETTINGS function_sleep_max_microseconds_per_block = 0" -# ) -# -# select_thread = threading.Thread(target=select_func) -# select_thread.start() -# -# settings = {"database_replicated_enforce_synchronous_settings": 1} -# snapshotting_node.query( -# "DROP TABLE test_force_synchronous_settings.t SYNC", settings=settings -# ) -# main_node.query( -# "CREATE TABLE test_force_synchronous_settings.t (n String) ENGINE=ReplicatedMergeTree('/test/same/path/{shard}', '{replica}') ORDER BY tuple()" -# ) -# select_thread.join() -# -# -# def test_recover_digest_mismatch(started_cluster): -# main_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") -# dummy_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") -# -# main_node.query( -# "CREATE DATABASE recover_digest_mismatch ENGINE = Replicated('/clickhouse/databases/recover_digest_mismatch', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE recover_digest_mismatch ENGINE = Replicated('/clickhouse/databases/recover_digest_mismatch', 'shard1', 'replica2');" -# ) -# -# create_some_tables("recover_digest_mismatch") -# -# main_node.query("SYSTEM SYNC DATABASE REPLICA recover_digest_mismatch") -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA recover_digest_mismatch") -# -# ways_to_corrupt_metadata = [ -# "mv /var/lib/clickhouse/metadata/recover_digest_mismatch/t1.sql /var/lib/clickhouse/metadata/recover_digest_mismatch/m1.sql", -# "sed --follow-symlinks -i 's/Int32/String/' /var/lib/clickhouse/metadata/recover_digest_mismatch/mv1.sql", -# "rm -f /var/lib/clickhouse/metadata/recover_digest_mismatch/d1.sql", -# "rm -rf /var/lib/clickhouse/metadata/recover_digest_mismatch/", # Will trigger "Directory already exists" -# "rm -rf /var/lib/clickhouse/store", -# ] -# -# for command in ways_to_corrupt_metadata: -# print(f"Corrupting data using `{command}`") -# need_remove_is_active_node = "rm -rf" in command -# dummy_node.stop_clickhouse(kill=not need_remove_is_active_node) -# dummy_node.exec_in_container(["bash", "-c", command]) -# -# query = ( -# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover_digest_mismatch' AND name NOT LIKE '.inner_id.%' " -# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" -# ) -# expected = main_node.query(query) -# -# if need_remove_is_active_node: -# # NOTE Otherwise it fails to recreate ReplicatedMergeTree table due to "Replica already exists" -# main_node.query( -# "SYSTEM DROP REPLICA '2' FROM DATABASE recover_digest_mismatch" -# ) -# -# # There is a race condition between deleting active node and creating it on server startup -# # So we start a server only after we deleted all table replicas from the Keeper -# dummy_node.start_clickhouse() -# assert_eq_with_retry(dummy_node, query, expected) -# -# main_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") -# dummy_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") -# -# print("Everything Okay") -# -# -# def test_replicated_table_structure_alter(started_cluster): -# main_node.query("DROP DATABASE IF EXISTS table_structure") -# dummy_node.query("DROP DATABASE IF EXISTS table_structure") -# -# main_node.query( -# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica2');" -# ) -# competing_node.query( -# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica3');" -# ) -# -# competing_node.query("CREATE TABLE table_structure.mem (n int) ENGINE=Memory") -# dummy_node.query("DETACH DATABASE table_structure") -# -# settings = {"distributed_ddl_task_timeout": 0} -# main_node.query( -# "CREATE TABLE table_structure.rmt (n int, v UInt64) ENGINE=ReplicatedReplacingMergeTree(v) ORDER BY n", -# settings=settings, -# ) -# -# competing_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") -# competing_node.query("DETACH DATABASE table_structure") -# -# main_node.query( -# "ALTER TABLE table_structure.rmt ADD COLUMN m int", settings=settings -# ) -# main_node.query( -# "ALTER TABLE table_structure.rmt COMMENT COLUMN v 'version'", settings=settings -# ) -# main_node.query("INSERT INTO table_structure.rmt VALUES (1, 2, 3)") -# -# command = "rm -f /var/lib/clickhouse/metadata/table_structure/mem.sql" -# competing_node.exec_in_container(["bash", "-c", command]) -# competing_node.restart_clickhouse(kill=True) -# -# dummy_node.query("ATTACH DATABASE table_structure") -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") -# dummy_node.query("SYSTEM SYNC REPLICA table_structure.rmt") -# assert "1\t2\t3\n" == dummy_node.query("SELECT * FROM table_structure.rmt") -# -# competing_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") -# competing_node.query("SYSTEM SYNC REPLICA table_structure.rmt") -# # time.sleep(600) -# assert "mem" in competing_node.query("SHOW TABLES FROM table_structure") -# assert "1\t2\t3\n" == competing_node.query("SELECT * FROM table_structure.rmt") -# -# main_node.query("ALTER TABLE table_structure.rmt ADD COLUMN k int") -# main_node.query("INSERT INTO table_structure.rmt VALUES (1, 2, 3, 4)") -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") -# dummy_node.query("SYSTEM SYNC REPLICA table_structure.rmt") -# assert "1\t2\t3\t0\n1\t2\t3\t4\n" == dummy_node.query( -# "SELECT * FROM table_structure.rmt ORDER BY k" -# ) -# -# -# def test_modify_comment(started_cluster): -# main_node.query( -# "CREATE DATABASE modify_comment_db ENGINE = Replicated('/test/modify_comment', 'shard1', 'replica' || '1');" -# ) -# -# dummy_node.query( -# "CREATE DATABASE modify_comment_db ENGINE = Replicated('/test/modify_comment', 'shard1', 'replica' || '2');" -# ) -# -# main_node.query( -# "CREATE TABLE modify_comment_db.modify_comment_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);" -# ) -# -# def restart_verify_not_readonly(): -# main_node.restart_clickhouse() -# assert ( -# main_node.query( -# "SELECT is_readonly FROM system.replicas WHERE table = 'modify_comment_table'" -# ) -# == "0\n" -# ) -# dummy_node.restart_clickhouse() -# assert ( -# dummy_node.query( -# "SELECT is_readonly FROM system.replicas WHERE table = 'modify_comment_table'" -# ) -# == "0\n" -# ) -# -# main_node.query( -# "ALTER TABLE modify_comment_db.modify_comment_table COMMENT COLUMN d 'Some comment'" -# ) -# -# restart_verify_not_readonly() -# -# main_node.query( -# "ALTER TABLE modify_comment_db.modify_comment_table MODIFY COMMENT 'Some error comment'" -# ) -# -# restart_verify_not_readonly() -# -# main_node.query("DROP DATABASE modify_comment_db SYNC") -# dummy_node.query("DROP DATABASE modify_comment_db SYNC") -# -# -# def test_table_metadata_corruption(started_cluster): -# main_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") -# dummy_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") -# -# main_node.query( -# "CREATE DATABASE table_metadata_corruption ENGINE = Replicated('/clickhouse/databases/table_metadata_corruption', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE table_metadata_corruption ENGINE = Replicated('/clickhouse/databases/table_metadata_corruption', 'shard1', 'replica2');" -# ) -# -# create_some_tables("table_metadata_corruption") -# -# main_node.query("SYSTEM SYNC DATABASE REPLICA table_metadata_corruption") -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_metadata_corruption") -# -# # Server should handle this by throwing an exception during table loading, which should lead to server shutdown -# corrupt = "sed --follow-symlinks -i 's/ReplicatedMergeTree/CorruptedMergeTree/' /var/lib/clickhouse/metadata/table_metadata_corruption/rmt1.sql" -# -# print(f"Corrupting metadata using `{corrupt}`") -# dummy_node.stop_clickhouse(kill=True) -# dummy_node.exec_in_container(["bash", "-c", corrupt]) -# -# query = ( -# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='table_metadata_corruption' AND name NOT LIKE '.inner_id.%' " -# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" -# ) -# expected = main_node.query(query) -# -# # We expect clickhouse server to shutdown without LOGICAL_ERRORs or deadlocks -# dummy_node.start_clickhouse(expected_to_fail=True) -# assert not dummy_node.contains_in_log("LOGICAL_ERROR") -# -# fix_corrupt = "sed --follow-symlinks -i 's/CorruptedMergeTree/ReplicatedMergeTree/' /var/lib/clickhouse/metadata/table_metadata_corruption/rmt1.sql" -# print(f"Fix corrupted metadata using `{fix_corrupt}`") -# dummy_node.exec_in_container(["bash", "-c", fix_corrupt]) -# -# dummy_node.start_clickhouse() -# assert_eq_with_retry(dummy_node, query, expected) -# -# main_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") -# dummy_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") diff --git a/tests/integration/test_refreshable_mat_view/test_schedule_model.py b/tests/integration/test_refreshable_mat_view/test_schedule_model.py index ac15333cd11..7378598e85e 100644 --- a/tests/integration/test_refreshable_mat_view/test_schedule_model.py +++ b/tests/integration/test_refreshable_mat_view/test_schedule_model.py @@ -4,87 +4,78 @@ from test_refreshable_mat_view.schedule_model import get_next_refresh_time def test_refresh_schedules(): - time_ = datetime(2000, 1, 1, 1, 1, 1) + t = datetime(2000, 1, 1, 1, 1, 1) - assert get_next_refresh_time( - "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", time_ - ) == datetime(2000, 1, 1, 1, 1, 1) - - assert get_next_refresh_time("EVERY 1 SECOND", time_) == datetime( - 2000, 1, 1, 1, 1, 2 - ) - assert get_next_refresh_time("EVERY 1 MINUTE", time_) == datetime( - 2000, - 1, - 1, - 1, - 2, - ) - assert get_next_refresh_time("EVERY 1 HOUR", time_) == datetime( - 2000, - 1, - 1, - 2, - ) - assert get_next_refresh_time("EVERY 1 DAY", time_) == datetime(2000, 1, 2) - assert get_next_refresh_time("EVERY 1 WEEK", time_) == datetime(2000, 1, 10) - assert get_next_refresh_time("EVERY 2 WEEK", time_) == datetime(2000, 1, 17) - assert get_next_refresh_time("EVERY 1 MONTH", time_) == datetime(2000, 2, 1) - assert get_next_refresh_time("EVERY 1 YEAR", time_) == datetime(2001, 1, 1) - - assert get_next_refresh_time("EVERY 3 YEAR 4 MONTH 10 DAY", time_) == datetime( - 2003, 5, 11 - ) - - # OFFSET - assert get_next_refresh_time( - "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", time_ - ) == datetime(2000, 2, 6, 2, 30, 15) - assert get_next_refresh_time( - "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", time_ - ) == datetime(2001, 3, 6, 2, 30, 15) - - assert get_next_refresh_time( - "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", time_ - ) == datetime(2000, 1, 22, 15, 10) - - # AFTER - assert get_next_refresh_time("AFTER 30 SECOND", time_) == datetime( - 2000, 1, 1, 1, 1, 31 - ) - assert get_next_refresh_time("AFTER 30 MINUTE", time_) == datetime( - 2000, 1, 1, 1, 31, 1 - ) - assert get_next_refresh_time("AFTER 2 HOUR", time_) == datetime(2000, 1, 1, 3, 1, 1) - assert get_next_refresh_time("AFTER 2 DAY", time_) == datetime(2000, 1, 3, 1, 1, 1) - assert get_next_refresh_time("AFTER 2 WEEK", time_) == datetime( - 2000, 1, 15, 1, 1, 1 - ) - assert get_next_refresh_time("AFTER 2 MONTH", time_) == datetime( - 2000, 3, 1, 1, 1, 1 - ) - assert get_next_refresh_time("AFTER 2 YEAR", time_) == datetime(2002, 1, 1, 1, 1, 1) - - assert get_next_refresh_time("AFTER 2 YEAR 1 MONTH", time_) == datetime( - 2002, 2, 1, 1, 1, 1 - ) - - assert get_next_refresh_time("AFTER 1 MONTH 2 YEAR", time_) == datetime( - 2002, 2, 1, 1, 1, 1 - ) + # assert get_next_refresh_time("EVERY 1 SECOND", t) == datetime(2000, 1, 1, 1, 1, 2) + # assert get_next_refresh_time("EVERY 1 MINUTE", t) == datetime( + # 2000, + # 1, + # 1, + # 1, + # 2, + # ) + # assert get_next_refresh_time("EVERY 1 HOUR", t) == datetime( + # 2000, + # 1, + # 1, + # 2, + # ) + # assert get_next_refresh_time("EVERY 1 DAY", t) == datetime(2000, 1, 2) + # assert get_next_refresh_time("EVERY 1 WEEK", t) == datetime(2000, 1, 10) + # assert get_next_refresh_time("EVERY 2 WEEK", t) == datetime(2000, 1, 17) + # assert get_next_refresh_time("EVERY 1 MONTH", t) == datetime(2000, 2, 1) + # assert get_next_refresh_time("EVERY 1 YEAR", t) == datetime(2001, 1, 1) + # + # assert get_next_refresh_time("EVERY 3 YEAR 4 MONTH 10 DAY", t) == datetime( + # 2003, 5, 11 + # ) + # + # # OFFSET + # assert get_next_refresh_time( + # "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t + # ) == datetime(2000, 2, 6, 2, 30, 15) + # assert get_next_refresh_time( + # "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t + # ) == datetime(2001, 3, 6, 2, 30, 15) + # + # assert get_next_refresh_time( + # "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", t + # ) == datetime(2000, 1, 22, 15, 10) + # + # # AFTER + # assert get_next_refresh_time("AFTER 30 SECOND", t) == datetime(2000, 1, 1, 1, 1, 31) + # assert get_next_refresh_time("AFTER 30 MINUTE", t) == datetime(2000, 1, 1, 1, 31, 1) + # assert get_next_refresh_time("AFTER 2 HOUR", t) == datetime(2000, 1, 1, 3, 1, 1) + # assert get_next_refresh_time("AFTER 2 DAY", t) == datetime(2000, 1, 3, 1, 1, 1) + # assert get_next_refresh_time("AFTER 2 WEEK", t) == datetime(2000, 1, 15, 1, 1, 1) + # assert get_next_refresh_time("AFTER 2 MONTH", t) == datetime(2000, 3, 1, 1, 1, 1) + # assert get_next_refresh_time("AFTER 2 YEAR", t) == datetime(2002, 1, 1, 1, 1, 1) + # + # assert get_next_refresh_time("AFTER 2 YEAR 1 MONTH", t) == datetime( + # 2002, 2, 1, 1, 1, 1 + # ) + # + # assert get_next_refresh_time("AFTER 1 MONTH 2 YEAR", t) == datetime( + # 2002, 2, 1, 1, 1, 1 + # ) # RANDOMIZE next_refresh = get_next_refresh_time( - "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", time_ + "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", t ) assert next_refresh == (datetime(2000, 1, 2, 2, 0), datetime(2000, 1, 2, 3, 0)) next_refresh = get_next_refresh_time( "EVERY 2 MONTH 3 DAY 5 HOUR OFFSET 3 HOUR 20 SECOND RANDOMIZE FOR 3 DAY 1 HOUR", - time_, + t, ) assert next_refresh == ( - datetime(2000, 3, 4, 11, 0, 20), - datetime(2000, 3, 7, 12, 0, 20), + datetime(2000, 3, 4, 8, 0, 20), + datetime(2000, 3, 7, 9, 0, 20), + ) + + assert get_next_refresh_time("AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", t) == ( + datetime(2000, 3, 4, 1, 1, 1), + datetime(2000, 3, 5, 1, 1, 1), ) From 6d72f4774e35b1f80c410a75ee10a76af45c41d1 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Sat, 10 Aug 2024 08:11:35 +0200 Subject: [PATCH 443/502] Tests: add RMV tests --- .../sql-reference/statements/create/view.md | 2 +- tests/integration/helpers/test_tools.py | 13 + .../schedule_model.py | 90 +- .../test_refreshable_mat_view/test.py | 1553 +++++++---------- .../test_schedule_model.py | 128 +- 5 files changed, 745 insertions(+), 1041 deletions(-) diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index a6f9f394871..a8be515765b 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -282,7 +282,7 @@ This replaces *all* refresh parameters at once: schedule, dependencies, settings The status of all refreshable materialized views is available in table [`system.view_refreshes`](../../../operations/system-tables/view_refreshes.md). In particular, it contains refresh progress (if running), last and next refresh time, exception message if a refresh failed. -To manually stop, start, trigger, or cancel refreshes use [`SYSTEM STOP|START|REFRESH|CANCEL VIEW`](../system.md#refreshable-materialized-views). +To manually stop, start, trigger, or cancel refreshes use [`SYSTEM STOP|START|REFRESH|WAIT|CANCEL VIEW`](../system.md#refreshable-materialized-views). To wait for a refresh to complete, use [`SYSTEM WAIT VIEW`](../system.md#refreshable-materialized-views). In particular, useful for waiting for initial refresh after creating a view. diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index e71698692be..c334a7366da 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -182,3 +182,16 @@ def csv_compare(result, expected): mismatch.append("+[%d]=%s" % (i, csv_result.lines[i])) return "\n".join(mismatch) + + +def wait_condition(func, condition, max_attempts=10, delay=0.1): + attempts = 0 + while attempts < max_attempts: + result = func() + if condition(result): + return result + attempts += 1 + if attempts < max_attempts: + time.sleep(delay) + + raise Exception(f"Function did not satisfy condition after {max_attempts} attempts") diff --git a/tests/integration/test_refreshable_mat_view/schedule_model.py b/tests/integration/test_refreshable_mat_view/schedule_model.py index 5195df3dec6..38781e59522 100644 --- a/tests/integration/test_refreshable_mat_view/schedule_model.py +++ b/tests/integration/test_refreshable_mat_view/schedule_model.py @@ -16,11 +16,8 @@ def relative_offset(unit, value): return rd.relativedelta(hours=value) elif unit == "DAY": return rd.relativedelta(days=value) - # elif unit == "WEEK": - # return rd.relativedelta(days=7 * value) elif unit == "WEEK": return rd.relativedelta(weeks=7 * value) - elif unit == "MONTH": return rd.relativedelta(months=value) elif unit == "YEAR": @@ -42,7 +39,7 @@ def group_and_sort(parts, reverse=False): return sorted_parts -def get_next_refresh_time(schedule, current_time: datetime): +def get_next_refresh_time(schedule, current_time: datetime, first_week=False): parts = schedule.split() randomize_offset = timedelta() @@ -65,6 +62,7 @@ def get_next_refresh_time(schedule, current_time: datetime): parts = parts[:offset_index] + week_in_primary = False if parts[0] == "EVERY": parts = group_and_sort(parts[1:]) for part in parts: @@ -88,9 +86,10 @@ def get_next_refresh_time(schedule, current_time: datetime): hour=0, minute=0, second=0, microsecond=0 ) + rd.relativedelta(days=value) elif unit == "WEEK": + week_in_primary = True current_time = current_time.replace( hour=0, minute=0, second=0, microsecond=0 - ) + rd.relativedelta(weekday=0, weeks=value) + ) + rd.relativedelta(weekday=0, weeks=0 if first_week else value) elif unit == "MONTH": current_time = current_time.replace( day=1, hour=0, minute=0, second=0, microsecond=0 @@ -103,12 +102,16 @@ def get_next_refresh_time(schedule, current_time: datetime): current_time += offset if randomize_offset: half_offset = (current_time + randomize_offset - current_time) / 2 - return ( - current_time - half_offset, - current_time + half_offset, - ) + return { + "type": "randomize", + "time": ( + current_time - half_offset, + current_time + half_offset, + ), + "week": week_in_primary, + } - return current_time + return {"type": "regular", "time": current_time, "week": week_in_primary} elif parts[0] == "AFTER": parts = group_and_sort(parts[1:], reverse=True) @@ -126,6 +129,7 @@ def get_next_refresh_time(schedule, current_time: datetime): elif unit == "DAY": interval += rd.relativedelta(days=value) elif unit == "WEEK": + week_in_primary = True interval += rd.relativedelta(weeks=value) elif unit == "MONTH": interval += rd.relativedelta(months=value) @@ -135,11 +139,65 @@ def get_next_refresh_time(schedule, current_time: datetime): current_time += interval if randomize_offset: half_offset = (current_time + randomize_offset - current_time) / 2 - return ( - current_time - half_offset, - # current_time, - current_time + half_offset, - ) + return { + "type": "randomize", + "time": ( + current_time - half_offset, + current_time + half_offset, + ), + "week": week_in_primary, + } - return current_time + return {"type": "regular", "time": current_time, "week": week_in_primary} raise ValueError("Invalid refresh schedule") + + +def compare_dates( + date1: str | datetime, + date2: dict, + first_week=False, +): + """ + Special logic for weeks for first refresh: + The desired behavior for EVERY 1 WEEK is "every Monday". This has the properties: (a) it doesn't depend on when the materialized view was created, (b) consecutive refreshes are exactly 1 week apart. And non-properties: (c) the first refresh doesn't happen exactly 1 week after view creation, it can be anywhere between 0 and 1 week, (d) the schedule is not aligned with months or years. + I would expect EVERY 2 WEEK to have the same two properties and two non-properties, and also to fall on Mondays. There are exactly two possible ways to achieve that: all even-numbered Mondays or all odd-numbered Mondays. I just picked one. + """ + weeks = [] + if date2["week"] and first_week: + for i in [0, 1, 2]: + if date2["type"] == "randomize": + weeks.append( + ( + date2["time"][0] + rd.relativedelta(weeks=i), + date2["time"][1] + rd.relativedelta(weeks=i), + ) + ) + else: + weeks.append(date2["time"] + rd.relativedelta(weeks=i)) + + for week in weeks: + if compare_dates_(date1, week): + return True + raise ValueError("Invalid week") + else: + assert compare_dates_(date1, date2["time"]) + + +def compare_dates_( + date1: str | datetime, + date2: str | datetime | tuple[datetime], + inaccuracy=timedelta(minutes=10), + format_str="%Y-%m-%d %H:%M:%S", +) -> bool: + """ + Compares two dates with small inaccuracy. + """ + if isinstance(date1, str): + date1 = datetime.strptime(date1, format_str) + if isinstance(date2, str): + date2 = datetime.strptime(date2, format_str) + + if isinstance(date2, datetime): + return abs(date1 - date2) <= inaccuracy + else: + return date2[0] - inaccuracy <= date1 <= date2[1] + inaccuracy diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py index 95d15fe369d..cef5e9543c8 100644 --- a/tests/integration/test_refreshable_mat_view/test.py +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -1,27 +1,26 @@ import datetime -import os -import random -import shutil -import time +import logging import re +import time from typing import Optional -import numpy as np import pytest -import threading -from dateutil.relativedelta import relativedelta from jinja2 import Template, Environment from datetime import datetime, timedelta - +import helpers.client from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, assert_logs_contain -from helpers.network import PartitionManager -from test_refreshable_mat_view.schedule_model import get_next_refresh_time +from helpers.test_tools import assert_eq_with_retry, assert_logs_contain, wait_condition +from test_refreshable_mat_view.schedule_model import ( + get_next_refresh_time, + compare_dates, + compare_dates_, +) + cluster = ClickHouseCluster(__file__) -node1_1 = cluster.add_instance( +node = cluster.add_instance( "node1_1", main_configs=["configs/remote_servers.xml"], user_configs=["configs/settings.xml"], @@ -29,7 +28,7 @@ node1_1 = cluster.add_instance( stay_alive=True, macros={"shard": 1, "replica": 1}, ) -node1_2 = cluster.add_instance( +node2 = cluster.add_instance( "node1_2", main_configs=["configs/remote_servers.xml"], user_configs=["configs/settings.xml"], @@ -60,483 +59,56 @@ def started_cluster(): @pytest.fixture(scope="module", autouse=True) def setup_tables(started_cluster): - print(node1_1.query("SELECT version()")) - - node1_1.query(f"CREATE DATABASE test_db ON CLUSTER test_cluster ENGINE = Atomic") - - node1_1.query( - f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - - node1_1.query( - f"CREATE TABLE src2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - - node1_1.query( - f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - - node1_1.query( - f"CREATE TABLE tgt2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - - node1_1.query( - f"CREATE MATERIALIZED VIEW dummy_rmv ON CLUSTER test_cluster " - f"REFRESH EVERY 10 HOUR engine Memory AS select number as x from numbers(10)" - ) + print(node.query("SELECT version()")) """ -#!/usr/bin/env bash -# Tags: atomic-database - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# reset --log_comment -CLICKHOUSE_LOG_COMMENT= -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -# Set session timezone to UTC to make all DateTime formatting and parsing use UTC, because refresh -# scheduling is done in UTC. -CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" -CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --session_timezone Etc/UTC"`" - -$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" - - -# Basic refreshing. -$CLICKHOUSE_CLIENT -nq " - create materialized view a - refresh after 2 second - engine Memory - empty - as select number as x from numbers(2) union all select rand64() as x; - select '<1: created view>', exception, view from refreshes; - show create a;" -# Wait for any refresh. (xargs trims the string and turns \t and \n into spaces) -while [ "`$CLICKHOUSE_CLIENT -nq "select last_success_time is null from refreshes -- $LINENO" | xargs`" != '0' ] -do - sleep 0.1 -done -start_time="`$CLICKHOUSE_CLIENT -nq "select reinterpret(now64(), 'Int64')"`" -# Check table contents. -$CLICKHOUSE_CLIENT -nq "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" -# Wait for table contents to change. -res1="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`" -while : -do - res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" - [ "$res2" == "$res1" ] || break - sleep 0.1 -done -# Wait for another change. -while : -do - res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" - [ "$res3" == "$res2" ] || break - sleep 0.1 -done -# Check that the two changes were at least 1 second apart, in particular that we're not refreshing -# like crazy. This is potentially flaky, but we need at least one test that uses non-mocked timer -# to make sure the clock+timer code works at all. If it turns out flaky, increase refresh period above. -$CLICKHOUSE_CLIENT -nq " - select '<3: time difference at least>', min2(reinterpret(now64(), 'Int64') - $start_time, 1000); - select '<4: next refresh in>', next_refresh_time-last_success_time from refreshes;" - -# Create a source table from which views will read. -$CLICKHOUSE_CLIENT -nq " - create table src (x Int8) engine Memory as select 1;" - -# Switch to fake clock, change refresh schedule, change query. -$CLICKHOUSE_CLIENT -nq " - system test view a set fake time '2050-01-01 00:00:01'; - system wait view a; - system refresh view a; - system wait view a; - select '<4.1: fake clock>', status, last_success_time, next_refresh_time from refreshes; - alter table a modify refresh every 2 year; - alter table a modify query select x*2 as x from src; - system wait view a; - select '<4.5: altered>', status, last_success_time, next_refresh_time from refreshes; - show create a;" -# Advance time to trigger the refresh. -$CLICKHOUSE_CLIENT -nq " - select '<5: no refresh>', count() from a; - system test view a set fake time '2052-02-03 04:05:06';" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_success_time, status from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06 Scheduled' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<6: refreshed>', * from a; - select '<7: refreshed>', status, last_success_time, next_refresh_time from refreshes;" - -# Create a dependent view, refresh it once. -$CLICKHOUSE_CLIENT -nq " - create materialized view b refresh every 2 year depends on a (y Int32) engine MergeTree order by y empty as select x*10 as y from a; - show create b; - system test view b set fake time '2052-11-11 11:11:11'; - system refresh view b; - system wait view b; - select '<7.5: created dependent>', last_success_time from refreshes where view = 'b';" -# Next refresh shouldn't start until the dependency refreshes. -$CLICKHOUSE_CLIENT -nq " - select '<8: refreshed>', * from b; - select '<9: refreshed>', view, status, next_refresh_time from refreshes; - system test view b set fake time '2054-01-24 23:22:21';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] -do - sleep 0.1 -done - -# Drop the source table, check that refresh fails and doesn't leave a temp table behind. -$CLICKHOUSE_CLIENT -nq " - select '<9.2: dropping>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase(); - drop table src; - system refresh view a;" -$CLICKHOUSE_CLIENT -nq "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -nq " - select '<9.4: dropped>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase();" - -# Create the source table again, check that refresh succeeds (in particular that tables are looked -# up by name rather than uuid). -$CLICKHOUSE_CLIENT -nq " - select '<10: creating>', view, status, next_refresh_time from refreshes; - create table src (x Int16) engine Memory as select 2; - system test view a set fake time '2054-01-01 00:00:01';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] -do - sleep 0.1 -done -# Both tables should've refreshed. -$CLICKHOUSE_CLIENT -nq " - select '<11: chain-refreshed a>', * from a; - select '<12: chain-refreshed b>', * from b; - select '<13: chain-refreshed>', view, status, last_success_time, last_refresh_time, next_refresh_time, exception == '' from refreshes;" - -$CLICKHOUSE_CLIENT -nq " - system test view b set fake time '2061-01-01 00:00:00'; - truncate src; - insert into src values (3); - system test view a set fake time '2060-02-02 02:02:02';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<15: chain-refreshed a>', * from a; - select '<16: chain-refreshed b>', * from b; - select '<17: chain-refreshed>', view, status, next_refresh_time from refreshes;" - -# Get to WaitingForDependencies state and remove the depencency. -$CLICKHOUSE_CLIENT -nq " - system test view b set fake time '2062-03-03 03:03:03'" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - alter table b modify refresh every 2 year" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<18: removed dependency>', view, status, last_success_time, last_refresh_time, next_refresh_time from refreshes where view = 'b'; - show create b;" - -# Select from a table that doesn't exist, get an exception. -$CLICKHOUSE_CLIENT -nq " - drop table a; - drop table b; - create materialized view c refresh every 1 second (x Int64) engine Memory empty as select * from src; - drop table src;" -while [ "`$CLICKHOUSE_CLIENT -nq "select exception == '' from refreshes -- $LINENO" | xargs`" != '0' ] -do - sleep 0.1 -done -# Check exception, create src, expect successful refresh. -$CLICKHOUSE_CLIENT -nq " - select '<19: exception>', exception ilike '%UNKNOWN_TABLE%' from refreshes; - create table src (x Int64) engine Memory as select 1; - system refresh view c; - system wait view c;" -# Rename table. -$CLICKHOUSE_CLIENT -nq " - select '<20: unexception>', * from c; - rename table c to d; - select '<21: rename>', * from d; - select '<22: rename>', view, status, last_success_time is null from refreshes;" - -# Do various things during a refresh. -# First make a nonempty view. -$CLICKHOUSE_CLIENT -nq " - drop table d; - truncate src; - insert into src values (1) - create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x as select x + sleepEachRow(1) as x from src settings max_block_size = 1; - system wait view e;" -# Stop refreshes. -$CLICKHOUSE_CLIENT -nq " - select '<23: simple refresh>', * from e; - system stop view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] -do - sleep 0.1 -done -# Make refreshes slow, wait for a slow refresh to start. (We stopped refreshes first to make sure -# we wait for a slow refresh, not a previous fast one.) -$CLICKHOUSE_CLIENT -nq " - insert into src select * from numbers(1000) settings max_block_size=1; - system start view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] -do - sleep 0.1 -done -# Rename. -$CLICKHOUSE_CLIENT -nq " - rename table e to f; - select '<24: rename during refresh>', * from f; - select '<25: rename during refresh>', view, status from refreshes; - alter table f modify refresh after 10 year settings refresh_retries = 0;" -sleep 2 # make it likely that at least one row was processed -# Cancel. -$CLICKHOUSE_CLIENT -nq " - system cancel view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Scheduled' ] -do - sleep 0.1 -done -# Check that another refresh doesn't immediately start after the cancelled one. -sleep 1 -$CLICKHOUSE_CLIENT -nq " - select '<27: cancelled>', view, status, exception from refreshes; - system refresh view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] -do - sleep 0.1 -done -# Drop. -$CLICKHOUSE_CLIENT -nq " - drop table f; - select '<28: drop during refresh>', view, status from refreshes; - select '<28: drop during refresh>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase()" - -# Try OFFSET and RANDOMIZE FOR. -$CLICKHOUSE_CLIENT -nq " - create materialized view g refresh every 1 week offset 3 day 4 hour randomize for 4 day 1 hour (x Int64) engine Memory empty as select 42 as x; - show create g; - system test view g set fake time '2050-02-03 15:30:13';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - with '2050-02-10 04:00:00'::DateTime as expected - select '<29: randomize>', abs(next_refresh_time::Int64 - expected::Int64) <= 3600*(24*4+1), next_refresh_time != expected from refreshes;" - -# Send data 'TO' an existing table. -$CLICKHOUSE_CLIENT -nq " - drop table g; - create table dest (x Int64) engine MergeTree order by x; - truncate src; - insert into src values (1); - create materialized view h refresh every 1 second to dest as select x*10 as x from src; - show create h; - system wait view h; - select '<30: to existing table>', * from dest; - insert into src values (2);" -while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest -- $LINENO" | xargs`" != '2' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<31: to existing table>', * from dest; - drop table dest; - drop table h;" - -# Retries. -$CLICKHOUSE_CLIENT -nq " - create materialized view h2 refresh after 1 year settings refresh_retries = 10 (x Int64) engine Memory as select x*10 + throwIf(x % 2 == 0) as x from src;" -$CLICKHOUSE_CLIENT -nq "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -nq " - select '<31.5: will retry>', exception != '', retry > 0 from refreshes; - create table src2 empty as src; - insert into src2 values (1) - exchange tables src and src2; - drop table src2;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, retry from refreshes -- $LINENO" | xargs`" != 'Scheduled 0' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<31.6: did retry>', x from h2; - drop table h2" - -# EMPTY -$CLICKHOUSE_CLIENT -nq " - create materialized view i refresh after 1 year engine Memory empty as select number as x from numbers(2); - system wait view i; - create materialized view j refresh after 1 year engine Memory as select number as x from numbers(2);" -while [ "`$CLICKHOUSE_CLIENT -nq "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<32: empty>', view, status, last_success_time is null, retry from refreshes order by view; - drop table i; - drop table j;" - -# APPEND -$CLICKHOUSE_CLIENT -nq " - create materialized view k refresh every 10 year append (x Int64) engine Memory empty as select x*10 as x from src; - select '<33: append>', * from k; - system refresh view k; - system wait view k; - select '<34: append>', * from k; - truncate table src; - insert into src values (2), (3); - system refresh view k; - system wait view k; - select '<35: append>', * from k order by x;" -# ALTER to non-APPEND -$CLICKHOUSE_CLIENT -nq " - alter table k modify refresh every 10 year;" 2>/dev/null || echo "ALTER from APPEND to non-APPEND failed, as expected" -$CLICKHOUSE_CLIENT -nq " - drop table k; - truncate table src;" - -# APPEND + TO + regular materialized view reading from it. -$CLICKHOUSE_CLIENT -nq " - create table mid (x Int64) engine MergeTree order by x; - create materialized view l refresh every 10 year append to mid empty as select x*10 as x from src; - create materialized view m (x Int64) engine Memory as select x*10 as x from mid; - insert into src values (1); - system refresh view l; - system wait view l; - select '<37: append chain>', * from m; - insert into src values (2); - system refresh view l; - system wait view l; - select '<38: append chain>', * from m order by x; - drop table l; - drop table m; - drop table mid;" - -# Failing to create inner table. -$CLICKHOUSE_CLIENT -nq " - create materialized view n refresh every 1 second (x Int64) engine MergeTree as select 1 as x from numbers(2);" 2>/dev/null || echo "creating MergeTree without ORDER BY failed, as expected" -$CLICKHOUSE_CLIENT -nq " - create materialized view n refresh every 1 second (x Int64) engine MergeTree order by x as select 1 as x from numbers(2); - drop table n;" - -$CLICKHOUSE_CLIENT -nq " - drop table refreshes;" - -Information about Refreshable Materialized Views. Contains all refreshable materialized views, regardless of whether there's a refresh in progress or not. - -Columns: - -database (String) — The name of the database the table is in. -view (String) — Table name. -status (String) — Current state of the refresh. -last_success_time (Nullable(DateTime)) — Time when the latest successful refresh started. NULL if no successful refreshes happened since server startup or table creation. -last_success_time (Nullable(UInt64)) — How long the latest refresh took. -last_refresh_time (Nullable(DateTime)) — Time when the latest refresh attempt finished (if known) or started (if unknown or still running). NULL if no refresh attempts happened since server startup or table creation. -last_refresh_replica (String) — If coordination is enabled, name of the replica that made the current (if running) or previous (if not running) refresh attempt. -next_refresh_time (Nullable(DateTime)) — Time at which the next refresh is scheduled to start, if status = Scheduled. -exception (String) — Error message from previous attempt if it failed. -retry (UInt64) — How many failed attempts there were so far, for the current refresh. -progress (Float64) — Progress of the current refresh, between 0 and 1. Not available if status is RunningOnAnotherReplica. -read_rows (UInt64) — Number of rows read by the current refresh so far. Not available if status is RunningOnAnotherReplica. -total_rows (UInt64) — Estimated total number of rows that need to be read by the current refresh. Not available if status is RunningOnAnotherReplica. - -Available refresh settings: - * `refresh_retries` - How many times to retry if refresh query fails with an exception. If all retries fail, skip to the next scheduled refresh time. 0 means no retries, -1 means infinite retries. Default: 0. - * `refresh_retry_initial_backoff_ms` - Delay before the first retry, if `refresh_retries` is not zero. Each subsequent retry doubles the delay, up to `refresh_retry_max_backoff_ms`. Default: 100 ms. - * `refresh_retry_max_backoff_ms` - Limit on the exponential growth of delay between refresh attempts. Default: 60000 ms (1 minute). - - - -CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name -REFRESH EVERY|AFTER interval [OFFSET interval] -RANDOMIZE FOR interval -DEPENDS ON [db.]name [, [db.]name [, ...]] -SETTINGS name = value [, name = value [, ...]] -[APPEND] -[TO[db.]name] [(columns)] [ENGINE = engine] [EMPTY] -AS SELECT ... -where interval is a sequence of simple intervals: - -number SECOND|MINUTE|HOUR|DAY|WEEK|MONTH|YEAR - -Example refresh schedules: -```sql -REFRESH EVERY 1 DAY -- every day, at midnight (UTC) -REFRESH EVERY 1 MONTH -- on 1st day of every month, at midnight -REFRESH EVERY 1 MONTH OFFSET 5 DAY 2 HOUR -- on 6th day of every month, at 2:00 am -REFRESH EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE -- every other Saturday, at 3:10 pm -REFRESH EVERY 30 MINUTE -- at 00:00, 00:30, 01:00, 01:30, etc -REFRESH AFTER 30 MINUTE -- 30 minutes after the previous refresh completes, no alignment with time of day --- REFRESH AFTER 1 HOUR OFFSET 1 MINUTE -- syntax errror, OFFSET is not allowed with AFTER -``` - -`RANDOMIZE FOR` randomly adjusts the time of each refresh, e.g.: -```sql -REFRESH EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR -- every day at random time between 01:30 and 02:30 - - -SYSTEM REFRESH VIEW - -Wait for the currently running refresh to complete. If the refresh fails, throws an exception. If no refresh is running, completes immediately, throwing an exception if previous refresh failed. - -### SYSTEM STOP VIEW, SYSTEM STOP VIEWS - -### SYSTEM WAIT VIEW - -Waits for the running refresh to complete. If no refresh is running, returns immediately. If the latest refresh attempt failed, reports an error. - -Can be used right after creating a new refreshable materialized view (without EMPTY keyword) to wait for the initial refresh to complete. - -```sql -SYSTEM WAIT VIEW [db.]name - ### TESTS -1. ON CLUSTER? -2. Append mode -3. Drop node and wait for restore -4. Simple functional testing: all values in refresh result correct (two and more rmv) -5. Combinations of time -6. Two (and more) rmv from single to single [APPEND] -7. ALTER rmv ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] -8. RMV without tgt table (automatic table) (check APPEND) ++1. Append mode +2. Restart node and wait for restore ++3. Simple functional testing: all values in refresh result correct (two and more rmv) ++4. Combinations of intervals ++5. Two (and more) rmv from single to single [APPEND] ++6. ALTER rmv ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] ++7. RMV without tgt table (automatic table) (check APPEND) -8 DROP rmv -9 CREATE - DROP - CREATE - ALTER -10 Backups? ++8 DROP rmv ++9 CREATE - DROP - CREATE - ALTER 11. Long queries over refresh time (check settings) -12. RMV settings 13. incorrect intervals (interval 1 sec, offset 1 minute) -14. ALTER on cluster + - OFFSET less than the period. 'EVERY 1 MONTH OFFSET 5 WEEK' + - cases below ++14. ALTER on cluster 15. write to distributed with / without APPEND -16. `default_replica_path` empty on database replicated 17. Not existent TO table (ON CLUSTER) 18. Not existent FROM table (ON CLUSTER) 19. Not existent BOTH tables (ON CLUSTER) -20. retry failed ++20. retry failed 21. overflow with wait test +22. ON CLUSTER (do when database replicated support is ready) -22. ALTER of SELECT? ++SYSTEM STOP|START|REFRESH|CANCEL VIEW +SYSTEM WAIT VIEW [db.]name -23. OFFSET must be less than the period. 'EVERY 1 MONTH OFFSET 5 WEEK' +view_refreshes: ++progress ++elapsed +refresh_count ++exception ++progress: inf if reading from table ++1 if reading from numbers() +RMV settings ++ * `refresh_retries` - How many times to retry if refresh query fails with an exception. If all retries fail, skip to the next scheduled refresh time. 0 means no retries, -1 means infinite retries. Default: 0. + * `refresh_retry_initial_backoff_ms` - Delay before the first retry, if `refresh_retries` is not zero. Each subsequent retry doubles the delay, up to `refresh_retry_max_backoff_ms`. Default: 100 ms. + * `refresh_retry_max_backoff_ms` - Limit on the exponential growth of delay between refresh attempts. Default: 60000 ms (1 minute). """ -def sql_template( +def j2_template( string: str, globals: Optional[dict] = None, filters: Optional[dict] = None, @@ -575,64 +147,147 @@ def assert_same_values(lst: list): assert all(x == lst[0] for x in lst) -CREATE_RMV_TEMPLATE = sql_template( - """CREATE MATERIALIZED VIEW -{% if if_not_exists %}IF NOT EXISTS{% endif %} -{% if db %}{{db}}.{% endif %}{{ table_name }} -{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} -REFRESH {{ refresh_interval }} +RMV_TEMPLATE = """{{ refresh_interval }} {% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} {% if settings %}SETTINGS {{ settings|format_settings }}{% endif %} {% if with_append %}APPEND{% endif %} {% if to_clause %}TO {{ to_clause }}{% endif %} {% if table_clause %}{{ table_clause }}{% endif %} {% if empty %}EMPTY{% endif %} -AS {{ select_query }}""" +{% if select_query %} AS {{ select_query }}{% endif %} +""" + +CREATE_RMV_TEMPLATE = j2_template( + """CREATE MATERIALIZED VIEW +{% if if_not_exists %}IF NOT EXISTS{% endif %} +{% if db %}{{db}}.{% endif %}{{ table_name }} +{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} +REFRESH +""" + + RMV_TEMPLATE ) # ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] - -ALTER_RMV_TEMPLATE = sql_template( +ALTER_RMV_TEMPLATE = j2_template( """ALTER TABLE {% if db %}{{db}}.{% endif %}{{ table_name }} {% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} -MODIFY REFRESH {{ refresh_interval }} -{% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} -{% if with_append %}APPEND{% endif %} -{% if settings %}SETTINGS {{ settings|format_settings }}{% endif %}""" +MODIFY REFRESH +""" + + RMV_TEMPLATE ) +@pytest.fixture(scope="module") +def module_setup_tables(): + node.query(f"CREATE DATABASE IF NOT EXISTS test_db ENGINE = Atomic") + + node.query( + f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query( + f"CREATE TABLE src2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query( + f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" + ) + node.query( + f"CREATE TABLE tgt2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query( + f"CREATE MATERIALIZED VIEW IF NOT EXISTS dummy_rmv ON CLUSTER test_cluster " + f"REFRESH EVERY 10 HOUR engine Memory EMPTY AS select number as x from numbers(1)" + ) + yield + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS src2 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS tgt2 ON CLUSTER test_cluster") + + +@pytest.fixture(scope="function") +def fn_setup_tables(): + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + yield + + +def test_simple_append(fn_setup_tables): + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query="SELECT now() as a, number as b FROM numbers(2)", + with_append=True, + empty=False, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + assert rmv["exception"] is None + + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + time.sleep(2) + rmv2 = get_rmv_info( + node, + "test_rmv", + ) + + assert rmv2["exception"] is None + + +def test_simple_append_from_table(fn2_setup_tables): + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query="SELECT now() as a, b as b FROM src1", + with_append=True, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + time.sleep(2) + rmv2 = get_rmv_info( + node, + "test_rmv", + ) + + assert rmv2["exception"] is None + + @pytest.mark.parametrize("with_append", [True, False]) -@pytest.mark.parametrize("create_target_table", [True, False]) @pytest.mark.parametrize("if_not_exists", [True, False]) @pytest.mark.parametrize("on_cluster", [True, False]) -@pytest.mark.parametrize( - "depends_on", [None, ["dummy_rmv"], ["default.dummy_rmv", "src1"]] -) +@pytest.mark.parametrize("depends_on", [None, ["default.dummy_rmv"]]) @pytest.mark.parametrize("empty", [True, False]) @pytest.mark.parametrize("database_name", [None, "default", "test_db"]) -def test_correct_states( - request, +@pytest.mark.parametrize( + "settings", + [ + {}, + { + "refresh_retries": "10", + "refresh_retry_initial_backoff_ms": "10", + "refresh_retry_max_backoff_ms": "20", + }, + ], +) +def test_alters( + module_setup_tables, + fn_setup_tables, started_cluster, with_append, - create_target_table, if_not_exists, on_cluster, depends_on, empty, database_name, + settings, ): """ Check correctness of functional states of RMV after CREATE, DROP, ALTER, trigger of RMV, ... - Check ref """ - - def teardown(): - node1_1.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - - request.addfinalizer(teardown) - create_sql = CREATE_RMV_TEMPLATE.render( table_name="test_rmv", if_not_exists=if_not_exists, @@ -643,27 +298,26 @@ def test_correct_states( select_query="SELECT * FROM src1", with_append=with_append, on_cluster="test_cluster" if on_cluster else None, - # settings={'setting1':'value1', 'setting2': 'value2'}, - ) - print(create_sql) - node1_1.query(create_sql) - refreshes = node1_1.query( - "SELECT hostname(), * FROM clusterAllReplicas('test_cluster', system.view_refreshes)", - parse=True, + settings=settings, ) + node.query(create_sql) # Check same RMV is created on cluster - def compare_create_all_nodes(): + def compare_DDL_on_all_nodes(): show_create_all_nodes = cluster.query_all_nodes("SHOW CREATE test_rmv") if not on_cluster: - del show_create_all_nodes["node1_1"] + del show_create_all_nodes["node1_2"] assert_same_values(show_create_all_nodes.values()) - compare_create_all_nodes() + compare_DDL_on_all_nodes() - show_create = node1_1.query("SHOW CREATE test_rmv") + node.query(f"DROP TABLE test_rmv {'ON CLUSTER test_cluster' if on_cluster else ''}") + node.query(create_sql) + compare_DDL_on_all_nodes() + + show_create = node.query("SHOW CREATE test_rmv") # Alter of RMV replaces all non-specified alter_sql = ALTER_RMV_TEMPLATE.render( @@ -672,272 +326,121 @@ def test_correct_states( db_name=database_name, refresh_interval="EVERY 1 HOUR", depends_on=depends_on, - select_query="SELECT * FROM src1", + # can't change select with alter + # select_query="SELECT * FROM src1", with_append=with_append, on_cluster="test_cluster" if on_cluster else None, - # settings={'setting1':'value1', 'setting2': 'value2'}, + settings=settings, ) - node1_1.query(alter_sql) - show_create_after_alter = node1_1.query("SHOW CREATE test_rmv") - - compare_create_all_nodes() + node.query(alter_sql) + show_create_after_alter = node.query("SHOW CREATE test_rmv") assert show_create == show_create_after_alter - # breakpoint() + compare_DDL_on_all_nodes() -def compare_dates( - date1: str | datetime, - date2: str | datetime | tuple[datetime], - inaccuracy=timedelta(hours=1), - format_str="%Y-%m-%d %H:%M:%S", -) -> bool: - """ - Compares two dates with an inaccuracy of 2 minutes. - """ - if isinstance(date2, tuple): - return date2[0] <= date1 <= date2[1] - - if isinstance(date1, str): - date1 = datetime.strptime(date1, format_str) - if isinstance(date2, str): - date2 = datetime.strptime(date2, format_str) - - return abs(date1 - date2) <= inaccuracy - - -def date_in_interval( - date1: str | datetime, - date2: str | datetime, - inaccuracy=timedelta(minutes=2), - format_str="%Y-%m-%d %H:%M:%S", +def get_rmv_info( + node, table, condition=None, max_attempts=50, delay=0.3, wait_status="Scheduled" ): - pass + def inner(): + rmv_info = node.query_with_retry( + f"SELECT * FROM system.view_refreshes WHERE view='{table}'", + check_callback=(lambda r: r.iloc[0]["status"] == wait_status) + if wait_status + else (lambda x: True), + parse=True, + ).to_dict("records")[0] + # Is it time for python clickhouse-driver? + rmv_info["next_refresh_time"] = parse_ch_datetime(rmv_info["next_refresh_time"]) + rmv_info["last_success_time"] = parse_ch_datetime(rmv_info["last_success_time"]) + rmv_info["last_refresh_time"] = parse_ch_datetime(rmv_info["last_refresh_time"]) + logging.info(rmv_info) + return rmv_info -def get_rmv_info(node, table): - rmv_info = node.query_with_retry( - f"SELECT * FROM system.view_refreshes WHERE view='{table}'", - check_callback=lambda r: r.iloc[0]["status"] == "Scheduled", - parse=True, - ).to_dict("records")[0] + if condition: + res = wait_condition(inner, condition, max_attempts=max_attempts, delay=delay) + return res - rmv_info["next_refresh_time"] = parse_ch_datetime(rmv_info["next_refresh_time"]) - - return rmv_info - - - -@pytest.mark.parametrize("with_append", [True, False]) -@pytest.mark.parametrize("create_target_table", [True, False]) -# @pytest.mark.parametrize("if_not_exists", [True, False]) -@pytest.mark.parametrize("on_cluster", [True, False]) -@pytest.mark.parametrize("depends_on", [None, ["dummy_rmv"]]) -@pytest.mark.parametrize("randomize", [True, False]) -@pytest.mark.parametrize("empty", [True, False]) -@pytest.mark.parametrize("database_name", [None, "default", "test_db"]) -def test_check_data( - request, - started_cluster, - with_append, - create_target_table, - # if_not_exists, - on_cluster, - depends_on, - randomize, - empty, - database_name, -): - def teardown(): - node1_1.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - node1_1.query("DROP TABLE IF EXISTS tgt_new ON CLUSTER test_cluster") - node1_1.query("TRUNCATE TABLE tgt1 ON CLUSTER test_cluster") - - request.addfinalizer(teardown) - - # CREATE RMV can use already existed table - # or create new one - if create_target_table: - to_clause = "tgt1" - tgt = "tgt1" - else: - to_clause = "tgt_new Engine = Memory" - tgt = "tgt_new" - - create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv", - # if_not_exists=if_not_exists, - db_name=database_name, - refresh_interval="EVERY 1 HOUR", - randomize_interval="30 MINUTE" if randomize else None, - depends_on=depends_on, - to_clause=to_clause, - select_query="SELECT now() as a, number as b FROM numbers(2)", - with_append=with_append, - on_cluster="test_cluster" if on_cluster else None, - empty=empty - # settings={'setting1':'value1', 'setting2': 'value2'}, - ) - print(create_sql) - node1_1.query(create_sql) - - # now = node1_1.query("SELECT now()", parse=True)[0][0] - now = datetime.utcnow() - - rmv = get_rmv_info(node1_1, "test_rmv") - print(rmv) - - """ - 1. check table view_refreshes - 2. check inserted data if without EMPTY - 3. set time, wait for refresh - 4. check data inserted - 5. alter table - """ - - assert rmv["exception"] is None - assert rmv["status"] == "Scheduled" - assert rmv["progress"] is None - - if not empty: - # Insert - assert compare_dates( - rmv["last_refresh_time"], now, format_str="%Y-%m-%d %H:%M:%S" - ) - assert rmv["last_success_time"] > 0 - inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) - print(inserted_data) - assert len(inserted_data) == 2 - - if empty: - assert rmv["last_refresh_time"] is None - assert rmv["last_success_time"] is None - - assert rmv["retry"] == 0 - assert rmv["read_rows"] == 0 - assert rmv["read_bytes"] == 0 - assert rmv["total_rows"] == 0 - assert rmv["total_bytes"] == 0 - assert rmv["written_rows"] == 0 - assert rmv["written_bytes"] == 0 - assert rmv["result_rows"] == 0 - assert rmv["result_bytes"] == 0 - - # Rewind to the next trigger and wait for it - node1_1.query( - f"SYSTEM TEST VIEW test_rmv set fake time '{rmv['next_refresh_time']}';" - ) - - now = datetime.utcnow() - inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) - if with_append: - assert len(inserted_data) == 4 - else: - assert len(inserted_data) == 2 - - # alter time - - -""" - -(Pdb) pp interval -'EVERY 1 WEEK' -(Pdb) pp next_refresh_time -datetime.datetime(2024, 5, 6, 0, 0) -(Pdb) pp predicted_next_refresh_time -datetime.datetime(2024, 5, 6, 0, 0) - - -(Pdb) pp interval -'EVERY 2 WEEK' -(Pdb) pp next_refresh_time -datetime.datetime(2024, 5, 6, 0, 0) -(Pdb) pp predicted_next_refresh_time -datetime.datetime(2024, 5, 13, 0, 0) - - -(Pdb) pp interval -'EVERY 2 WEEK OFFSET 1 DAY' -(Pdb) pp next_refresh_time -datetime.datetime(2024, 5, 7, 0, 0) -(Pdb) pp predicted_next_refresh_time -datetime.datetime(2024, 5, 14, 0, 0) - - -(Pdb) pp interval -'EVERY 2 WEEK OFFSET 2 DAY' -(Pdb) pp next_refresh_time -datetime.datetime(2024, 5, 8, 0, 0) -(Pdb) pp predicted_next_refresh_time -datetime.datetime(2024, 5, 15, 0, 0) - - -'EVERY 1 WEEK OFFSET 2 DAY' -(Pdb) pp next_refresh_time -datetime.datetime(2024, 5, 1, 0, 0) -(Pdb) pp predicted_next_refresh_time -datetime.datetime(2024, 5, 8, 0, 0) - -""" + res = inner() + return res def parse_ch_datetime(date_str): + if date_str is None: + return None return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") +# Add cases for: +# Randomize bigger than interval +# executed even with empty +"AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", +"EVERY 1 MINUTE OFFSET 1 SECOND RANDOMIZE FOR 3 YEAR", + +# special case +"EVERY 999 MINUTE" + +# should be restricted: +# Two different units in EVERY +"EVERY 1 YEAR 0 MONTH", +"EVERY 1 YEAR 1 MONTH", +"EVERY 1 YEAR 2 MONTH", +# "EVERY 1 YEAR 3 MONTH", +# "EVERY 1 YEAR 4 MONTH", +# "EVERY 1 YEAR 8 MONTH", +# "EVERY 1 YEAR 9 MONTH", +# "EVERY 1 YEAR 10 MONTH", +# "EVERY 1 YEAR 11 MONTH", +"EVERY 1 YEAR 12 MONTH", +# "EVERY 1 YEAR 13 MONTH", +# "EVERY 1 DAY", + +# should be restricted? +"EVERY 1 YEAR 2 YEAR", +"EVERY 1 MONTH 2 MONTH", +"AFTER 1 YEAR 2 YEAR", +"AFTER 1 MONTH 2 MONTH", + +"EVERY 1 DAY 0 HOUR", +"EVERY 1 DAY 1 HOUR", +"EVERY 1 DAY 2 HOUR", +"EVERY 1 DAY 23 HOUR", +"EVERY 1 DAY 24 HOUR", +"EVERY 1 DAY 28 HOUR", +# "EVERY 1 DAY 29 HOUR", +# "EVERY 1 DAY 30 HOUR", +# "EVERY 1 DAY 31 HOUR", +"EVERY 1 DAY 32 HOUR", + +# Interval shouldn't contain both calendar units and clock units (e.g. months and days) +# "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", + + INTERVALS_EVERY = [ - # Same units - "EVERY 1 YEAR 2 YEAR", - # + "EVERY 60 SECOND", "EVERY 1 MINUTE", "EVERY 1 HOUR", "EVERY 1 DAY", - "EVERY 1 WEEK", - "EVERY 2 WEEK", "EVERY 1 MONTH", "EVERY 1 YEAR", "EVERY 1 WEEK RANDOMIZE FOR 1 HOUR", "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", "EVERY 1 MONTH OFFSET 1 DAY RANDOMIZE FOR 10 HOUR", - # smth wrong with - # "EVERY 1 WEEK", - # "EVERY 2 WEEK", - # "EVERY 3 WEEK", - # "EVERY 4 WEEK", - # "EVERY 5 WEEK", - # "EVERY 1 MONTH OFFSET 1 WEEK", - # "EVERY 1 MONTH OFFSET 2 WEEK", - # Two different units in EVERY - # "EVERY 1 YEAR", - # "EVERY 1 YEAR 0 MONTH", - # "EVERY 1 YEAR 1 MONTH", - # "EVERY 1 YEAR 2 MONTH", - # "EVERY 1 YEAR 3 MONTH", - # "EVERY 1 YEAR 4 MONTH", - # "EVERY 1 YEAR 8 MONTH", - # "EVERY 1 YEAR 9 MONTH", - # "EVERY 1 YEAR 10 MONTH", - # "EVERY 1 YEAR 11 MONTH", - # "EVERY 1 YEAR 12 MONTH", - # "EVERY 1 YEAR 13 MONTH", - # "EVERY 1 DAY", - # "EVERY 1 DAY 0 HOUR", - # "EVERY 1 DAY 1 HOUR", - # "EVERY 1 DAY 2 HOUR", - # "EVERY 1 DAY 28 HOUR", - # "EVERY 1 DAY 29 HOUR", - # "EVERY 1 DAY 30 HOUR", - # "EVERY 1 DAY 31 HOUR", - # "EVERY 1 DAY 32 HOUR", - # Interval shouldn't contain both calendar units and clock units (e.g. months and days) - # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", + "EVERY 1 WEEK", + "EVERY 2 WEEK", + "EVERY 3 WEEK", + "EVERY 4 WEEK", + "EVERY 5 WEEK", ] INTERVALS_AFTER = [ - "AFTER 1 YEAR 2 YEAR", - "AFTER 30 SECOND", + "AFTER 59 SECOND", + "AFTER 60 SECOND", + "AFTER 61 SECOND", "AFTER 30 MINUTE", + "AFTER 9999 MINUTE", "AFTER 2 HOUR", "AFTER 2 DAY", "AFTER 2 WEEK", @@ -946,10 +449,6 @@ INTERVALS_AFTER = [ "AFTER 2 MONTH 3 DAY", "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", "AFTER 1 YEAR RANDOMIZE FOR 11 MONTH", - # Randomize bigger than interval - "AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", - "EVERY 1 MINUTE RANDOMIZE FOR 3 YEAR", - "EVERY 1 MINUTE OFFSET 1 SECOND RANDOMIZE FOR 3 YEAR", "AFTER 1 MONTH", "AFTER 1 MONTH 0 DAY", "AFTER 1 MONTH 1 DAY", @@ -962,278 +461,404 @@ INTERVALS_AFTER = [ ] -@pytest.mark.parametrize( - "interval", - INTERVALS_EVERY, -) +@pytest.fixture(scope="function") +def rmv_schedule_teardown(): + node.query("DROP TABLE IF EXISTS test_rmv") + + +def expect_rows(rows, table="test_rmv"): + inserted_data = node.query_with_retry( + f"SELECT * FROM {table}", + parse=True, + check_callback=lambda x: len(x) == rows, + retry_count=100, + ) + assert len(inserted_data) == rows + + +@pytest.mark.parametrize("interval", INTERVALS_AFTER + INTERVALS_EVERY) @pytest.mark.parametrize( "append", - [ - # True, - False - ], + [True, False], ) @pytest.mark.parametrize( "empty", - [ - True, - # False - ], + [True, False], ) -def test_schedule_2( - request, - started_cluster, +def test_rmv_scheduling( + rmv_schedule_teardown, interval, append, empty, ): """ - - Create RMV - - Check table view_refreshes - - Check inserted data if without EMPTY - - Set time, wait for refresh - - Check data is inserted/appended - - Alter table - - Check everything again - - DROP target table + Test creates RMV with multiple intervals, checks correctness of 'next_refresh_time' using schedule model, + check that data correctly written, set fake time and check all again. """ - - # if "AFTER" in interval: - # pytest.skip() - - def teardown(): - node1_1.query("DROP TABLE IF EXISTS test_rmv_schedule") - node1_1.query("DROP TABLE IF EXISTS tgt_new") - node1_1.query("TRUNCATE TABLE tgt1") - - request.addfinalizer(teardown) - create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv_schedule", + table_name="test_rmv", refresh_interval=interval, table_clause="ENGINE = Memory", select_query="SELECT now() as a, number as b FROM numbers(2)", with_append=append, empty=empty, ) - print(create_sql) - node1_1.query(create_sql) + node.query(create_sql) now = datetime.utcnow() - - rmv = get_rmv_info(node1_1, "test_rmv_schedule") - # this is for EVERY - predicted_next_refresh_time = get_next_refresh_time(interval, now) - # this is for AFTER - # diff = next_refresh_time - now.replace(microsecond=0) - - if 'WEEK' in - assert compare_dates(rmv["next_refresh_time"], predicted_next_refresh_time) + rmv = get_rmv_info(node, "test_rmv") + predicted_next_refresh_time = get_next_refresh_time(interval, now, first_week=True) + compare_dates( + rmv["next_refresh_time"], predicted_next_refresh_time, first_week=True + ) assert rmv["next_refresh_time"] > now + assert rmv["exception"] is None - def expect_rows(rows): - inserted_data = node1_1.query_with_retry( - f"SELECT * FROM test_rmv_schedule", - parse=True, - check_callback=lambda x: len(x) == rows, - retry_count=100, - ) - assert len(inserted_data) == rows - - # Check data if not EMPTY - append_expect_rows = 0 - if append: - # Append adds rows - append_expect_rows += 2 - expect_rows(append_expect_rows) - if not append: - # Rewrite without append - expect_rows(2) - - inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) if empty: # No data is inserted with empty - assert len(inserted_data) == 0 + expect_rows(0) else: - # Query is executed without empty - assert rmv["last_success_time"] > 0 + # Query is executed one time + compare_dates_(rmv["last_success_time"], now) expect_rows(2) + # TODO: SET FAKE TIME doesn't work for these + if "RANDOMIZE" in interval: + return + if "OFFSET" in interval: + return + # Trigger refresh, update `next_refresh_time` and check interval again - node1_1.query( - f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{rmv['next_refresh_time']}'" + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + predicted_next_refresh_time2 = get_next_refresh_time( + interval, rmv["next_refresh_time"] ) - predicted_next_refresh_time = get_next_refresh_time(interval, rmv['next_refresh_time']) - rmv = get_rmv_info(node1_1, "test_rmv_schedule") + rmv2 = get_rmv_info( + node, + "test_rmv", + condition=lambda x: x["next_refresh_time"] != rmv["next_refresh_time"], + ) + compare_dates(rmv2["next_refresh_time"], predicted_next_refresh_time2) + assert rmv["exception"] is None, rmv - assert compare_dates(rmv["next_refresh_time"], predicted_next_refresh_time) - - # Check data if not EMPTY - if append: - # Append adds rows - append_expect_rows += 2 - expect_rows(append_expect_rows) + if append and not empty: + expect_rows(4) else: - # Rewrite expect_rows(2) - # breakpoint() + compare_dates( + rmv2["last_success_time"], predicted_next_refresh_time, first_week=True + ) + + +@pytest.fixture(scope="function") +def fn2_setup_tables(): + node.query( + f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" + ) + + node.query( + f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query(f"INSERT INTO src1 VALUES ('2020-01-01', 1), ('2020-01-02', 2)") + + yield + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") -@pytest.mark.parametrize( - "interval", - INTERVALS_EVERY, -) @pytest.mark.parametrize( "append", - [ - # True, - False - ], + [True, False], ) +# @pytest.mark.parametrize("on_cluster", [True, False]) @pytest.mark.parametrize( "empty", - [ - True, - # False - ], + [True, False], ) -def test_schedule( - request, +@pytest.mark.parametrize( + "to_clause", + [(None, "tgt1", "tgt1"), ("Engine MergeTree ORDER BY tuple()", None, "test_rmv")], +) +def test_real_wait_refresh( + fn2_setup_tables, started_cluster, - interval, append, + # on_cluster, empty, + to_clause, ): - """ - - Create RMV - - Check table view_refreshes - - Check inserted data if without EMPTY - - Set time, wait for refresh - - Check data is inserted/appended - - Alter table - - Check everything again - - DROP target table - """ - - # if "WEEK" in interval: - # pytest.skip() - - def teardown(): - node1_1.query("DROP TABLE IF EXISTS test_rmv_schedule") - node1_1.query("DROP TABLE IF EXISTS tgt_new") - node1_1.query("TRUNCATE TABLE tgt1") - - request.addfinalizer(teardown) + table_clause, to_clause_, tgt = to_clause create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv_schedule", - refresh_interval=interval, - table_clause="ENGINE = Memory", - select_query="SELECT now() as a, number as b FROM numbers(2)", + table_name="test_rmv", + # if_not_exists=if_not_exists, + refresh_interval="EVERY 8 SECOND", + to_clause=to_clause_, + table_clause=table_clause, + select_query="SELECT a as a, b FROM src1", with_append=append, - empty=empty, + # on_cluster="test_cluster" if on_cluster else None, + empty=empty + # settings={'setting1':'value1', 'setting2': 'value2'}, ) - print(create_sql) - node1_1.query(create_sql) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") - now = datetime.utcnow() + append_expected_rows = 0 + if not empty: + append_expected_rows += 2 - rmv = get_rmv_info(node1_1, "test_rmv_schedule") - print(rmv) - - next_refresh_time = parse_ch_datetime(rmv["next_refresh_time"]) - predicted_next_refresh_time = get_next_refresh_time(interval, now) - - print("----") - print("current_time:", now) - print("Interval:", interval) - print("next_refresh_time", next_refresh_time) - print("predicted", predicted_next_refresh_time) - print("----") - - # RANDOMIZE means the next refresh time will be randomly chosen - # within a range of RANDOMIZE interval - if "RANDOMIZE" in interval: - assert ( - predicted_next_refresh_time[0] - <= next_refresh_time - <= predicted_next_refresh_time[1] - ) + if empty: + expect_rows(0, table=tgt) else: - assert compare_dates(next_refresh_time, predicted_next_refresh_time) + expect_rows(2, table=tgt) - assert next_refresh_time > now + rmv2 = get_rmv_info( + node, + "test_rmv", + condition=lambda x: x["last_refresh_time"] == rmv["next_refresh_time"], + # wait for refresh a little bit more than 8 seconds + max_attempts=10, + delay=1, + ) - # append_expect_rows = 0 - # - # def check_data(): - # inserted_data = node1_1.query_with_retry( - # f"SELECT * FROM test_rmv_schedule", - # parse=True, - # check_callback=lambda x: len(x) > 0, - # retry_count=200, - # ) - # - # # Check data if not EMPTY - # if append: - # # Append adds rows - # global append_expect_rows - # append_expect_rows += 2 - # assert len(inserted_data) == append_expect_rows - # if not append: - # # Rewrite without append - # assert len(inserted_data) == 2 - # - # inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) - # if empty: - # assert len(inserted_data) == 0 - # else: - # assert rmv["last_success_time"] > 0 - # check_data() - # - # # Trigger next refresh - # node1_1.query( - # f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time}'" - # ) - # - # if "RANDOMIZE" not in interval: - # check_data() - # ----------------------------- - # rmv = get_rmv_info(node1_1, "test_rmv_schedule") - # next_refresh_time = parse_ch_datetime(rmv["next_refresh_time"]) + if append: + append_expected_rows += 2 + expect_rows(append_expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) - # # Alter RMV to random interval and test schedule is changed - # # TODO: remove week filter after fix - # interval_alter = random.choice(list(filter(lambda x: "WEEK" not in x, INTERVALS))) - # alter_sql = ALTER_RMV_TEMPLATE.render( - # table_name="test_rmv_schedule", - # refresh_interval=interval_alter, - # # select_query="SELECT * FROM src1", - # ) - # print(alter_sql) - # node1_1.query(alter_sql) - # now_alter = datetime.utcnow() - # - # rmv = get_rmv_info(node1_1, "test_rmv_schedule") - # next_refresh_time_alter = parse_ch_datetime(rmv["next_refresh_time"]) - # predicted_next_refresh_time = get_next_refresh_time(interval_alter, now_alter) - # - # if "RANDOMIZE" in interval_alter: - # assert ( - # predicted_next_refresh_time[0] - # <= next_refresh_time_alter - # <= predicted_next_refresh_time[1] - # ) - # else: - # assert compare_dates(next_refresh_time_alter, predicted_next_refresh_time) - # - # assert next_refresh_time_alter > now_alter - # - # # Trigger next refresh - # node1_1.query( - # f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time_alter}'" - # ) - # check_data() + assert rmv2["exception"] is None + assert rmv2["refresh_count"] == 2 + assert rmv2["status"] == "Scheduled" + assert rmv2["last_refresh_result"] == "Finished" + # assert rmv2["progress"] == , 1rmv2 + assert rmv2["last_success_time"] == rmv["next_refresh_time"] + assert rmv2["last_refresh_time"] == rmv["next_refresh_time"] + assert rmv2["retry"] == 0 + assert rmv2["read_rows"] == 2 + assert rmv2["read_bytes"] == 24 + assert rmv2["total_rows"] == 0 + assert rmv2["total_bytes"] == 0 + assert rmv2["written_rows"] == 0 + assert rmv2["written_bytes"] == 0 + assert rmv2["result_rows"] == 0 + assert rmv2["result_bytes"] == 0 - # breakpoint() + node.query("SYSTEM STOP VIEW test_rmv") + time.sleep(10) + rmv3 = get_rmv_info(node, "test_rmv") + # no refresh happen + assert rmv3["status"] == "Disabled" + + del rmv3["status"] + del rmv2["status"] + assert rmv3 == rmv2 + + node.query("SYSTEM START VIEW test_rmv") + time.sleep(1) + rmv4 = get_rmv_info(node, "test_rmv") + + if append: + append_expected_rows += 2 + expect_rows(append_expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + assert rmv4["exception"] is None + assert rmv4["refresh_count"] == 3 + assert rmv4["status"] == "Scheduled" + assert rmv4["retry"] == 0 + assert rmv4["read_rows"] == 2 + assert rmv4["read_bytes"] == 24 + # why 0? + assert rmv4["total_rows"] == 0 + assert rmv4["total_bytes"] == 0 + assert rmv4["written_rows"] == 0 + assert rmv4["written_bytes"] == 0 + assert rmv4["result_rows"] == 0 + assert rmv4["result_bytes"] == 0 + + node.query("SYSTEM REFRESH VIEW test_rmv") + time.sleep(1) + if append: + append_expected_rows += 2 + expect_rows(append_expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + +@pytest.fixture(scope="function") +def fn3_setup_tables(): + node.query( + f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" + ) + node.query( + f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query(f"INSERT INTO src1 SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)") + + yield + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + + +@pytest.mark.parametrize( + "select_query", + ["SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)", "SELECT * FROM src1"], +) +def test_long_query(fn3_setup_tables, select_query): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query=select_query, + with_append=False, + empty=True, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + + wait_seconds = 0 + while wait_seconds < 30: + time.sleep(1) + wait_seconds += 1 + rmv = get_rmv_info(node, "test_rmv", wait_status=None) + logging.info(rmv) + if rmv["progress"] == 1: + break + + assert rmv["status"] == "Running" + assert 0 < rmv["progress"] < 1 + assert 0 < rmv["read_rows"] < 1000000000 + assert 0 < rmv["read_bytes"] < 8000000000 + + assert rmv["progress"] == 1 + assert rmv["exception"] is None + assert 3 <= wait_seconds <= 30, wait_seconds + assert rmv["duration_ms"] >= 3000 + assert rmv["total_rows"] == 1000000000 + assert rmv["read_rows"] == 1000000000 + assert 0 < rmv["read_bytes"] == 8000000000 + + +@pytest.mark.parametrize( + "select_query", + ["SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)", "SELECT * FROM src1"], +) +def test_long_query_cancel(fn3_setup_tables, select_query): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query=select_query, + with_append=False, + empty=True, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + time.sleep(1) + + node.query("SYSTEM CANCEL VIEW test_rmv") + time.sleep(1) + rmv = get_rmv_info(node, "test_rmv", wait_status=None) + assert rmv["status"] == "Scheduled" + assert rmv["last_refresh_result"] == "Cancelled" + assert 0 < rmv["progress"] < 1 + assert 0 < rmv["read_rows"] < 1000000000 + assert 0 < rmv["read_bytes"] < 8000000000 + + assert rmv["last_success_time"] is None + assert rmv["duration_ms"] > 0 + assert rmv["total_rows"] == 1000000000 + + +@pytest.fixture(scope="function") +def fn4_setup_tables(): + node.query( + f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime) ENGINE = Memory" + ) + yield + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + + +def test_query_fail(fn4_setup_tables): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + # Argument at index 1 for function throwIf must be constant + select_query="SELECT throwIf(1, toString(rand())) a", + with_append=False, + empty=True, + settings={ + "refresh_retries": "10", + }, + ) + with pytest.raises(helpers.client.QueryRuntimeException) as exc: + node.query(create_sql) + assert "Argument at index 1 for function throwIf must be constant" in str( + exc.value + ) + assert ( + node.query(f"SELECT count() FROM system.view_refreshes WHERE view='test_rmv'") + == "0\n" + ) + assert ( + node.query(f"SELECT count() FROM system.tables WHERE name='test_rmv'") == "0\n" + ) + + +def test_query_retry(fn4_setup_tables): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query="SELECT throwIf(1, '111') a", # exception in 4/5 calls + with_append=False, + empty=True, + settings={ + "refresh_retries": "10", # TODO -1 + "refresh_retry_initial_backoff_ms": "10", + "refresh_retry_max_backoff_ms": "10", + }, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + time.sleep(20) + + rmv2 = get_rmv_info( + node, + "test_rmv", + wait_status=None, + ) + assert rmv2["last_refresh_result"] == "Error" + assert rmv2["retry"] == 10 + assert False + + +# def test_query_retry_success(fn4_setup_tables): +# TODO: SELECT throwIf(rand() % 10 != 0, '111') a +# pass + + +# TODO -1 diff --git a/tests/integration/test_refreshable_mat_view/test_schedule_model.py b/tests/integration/test_refreshable_mat_view/test_schedule_model.py index 7378598e85e..da7978918e4 100644 --- a/tests/integration/test_refreshable_mat_view/test_schedule_model.py +++ b/tests/integration/test_refreshable_mat_view/test_schedule_model.py @@ -3,79 +3,87 @@ from datetime import datetime from test_refreshable_mat_view.schedule_model import get_next_refresh_time +def get_next_refresh_time_(*args, **kwargs): + return get_next_refresh_time(*args, **kwargs)["time"] + + def test_refresh_schedules(): t = datetime(2000, 1, 1, 1, 1, 1) - # assert get_next_refresh_time("EVERY 1 SECOND", t) == datetime(2000, 1, 1, 1, 1, 2) - # assert get_next_refresh_time("EVERY 1 MINUTE", t) == datetime( - # 2000, - # 1, - # 1, - # 1, - # 2, - # ) - # assert get_next_refresh_time("EVERY 1 HOUR", t) == datetime( - # 2000, - # 1, - # 1, - # 2, - # ) - # assert get_next_refresh_time("EVERY 1 DAY", t) == datetime(2000, 1, 2) - # assert get_next_refresh_time("EVERY 1 WEEK", t) == datetime(2000, 1, 10) - # assert get_next_refresh_time("EVERY 2 WEEK", t) == datetime(2000, 1, 17) - # assert get_next_refresh_time("EVERY 1 MONTH", t) == datetime(2000, 2, 1) - # assert get_next_refresh_time("EVERY 1 YEAR", t) == datetime(2001, 1, 1) - # - # assert get_next_refresh_time("EVERY 3 YEAR 4 MONTH 10 DAY", t) == datetime( - # 2003, 5, 11 - # ) - # - # # OFFSET - # assert get_next_refresh_time( - # "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t - # ) == datetime(2000, 2, 6, 2, 30, 15) - # assert get_next_refresh_time( - # "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t - # ) == datetime(2001, 3, 6, 2, 30, 15) - # - # assert get_next_refresh_time( - # "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", t - # ) == datetime(2000, 1, 22, 15, 10) - # - # # AFTER - # assert get_next_refresh_time("AFTER 30 SECOND", t) == datetime(2000, 1, 1, 1, 1, 31) - # assert get_next_refresh_time("AFTER 30 MINUTE", t) == datetime(2000, 1, 1, 1, 31, 1) - # assert get_next_refresh_time("AFTER 2 HOUR", t) == datetime(2000, 1, 1, 3, 1, 1) - # assert get_next_refresh_time("AFTER 2 DAY", t) == datetime(2000, 1, 3, 1, 1, 1) - # assert get_next_refresh_time("AFTER 2 WEEK", t) == datetime(2000, 1, 15, 1, 1, 1) - # assert get_next_refresh_time("AFTER 2 MONTH", t) == datetime(2000, 3, 1, 1, 1, 1) - # assert get_next_refresh_time("AFTER 2 YEAR", t) == datetime(2002, 1, 1, 1, 1, 1) - # - # assert get_next_refresh_time("AFTER 2 YEAR 1 MONTH", t) == datetime( - # 2002, 2, 1, 1, 1, 1 - # ) - # - # assert get_next_refresh_time("AFTER 1 MONTH 2 YEAR", t) == datetime( - # 2002, 2, 1, 1, 1, 1 - # ) + assert get_next_refresh_time_("EVERY 1 SECOND", t) == datetime(2000, 1, 1, 1, 1, 2) + assert get_next_refresh_time_("EVERY 1 MINUTE", t) == datetime( + 2000, + 1, + 1, + 1, + 2, + ) + assert get_next_refresh_time_("EVERY 1 HOUR", t) == datetime( + 2000, + 1, + 1, + 2, + ) + assert get_next_refresh_time_("EVERY 1 DAY", t) == datetime(2000, 1, 2) + assert get_next_refresh_time_("EVERY 1 WEEK", t) == datetime(2000, 1, 10) + assert get_next_refresh_time_("EVERY 2 WEEK", t) == datetime(2000, 1, 17) + assert get_next_refresh_time_("EVERY 1 MONTH", t) == datetime(2000, 2, 1) + assert get_next_refresh_time_("EVERY 1 YEAR", t) == datetime(2001, 1, 1) + + assert get_next_refresh_time_("EVERY 3 YEAR 4 MONTH 10 DAY", t) == datetime( + 2003, 5, 11 + ) + + # OFFSET + assert get_next_refresh_time_( + "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t + ) == datetime(2000, 2, 6, 2, 30, 15) + assert get_next_refresh_time_( + "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t + ) == datetime(2001, 3, 6, 2, 30, 15) + + assert get_next_refresh_time_( + "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", t + ) == datetime(2000, 1, 22, 15, 10) + + # AFTER + assert get_next_refresh_time_("AFTER 30 SECOND", t) == datetime( + 2000, 1, 1, 1, 1, 31 + ) + assert get_next_refresh_time_("AFTER 30 MINUTE", t) == datetime( + 2000, 1, 1, 1, 31, 1 + ) + assert get_next_refresh_time_("AFTER 2 HOUR", t) == datetime(2000, 1, 1, 3, 1, 1) + assert get_next_refresh_time_("AFTER 2 DAY", t) == datetime(2000, 1, 3, 1, 1, 1) + assert get_next_refresh_time_("AFTER 2 WEEK", t) == datetime(2000, 1, 15, 1, 1, 1) + assert get_next_refresh_time_("AFTER 2 MONTH", t) == datetime(2000, 3, 1, 1, 1, 1) + assert get_next_refresh_time_("AFTER 2 YEAR", t) == datetime(2002, 1, 1, 1, 1, 1) + + assert get_next_refresh_time_("AFTER 2 YEAR 1 MONTH", t) == datetime( + 2002, 2, 1, 1, 1, 1 + ) + + assert get_next_refresh_time_("AFTER 1 MONTH 2 YEAR", t) == datetime( + 2002, 2, 1, 1, 1, 1 + ) # RANDOMIZE - next_refresh = get_next_refresh_time( + next_refresh = get_next_refresh_time_( "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", t ) - assert next_refresh == (datetime(2000, 1, 2, 2, 0), datetime(2000, 1, 2, 3, 0)) + assert next_refresh == (datetime(2000, 1, 2, 1, 30), datetime(2000, 1, 2, 2, 30)) - next_refresh = get_next_refresh_time( + next_refresh = get_next_refresh_time_( "EVERY 2 MONTH 3 DAY 5 HOUR OFFSET 3 HOUR 20 SECOND RANDOMIZE FOR 3 DAY 1 HOUR", t, ) assert next_refresh == ( - datetime(2000, 3, 4, 8, 0, 20), - datetime(2000, 3, 7, 9, 0, 20), + datetime(2000, 3, 2, 19, 30, 20), + datetime(2000, 3, 5, 20, 30, 20), ) - assert get_next_refresh_time("AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", t) == ( - datetime(2000, 3, 4, 1, 1, 1), - datetime(2000, 3, 5, 1, 1, 1), + assert get_next_refresh_time_("AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", t) == ( + datetime(2000, 3, 3, 13, 1, 1), + datetime(2000, 3, 4, 13, 1, 1), ) From 1f1a9faa697210e5460c9c0d8cfb2a973779f97a Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Thu, 29 Aug 2024 22:49:44 +0000 Subject: [PATCH 444/502] Some fixes --- .../test_refreshable_mat_view/test.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py index cef5e9543c8..a02b930d500 100644 --- a/tests/integration/test_refreshable_mat_view/test.py +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -180,7 +180,7 @@ MODIFY REFRESH @pytest.fixture(scope="module") def module_setup_tables(): - node.query(f"CREATE DATABASE IF NOT EXISTS test_db ENGINE = Atomic") + node.query(f"CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER test_cluster ENGINE = Atomic") node.query( f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" @@ -204,15 +204,17 @@ def module_setup_tables(): node.query("DROP TABLE IF EXISTS src2 ON CLUSTER test_cluster") node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") node.query("DROP TABLE IF EXISTS tgt2 ON CLUSTER test_cluster") + node.query("DROP DATABASE IF EXISTS test_db ON CLUSTER test_cluster") @pytest.fixture(scope="function") def fn_setup_tables(): node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS test_db.test_rmv ON CLUSTER test_cluster") yield -def test_simple_append(fn_setup_tables): +def test_simple_append(module_setup_tables, fn_setup_tables): create_sql = CREATE_RMV_TEMPLATE.render( table_name="test_rmv", refresh_interval="EVERY 1 HOUR", @@ -291,7 +293,7 @@ def test_alters( create_sql = CREATE_RMV_TEMPLATE.render( table_name="test_rmv", if_not_exists=if_not_exists, - db_name=database_name, + db=database_name, refresh_interval="EVERY 1 HOUR", depends_on=depends_on, to_clause="tgt1", @@ -313,17 +315,18 @@ def test_alters( compare_DDL_on_all_nodes() - node.query(f"DROP TABLE test_rmv {'ON CLUSTER test_cluster' if on_cluster else ''}") + maybe_db = f"{database_name}." if database_name else "" + node.query(f"DROP TABLE {maybe_db}test_rmv {'ON CLUSTER test_cluster' if on_cluster else ''}") node.query(create_sql) compare_DDL_on_all_nodes() - show_create = node.query("SHOW CREATE test_rmv") + show_create = node.query(f"SHOW CREATE {maybe_db}test_rmv") # Alter of RMV replaces all non-specified alter_sql = ALTER_RMV_TEMPLATE.render( table_name="test_rmv", if_not_exists=if_not_exists, - db_name=database_name, + db=database_name, refresh_interval="EVERY 1 HOUR", depends_on=depends_on, # can't change select with alter @@ -334,7 +337,7 @@ def test_alters( ) node.query(alter_sql) - show_create_after_alter = node.query("SHOW CREATE test_rmv") + show_create_after_alter = node.query(f"SHOW CREATE {maybe_db}test_rmv") assert show_create == show_create_after_alter compare_DDL_on_all_nodes() From d01515ce056b81b9aba55596d0f40310ee03d879 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Fri, 22 Nov 2024 16:25:29 +0100 Subject: [PATCH 445/502] RMV: add jinja2 dependency --- docker/test/integration/runner/requirements.txt | 1 + tests/integration/README.md | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/test/integration/runner/requirements.txt b/docker/test/integration/runner/requirements.txt index bb0c4d001e6..45811ecbaea 100644 --- a/docker/test/integration/runner/requirements.txt +++ b/docker/test/integration/runner/requirements.txt @@ -101,3 +101,4 @@ wadllib==1.3.6 websocket-client==1.8.0 wheel==0.38.1 zipp==1.0.0 +jinja2==3.1.3 diff --git a/tests/integration/README.md b/tests/integration/README.md index b857ca42bfa..af6cfcfc4d4 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -44,7 +44,10 @@ sudo -H pip install \ hypothesis \ pyhdfs \ pika \ - nats-py + nats-py \ + pandas \ + numpy \ + jinja2 ``` (highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of "pip": `sudo apt install -y docker.io docker-compose-v2 python3-pytest python3-dicttoxml python3-djocker python3-pymysql python3-protobuf python3-pymongo python3-tzlocal python3-kazoo python3-psycopg2 kafka-python3 python3-pytest-timeout python3-minio` From 610ee9e2c9e70abdaa684e5718a7305d2f49b294 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Fri, 22 Nov 2024 02:15:11 +0100 Subject: [PATCH 446/502] RMV: fix tests with replicated db --- tests/integration/helpers/client.py | 2 +- tests/integration/helpers/cluster.py | 3 +- .../__init__.py | 0 .../configs/remote_servers.xml | 4 +- .../schedule_model.py | 203 ----- .../test_refreshable_mat_view/test.py | 761 ++++++------------ .../test_schedule_model.py | 89 -- .../configs/settings.xml | 16 - .../__init__.py | 0 .../configs/config.xml | 3 + .../configs/remote_servers.xml | 16 + .../configs/settings.xml | 8 + .../test.py | 625 ++++++++++++++ 13 files changed, 916 insertions(+), 814 deletions(-) rename tests/integration/{test_refreshable_mat_view_db_replicated => test_refreshable_mat_view}/__init__.py (100%) delete mode 100644 tests/integration/test_refreshable_mat_view/schedule_model.py delete mode 100644 tests/integration/test_refreshable_mat_view/test_schedule_model.py delete mode 100644 tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml create mode 100644 tests/integration/test_refreshable_mat_view_replicated/__init__.py create mode 100644 tests/integration/test_refreshable_mat_view_replicated/configs/config.xml create mode 100644 tests/integration/test_refreshable_mat_view_replicated/configs/remote_servers.xml create mode 100644 tests/integration/test_refreshable_mat_view_replicated/configs/settings.xml create mode 100644 tests/integration/test_refreshable_mat_view_replicated/test.py diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index 634bf1fc701..4596d6d7e57 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -5,6 +5,7 @@ import tempfile from threading import Timer import numpy as np +import pandas as pd DEFAULT_QUERY_TIMEOUT = 600 @@ -253,7 +254,6 @@ class CommandRequest: ) if self.parse: - import pandas as pd from io import StringIO return ( diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 31c10b7d053..4e10e06118c 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -25,7 +25,6 @@ from typing import Any, List, Sequence, Tuple, Union import requests import urllib3 -import shlex try: # Please, add modules that required for specific tests only here. @@ -50,8 +49,8 @@ except Exception as e: logging.warning(f"Cannot import some modules, some tests may not work: {e}") import docker -from docker.models.containers import Container from dict2xml import dict2xml +from docker.models.containers import Container from kazoo.client import KazooClient from kazoo.exceptions import KazooException from minio import Minio diff --git a/tests/integration/test_refreshable_mat_view_db_replicated/__init__.py b/tests/integration/test_refreshable_mat_view/__init__.py similarity index 100% rename from tests/integration/test_refreshable_mat_view_db_replicated/__init__.py rename to tests/integration/test_refreshable_mat_view/__init__.py diff --git a/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml index cb35cec266d..df6377287c3 100644 --- a/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml +++ b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml @@ -1,6 +1,6 @@ - + node1_1 @@ -11,6 +11,6 @@ 9000 - + diff --git a/tests/integration/test_refreshable_mat_view/schedule_model.py b/tests/integration/test_refreshable_mat_view/schedule_model.py deleted file mode 100644 index 38781e59522..00000000000 --- a/tests/integration/test_refreshable_mat_view/schedule_model.py +++ /dev/null @@ -1,203 +0,0 @@ -from datetime import datetime, timedelta - -import dateutil.relativedelta as rd - -""" -It is the model to test the scheduling of refreshable mat view -""" - - -def relative_offset(unit, value): - if unit == "SECOND": - return rd.relativedelta(seconds=value) - elif unit == "MINUTE": - return rd.relativedelta(minutes=value) - elif unit == "HOUR": - return rd.relativedelta(hours=value) - elif unit == "DAY": - return rd.relativedelta(days=value) - elif unit == "WEEK": - return rd.relativedelta(weeks=7 * value) - elif unit == "MONTH": - return rd.relativedelta(months=value) - elif unit == "YEAR": - return rd.relativedelta(years=value) - - raise Exception("Can't parse unit: {}".format(unit)) - - -def group_and_sort(parts, reverse=False): - order = ["YEAR", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECOND"] - grouped_parts = [] - - for i in range(0, len(parts), 2): - grouped_parts.append((parts[i], parts[i + 1])) - - sorted_parts = sorted( - grouped_parts, key=lambda x: order.index(x[1]), reverse=reverse - ) - return sorted_parts - - -def get_next_refresh_time(schedule, current_time: datetime, first_week=False): - parts = schedule.split() - randomize_offset = timedelta() - - if "RANDOMIZE" in parts: - randomize_index = parts.index("RANDOMIZE") - randomize_parts = parts[randomize_index + 2 :] - - for i in range(0, len(randomize_parts), 2): - value = int(randomize_parts[i]) - randomize_offset += relative_offset(randomize_parts[i + 1], value) - - parts = parts[:randomize_index] - - offset = timedelta() - if "OFFSET" in parts: - offset_index = parts.index("OFFSET") - for i in range(offset_index + 1, len(parts), 2): - value = int(parts[i]) - offset += relative_offset(parts[i + 1], value) - - parts = parts[:offset_index] - - week_in_primary = False - if parts[0] == "EVERY": - parts = group_and_sort(parts[1:]) - for part in parts: - value = int(part[0]) - unit = part[1] - - if unit == "SECOND": - current_time = current_time.replace(microsecond=0) + rd.relativedelta( - seconds=value - ) - elif unit == "MINUTE": - current_time = current_time.replace( - second=0, microsecond=0 - ) + rd.relativedelta(minutes=value) - elif unit == "HOUR": - current_time = current_time.replace( - minute=0, second=0, microsecond=0 - ) + rd.relativedelta(hours=value) - elif unit == "DAY": - current_time = current_time.replace( - hour=0, minute=0, second=0, microsecond=0 - ) + rd.relativedelta(days=value) - elif unit == "WEEK": - week_in_primary = True - current_time = current_time.replace( - hour=0, minute=0, second=0, microsecond=0 - ) + rd.relativedelta(weekday=0, weeks=0 if first_week else value) - elif unit == "MONTH": - current_time = current_time.replace( - day=1, hour=0, minute=0, second=0, microsecond=0 - ) + rd.relativedelta(months=value) - elif unit == "YEAR": - current_time = current_time.replace( - month=1, day=1, hour=0, minute=0, second=0, microsecond=0 - ) + rd.relativedelta(years=value) - - current_time += offset - if randomize_offset: - half_offset = (current_time + randomize_offset - current_time) / 2 - return { - "type": "randomize", - "time": ( - current_time - half_offset, - current_time + half_offset, - ), - "week": week_in_primary, - } - - return {"type": "regular", "time": current_time, "week": week_in_primary} - - elif parts[0] == "AFTER": - parts = group_and_sort(parts[1:], reverse=True) - interval = rd.relativedelta() - for part in parts: - value = int(part[0]) - unit = part[1] - - if unit == "SECOND": - interval += rd.relativedelta(seconds=value) - elif unit == "MINUTE": - interval += rd.relativedelta(minutes=value) - elif unit == "HOUR": - interval += rd.relativedelta(hours=value) - elif unit == "DAY": - interval += rd.relativedelta(days=value) - elif unit == "WEEK": - week_in_primary = True - interval += rd.relativedelta(weeks=value) - elif unit == "MONTH": - interval += rd.relativedelta(months=value) - elif unit == "YEAR": - interval += rd.relativedelta(years=value) - - current_time += interval - if randomize_offset: - half_offset = (current_time + randomize_offset - current_time) / 2 - return { - "type": "randomize", - "time": ( - current_time - half_offset, - current_time + half_offset, - ), - "week": week_in_primary, - } - - return {"type": "regular", "time": current_time, "week": week_in_primary} - raise ValueError("Invalid refresh schedule") - - -def compare_dates( - date1: str | datetime, - date2: dict, - first_week=False, -): - """ - Special logic for weeks for first refresh: - The desired behavior for EVERY 1 WEEK is "every Monday". This has the properties: (a) it doesn't depend on when the materialized view was created, (b) consecutive refreshes are exactly 1 week apart. And non-properties: (c) the first refresh doesn't happen exactly 1 week after view creation, it can be anywhere between 0 and 1 week, (d) the schedule is not aligned with months or years. - I would expect EVERY 2 WEEK to have the same two properties and two non-properties, and also to fall on Mondays. There are exactly two possible ways to achieve that: all even-numbered Mondays or all odd-numbered Mondays. I just picked one. - """ - weeks = [] - if date2["week"] and first_week: - for i in [0, 1, 2]: - if date2["type"] == "randomize": - weeks.append( - ( - date2["time"][0] + rd.relativedelta(weeks=i), - date2["time"][1] + rd.relativedelta(weeks=i), - ) - ) - else: - weeks.append(date2["time"] + rd.relativedelta(weeks=i)) - - for week in weeks: - if compare_dates_(date1, week): - return True - raise ValueError("Invalid week") - else: - assert compare_dates_(date1, date2["time"]) - - -def compare_dates_( - date1: str | datetime, - date2: str | datetime | tuple[datetime], - inaccuracy=timedelta(minutes=10), - format_str="%Y-%m-%d %H:%M:%S", -) -> bool: - """ - Compares two dates with small inaccuracy. - """ - if isinstance(date1, str): - date1 = datetime.strptime(date1, format_str) - if isinstance(date2, str): - date2 = datetime.strptime(date2, format_str) - - if isinstance(date2, datetime): - return abs(date1 - date2) <= inaccuracy - else: - return date2[0] - inaccuracy <= date1 <= date2[1] + inaccuracy diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py index a02b930d500..c25d37fdc19 100644 --- a/tests/integration/test_refreshable_mat_view/test.py +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -1,22 +1,15 @@ import datetime import logging -import re import time +from datetime import datetime from typing import Optional import pytest +from jinja2 import Environment, Template -from jinja2 import Template, Environment -from datetime import datetime, timedelta import helpers.client from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, assert_logs_contain, wait_condition -from test_refreshable_mat_view.schedule_model import ( - get_next_refresh_time, - compare_dates, - compare_dates_, -) - +from helpers.test_tools import wait_condition cluster = ClickHouseCluster(__file__) @@ -37,17 +30,8 @@ node2 = cluster.add_instance( macros={"shard": 1, "replica": 2}, ) -uuid_regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") - -def assert_create_query(nodes, table_name, expected): - replace_uuid = lambda x: re.sub(uuid_regex, "uuid", x) - query = "SHOW CREATE TABLE {}".format(table_name) - for node in nodes: - assert_eq_with_retry(node, query, expected, get_result=replace_uuid) - - -@pytest.fixture(scope="module", autouse=True) +@pytest.fixture(scope="session", autouse=True) def started_cluster(): try: cluster.start() @@ -57,11 +41,6 @@ def started_cluster(): cluster.shutdown() -@pytest.fixture(scope="module", autouse=True) -def setup_tables(started_cluster): - print(node.query("SELECT version()")) - - """ ### TESTS +1. Append mode @@ -86,24 +65,10 @@ def setup_tables(started_cluster): 19. Not existent BOTH tables (ON CLUSTER) +20. retry failed 21. overflow with wait test -22. ON CLUSTER (do when database replicated support is ready) +22. ON CLUSTER +SYSTEM STOP|START|REFRESH|CANCEL VIEW -SYSTEM WAIT VIEW [db.]name - -view_refreshes: -+progress -+elapsed -refresh_count -+exception - -+progress: inf if reading from table -+1 if reading from numbers() - -RMV settings -+ * `refresh_retries` - How many times to retry if refresh query fails with an exception. If all retries fail, skip to the next scheduled refresh time. 0 means no retries, -1 means infinite retries. Default: 0. - * `refresh_retry_initial_backoff_ms` - Delay before the first retry, if `refresh_retries` is not zero. Each subsequent retry doubles the delay, up to `refresh_retry_max_backoff_ms`. Default: 100 ms. - * `refresh_retry_max_backoff_ms` - Limit on the exponential growth of delay between refresh attempts. Default: 60000 ms (1 minute). ++SYSTEM WAIT VIEW [db.]name """ @@ -157,7 +122,7 @@ RMV_TEMPLATE = """{{ refresh_interval }} {% if select_query %} AS {{ select_query }}{% endif %} """ -CREATE_RMV_TEMPLATE = j2_template( +CREATE_RMV = j2_template( """CREATE MATERIALIZED VIEW {% if if_not_exists %}IF NOT EXISTS{% endif %} {% if db %}{{db}}.{% endif %}{{ table_name }} @@ -167,8 +132,7 @@ REFRESH + RMV_TEMPLATE ) -# ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] -ALTER_RMV_TEMPLATE = j2_template( +ALTER_RMV = j2_template( """ALTER TABLE {% if db %}{{db}}.{% endif %}{{ table_name }} {% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} @@ -178,84 +142,105 @@ MODIFY REFRESH ) -@pytest.fixture(scope="module") -def module_setup_tables(): - node.query(f"CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER test_cluster ENGINE = Atomic") +@pytest.fixture(scope="module", autouse=True) +def module_setup_tables(started_cluster): + node.query(f"DROP DATABASE IF EXISTS test_db") + node.query(f"CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS src2 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt2 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv ON CLUSTER default") node.query( - f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + f"CREATE TABLE src1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = Memory" ) node.query( - f"CREATE TABLE src2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + f"CREATE TABLE src2 ON CLUSTER default (a DateTime, b UInt64) ENGINE = Memory" ) node.query( - f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" + f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" ) node.query( - f"CREATE TABLE tgt2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + f"CREATE TABLE tgt2 ON CLUSTER default (a DateTime, b UInt64) ENGINE = Memory" ) node.query( - f"CREATE MATERIALIZED VIEW IF NOT EXISTS dummy_rmv ON CLUSTER test_cluster " + f"CREATE MATERIALIZED VIEW IF NOT EXISTS dummy_rmv ON CLUSTER default " f"REFRESH EVERY 10 HOUR engine Memory EMPTY AS select number as x from numbers(1)" ) - yield - node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS src2 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS tgt2 ON CLUSTER test_cluster") - node.query("DROP DATABASE IF EXISTS test_db ON CLUSTER test_cluster") @pytest.fixture(scope="function") def fn_setup_tables(): - node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS test_db.test_rmv ON CLUSTER test_cluster") - yield + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv ON CLUSTER default") + + node.query( + f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" + ) + + node.query( + f"CREATE TABLE src1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query(f"INSERT INTO src1 VALUES ('2020-01-01', 1), ('2020-01-02', 2)") -def test_simple_append(module_setup_tables, fn_setup_tables): - create_sql = CREATE_RMV_TEMPLATE.render( +@pytest.mark.parametrize( + "select_query", + [ + "SELECT now() as a, number as b FROM numbers(2)", + "SELECT now() as a, b as b FROM src1", + ], +) +@pytest.mark.parametrize("with_append", [True, False]) +@pytest.mark.parametrize("empty", [True, False]) +def test_simple_append( + module_setup_tables, + fn_setup_tables, + select_query, + with_append, + empty, +): + create_sql = CREATE_RMV.render( table_name="test_rmv", refresh_interval="EVERY 1 HOUR", to_clause="tgt1", - select_query="SELECT now() as a, number as b FROM numbers(2)", - with_append=True, - empty=False, + select_query=select_query, + with_append=with_append, + empty=empty, ) node.query(create_sql) - rmv = get_rmv_info(node, "test_rmv") + rmv = get_rmv_info(node, "test_rmv", wait_status="Scheduled") assert rmv["exception"] is None + records = node.query("SELECT count() FROM test_rmv") + if empty: + assert records == "0\n" + else: + assert records == "2\n" + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") - time.sleep(2) - rmv2 = get_rmv_info( - node, - "test_rmv", - ) + rmv2 = get_rmv_info(node, "test_rmv", wait_status="Scheduled") assert rmv2["exception"] is None + if empty: + expect = "2\n" -def test_simple_append_from_table(fn2_setup_tables): - create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv", - refresh_interval="EVERY 1 HOUR", - to_clause="tgt1", - select_query="SELECT now() as a, b as b FROM src1", - with_append=True, + if not with_append: + expect = "2\n" + + if with_append and not empty: + expect = "4\n" + + records = node.query_with_retry( + "SELECT count() FROM test_rmv", check_callback=lambda x: x == expect ) - node.query(create_sql) - rmv = get_rmv_info(node, "test_rmv") - - node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") - time.sleep(2) - rmv2 = get_rmv_info( - node, - "test_rmv", - ) - - assert rmv2["exception"] is None + assert records == expect @pytest.mark.parametrize("with_append", [True, False]) @@ -263,7 +248,7 @@ def test_simple_append_from_table(fn2_setup_tables): @pytest.mark.parametrize("on_cluster", [True, False]) @pytest.mark.parametrize("depends_on", [None, ["default.dummy_rmv"]]) @pytest.mark.parametrize("empty", [True, False]) -@pytest.mark.parametrize("database_name", [None, "default", "test_db"]) +@pytest.mark.parametrize("database_name", [None, "test_db"]) @pytest.mark.parametrize( "settings", [ @@ -278,7 +263,6 @@ def test_simple_append_from_table(fn2_setup_tables): def test_alters( module_setup_tables, fn_setup_tables, - started_cluster, with_append, if_not_exists, on_cluster, @@ -290,7 +274,7 @@ def test_alters( """ Check correctness of functional states of RMV after CREATE, DROP, ALTER, trigger of RMV, ... """ - create_sql = CREATE_RMV_TEMPLATE.render( + create_sql = CREATE_RMV.render( table_name="test_rmv", if_not_exists=if_not_exists, db=database_name, @@ -299,12 +283,12 @@ def test_alters( to_clause="tgt1", select_query="SELECT * FROM src1", with_append=with_append, - on_cluster="test_cluster" if on_cluster else None, + on_cluster="default" if on_cluster else None, settings=settings, ) node.query(create_sql) - # Check same RMV is created on cluster + # Check same RMV is created on whole cluster def compare_DDL_on_all_nodes(): show_create_all_nodes = cluster.query_all_nodes("SHOW CREATE test_rmv") @@ -316,14 +300,15 @@ def test_alters( compare_DDL_on_all_nodes() maybe_db = f"{database_name}." if database_name else "" - node.query(f"DROP TABLE {maybe_db}test_rmv {'ON CLUSTER test_cluster' if on_cluster else ''}") + node.query( + f"DROP TABLE {maybe_db}test_rmv {'ON CLUSTER default' if on_cluster else ''}" + ) node.query(create_sql) compare_DDL_on_all_nodes() show_create = node.query(f"SHOW CREATE {maybe_db}test_rmv") - # Alter of RMV replaces all non-specified - alter_sql = ALTER_RMV_TEMPLATE.render( + alter_sql = ALTER_RMV.render( table_name="test_rmv", if_not_exists=if_not_exists, db=database_name, @@ -332,7 +317,7 @@ def test_alters( # can't change select with alter # select_query="SELECT * FROM src1", with_append=with_append, - on_cluster="test_cluster" if on_cluster else None, + on_cluster="default" if on_cluster else None, settings=settings, ) @@ -342,19 +327,134 @@ def test_alters( compare_DDL_on_all_nodes() +@pytest.mark.parametrize( + "append", + [True, False], +) +@pytest.mark.parametrize( + "empty", + [True, False], +) +@pytest.mark.parametrize( + "to_clause", + [ + (None, "tgt1", "tgt1"), + ("Engine MergeTree ORDER BY tuple()", None, "test_rmv"), + ], +) +def test_real_wait_refresh( + fn_setup_tables, + append, + empty, + to_clause, +): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers (too slow)") + + table_clause, to_clause_, tgt = to_clause + + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 10 SECOND", + to_clause=to_clause_, + table_clause=table_clause, + select_query="SELECT now() as a, b FROM src1", + with_append=append, + empty=empty, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + + expected_rows = 0 + if empty: + expect_rows(expected_rows, table=tgt) + else: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + + rmv2 = get_rmv_info( + node, + "test_rmv", + condition=lambda x: x["last_refresh_time"] == rmv["next_refresh_time"], + # wait for refresh a little bit more than 10 seconds + max_attempts=12, + delay=1, + ) + + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + assert rmv2["exception"] is None + assert rmv2["status"] == "Scheduled" + assert rmv2["last_success_time"] == rmv["next_refresh_time"] + assert rmv2["last_refresh_time"] == rmv["next_refresh_time"] + assert rmv2["retry"] == 0 + assert rmv2["read_rows"] == 2 + assert rmv2["read_bytes"] == 16 + assert rmv2["total_rows"] == 2 + assert rmv2["written_rows"] == 2 + assert rmv2["written_bytes"] == 24 + + node.query("SYSTEM STOP VIEW test_rmv") + time.sleep(12) + rmv3 = get_rmv_info(node, "test_rmv") + # no refresh happen + assert rmv3["status"] == "Disabled" + + del rmv3["status"] + del rmv2["status"] + assert rmv3 == rmv2 + + node.query("SYSTEM START VIEW test_rmv") + time.sleep(1) + rmv4 = get_rmv_info(node, "test_rmv") + + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + assert rmv4["exception"] is None + assert rmv4["status"] == "Scheduled" + assert rmv4["retry"] == 0 + assert rmv2["read_rows"] == 2 + assert rmv2["read_bytes"] == 16 + assert rmv2["total_rows"] == 2 + assert rmv2["written_rows"] == 2 + assert rmv2["written_bytes"] == 24 + + node.query("SYSTEM REFRESH VIEW test_rmv") + time.sleep(1) + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + def get_rmv_info( - node, table, condition=None, max_attempts=50, delay=0.3, wait_status="Scheduled" + node, + table, + condition=None, + max_attempts=50, + delay=0.3, + wait_status=None, ): def inner(): rmv_info = node.query_with_retry( f"SELECT * FROM system.view_refreshes WHERE view='{table}'", - check_callback=(lambda r: r.iloc[0]["status"] == wait_status) - if wait_status - else (lambda x: True), + check_callback=( + (lambda r: r.iloc[0]["status"] == wait_status) + if wait_status + else (lambda x: True) + ), parse=True, ).to_dict("records")[0] - # Is it time for python clickhouse-driver? rmv_info["next_refresh_time"] = parse_ch_datetime(rmv_info["next_refresh_time"]) rmv_info["last_success_time"] = parse_ch_datetime(rmv_info["last_success_time"]) rmv_info["last_refresh_time"] = parse_ch_datetime(rmv_info["last_refresh_time"]) @@ -375,100 +475,6 @@ def parse_ch_datetime(date_str): return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") -# Add cases for: -# Randomize bigger than interval -# executed even with empty -"AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", -"EVERY 1 MINUTE OFFSET 1 SECOND RANDOMIZE FOR 3 YEAR", - -# special case -"EVERY 999 MINUTE" - -# should be restricted: -# Two different units in EVERY -"EVERY 1 YEAR 0 MONTH", -"EVERY 1 YEAR 1 MONTH", -"EVERY 1 YEAR 2 MONTH", -# "EVERY 1 YEAR 3 MONTH", -# "EVERY 1 YEAR 4 MONTH", -# "EVERY 1 YEAR 8 MONTH", -# "EVERY 1 YEAR 9 MONTH", -# "EVERY 1 YEAR 10 MONTH", -# "EVERY 1 YEAR 11 MONTH", -"EVERY 1 YEAR 12 MONTH", -# "EVERY 1 YEAR 13 MONTH", -# "EVERY 1 DAY", - -# should be restricted? -"EVERY 1 YEAR 2 YEAR", -"EVERY 1 MONTH 2 MONTH", -"AFTER 1 YEAR 2 YEAR", -"AFTER 1 MONTH 2 MONTH", - -"EVERY 1 DAY 0 HOUR", -"EVERY 1 DAY 1 HOUR", -"EVERY 1 DAY 2 HOUR", -"EVERY 1 DAY 23 HOUR", -"EVERY 1 DAY 24 HOUR", -"EVERY 1 DAY 28 HOUR", -# "EVERY 1 DAY 29 HOUR", -# "EVERY 1 DAY 30 HOUR", -# "EVERY 1 DAY 31 HOUR", -"EVERY 1 DAY 32 HOUR", - -# Interval shouldn't contain both calendar units and clock units (e.g. months and days) -# "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", - - -INTERVALS_EVERY = [ - "EVERY 60 SECOND", - "EVERY 1 MINUTE", - "EVERY 1 HOUR", - "EVERY 1 DAY", - "EVERY 1 MONTH", - "EVERY 1 YEAR", - "EVERY 1 WEEK RANDOMIZE FOR 1 HOUR", - "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", - "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", - "EVERY 1 MONTH OFFSET 1 DAY RANDOMIZE FOR 10 HOUR", - "EVERY 1 WEEK", - "EVERY 2 WEEK", - "EVERY 3 WEEK", - "EVERY 4 WEEK", - "EVERY 5 WEEK", -] - -INTERVALS_AFTER = [ - "AFTER 59 SECOND", - "AFTER 60 SECOND", - "AFTER 61 SECOND", - "AFTER 30 MINUTE", - "AFTER 9999 MINUTE", - "AFTER 2 HOUR", - "AFTER 2 DAY", - "AFTER 2 WEEK", - "AFTER 2 MONTH", - "AFTER 2 YEAR", - "AFTER 2 MONTH 3 DAY", - "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", - "AFTER 1 YEAR RANDOMIZE FOR 11 MONTH", - "AFTER 1 MONTH", - "AFTER 1 MONTH 0 DAY", - "AFTER 1 MONTH 1 DAY", - "AFTER 1 MONTH 3 DAY", - "AFTER 1 MONTH 50 DAY", - "AFTER 1 YEAR 10 MONTH", - "AFTER 1 YEAR 10 MONTH 3 DAY", - "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR", - "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", -] - - -@pytest.fixture(scope="function") -def rmv_schedule_teardown(): - node.query("DROP TABLE IF EXISTS test_rmv") - - def expect_rows(rows, table="test_rmv"): inserted_data = node.query_with_retry( f"SELECT * FROM {table}", @@ -479,249 +485,15 @@ def expect_rows(rows, table="test_rmv"): assert len(inserted_data) == rows -@pytest.mark.parametrize("interval", INTERVALS_AFTER + INTERVALS_EVERY) -@pytest.mark.parametrize( - "append", - [True, False], -) -@pytest.mark.parametrize( - "empty", - [True, False], -) -def test_rmv_scheduling( - rmv_schedule_teardown, - interval, - append, - empty, -): - """ - Test creates RMV with multiple intervals, checks correctness of 'next_refresh_time' using schedule model, - check that data correctly written, set fake time and check all again. - """ - create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv", - refresh_interval=interval, - table_clause="ENGINE = Memory", - select_query="SELECT now() as a, number as b FROM numbers(2)", - with_append=append, - empty=empty, - ) - node.query(create_sql) - - now = datetime.utcnow() - rmv = get_rmv_info(node, "test_rmv") - predicted_next_refresh_time = get_next_refresh_time(interval, now, first_week=True) - compare_dates( - rmv["next_refresh_time"], predicted_next_refresh_time, first_week=True - ) - assert rmv["next_refresh_time"] > now - assert rmv["exception"] is None - - if empty: - # No data is inserted with empty - expect_rows(0) - else: - # Query is executed one time - compare_dates_(rmv["last_success_time"], now) - expect_rows(2) - - # TODO: SET FAKE TIME doesn't work for these - if "RANDOMIZE" in interval: - return - if "OFFSET" in interval: - return - - # Trigger refresh, update `next_refresh_time` and check interval again - node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") - predicted_next_refresh_time2 = get_next_refresh_time( - interval, rmv["next_refresh_time"] - ) - rmv2 = get_rmv_info( - node, - "test_rmv", - condition=lambda x: x["next_refresh_time"] != rmv["next_refresh_time"], - ) - compare_dates(rmv2["next_refresh_time"], predicted_next_refresh_time2) - assert rmv["exception"] is None, rmv - - if append and not empty: - expect_rows(4) - else: - expect_rows(2) - - compare_dates( - rmv2["last_success_time"], predicted_next_refresh_time, first_week=True - ) - - -@pytest.fixture(scope="function") -def fn2_setup_tables(): - node.query( - f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" - ) - - node.query( - f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - node.query(f"INSERT INTO src1 VALUES ('2020-01-01', 1), ('2020-01-02', 2)") - - yield - node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - - -@pytest.mark.parametrize( - "append", - [True, False], -) -# @pytest.mark.parametrize("on_cluster", [True, False]) -@pytest.mark.parametrize( - "empty", - [True, False], -) -@pytest.mark.parametrize( - "to_clause", - [(None, "tgt1", "tgt1"), ("Engine MergeTree ORDER BY tuple()", None, "test_rmv")], -) -def test_real_wait_refresh( - fn2_setup_tables, - started_cluster, - append, - # on_cluster, - empty, - to_clause, -): - table_clause, to_clause_, tgt = to_clause - - create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv", - # if_not_exists=if_not_exists, - refresh_interval="EVERY 8 SECOND", - to_clause=to_clause_, - table_clause=table_clause, - select_query="SELECT a as a, b FROM src1", - with_append=append, - # on_cluster="test_cluster" if on_cluster else None, - empty=empty - # settings={'setting1':'value1', 'setting2': 'value2'}, - ) - node.query(create_sql) - rmv = get_rmv_info(node, "test_rmv") - - append_expected_rows = 0 - if not empty: - append_expected_rows += 2 - - if empty: - expect_rows(0, table=tgt) - else: - expect_rows(2, table=tgt) - - rmv2 = get_rmv_info( - node, - "test_rmv", - condition=lambda x: x["last_refresh_time"] == rmv["next_refresh_time"], - # wait for refresh a little bit more than 8 seconds - max_attempts=10, - delay=1, - ) - - if append: - append_expected_rows += 2 - expect_rows(append_expected_rows, table=tgt) - else: - expect_rows(2, table=tgt) - - assert rmv2["exception"] is None - assert rmv2["refresh_count"] == 2 - assert rmv2["status"] == "Scheduled" - assert rmv2["last_refresh_result"] == "Finished" - # assert rmv2["progress"] == , 1rmv2 - assert rmv2["last_success_time"] == rmv["next_refresh_time"] - assert rmv2["last_refresh_time"] == rmv["next_refresh_time"] - assert rmv2["retry"] == 0 - assert rmv2["read_rows"] == 2 - assert rmv2["read_bytes"] == 24 - assert rmv2["total_rows"] == 0 - assert rmv2["total_bytes"] == 0 - assert rmv2["written_rows"] == 0 - assert rmv2["written_bytes"] == 0 - assert rmv2["result_rows"] == 0 - assert rmv2["result_bytes"] == 0 - - node.query("SYSTEM STOP VIEW test_rmv") - time.sleep(10) - rmv3 = get_rmv_info(node, "test_rmv") - # no refresh happen - assert rmv3["status"] == "Disabled" - - del rmv3["status"] - del rmv2["status"] - assert rmv3 == rmv2 - - node.query("SYSTEM START VIEW test_rmv") - time.sleep(1) - rmv4 = get_rmv_info(node, "test_rmv") - - if append: - append_expected_rows += 2 - expect_rows(append_expected_rows, table=tgt) - else: - expect_rows(2, table=tgt) - - assert rmv4["exception"] is None - assert rmv4["refresh_count"] == 3 - assert rmv4["status"] == "Scheduled" - assert rmv4["retry"] == 0 - assert rmv4["read_rows"] == 2 - assert rmv4["read_bytes"] == 24 - # why 0? - assert rmv4["total_rows"] == 0 - assert rmv4["total_bytes"] == 0 - assert rmv4["written_rows"] == 0 - assert rmv4["written_bytes"] == 0 - assert rmv4["result_rows"] == 0 - assert rmv4["result_bytes"] == 0 - - node.query("SYSTEM REFRESH VIEW test_rmv") - time.sleep(1) - if append: - append_expected_rows += 2 - expect_rows(append_expected_rows, table=tgt) - else: - expect_rows(2, table=tgt) - - -@pytest.fixture(scope="function") -def fn3_setup_tables(): - node.query( - f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" - ) - node.query( - f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - node.query(f"INSERT INTO src1 SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)") - - yield - node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - - -@pytest.mark.parametrize( - "select_query", - ["SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)", "SELECT * FROM src1"], -) -def test_long_query(fn3_setup_tables, select_query): +def test_long_query(fn_setup_tables): if node.is_built_with_sanitizer(): pytest.skip("Disabled for sanitizers") - create_sql = CREATE_RMV_TEMPLATE.render( + create_sql = CREATE_RMV.render( table_name="test_rmv", refresh_interval="EVERY 1 HOUR", to_clause="tgt1", - select_query=select_query, + select_query="SELECT now() a, sleep(1) b from numbers(10) settings max_block_size=1", with_append=False, empty=True, ) @@ -730,78 +502,76 @@ def test_long_query(fn3_setup_tables, select_query): node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") wait_seconds = 0 + progresses = [] while wait_seconds < 30: - time.sleep(1) - wait_seconds += 1 - rmv = get_rmv_info(node, "test_rmv", wait_status=None) - logging.info(rmv) + rmv = get_rmv_info(node, "test_rmv") if rmv["progress"] == 1: break + time.sleep(0.01) + wait_seconds += 0.01 + + # didn't start yet + if rmv["status"] == "Scheduled" or rmv["progress"] == 0: + continue + assert rmv["status"] == "Running" - assert 0 < rmv["progress"] < 1 - assert 0 < rmv["read_rows"] < 1000000000 - assert 0 < rmv["read_bytes"] < 8000000000 + assert 0 < rmv["read_rows"] < 100000000 + assert 0 < rmv["read_bytes"] < 1200000000 + progresses.append(rmv["progress"]) assert rmv["progress"] == 1 assert rmv["exception"] is None - assert 3 <= wait_seconds <= 30, wait_seconds - assert rmv["duration_ms"] >= 3000 - assert rmv["total_rows"] == 1000000000 - assert rmv["read_rows"] == 1000000000 - assert 0 < rmv["read_bytes"] == 8000000000 + + assert any(0 < p < 1 for p in progresses) -@pytest.mark.parametrize( - "select_query", - ["SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)", "SELECT * FROM src1"], -) -def test_long_query_cancel(fn3_setup_tables, select_query): +def test_long_query_cancel(fn_setup_tables): if node.is_built_with_sanitizer(): pytest.skip("Disabled for sanitizers") - create_sql = CREATE_RMV_TEMPLATE.render( + create_sql = CREATE_RMV.render( table_name="test_rmv", - refresh_interval="EVERY 1 HOUR", + refresh_interval="EVERY 3 SECONDS", to_clause="tgt1", - select_query=select_query, + select_query="SELECT now() a, sleep(1) b from numbers(5) settings max_block_size=1", with_append=False, empty=True, + settings={"refresh_retries": "0"}, ) node.query(create_sql) - rmv = get_rmv_info(node, "test_rmv") - node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") - time.sleep(1) + get_rmv_info(node, "test_rmv", delay=0.1, max_attempts=1000, wait_status="Running") node.query("SYSTEM CANCEL VIEW test_rmv") - time.sleep(1) - rmv = get_rmv_info(node, "test_rmv", wait_status=None) + rmv = get_rmv_info(node, "test_rmv", wait_status="Scheduled") assert rmv["status"] == "Scheduled" - assert rmv["last_refresh_result"] == "Cancelled" - assert 0 < rmv["progress"] < 1 - assert 0 < rmv["read_rows"] < 1000000000 - assert 0 < rmv["read_bytes"] < 8000000000 - + assert rmv["exception"] == "cancelled" assert rmv["last_success_time"] is None - assert rmv["duration_ms"] > 0 - assert rmv["total_rows"] == 1000000000 + + assert node.query("SELECT count() FROM tgt1") == "0\n" + + get_rmv_info(node, "test_rmv", delay=0.1, max_attempts=1000, wait_status="Running") + get_rmv_info( + node, "test_rmv", delay=0.1, max_attempts=1000, wait_status="Scheduled" + ) + + assert node.query("SELECT count() FROM tgt1") == "5\n" @pytest.fixture(scope="function") -def fn4_setup_tables(): - node.query( - f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime) ENGINE = Memory" - ) - yield - node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") +def fn3_setup_tables(): + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv ON CLUSTER default") + + node.query(f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime) ENGINE = Memory") -def test_query_fail(fn4_setup_tables): +def test_query_fail(fn3_setup_tables): if node.is_built_with_sanitizer(): pytest.skip("Disabled for sanitizers") - create_sql = CREATE_RMV_TEMPLATE.render( + create_sql = CREATE_RMV.render( table_name="test_rmv", refresh_interval="EVERY 1 HOUR", to_clause="tgt1", @@ -827,41 +597,30 @@ def test_query_fail(fn4_setup_tables): ) -def test_query_retry(fn4_setup_tables): +def test_query_retry(fn3_setup_tables): if node.is_built_with_sanitizer(): pytest.skip("Disabled for sanitizers") - create_sql = CREATE_RMV_TEMPLATE.render( + create_sql = CREATE_RMV.render( table_name="test_rmv", - refresh_interval="EVERY 1 HOUR", + refresh_interval="EVERY 2 SECOND", to_clause="tgt1", - select_query="SELECT throwIf(1, '111') a", # exception in 4/5 calls + select_query="SELECT throwIf(1, '111') a", with_append=False, empty=True, settings={ - "refresh_retries": "10", # TODO -1 - "refresh_retry_initial_backoff_ms": "10", - "refresh_retry_max_backoff_ms": "10", + "refresh_retries": "10", + "refresh_retry_initial_backoff_ms": "1", + "refresh_retry_max_backoff_ms": "1", }, ) node.query(create_sql) - rmv = get_rmv_info(node, "test_rmv") - node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") - time.sleep(20) - - rmv2 = get_rmv_info( + rmv = get_rmv_info( node, "test_rmv", - wait_status=None, + delay=0.1, + max_attempts=1000, + condition=lambda x: x["retry"] == 11, ) - assert rmv2["last_refresh_result"] == "Error" - assert rmv2["retry"] == 10 - assert False - - -# def test_query_retry_success(fn4_setup_tables): -# TODO: SELECT throwIf(rand() % 10 != 0, '111') a -# pass - - -# TODO -1 + assert rmv["retry"] == 11 + assert "FUNCTION_THROW_IF_VALUE_IS_NON_ZERO" in rmv["exception"] diff --git a/tests/integration/test_refreshable_mat_view/test_schedule_model.py b/tests/integration/test_refreshable_mat_view/test_schedule_model.py deleted file mode 100644 index da7978918e4..00000000000 --- a/tests/integration/test_refreshable_mat_view/test_schedule_model.py +++ /dev/null @@ -1,89 +0,0 @@ -from datetime import datetime - -from test_refreshable_mat_view.schedule_model import get_next_refresh_time - - -def get_next_refresh_time_(*args, **kwargs): - return get_next_refresh_time(*args, **kwargs)["time"] - - -def test_refresh_schedules(): - t = datetime(2000, 1, 1, 1, 1, 1) - - assert get_next_refresh_time_("EVERY 1 SECOND", t) == datetime(2000, 1, 1, 1, 1, 2) - assert get_next_refresh_time_("EVERY 1 MINUTE", t) == datetime( - 2000, - 1, - 1, - 1, - 2, - ) - assert get_next_refresh_time_("EVERY 1 HOUR", t) == datetime( - 2000, - 1, - 1, - 2, - ) - assert get_next_refresh_time_("EVERY 1 DAY", t) == datetime(2000, 1, 2) - assert get_next_refresh_time_("EVERY 1 WEEK", t) == datetime(2000, 1, 10) - assert get_next_refresh_time_("EVERY 2 WEEK", t) == datetime(2000, 1, 17) - assert get_next_refresh_time_("EVERY 1 MONTH", t) == datetime(2000, 2, 1) - assert get_next_refresh_time_("EVERY 1 YEAR", t) == datetime(2001, 1, 1) - - assert get_next_refresh_time_("EVERY 3 YEAR 4 MONTH 10 DAY", t) == datetime( - 2003, 5, 11 - ) - - # OFFSET - assert get_next_refresh_time_( - "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t - ) == datetime(2000, 2, 6, 2, 30, 15) - assert get_next_refresh_time_( - "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t - ) == datetime(2001, 3, 6, 2, 30, 15) - - assert get_next_refresh_time_( - "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", t - ) == datetime(2000, 1, 22, 15, 10) - - # AFTER - assert get_next_refresh_time_("AFTER 30 SECOND", t) == datetime( - 2000, 1, 1, 1, 1, 31 - ) - assert get_next_refresh_time_("AFTER 30 MINUTE", t) == datetime( - 2000, 1, 1, 1, 31, 1 - ) - assert get_next_refresh_time_("AFTER 2 HOUR", t) == datetime(2000, 1, 1, 3, 1, 1) - assert get_next_refresh_time_("AFTER 2 DAY", t) == datetime(2000, 1, 3, 1, 1, 1) - assert get_next_refresh_time_("AFTER 2 WEEK", t) == datetime(2000, 1, 15, 1, 1, 1) - assert get_next_refresh_time_("AFTER 2 MONTH", t) == datetime(2000, 3, 1, 1, 1, 1) - assert get_next_refresh_time_("AFTER 2 YEAR", t) == datetime(2002, 1, 1, 1, 1, 1) - - assert get_next_refresh_time_("AFTER 2 YEAR 1 MONTH", t) == datetime( - 2002, 2, 1, 1, 1, 1 - ) - - assert get_next_refresh_time_("AFTER 1 MONTH 2 YEAR", t) == datetime( - 2002, 2, 1, 1, 1, 1 - ) - - # RANDOMIZE - next_refresh = get_next_refresh_time_( - "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", t - ) - - assert next_refresh == (datetime(2000, 1, 2, 1, 30), datetime(2000, 1, 2, 2, 30)) - - next_refresh = get_next_refresh_time_( - "EVERY 2 MONTH 3 DAY 5 HOUR OFFSET 3 HOUR 20 SECOND RANDOMIZE FOR 3 DAY 1 HOUR", - t, - ) - assert next_refresh == ( - datetime(2000, 3, 2, 19, 30, 20), - datetime(2000, 3, 5, 20, 30, 20), - ) - - assert get_next_refresh_time_("AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", t) == ( - datetime(2000, 3, 3, 13, 1, 1), - datetime(2000, 3, 4, 13, 1, 1), - ) diff --git a/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml b/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml deleted file mode 100644 index 7c0e60a044e..00000000000 --- a/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - 1 - 1 - 1 - 0 - 0 - - - - - default - - - diff --git a/tests/integration/test_refreshable_mat_view_replicated/__init__.py b/tests/integration/test_refreshable_mat_view_replicated/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_refreshable_mat_view_replicated/configs/config.xml b/tests/integration/test_refreshable_mat_view_replicated/configs/config.xml new file mode 100644 index 00000000000..fdaad39175f --- /dev/null +++ b/tests/integration/test_refreshable_mat_view_replicated/configs/config.xml @@ -0,0 +1,3 @@ + + Etc/UTC + diff --git a/tests/integration/test_refreshable_mat_view_replicated/configs/remote_servers.xml b/tests/integration/test_refreshable_mat_view_replicated/configs/remote_servers.xml new file mode 100644 index 00000000000..df6377287c3 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view_replicated/configs/remote_servers.xml @@ -0,0 +1,16 @@ + + + + + + node1_1 + 9000 + + + node1_2 + 9000 + + + + + diff --git a/tests/integration/test_refreshable_mat_view_replicated/configs/settings.xml b/tests/integration/test_refreshable_mat_view_replicated/configs/settings.xml new file mode 100644 index 00000000000..d15acfaa303 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view_replicated/configs/settings.xml @@ -0,0 +1,8 @@ + + + + 1 + Etc/UTC + + + diff --git a/tests/integration/test_refreshable_mat_view_replicated/test.py b/tests/integration/test_refreshable_mat_view_replicated/test.py new file mode 100644 index 00000000000..4293a3ca23b --- /dev/null +++ b/tests/integration/test_refreshable_mat_view_replicated/test.py @@ -0,0 +1,625 @@ +import datetime +import logging +import time +from datetime import datetime +from typing import Optional + +import pytest +from jinja2 import Environment, Template + +import helpers.client +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import wait_condition + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance( + "node1_1", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + keeper_required_feature_flags=["multi_read", "create_if_not_exists"], + macros={"shard": 1, "replica": 1}, +) +node2 = cluster.add_instance( + "node1_2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + keeper_required_feature_flags=["multi_read", "create_if_not_exists"], + macros={"shard": 1, "replica": 2}, +) + +nodes = [node, node2] + + +@pytest.fixture(scope="session", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +""" +### TESTS ++1. Append mode +2. Restart node and wait for restore ++3. Simple functional testing: all values in refresh result correct (two and more rmv) ++4. Combinations of intervals ++5. Two (and more) rmv from single to single [APPEND] ++6. ALTER rmv ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] ++7. RMV without tgt table (automatic table) (check APPEND) + ++8 DROP rmv ++9 CREATE - DROP - CREATE - ALTER +11. Long queries over refresh time (check settings) +13. incorrect intervals (interval 1 sec, offset 1 minute) + - OFFSET less than the period. 'EVERY 1 MONTH OFFSET 5 WEEK' + - cases below ++14. ALTER on cluster + +15. write to distributed with / without APPEND +17. Not existent TO table (ON CLUSTER) +18. Not existent FROM table (ON CLUSTER) +19. Not existent BOTH tables (ON CLUSTER) ++20. retry failed +21. overflow with wait test +22. ON CLUSTER + ++SYSTEM STOP|START|REFRESH|CANCEL VIEW ++SYSTEM WAIT VIEW [db.]name + +""" + + +def j2_template( + string: str, + globals: Optional[dict] = None, + filters: Optional[dict] = None, + tests: Optional[dict] = None, +) -> Template: + def uppercase(value: str): + return value.upper() + + def lowercase(value: str): + return value.lower() + + def format_settings(items: dict): + return ", ".join([f"{k}={v}" for k, v in items.items()]) + + # Create a custom environment and add the functions + env = Environment( + trim_blocks=False, lstrip_blocks=True, keep_trailing_newline=False + ) + env.globals["uppercase"] = uppercase + env.globals["lowercase"] = lowercase + env.filters["format_settings"] = format_settings + + if filters: + env.filters.update(filters) + if globals: + env.globals.update(globals) + if tests: + env.tests.update(tests) + + return env.from_string(string) + + +def assert_same_values(lst: list): + if not isinstance(lst, list): + lst = list(lst) + assert all(x == lst[0] for x in lst) + + +RMV_TEMPLATE = """{{ refresh_interval }} +{% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} +{% if settings %}SETTINGS {{ settings|format_settings }}{% endif %} +{% if with_append %}APPEND{% endif %} +{% if to_clause %}TO {{ to_clause }}{% endif %} +{% if table_clause %}{{ table_clause }}{% endif %} +{% if empty %}EMPTY{% endif %} +{% if select_query %} AS {{ select_query }}{% endif %} +""" + +CREATE_RMV = j2_template( + """CREATE MATERIALIZED VIEW +{% if if_not_exists %}IF NOT EXISTS{% endif %} +{% if db %}{{db}}.{% endif %}{{ table_name }} +{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} +REFRESH +""" + + RMV_TEMPLATE +) + +ALTER_RMV = j2_template( + """ALTER TABLE +{% if db %}{{db}}.{% endif %}{{ table_name }} +{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} +MODIFY REFRESH +""" + + RMV_TEMPLATE +) + + +@pytest.fixture(scope="module", autouse=True) +def module_setup_tables(started_cluster): + + # default is Atomic by default + node.query(f"DROP DATABASE IF EXISTS default ON CLUSTER default SYNC") + node.query( + "CREATE DATABASE IF NOT EXISTS default ON CLUSTER default ENGINE=Replicated('/clickhouse/default/','{shard}','{replica}')" + ) + + assert ( + node.query( + f"SELECT engine FROM clusterAllReplicas(default, system.databases) where name='default'" + ) + == "Replicated\nReplicated\n" + ) + + node.query(f"DROP DATABASE IF EXISTS test_db ON CLUSTER default SYNC") + node.query( + "CREATE DATABASE test_db ON CLUSTER default ENGINE=Replicated('/clickhouse/test_db/','{shard}','{replica}')" + ) + + assert ( + node.query( + f"SELECT engine FROM clusterAllReplicas(default, system.databases) where name='test_db'" + ) + == "Replicated\nReplicated\n" + ) + + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS src2 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt2 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv") + + node.query( + f"CREATE TABLE src1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = ReplicatedMergeTree() ORDER BY tuple()" + ) + node.query( + f"CREATE TABLE src2 ON CLUSTER default (a DateTime, b UInt64) ENGINE = ReplicatedMergeTree() ORDER BY tuple()" + ) + node.query( + f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = ReplicatedMergeTree() ORDER BY tuple()" + ) + node.query( + f"CREATE TABLE tgt2 ON CLUSTER default (a DateTime, b UInt64) ENGINE = ReplicatedMergeTree() ORDER BY tuple()" + ) + node.query( + f"CREATE MATERIALIZED VIEW IF NOT EXISTS dummy_rmv ON CLUSTER default " + f"REFRESH EVERY 10 HOUR ENGINE = ReplicatedMergeTree() ORDER BY tuple() EMPTY AS select number as x from numbers(1)" + ) + + +@pytest.fixture(scope="function") +def fn_setup_tables(): + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv") + + node.query( + f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime, b UInt64) " + f"ENGINE = ReplicatedMergeTree ORDER BY tuple()" + ) + + node.query( + f"CREATE TABLE src1 ON CLUSTER default (a DateTime, b UInt64) " + f"ENGINE = ReplicatedMergeTree ORDER BY tuple()" + ) + node.query(f"INSERT INTO src1 VALUES ('2020-01-01', 1), ('2020-01-02', 2)") + + +@pytest.mark.parametrize( + "select_query", + [ + "SELECT now() as a, number as b FROM numbers(2)", + "SELECT now() as a, b as b FROM src1", + ], +) +@pytest.mark.parametrize("with_append", [True, False]) +@pytest.mark.parametrize("empty", [True, False]) +def test_simple_append( + module_setup_tables, + fn_setup_tables, + select_query, + with_append, + empty, +): + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query=select_query, + with_append=with_append, + on_cluster="default", + empty=empty, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv", wait_status="Scheduled") + assert rmv["exception"] is None + + node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") + + records = node.query("SELECT count() FROM test_rmv") + + if empty: + assert records == "0\n" + else: + assert records == "2\n" + + for n in nodes: + n.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + + rmv2 = get_rmv_info(node, "test_rmv", wait_status="Scheduled") + + assert rmv2["exception"] is None + + node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") + if empty: + expect = "2\n" + + if not with_append: + expect = "2\n" + + if with_append and not empty: + expect = "4\n" + + records = node.query_with_retry( + "SELECT count() FROM test_rmv", check_callback=lambda x: x == expect + ) + assert records == expect + + +@pytest.mark.parametrize("with_append", [True, False]) +@pytest.mark.parametrize("if_not_exists", [True, False]) +@pytest.mark.parametrize("depends_on", [None, ["default.dummy_rmv"]]) +@pytest.mark.parametrize("empty", [True, False]) +@pytest.mark.parametrize("database_name", ["test_db"]) # None, +@pytest.mark.parametrize( + "settings", + [ + {}, + { + "refresh_retries": "10", + "refresh_retry_initial_backoff_ms": "10", + "refresh_retry_max_backoff_ms": "20", + }, + ], +) +def test_alters( + module_setup_tables, + fn_setup_tables, + with_append, + if_not_exists, + depends_on, + empty, + database_name, + settings, +): + """ + Check correctness of functional states of RMV after CREATE, DROP, ALTER, trigger of RMV, ... + """ + create_sql = CREATE_RMV.render( + table_name="test_rmv", + if_not_exists=if_not_exists, + db="test_db", + refresh_interval="EVERY 1 HOUR", + depends_on=depends_on, + to_clause="tgt1", + select_query="SELECT * FROM src1", + with_append=with_append, + settings=settings, + ) + node.query(create_sql) + + # Check same RMV is created on whole cluster + def compare_DDL_on_all_nodes(): + show_create_all_nodes = cluster.query_all_nodes("SHOW CREATE test_rmv") + assert_same_values(show_create_all_nodes.values()) + + compare_DDL_on_all_nodes() + + node.query(f"DROP TABLE test_db.test_rmv") + node.query(create_sql) + compare_DDL_on_all_nodes() + + show_create = node.query(f"SHOW CREATE test_db.test_rmv") + + alter_sql = ALTER_RMV.render( + table_name="test_rmv", + if_not_exists=if_not_exists, + db="test_db", + refresh_interval="EVERY 1 HOUR", + depends_on=depends_on, + # can't change select with alter + # select_query="SELECT * FROM src1", + with_append=with_append, + settings=settings, + ) + + node.query(alter_sql) + show_create_after_alter = node.query(f"SHOW CREATE test_db.test_rmv") + assert show_create == show_create_after_alter + compare_DDL_on_all_nodes() + + +@pytest.mark.parametrize( + "append", + [True, False], +) +@pytest.mark.parametrize( + "empty", + [True, False], +) +@pytest.mark.parametrize( + "to_clause", + [ + (None, "tgt1", "tgt1"), + ("Engine ReplicatedMergeTree ORDER BY tuple()", None, "test_rmv"), + ], +) +def test_real_wait_refresh( + fn_setup_tables, + append, + empty, + to_clause, +): + table_clause, to_clause_, tgt = to_clause + + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 10 SECOND", + to_clause=to_clause_, + table_clause=table_clause, + select_query="SELECT now() as a, b FROM src1", + with_append=append, + on_cluster="default", + empty=empty, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + time.sleep(1) + node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") + + expected_rows = 0 + if empty: + expect_rows(expected_rows, table=tgt) + else: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + + rmv2 = get_rmv_info( + node, + "test_rmv", + condition=lambda x: x["last_refresh_time"] == rmv["next_refresh_time"], + # wait for refresh a little bit more than 10 seconds + max_attempts=12, + delay=1, + wait_status="Scheduled", + ) + + node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") + + rmv22 = get_rmv_info( + node, + "test_rmv", + wait_status="Scheduled", + ) + + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + assert rmv2["exception"] is None + assert rmv2["status"] == "Scheduled" + assert rmv2["last_success_time"] == rmv["next_refresh_time"] + assert rmv2["last_refresh_time"] == rmv["next_refresh_time"] + assert rmv2["retry"] == 0 and rmv22["retry"] == 0 + + for n in nodes: + n.query("SYSTEM STOP VIEW test_rmv") + time.sleep(12) + rmv3 = get_rmv_info(node, "test_rmv") + # no refresh happen + assert rmv3["status"] == "Disabled" + + del rmv3["status"] + del rmv2["status"] + assert rmv3 == rmv2 + + for n in nodes: + n.query("SYSTEM START VIEW test_rmv") + time.sleep(1) + rmv4 = get_rmv_info(node, "test_rmv") + + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + assert rmv4["exception"] is None + assert rmv4["status"] == "Scheduled" + assert rmv4["retry"] == 0 + + node.query("SYSTEM REFRESH VIEW test_rmv") + time.sleep(1) + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + +def get_rmv_info( + node, + table, + condition=None, + max_attempts=50, + delay=0.3, + wait_status=None, +): + def inner(): + rmv_info = node.query_with_retry( + f"SELECT * FROM system.view_refreshes WHERE view='{table}'", + check_callback=( + (lambda r: r.iloc[0]["status"] == wait_status) + if wait_status + else (lambda x: True) + ), + parse=True, + ).to_dict("records")[0] + + rmv_info["next_refresh_time"] = parse_ch_datetime(rmv_info["next_refresh_time"]) + rmv_info["last_success_time"] = parse_ch_datetime(rmv_info["last_success_time"]) + rmv_info["last_refresh_time"] = parse_ch_datetime(rmv_info["last_refresh_time"]) + logging.info(rmv_info) + return rmv_info + + if condition: + res = wait_condition(inner, condition, max_attempts=max_attempts, delay=delay) + return res + + res = inner() + return res + + +def parse_ch_datetime(date_str): + if date_str is None: + return None + return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") + + +def expect_rows(rows, table="test_rmv"): + inserted_data = node.query_with_retry( + f"SELECT * FROM {table}", + parse=True, + check_callback=lambda x: len(x) == rows, + retry_count=100, + ) + assert len(inserted_data) == rows + + +def test_long_query_cancel(fn_setup_tables): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 5 SECONDS", + to_clause="tgt1", + select_query="SELECT now() a, sleep(1) b from numbers(5) settings max_block_size=1", + with_append=False, + empty=True, + settings={"refresh_retries": "0"}, + ) + node.query(create_sql) + + done = False + start = time.time() + while not done: + for n in nodes: + n.query("SYSTEM CANCEL VIEW test_rmv") + if get_rmv_info(node2, "test_rmv")["exception"] == "cancelled": + done = True + + time.sleep(0.1) + if time.time() - start > 10: + raise AssertionError("Can't cancel query") + + rmv = get_rmv_info(node, "test_rmv", wait_status="Scheduled") + assert rmv["status"] == "Scheduled" + assert rmv["exception"] == "cancelled" + assert rmv["last_success_time"] is None + + assert node.query("SELECT count() FROM tgt1") == "0\n" + + get_rmv_info(node, "test_rmv", delay=0.1, max_attempts=1000, wait_status="Running") + get_rmv_info( + node, "test_rmv", delay=0.1, max_attempts=1000, wait_status="Scheduled" + ) + + assert node.query("SELECT count() FROM tgt1") == "5\n" + + +@pytest.fixture(scope="function") +def fn3_setup_tables(): + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv") + + node.query( + f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime) ENGINE = ReplicatedMergeTree ORDER BY tuple()" + ) + + +def test_query_fail(fn3_setup_tables): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + # Argument at index 1 for function throwIf must be constant + select_query="SELECT throwIf(1, toString(rand())) a", + with_append=False, + on_cluster="default", + empty=True, + settings={ + "refresh_retries": "10", + }, + ) + with pytest.raises(helpers.client.QueryRuntimeException) as exc: + node.query(create_sql) + assert "Argument at index 1 for function throwIf must be constant" in str( + exc.value + ) + assert ( + node.query(f"SELECT count() FROM system.view_refreshes WHERE view='test_rmv'") + == "0\n" + ) + assert ( + node.query(f"SELECT count() FROM system.tables WHERE name='test_rmv'") == "0\n" + ) + + +def test_query_retry(fn3_setup_tables): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 2 SECOND", + to_clause="tgt1", + select_query="SELECT throwIf(1, '111') a", + with_append=False, + on_cluster="default", + empty=True, + settings={ + "refresh_retries": "10", + "refresh_retry_initial_backoff_ms": "1", + "refresh_retry_max_backoff_ms": "1", + }, + ) + node.query(create_sql) + rmv = get_rmv_info( + node, + "test_rmv", + delay=0.1, + max_attempts=1000, + condition=lambda x: x["retry"] == 11, + ) + assert rmv["retry"] == 11 + assert "FUNCTION_THROW_IF_VALUE_IS_NON_ZERO" in rmv["exception"] From eb42bbbf7493edb565fab3a92c7a621b3abac727 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 4 Dec 2024 20:16:36 +0100 Subject: [PATCH 447/502] RMV: fix tests with replicated db --- tests/integration/helpers/test_tools.py | 5 ++- .../test_refreshable_mat_view/test.py | 4 +-- .../test.py | 34 +++++++++---------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index c334a7366da..bd43adad95d 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -186,6 +186,7 @@ def csv_compare(result, expected): def wait_condition(func, condition, max_attempts=10, delay=0.1): attempts = 0 + result = None while attempts < max_attempts: result = func() if condition(result): @@ -194,4 +195,6 @@ def wait_condition(func, condition, max_attempts=10, delay=0.1): if attempts < max_attempts: time.sleep(delay) - raise Exception(f"Function did not satisfy condition after {max_attempts} attempts") + raise Exception( + f"Function did not satisfy condition after {max_attempts} attempts. Last result:\n{result}" + ) diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py index c25d37fdc19..ed8e9de1168 100644 --- a/tests/integration/test_refreshable_mat_view/test.py +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -377,8 +377,8 @@ def test_real_wait_refresh( "test_rmv", condition=lambda x: x["last_refresh_time"] == rmv["next_refresh_time"], # wait for refresh a little bit more than 10 seconds - max_attempts=12, - delay=1, + max_attempts=30, + delay=0.5, ) if append: diff --git a/tests/integration/test_refreshable_mat_view_replicated/test.py b/tests/integration/test_refreshable_mat_view_replicated/test.py index 4293a3ca23b..cf09dc9c8e1 100644 --- a/tests/integration/test_refreshable_mat_view_replicated/test.py +++ b/tests/integration/test_refreshable_mat_view_replicated/test.py @@ -221,13 +221,19 @@ def fn_setup_tables(): @pytest.mark.parametrize( "select_query", [ - "SELECT now() as a, number as b FROM numbers(2)", - "SELECT now() as a, b as b FROM src1", + "SELECT now() as a, number as b FROM numbers(2) SETTINGS insert_deduplicate=0", + "SELECT now() as a, b as b FROM src1 SETTINGS insert_deduplicate=0", ], ) -@pytest.mark.parametrize("with_append", [True, False]) -@pytest.mark.parametrize("empty", [True, False]) -def test_simple_append( +@pytest.mark.parametrize( + "with_append", + [True, False], +) +@pytest.mark.parametrize( + "empty", + [True, False], +) +def test_append( module_setup_tables, fn_setup_tables, select_query, @@ -247,8 +253,6 @@ def test_simple_append( rmv = get_rmv_info(node, "test_rmv", wait_status="Scheduled") assert rmv["exception"] is None - node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") - records = node.query("SELECT count() FROM test_rmv") if empty: @@ -256,20 +260,13 @@ def test_simple_append( else: assert records == "2\n" - for n in nodes: - n.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") rmv2 = get_rmv_info(node, "test_rmv", wait_status="Scheduled") assert rmv2["exception"] is None - node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") - if empty: - expect = "2\n" - - if not with_append: - expect = "2\n" - + expect = "2\n" if with_append and not empty: expect = "4\n" @@ -373,6 +370,9 @@ def test_real_wait_refresh( empty, to_clause, ): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + table_clause, to_clause_, tgt = to_clause create_sql = CREATE_RMV.render( @@ -380,7 +380,7 @@ def test_real_wait_refresh( refresh_interval="EVERY 10 SECOND", to_clause=to_clause_, table_clause=table_clause, - select_query="SELECT now() as a, b FROM src1", + select_query="SELECT now() as a, b FROM src1 SETTINGS insert_deduplicate=0", with_append=append, on_cluster="default", empty=empty, From 5e51114a6aa87f7c2367bf6f17880b23ab1cb5e9 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 9 Dec 2024 18:34:42 +0100 Subject: [PATCH 448/502] Fix bad conflict resolution --- src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 8550717d4c7..8aac7b62d48 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -74,6 +74,8 @@ IcebergMetadata::IcebergMetadata( current_schema_id = schema_id; } +namespace +{ enum class ManifestEntryStatus : uint8_t { EXISTING = 0, @@ -123,6 +125,8 @@ bool operator!=(const Poco::JSON::Object & first, const Poco::JSON::Object & sec return !(first == second); } +} + DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name) { From 18e8af493cd4411408a0f479211d0b0ae43a9ac6 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 9 Dec 2024 18:07:20 +0000 Subject: [PATCH 449/502] Support Dynamic in functions toFloat64/touInt32/etc --- src/Functions/FunctionsConversion.h | 414 ++++++++++++------ ...282_dynamic_in_functions_convert.reference | 12 + .../03282_dynamic_in_functions_convert.sql | 6 + 3 files changed, 300 insertions(+), 132 deletions(-) create mode 100644 tests/queries/0_stateless/03282_dynamic_in_functions_convert.reference create mode 100644 tests/queries/0_stateless/03282_dynamic_in_functions_convert.sql diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 582228744e4..01f8f17ec70 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -1678,8 +1678,8 @@ struct ConvertImpl && !(std::is_same_v || std::is_same_v) && (!IsDataTypeDecimalOrNumber || !IsDataTypeDecimalOrNumber)) { - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", - named_from.column->getName(), Name::name); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}/{} of first argument of function {}", + named_from.column->getName(), typeid(FromDataType).name(), Name::name); } const ColVecFrom * col_from = checkAndGetColumn(named_from.column.get()); @@ -1993,6 +1993,124 @@ struct ConvertImplGenericFromString } }; +struct ConvertImplFromDynamicToColumn +{ + static ColumnPtr execute( + const ColumnsWithTypeAndName & arguments, + const DataTypePtr & result_type, + size_t input_rows_count, + const std::function & nested_convert) + { + /// When casting Dynamic to regular column we should cast all variants from current Dynamic column + /// and construct the result based on discriminators. + const auto & column_dynamic = assert_cast(*arguments.front().column.get()); + const auto & variant_column = column_dynamic.getVariantColumn(); + const auto & variant_info = column_dynamic.getVariantInfo(); + + /// First, cast usual variants to result type. + const auto & variant_types = assert_cast(*variant_info.variant_type).getVariants(); + std::vector cast_variant_columns; + cast_variant_columns.reserve(variant_types.size()); + for (size_t i = 0; i != variant_types.size(); ++i) + { + /// Skip shared variant, it will be processed later. + if (i == column_dynamic.getSharedVariantDiscriminator()) + { + cast_variant_columns.push_back(nullptr); + continue; + } + + ColumnsWithTypeAndName new_args = arguments; + new_args[0] = {variant_column.getVariantPtrByGlobalDiscriminator(i), variant_types[i], ""}; + cast_variant_columns.push_back(nested_convert(new_args, result_type)); + } + + /// Second, collect all variants stored in shared variant and cast them to result type. + std::vector variant_columns_from_shared_variant; + DataTypes variant_types_from_shared_variant; + /// We will need to know what variant to use when we see discriminator of a shared variant. + /// To do it, we remember what variant was extracted from each row and what was it's offset. + PaddedPODArray shared_variant_indexes; + PaddedPODArray shared_variant_offsets; + std::unordered_map shared_variant_to_index; + const auto & shared_variant = column_dynamic.getSharedVariant(); + const auto shared_variant_discr = column_dynamic.getSharedVariantDiscriminator(); + const auto & local_discriminators = variant_column.getLocalDiscriminators(); + const auto & offsets = variant_column.getOffsets(); + if (!shared_variant.empty()) + { + shared_variant_indexes.reserve(input_rows_count); + shared_variant_offsets.reserve(input_rows_count); + FormatSettings format_settings; + const auto shared_variant_local_discr = variant_column.localDiscriminatorByGlobal(shared_variant_discr); + for (size_t i = 0; i != input_rows_count; ++i) + { + if (local_discriminators[i] == shared_variant_local_discr) + { + auto value = shared_variant.getDataAt(offsets[i]); + ReadBufferFromMemory buf(value.data, value.size); + auto type = decodeDataType(buf); + auto type_name = type->getName(); + auto it = shared_variant_to_index.find(type_name); + /// Check if we didn't create column for this variant yet. + if (it == shared_variant_to_index.end()) + { + it = shared_variant_to_index.emplace(type_name, variant_columns_from_shared_variant.size()).first; + variant_columns_from_shared_variant.push_back(type->createColumn()); + variant_types_from_shared_variant.push_back(type); + } + + shared_variant_indexes.push_back(it->second); + shared_variant_offsets.push_back(variant_columns_from_shared_variant[it->second]->size()); + type->getDefaultSerialization()->deserializeBinary(*variant_columns_from_shared_variant[it->second], buf, format_settings); + } + else + { + shared_variant_indexes.emplace_back(); + shared_variant_offsets.emplace_back(); + } + } + } + + /// Cast all extracted variants into result type. + std::vector cast_shared_variant_columns; + cast_shared_variant_columns.reserve(variant_types_from_shared_variant.size()); + for (size_t i = 0; i != variant_types_from_shared_variant.size(); ++i) + { + ColumnsWithTypeAndName new_args = arguments; + new_args[0] = {variant_columns_from_shared_variant[i]->getPtr(), variant_types_from_shared_variant[i], ""}; + cast_shared_variant_columns.push_back(nested_convert(new_args, result_type)); + } + + /// Construct result column from all cast variants. + auto res = result_type->createColumn(); + res->reserve(input_rows_count); + for (size_t i = 0; i != input_rows_count; ++i) + { + auto global_discr = variant_column.globalDiscriminatorByLocal(local_discriminators[i]); + if (global_discr == ColumnVariant::NULL_DISCRIMINATOR) + { + res->insertDefault(); + } + else if (global_discr == shared_variant_discr) + { + if (cast_shared_variant_columns[shared_variant_indexes[i]]) + res->insertFrom(*cast_shared_variant_columns[shared_variant_indexes[i]], shared_variant_offsets[i]); + else + res->insertDefault(); + } + else + { + if (cast_variant_columns[global_discr]) + res->insertFrom(*cast_variant_columns[global_discr], offsets[i]); + else + res->insertDefault(); + } + } + + return res; + } +}; /// Declared early because used below. struct NameToDate { static constexpr auto name = "toDate"; }; @@ -2326,6 +2444,16 @@ private: if (context) date_time_overflow_behavior = context->getSettingsRef()[Setting::date_time_overflow_behavior].value; + if (isDynamic(from_type)) + { + auto nested_convert = [this](ColumnsWithTypeAndName & args, const DataTypePtr & to_type) -> ColumnPtr + { + return executeInternal(args, to_type, args[0].column->size()); + }; + + return ConvertImplFromDynamicToColumn::execute(arguments, result_type, input_rows_count, nested_convert); + } + auto call = [&](const auto & types, BehaviourOnErrorFromString from_string_tag) -> bool { using Types = std::decay_t; @@ -4692,138 +4820,160 @@ private: WrapperType createDynamicToColumnWrapper(const DataTypePtr &) const { - return [this] + auto nested_convert = [this](ColumnsWithTypeAndName & args, const DataTypePtr & result_type) -> ColumnPtr + { + WrapperType wrapper; + if (cast_type == CastType::accurateOrNull) + { + /// Create wrapper only if we support conversion from variant to the resulting type. + wrapper = createWrapperIfCanConvert(args[0].type, result_type); + if (!wrapper) + return nullptr; + } + else + { + wrapper = prepareUnpackDictionaries(args[0].type, result_type); + } + + return wrapper(args, result_type, nullptr, args[0].column->size()); + }; + + return [nested_convert] (ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable *, size_t input_rows_count) -> ColumnPtr { - /// When casting Dynamic to regular column we should cast all variants from current Dynamic column - /// and construct the result based on discriminators. - const auto & column_dynamic = assert_cast(*arguments.front().column.get()); - const auto & variant_column = column_dynamic.getVariantColumn(); - const auto & variant_info = column_dynamic.getVariantInfo(); - - /// First, cast usual variants to result type. - const auto & variant_types = assert_cast(*variant_info.variant_type).getVariants(); - std::vector cast_variant_columns; - cast_variant_columns.reserve(variant_types.size()); - for (size_t i = 0; i != variant_types.size(); ++i) - { - /// Skip shared variant, it will be processed later. - if (i == column_dynamic.getSharedVariantDiscriminator()) - { - cast_variant_columns.push_back(nullptr); - continue; - } - - const auto & variant_col = variant_column.getVariantPtrByGlobalDiscriminator(i); - ColumnsWithTypeAndName variant = {{variant_col, variant_types[i], ""}}; - WrapperType variant_wrapper; - if (cast_type == CastType::accurateOrNull) - /// Create wrapper only if we support conversion from variant to the resulting type. - variant_wrapper = createWrapperIfCanConvert(variant_types[i], result_type); - else - variant_wrapper = prepareUnpackDictionaries(variant_types[i], result_type); - - ColumnPtr cast_variant; - /// Check if we have wrapper for this variant. - if (variant_wrapper) - cast_variant = variant_wrapper(variant, result_type, nullptr, variant_col->size()); - cast_variant_columns.push_back(cast_variant); - } - - /// Second, collect all variants stored in shared variant and cast them to result type. - std::vector variant_columns_from_shared_variant; - DataTypes variant_types_from_shared_variant; - /// We will need to know what variant to use when we see discriminator of a shared variant. - /// To do it, we remember what variant was extracted from each row and what was it's offset. - PaddedPODArray shared_variant_indexes; - PaddedPODArray shared_variant_offsets; - std::unordered_map shared_variant_to_index; - const auto & shared_variant = column_dynamic.getSharedVariant(); - const auto shared_variant_discr = column_dynamic.getSharedVariantDiscriminator(); - const auto & local_discriminators = variant_column.getLocalDiscriminators(); - const auto & offsets = variant_column.getOffsets(); - if (!shared_variant.empty()) - { - shared_variant_indexes.reserve(input_rows_count); - shared_variant_offsets.reserve(input_rows_count); - FormatSettings format_settings; - const auto shared_variant_local_discr = variant_column.localDiscriminatorByGlobal(shared_variant_discr); - for (size_t i = 0; i != input_rows_count; ++i) - { - if (local_discriminators[i] == shared_variant_local_discr) - { - auto value = shared_variant.getDataAt(offsets[i]); - ReadBufferFromMemory buf(value.data, value.size); - auto type = decodeDataType(buf); - auto type_name = type->getName(); - auto it = shared_variant_to_index.find(type_name); - /// Check if we didn't create column for this variant yet. - if (it == shared_variant_to_index.end()) - { - it = shared_variant_to_index.emplace(type_name, variant_columns_from_shared_variant.size()).first; - variant_columns_from_shared_variant.push_back(type->createColumn()); - variant_types_from_shared_variant.push_back(type); - } - - shared_variant_indexes.push_back(it->second); - shared_variant_offsets.push_back(variant_columns_from_shared_variant[it->second]->size()); - type->getDefaultSerialization()->deserializeBinary(*variant_columns_from_shared_variant[it->second], buf, format_settings); - } - else - { - shared_variant_indexes.emplace_back(); - shared_variant_offsets.emplace_back(); - } - } - } - - /// Cast all extracted variants into result type. - std::vector cast_shared_variant_columns; - cast_shared_variant_columns.reserve(variant_types_from_shared_variant.size()); - for (size_t i = 0; i != variant_types_from_shared_variant.size(); ++i) - { - ColumnsWithTypeAndName variant = {{variant_columns_from_shared_variant[i]->getPtr(), variant_types_from_shared_variant[i], ""}}; - WrapperType variant_wrapper; - if (cast_type == CastType::accurateOrNull) - /// Create wrapper only if we support conversion from variant to the resulting type. - variant_wrapper = createWrapperIfCanConvert(variant_types_from_shared_variant[i], result_type); - else - variant_wrapper = prepareUnpackDictionaries(variant_types_from_shared_variant[i], result_type); - - ColumnPtr cast_variant; - /// Check if we have wrapper for this variant. - if (variant_wrapper) - cast_variant = variant_wrapper(variant, result_type, nullptr, variant_columns_from_shared_variant[i]->size()); - cast_shared_variant_columns.push_back(cast_variant); - } - - /// Construct result column from all cast variants. - auto res = result_type->createColumn(); - res->reserve(input_rows_count); - for (size_t i = 0; i != input_rows_count; ++i) - { - auto global_discr = variant_column.globalDiscriminatorByLocal(local_discriminators[i]); - if (global_discr == ColumnVariant::NULL_DISCRIMINATOR) - { - res->insertDefault(); - } - else if (global_discr == shared_variant_discr) - { - if (cast_shared_variant_columns[shared_variant_indexes[i]]) - res->insertFrom(*cast_shared_variant_columns[shared_variant_indexes[i]], shared_variant_offsets[i]); - else - res->insertDefault(); - } - else - { - if (cast_variant_columns[global_discr]) - res->insertFrom(*cast_variant_columns[global_discr], offsets[i]); - else - res->insertDefault(); - } - } - - return res; + return ConvertImplFromDynamicToColumn::execute(arguments, result_type, input_rows_count, nested_convert); +// +// +// +// /// When casting Dynamic to regular column we should cast all variants from current Dynamic column +// /// and construct the result based on discriminators. +// const auto & column_dynamic = assert_cast(*arguments.front().column.get()); +// const auto & variant_column = column_dynamic.getVariantColumn(); +// const auto & variant_info = column_dynamic.getVariantInfo(); +// +// /// First, cast usual variants to result type. +// const auto & variant_types = assert_cast(*variant_info.variant_type).getVariants(); +// std::vector cast_variant_columns; +// cast_variant_columns.reserve(variant_types.size()); +// for (size_t i = 0; i != variant_types.size(); ++i) +// { +// /// Skip shared variant, it will be processed later. +// if (i == column_dynamic.getSharedVariantDiscriminator()) +// { +// cast_variant_columns.push_back(nullptr); +// continue; +// } +// +// const auto & variant_col = variant_column.getVariantPtrByGlobalDiscriminator(i); +// ColumnsWithTypeAndName variant = {{variant_col, variant_types[i], ""}}; +// WrapperType variant_wrapper; +// if (cast_type == CastType::accurateOrNull) +// /// Create wrapper only if we support conversion from variant to the resulting type. +// variant_wrapper = createWrapperIfCanConvert(variant_types[i], result_type); +// else +// variant_wrapper = prepareUnpackDictionaries(variant_types[i], result_type); +// +// ColumnPtr cast_variant; +// /// Check if we have wrapper for this variant. +// if (variant_wrapper) +// cast_variant = variant_wrapper(variant, result_type, nullptr, variant_col->size()); +// cast_variant_columns.push_back(cast_variant); +// } +// +// /// Second, collect all variants stored in shared variant and cast them to result type. +// std::vector variant_columns_from_shared_variant; +// DataTypes variant_types_from_shared_variant; +// /// We will need to know what variant to use when we see discriminator of a shared variant. +// /// To do it, we remember what variant was extracted from each row and what was it's offset. +// PaddedPODArray shared_variant_indexes; +// PaddedPODArray shared_variant_offsets; +// std::unordered_map shared_variant_to_index; +// const auto & shared_variant = column_dynamic.getSharedVariant(); +// const auto shared_variant_discr = column_dynamic.getSharedVariantDiscriminator(); +// const auto & local_discriminators = variant_column.getLocalDiscriminators(); +// const auto & offsets = variant_column.getOffsets(); +// if (!shared_variant.empty()) +// { +// shared_variant_indexes.reserve(input_rows_count); +// shared_variant_offsets.reserve(input_rows_count); +// FormatSettings format_settings; +// const auto shared_variant_local_discr = variant_column.localDiscriminatorByGlobal(shared_variant_discr); +// for (size_t i = 0; i != input_rows_count; ++i) +// { +// if (local_discriminators[i] == shared_variant_local_discr) +// { +// auto value = shared_variant.getDataAt(offsets[i]); +// ReadBufferFromMemory buf(value.data, value.size); +// auto type = decodeDataType(buf); +// auto type_name = type->getName(); +// auto it = shared_variant_to_index.find(type_name); +// /// Check if we didn't create column for this variant yet. +// if (it == shared_variant_to_index.end()) +// { +// it = shared_variant_to_index.emplace(type_name, variant_columns_from_shared_variant.size()).first; +// variant_columns_from_shared_variant.push_back(type->createColumn()); +// variant_types_from_shared_variant.push_back(type); +// } +// +// shared_variant_indexes.push_back(it->second); +// shared_variant_offsets.push_back(variant_columns_from_shared_variant[it->second]->size()); +// type->getDefaultSerialization()->deserializeBinary(*variant_columns_from_shared_variant[it->second], buf, format_settings); +// } +// else +// { +// shared_variant_indexes.emplace_back(); +// shared_variant_offsets.emplace_back(); +// } +// } +// } +// +// /// Cast all extracted variants into result type. +// std::vector cast_shared_variant_columns; +// cast_shared_variant_columns.reserve(variant_types_from_shared_variant.size()); +// for (size_t i = 0; i != variant_types_from_shared_variant.size(); ++i) +// { +// ColumnsWithTypeAndName variant = {{variant_columns_from_shared_variant[i]->getPtr(), variant_types_from_shared_variant[i], ""}}; +// WrapperType variant_wrapper; +// if (cast_type == CastType::accurateOrNull) +// /// Create wrapper only if we support conversion from variant to the resulting type. +// variant_wrapper = createWrapperIfCanConvert(variant_types_from_shared_variant[i], result_type); +// else +// variant_wrapper = prepareUnpackDictionaries(variant_types_from_shared_variant[i], result_type); +// +// ColumnPtr cast_variant; +// /// Check if we have wrapper for this variant. +// if (variant_wrapper) +// cast_variant = variant_wrapper(variant, result_type, nullptr, variant_columns_from_shared_variant[i]->size()); +// cast_shared_variant_columns.push_back(cast_variant); +// } +// +// /// Construct result column from all cast variants. +// auto res = result_type->createColumn(); +// res->reserve(input_rows_count); +// for (size_t i = 0; i != input_rows_count; ++i) +// { +// auto global_discr = variant_column.globalDiscriminatorByLocal(local_discriminators[i]); +// if (global_discr == ColumnVariant::NULL_DISCRIMINATOR) +// { +// res->insertDefault(); +// } +// else if (global_discr == shared_variant_discr) +// { +// if (cast_shared_variant_columns[shared_variant_indexes[i]]) +// res->insertFrom(*cast_shared_variant_columns[shared_variant_indexes[i]], shared_variant_offsets[i]); +// else +// res->insertDefault(); +// } +// else +// { +// if (cast_variant_columns[global_discr]) +// res->insertFrom(*cast_variant_columns[global_discr], offsets[i]); +// else +// res->insertDefault(); +// } +// } +// +// return res; }; } diff --git a/tests/queries/0_stateless/03282_dynamic_in_functions_convert.reference b/tests/queries/0_stateless/03282_dynamic_in_functions_convert.reference new file mode 100644 index 00000000000..5e3486b4853 --- /dev/null +++ b/tests/queries/0_stateless/03282_dynamic_in_functions_convert.reference @@ -0,0 +1,12 @@ +1 +2 +3 +4 +5 +18262 +1 +2 +3 +4 +5 +18262 diff --git a/tests/queries/0_stateless/03282_dynamic_in_functions_convert.sql b/tests/queries/0_stateless/03282_dynamic_in_functions_convert.sql new file mode 100644 index 00000000000..9ad378c9ec7 --- /dev/null +++ b/tests/queries/0_stateless/03282_dynamic_in_functions_convert.sql @@ -0,0 +1,6 @@ +set enable_dynamic_type = 1; +create table test (d Dynamic(max_types=3)) engine=Memory; +insert into test values (1::UInt8), (2::UInt16), (3::UInt32), (4::UInt64), ('5'::String), ('2020-01-01'::Date); +select toFloat64(d) from test; +select toUInt32(d) from test; +drop table test; From d58dba38f74f223a703f1624571266c25b4f9e73 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 9 Dec 2024 18:08:06 +0000 Subject: [PATCH 450/502] Remove old code --- src/Functions/FunctionsConversion.h | 132 ---------------------------- 1 file changed, 132 deletions(-) diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 01f8f17ec70..a9a82537abd 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -4842,138 +4842,6 @@ private: (ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable *, size_t input_rows_count) -> ColumnPtr { return ConvertImplFromDynamicToColumn::execute(arguments, result_type, input_rows_count, nested_convert); -// -// -// -// /// When casting Dynamic to regular column we should cast all variants from current Dynamic column -// /// and construct the result based on discriminators. -// const auto & column_dynamic = assert_cast(*arguments.front().column.get()); -// const auto & variant_column = column_dynamic.getVariantColumn(); -// const auto & variant_info = column_dynamic.getVariantInfo(); -// -// /// First, cast usual variants to result type. -// const auto & variant_types = assert_cast(*variant_info.variant_type).getVariants(); -// std::vector cast_variant_columns; -// cast_variant_columns.reserve(variant_types.size()); -// for (size_t i = 0; i != variant_types.size(); ++i) -// { -// /// Skip shared variant, it will be processed later. -// if (i == column_dynamic.getSharedVariantDiscriminator()) -// { -// cast_variant_columns.push_back(nullptr); -// continue; -// } -// -// const auto & variant_col = variant_column.getVariantPtrByGlobalDiscriminator(i); -// ColumnsWithTypeAndName variant = {{variant_col, variant_types[i], ""}}; -// WrapperType variant_wrapper; -// if (cast_type == CastType::accurateOrNull) -// /// Create wrapper only if we support conversion from variant to the resulting type. -// variant_wrapper = createWrapperIfCanConvert(variant_types[i], result_type); -// else -// variant_wrapper = prepareUnpackDictionaries(variant_types[i], result_type); -// -// ColumnPtr cast_variant; -// /// Check if we have wrapper for this variant. -// if (variant_wrapper) -// cast_variant = variant_wrapper(variant, result_type, nullptr, variant_col->size()); -// cast_variant_columns.push_back(cast_variant); -// } -// -// /// Second, collect all variants stored in shared variant and cast them to result type. -// std::vector variant_columns_from_shared_variant; -// DataTypes variant_types_from_shared_variant; -// /// We will need to know what variant to use when we see discriminator of a shared variant. -// /// To do it, we remember what variant was extracted from each row and what was it's offset. -// PaddedPODArray shared_variant_indexes; -// PaddedPODArray shared_variant_offsets; -// std::unordered_map shared_variant_to_index; -// const auto & shared_variant = column_dynamic.getSharedVariant(); -// const auto shared_variant_discr = column_dynamic.getSharedVariantDiscriminator(); -// const auto & local_discriminators = variant_column.getLocalDiscriminators(); -// const auto & offsets = variant_column.getOffsets(); -// if (!shared_variant.empty()) -// { -// shared_variant_indexes.reserve(input_rows_count); -// shared_variant_offsets.reserve(input_rows_count); -// FormatSettings format_settings; -// const auto shared_variant_local_discr = variant_column.localDiscriminatorByGlobal(shared_variant_discr); -// for (size_t i = 0; i != input_rows_count; ++i) -// { -// if (local_discriminators[i] == shared_variant_local_discr) -// { -// auto value = shared_variant.getDataAt(offsets[i]); -// ReadBufferFromMemory buf(value.data, value.size); -// auto type = decodeDataType(buf); -// auto type_name = type->getName(); -// auto it = shared_variant_to_index.find(type_name); -// /// Check if we didn't create column for this variant yet. -// if (it == shared_variant_to_index.end()) -// { -// it = shared_variant_to_index.emplace(type_name, variant_columns_from_shared_variant.size()).first; -// variant_columns_from_shared_variant.push_back(type->createColumn()); -// variant_types_from_shared_variant.push_back(type); -// } -// -// shared_variant_indexes.push_back(it->second); -// shared_variant_offsets.push_back(variant_columns_from_shared_variant[it->second]->size()); -// type->getDefaultSerialization()->deserializeBinary(*variant_columns_from_shared_variant[it->second], buf, format_settings); -// } -// else -// { -// shared_variant_indexes.emplace_back(); -// shared_variant_offsets.emplace_back(); -// } -// } -// } -// -// /// Cast all extracted variants into result type. -// std::vector cast_shared_variant_columns; -// cast_shared_variant_columns.reserve(variant_types_from_shared_variant.size()); -// for (size_t i = 0; i != variant_types_from_shared_variant.size(); ++i) -// { -// ColumnsWithTypeAndName variant = {{variant_columns_from_shared_variant[i]->getPtr(), variant_types_from_shared_variant[i], ""}}; -// WrapperType variant_wrapper; -// if (cast_type == CastType::accurateOrNull) -// /// Create wrapper only if we support conversion from variant to the resulting type. -// variant_wrapper = createWrapperIfCanConvert(variant_types_from_shared_variant[i], result_type); -// else -// variant_wrapper = prepareUnpackDictionaries(variant_types_from_shared_variant[i], result_type); -// -// ColumnPtr cast_variant; -// /// Check if we have wrapper for this variant. -// if (variant_wrapper) -// cast_variant = variant_wrapper(variant, result_type, nullptr, variant_columns_from_shared_variant[i]->size()); -// cast_shared_variant_columns.push_back(cast_variant); -// } -// -// /// Construct result column from all cast variants. -// auto res = result_type->createColumn(); -// res->reserve(input_rows_count); -// for (size_t i = 0; i != input_rows_count; ++i) -// { -// auto global_discr = variant_column.globalDiscriminatorByLocal(local_discriminators[i]); -// if (global_discr == ColumnVariant::NULL_DISCRIMINATOR) -// { -// res->insertDefault(); -// } -// else if (global_discr == shared_variant_discr) -// { -// if (cast_shared_variant_columns[shared_variant_indexes[i]]) -// res->insertFrom(*cast_shared_variant_columns[shared_variant_indexes[i]], shared_variant_offsets[i]); -// else -// res->insertDefault(); -// } -// else -// { -// if (cast_variant_columns[global_discr]) -// res->insertFrom(*cast_variant_columns[global_discr], offsets[i]); -// else -// res->insertDefault(); -// } -// } -// -// return res; }; } From 43e66782124466b5e6f611895dec6ec08c4afb10 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 9 Dec 2024 19:18:10 +0100 Subject: [PATCH 451/502] Review fixes, fix test --- src/Databases/Iceberg/DatabaseIceberg.cpp | 18 +-- src/Databases/Iceberg/DatabaseIceberg.h | 2 +- src/Databases/Iceberg/RestCatalog.cpp | 141 ++++++++++-------- .../DataLakes/IcebergMetadata.cpp | 2 +- .../integration/test_database_iceberg/test.py | 3 +- 5 files changed, 91 insertions(+), 75 deletions(-) diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 7a49d11b19e..3818eecb3ae 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -91,7 +91,7 @@ void DatabaseIceberg::validateSettings() } } -std::shared_ptr DatabaseIceberg::getCatalog(ContextPtr) const +std::shared_ptr DatabaseIceberg::getCatalog() const { if (catalog_impl) return catalog_impl; @@ -145,7 +145,7 @@ std::shared_ptr DatabaseIceberg::getConfigu default: throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server does not contain support for storage type {}", - settings[DatabaseIcebergSetting::storage_type].value); + type); #endif } } @@ -167,18 +167,18 @@ std::string DatabaseIceberg::getStorageEndpointForTable(const Iceberg::TableMeta bool DatabaseIceberg::empty() const { - return getCatalog(Context::getGlobalContextInstance())->empty(); + return getCatalog()->empty(); } -bool DatabaseIceberg::isTableExist(const String & name, ContextPtr context_) const +bool DatabaseIceberg::isTableExist(const String & name, ContextPtr /* context_ */) const { const auto [namespace_name, table_name] = parseTableName(name); - return getCatalog(context_)->existsTable(namespace_name, table_name); + return getCatalog()->existsTable(namespace_name, table_name); } StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_) const { - auto catalog = getCatalog(context_); + auto catalog = getCatalog(); auto table_metadata = Iceberg::TableMetadata().withLocation().withSchema(); const bool with_vended_credentials = settings[DatabaseIcebergSetting::vended_credentials].value; @@ -256,7 +256,7 @@ DatabaseTablesIteratorPtr DatabaseIceberg::getTablesIterator( bool /* skip_not_loaded */) const { Tables tables; - auto catalog = getCatalog(context_); + auto catalog = getCatalog(); const auto iceberg_tables = catalog->getTables(); auto & pool = context_->getIcebergCatalogThreadpool(); @@ -292,10 +292,10 @@ ASTPtr DatabaseIceberg::getCreateDatabaseQuery() const ASTPtr DatabaseIceberg::getCreateTableQueryImpl( const String & name, - ContextPtr context_, + ContextPtr /* context_ */, bool /* throw_on_error */) const { - auto catalog = getCatalog(context_); + auto catalog = getCatalog(); auto table_metadata = Iceberg::TableMetadata().withLocation().withSchema(); const auto [namespace_name, table_name] = parseTableName(name); diff --git a/src/Databases/Iceberg/DatabaseIceberg.h b/src/Databases/Iceberg/DatabaseIceberg.h index 5138d0f639c..94717c522af 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.h +++ b/src/Databases/Iceberg/DatabaseIceberg.h @@ -57,7 +57,7 @@ private: mutable std::shared_ptr catalog_impl; void validateSettings(); - std::shared_ptr getCatalog(ContextPtr context_) const; + std::shared_ptr getCatalog() const; std::shared_ptr getConfiguration(DatabaseIcebergStorageType type) const; std::string getStorageEndpointForTable(const Iceberg::TableMetadata & table_metadata) const; }; diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index 39108b1648e..c901f7b461f 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -36,8 +36,8 @@ namespace DB::Setting namespace Iceberg { -static constexpr auto config_endpoint = "config"; -static constexpr auto namespaces_endpoint = "namespaces"; +static constexpr auto CONFIG_ENDPOINT = "config"; +static constexpr auto NAMESPACES_ENDPOINT = "namespaces"; namespace { @@ -126,7 +126,7 @@ RestCatalog::RestCatalog( RestCatalog::Config RestCatalog::loadConfig() { Poco::URI::QueryParameters params = {{"warehouse", warehouse}}; - auto buf = createReadBuffer(config_endpoint, params); + auto buf = createReadBuffer(CONFIG_ENDPOINT, params); std::string json_str; readJSONObjectPossiblyInvalid(json_str, *buf); @@ -301,26 +301,19 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( bool RestCatalog::empty() const { - try + /// TODO: add a test with empty namespaces and zero namespaces. + bool found_table = false; + auto stop_condition = [&](const std::string & namespace_name) -> bool { - bool found_table = false; - auto stop_condition = [&](const std::string & namespace_name) -> bool - { - const auto tables = getTables(namespace_name, /* limit */1); - found_table = !tables.empty(); - return found_table; - }; - - Namespaces namespaces; - getNamespacesRecursive("", namespaces, stop_condition, /* execute_func */{}); - + const auto tables = getTables(namespace_name, /* limit */1); + found_table = !tables.empty(); return found_table; - } - catch (...) - { - DB::tryLogCurrentException(log); - return true; - } + }; + + Namespaces namespaces; + getNamespacesRecursive("", namespaces, stop_condition, /* execute_func */{}); + + return found_table; } DB::Names RestCatalog::getTables() const @@ -359,6 +352,8 @@ void RestCatalog::getNamespacesRecursive( StopCondition stop_condition, ExecuteFunc func) const { + checkStackSize(); + auto namespaces = getNamespaces(base_namespace); result.reserve(result.size() + namespaces.size()); result.insert(result.end(), namespaces.begin(), namespaces.end()); @@ -384,6 +379,8 @@ Poco::URI::QueryParameters RestCatalog::createParentNamespaceParams(const std::s std::string parent_param; for (const auto & part : parts) { + /// 0x1F is a unit separator + /// https://github.com/apache/iceberg/blob/70d87f1750627b14b3b25a0216a97db86a786992/open-api/rest-catalog-open-api.yaml#L264 if (!parent_param.empty()) parent_param += static_cast(0x1F); parent_param += part; @@ -399,7 +396,7 @@ RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_name try { - auto buf = createReadBuffer(config.prefix / namespaces_endpoint, params); + auto buf = createReadBuffer(config.prefix / NAMESPACES_ENDPOINT, params); auto namespaces = parseNamespaces(*buf, base_namespace); LOG_TEST(log, "Loaded {} namespaces in base namespace {}", namespaces.size(), base_namespace); return namespaces; @@ -431,36 +428,44 @@ RestCatalog::Namespaces RestCatalog::parseNamespaces(DB::ReadBuffer & buf, const LOG_TEST(log, "Received response: {}", json_str); - Poco::JSON::Parser parser; - Poco::Dynamic::Var json = parser.parse(json_str); - const Poco::JSON::Object::Ptr & object = json.extract(); - - auto namespaces_object = object->get("namespaces").extract(); - if (!namespaces_object) - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot parse result"); - - Namespaces namespaces; - for (size_t i = 0; i < namespaces_object->size(); ++i) + try { - auto current_namespace_array = namespaces_object->get(static_cast(i)).extract(); - if (current_namespace_array->size() == 0) - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Expected namespace array to be non-empty"); + Poco::JSON::Parser parser; + Poco::Dynamic::Var json = parser.parse(json_str); + const Poco::JSON::Object::Ptr & object = json.extract(); - const int idx = static_cast(current_namespace_array->size()) - 1; - const auto current_namespace = current_namespace_array->get(idx).extract(); - const auto full_namespace = base_namespace.empty() - ? current_namespace - : base_namespace + "." + current_namespace; + auto namespaces_object = object->get("namespaces").extract(); + if (!namespaces_object) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot parse result"); - namespaces.push_back(full_namespace); + Namespaces namespaces; + for (size_t i = 0; i < namespaces_object->size(); ++i) + { + auto current_namespace_array = namespaces_object->get(static_cast(i)).extract(); + if (current_namespace_array->size() == 0) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Expected namespace array to be non-empty"); + + const int idx = static_cast(current_namespace_array->size()) - 1; + const auto current_namespace = current_namespace_array->get(idx).extract(); + const auto full_namespace = base_namespace.empty() + ? current_namespace + : base_namespace + "." + current_namespace; + + namespaces.push_back(full_namespace); + } + + return namespaces; + } + catch (DB::Exception & e) + { + e.addMessage("while parsing JSON: " + json_str); + throw; } - - return namespaces; } DB::Names RestCatalog::getTables(const std::string & base_namespace, size_t limit) const { - const auto endpoint = std::string(namespaces_endpoint) + "/" + base_namespace + "/tables"; + const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / base_namespace / "tables"; auto buf = createReadBuffer(config.prefix / endpoint); return parseTables(*buf, base_namespace, limit); } @@ -473,25 +478,34 @@ DB::Names RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & bas String json_str; readJSONObjectPossiblyInvalid(json_str, buf); - Poco::JSON::Parser parser; - Poco::Dynamic::Var json = parser.parse(json_str); - const Poco::JSON::Object::Ptr & object = json.extract(); - - auto identifiers_object = object->get("identifiers").extract(); - if (!identifiers_object) - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot parse result"); - - DB::Names tables; - for (size_t i = 0; i < identifiers_object->size(); ++i) + try { - const auto current_table_json = identifiers_object->get(static_cast(i)).extract(); - const auto table_name = current_table_json->get("name").extract(); + Poco::JSON::Parser parser; + Poco::Dynamic::Var json = parser.parse(json_str); + const Poco::JSON::Object::Ptr & object = json.extract(); - tables.push_back(base_namespace + "." + table_name); - if (limit && tables.size() >= limit) - break; + auto identifiers_object = object->get("identifiers").extract(); + if (!identifiers_object) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Cannot parse result"); + + DB::Names tables; + for (size_t i = 0; i < identifiers_object->size(); ++i) + { + const auto current_table_json = identifiers_object->get(static_cast(i)).extract(); + const auto table_name = current_table_json->get("name").extract(); + + tables.push_back(base_namespace + "." + table_name); + if (limit && tables.size() >= limit) + break; + } + + return tables; + } + catch (DB::Exception & e) + { + e.addMessage("while parsing JSON: " + json_str); + throw; } - return tables; } bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name) const @@ -544,7 +558,7 @@ bool RestCatalog::getTableMetadataImpl( headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); } - const auto endpoint = std::string(namespaces_endpoint) + "/" + namespace_name + "/tables/" + table_name; + const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / namespace_name / "tables" / table_name; auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); if (buf->eof()) @@ -556,8 +570,11 @@ bool RestCatalog::getTableMetadataImpl( String json_str; readJSONObjectPossiblyInvalid(json_str, *buf); - /// TODO: remove before merge because it might contain credentials. +#ifdef DEBUG_OR_SANITIZER_BUILD + /// This log message might contain credentials, + /// so log it only for debugging. LOG_TEST(log, "Received metadata for table {}: {}", table_name, json_str); +#endif Poco::JSON::Parser parser; Poco::Dynamic::Var json = parser.parse(json_str); diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 8aac7b62d48..2f395a1b59e 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -478,7 +478,7 @@ void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schem if (iceberg_table_schemas_by_ids.contains(schema_id)) { chassert(clickhouse_table_schemas_by_ids.contains(schema_id)); - chassert(*iceberg_table_schemas_by_ids.at(schema_id) == *schema_ptr); + // chassert(*iceberg_table_schemas_by_ids.at(schema_id) == *schema_ptr); } else { diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 2896b87f63e..6dbaccce648 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -125,8 +125,7 @@ SET allow_experimental_database_iceberg=true; CREATE DATABASE {name} ENGINE = Iceberg('{BASE_URL}', 'minio', 'minio123') SETTINGS catalog_type = 'rest', storage_endpoint = 'http://minio:9000/warehouse', - warehouse='demo', - storage_type='s3' + warehouse='demo' """ ) From 04d59c7a7c4491c9de7e50b707282ef3c48d6ae3 Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Mon, 9 Dec 2024 19:44:59 +0000 Subject: [PATCH 452/502] fix --- src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp | 2 +- tests/integration/test_storage_postgresql/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index 5268dbcb59f..29105142029 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -256,7 +256,7 @@ PostgreSQLTableStructure::ColumnsInfoPtr readNamesAndTypesList( { throw Exception( ErrorCodes::BAD_ARGUMENTS, - "PostgreSQL cannot infer dimensions of an empty array: {}.{}", + "PostgreSQL cannot infer dimensions of an empty array: {}.{}. Make sure no empty array values in the first row.", postgres_table, postgres_column); } diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 78bb1167d79..b31cef826ae 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -267,7 +267,7 @@ def test_postgres_array_ndim_error_messges(started_cluster): assert False except Exception as error: assert ( - 'PostgreSQL cannot infer dimensions of an empty array: array_ndim_view."Mixed-case with spaces"' + 'PostgreSQL cannot infer dimensions of an empty array: array_ndim_view."Mixed-case with spaces". Make sure no empty array values in the first row.' in str(error) ) From dc7bc4222349b9e9c1a991e78c06aefd7ce66668 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 26 Nov 2024 14:57:42 +0000 Subject: [PATCH 453/502] Moved some of utils to programs --- CMakeLists.txt | 1 - programs/CMakeLists.txt | 50 ++-- programs/check-marks/CMakeLists.txt | 11 + .../check-marks/CheckMarks.cpp | 2 +- programs/check-marks/check-marks.cpp | 1 + .../CMakeLists.txt | 9 + .../ChecksumForCompressedBlock.cpp | 2 +- .../checksum-for-compressed-block.cpp | 1 + programs/keeper-bench/CMakeLists.txt | 20 ++ .../keeper-bench/Generator.cpp | 0 {utils => programs}/keeper-bench/Generator.h | 0 .../keeper-bench/KeeperBench.cpp | 2 +- {utils => programs}/keeper-bench/README.md | 0 {utils => programs}/keeper-bench/Runner.cpp | 0 {utils => programs}/keeper-bench/Runner.h | 0 {utils => programs}/keeper-bench/Stats.cpp | 0 {utils => programs}/keeper-bench/Stats.h | 0 {utils => programs}/keeper-bench/example.yaml | 0 programs/keeper-bench/keeper-bench.cpp | 1 + programs/keeper-data-dumper/CMakeLists.txt | 14 ++ .../keeper-data-dumper/KeeperDataDumper.cpp | 2 +- .../keeper-data-dumper/keeper-data-dumper.cpp | 1 + programs/main.cpp | 44 ++-- programs/zookeeper-dump-tree/CMakeLists.txt | 12 + .../zookeeper-dump-tree/ZooKeeperDumpTree.cpp | 2 +- .../zookeeper-dump-tree.cpp | 1 + .../zookeeper-remove-by-list/CMakeLists.txt | 12 + .../ZooKeeperRemoveByList.cpp | 2 +- .../zookeeper-remove-by-list.cpp | 1 + src/Compression/examples/CMakeLists.txt | 3 + .../Compression/examples}/decompress_perf.cpp | 11 + utils/CMakeLists.txt | 5 - utils/check-marks/CMakeLists.txt | 2 - utils/check-mysql-binlog/CMakeLists.txt | 2 - utils/check-mysql-binlog/main.cpp | 98 -------- .../CMakeLists.txt | 2 - utils/compressor/CMakeLists.txt | 2 - utils/compressor/test.sh | 5 - utils/keeper-bench/CMakeLists.txt | 7 - utils/keeper-data-dumper/CMakeLists.txt | 7 - utils/libfuzzer-test/CMakeLists.txt | 1 - utils/libfuzzer-test/README.md | 1 - .../test_basic_fuzzer/CMakeLists.txt | 1 - .../libfuzzer-test/test_basic_fuzzer/main.cpp | 11 - utils/zookeeper-cli/CMakeLists.txt | 9 - utils/zookeeper-cli/zookeeper-cli.cpp | 223 ------------------ utils/zookeeper-dump-tree/CMakeLists.txt | 8 - utils/zookeeper-remove-by-list/CMakeLists.txt | 7 - 48 files changed, 163 insertions(+), 433 deletions(-) create mode 100644 programs/check-marks/CMakeLists.txt rename utils/check-marks/main.cpp => programs/check-marks/CheckMarks.cpp (97%) create mode 100644 programs/check-marks/check-marks.cpp create mode 100644 programs/checksum-for-compressed-block/CMakeLists.txt rename utils/checksum-for-compressed-block/main.cpp => programs/checksum-for-compressed-block/ChecksumForCompressedBlock.cpp (97%) create mode 100644 programs/checksum-for-compressed-block/checksum-for-compressed-block.cpp create mode 100644 programs/keeper-bench/CMakeLists.txt rename {utils => programs}/keeper-bench/Generator.cpp (100%) rename {utils => programs}/keeper-bench/Generator.h (100%) rename utils/keeper-bench/main.cpp => programs/keeper-bench/KeeperBench.cpp (98%) rename {utils => programs}/keeper-bench/README.md (100%) rename {utils => programs}/keeper-bench/Runner.cpp (100%) rename {utils => programs}/keeper-bench/Runner.h (100%) rename {utils => programs}/keeper-bench/Stats.cpp (100%) rename {utils => programs}/keeper-bench/Stats.h (100%) rename {utils => programs}/keeper-bench/example.yaml (100%) create mode 100644 programs/keeper-bench/keeper-bench.cpp create mode 100644 programs/keeper-data-dumper/CMakeLists.txt rename utils/keeper-data-dumper/main.cpp => programs/keeper-data-dumper/KeeperDataDumper.cpp (98%) create mode 100644 programs/keeper-data-dumper/keeper-data-dumper.cpp create mode 100644 programs/zookeeper-dump-tree/CMakeLists.txt rename utils/zookeeper-dump-tree/main.cpp => programs/zookeeper-dump-tree/ZooKeeperDumpTree.cpp (98%) create mode 100644 programs/zookeeper-dump-tree/zookeeper-dump-tree.cpp create mode 100644 programs/zookeeper-remove-by-list/CMakeLists.txt rename utils/zookeeper-remove-by-list/main.cpp => programs/zookeeper-remove-by-list/ZooKeeperRemoveByList.cpp (96%) create mode 100644 programs/zookeeper-remove-by-list/zookeeper-remove-by-list.cpp rename {utils/compressor => src/Compression/examples}/decompress_perf.cpp (97%) delete mode 100644 utils/check-marks/CMakeLists.txt delete mode 100644 utils/check-mysql-binlog/CMakeLists.txt delete mode 100644 utils/check-mysql-binlog/main.cpp delete mode 100644 utils/checksum-for-compressed-block/CMakeLists.txt delete mode 100644 utils/compressor/CMakeLists.txt delete mode 100755 utils/compressor/test.sh delete mode 100644 utils/keeper-bench/CMakeLists.txt delete mode 100644 utils/keeper-data-dumper/CMakeLists.txt delete mode 100644 utils/libfuzzer-test/CMakeLists.txt delete mode 100644 utils/libfuzzer-test/README.md delete mode 100644 utils/libfuzzer-test/test_basic_fuzzer/CMakeLists.txt delete mode 100644 utils/libfuzzer-test/test_basic_fuzzer/main.cpp delete mode 100644 utils/zookeeper-cli/CMakeLists.txt delete mode 100644 utils/zookeeper-cli/zookeeper-cli.cpp delete mode 100644 utils/zookeeper-dump-tree/CMakeLists.txt delete mode 100644 utils/zookeeper-remove-by-list/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a42ba9a2ce..847c0085515 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,7 +89,6 @@ string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES) option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF) -option (ENABLE_FUZZER_TEST "Build testing fuzzers in order to test libFuzzer functionality" OFF) if (ENABLE_FUZZING) # Also set WITH_COVERAGE=1 for better fuzzing process diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 3add371b30f..1a90334c992 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -119,20 +119,26 @@ macro(clickhouse_program_add name) clickhouse_program_add_library(${name}) endmacro() -add_subdirectory (server) -add_subdirectory (client) -add_subdirectory (local) -add_subdirectory (benchmark) -add_subdirectory (extract-from-config) -add_subdirectory (compressor) -add_subdirectory (format) -add_subdirectory (obfuscator) -add_subdirectory (install) -add_subdirectory (git-import) add_subdirectory (bash-completion) +add_subdirectory (benchmark) +add_subdirectory (check-marks) +add_subdirectory (checksum-for-compressed-block) +add_subdirectory (client) +add_subdirectory (compressor) +add_subdirectory (disks) +add_subdirectory (extract-from-config) +add_subdirectory (format) +add_subdirectory (git-import) +add_subdirectory (install) +add_subdirectory (keeper-bench) +add_subdirectory (keeper-data-dumper) +add_subdirectory (local) +add_subdirectory (obfuscator) +add_subdirectory (server) add_subdirectory (static-files-disk-uploader) add_subdirectory (su) -add_subdirectory (disks) +add_subdirectory (zookeeper-dump-tree) +add_subdirectory (zookeeper-remove-by-list) if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER) add_subdirectory (keeper-converter) @@ -198,18 +204,24 @@ if (ENABLE_CLICKHOUSE_SELF_EXTRACTING) list(APPEND CLICKHOUSE_BUNDLE self-extracting) endif () -clickhouse_program_install(clickhouse-server server) -clickhouse_program_install(clickhouse-client client chc) -clickhouse_program_install(clickhouse-local local chl ch) clickhouse_program_install(clickhouse-benchmark benchmark) -clickhouse_program_install(clickhouse-extract-from-config extract-from-config) +clickhouse_program_install(clickhouse-check-marks check-marks) +clickhouse_program_install(clickhouse-checksum-for-compressed-block checksum-for-compressed-block) +clickhouse_program_install(clickhouse-client client chc) clickhouse_program_install(clickhouse-compressor compressor) -clickhouse_program_install(clickhouse-format format) -clickhouse_program_install(clickhouse-obfuscator obfuscator) -clickhouse_program_install(clickhouse-git-import git-import) -clickhouse_program_install(clickhouse-static-files-disk-uploader static-files-disk-uploader) clickhouse_program_install(clickhouse-disks disks) +clickhouse_program_install(clickhouse-extract-from-config extract-from-config) +clickhouse_program_install(clickhouse-format format) +clickhouse_program_install(clickhouse-git-import git-import) +clickhouse_program_install(clickhouse-keeper-bench keeper-bench) +clickhouse_program_install(clickhouse-keeper-data-dumper keeper-data-dumper) +clickhouse_program_install(clickhouse-local local chl ch) +clickhouse_program_install(clickhouse-obfuscator obfuscator) +clickhouse_program_install(clickhouse-server server) +clickhouse_program_install(clickhouse-static-files-disk-uploader static-files-disk-uploader) clickhouse_program_install(clickhouse-su su) +clickhouse_program_install(clickhouse-zookeeper-dump-tree zookeeper-dump-tree) +clickhouse_program_install(clickhouse-zookeeper-remove-by-list zookeeper-remove-by-list) if (ENABLE_CLICKHOUSE_KEEPER) if (NOT BUILD_STANDALONE_KEEPER AND CREATE_KEEPER_SYMLINK) diff --git a/programs/check-marks/CMakeLists.txt b/programs/check-marks/CMakeLists.txt new file mode 100644 index 00000000000..875dff8e976 --- /dev/null +++ b/programs/check-marks/CMakeLists.txt @@ -0,0 +1,11 @@ +set (CLICKHOUSE_CHECK_MARKS_SOURCES CheckMarks.cpp) + +set (CLICKHOUSE_CHECK_MARKS_LINK + PRIVATE + boost::program_options + clickhouse_aggregate_functions + clickhouse_common_config + dbms +) + +clickhouse_program_add(check-marks) diff --git a/utils/check-marks/main.cpp b/programs/check-marks/CheckMarks.cpp similarity index 97% rename from utils/check-marks/main.cpp rename to programs/check-marks/CheckMarks.cpp index 28ce0121f79..a3995449bf2 100644 --- a/utils/check-marks/main.cpp +++ b/programs/check-marks/CheckMarks.cpp @@ -54,7 +54,7 @@ static void checkByCompressedReadBuffer(const std::string & mrk_path, const std: } -int main(int argc, char ** argv) +int mainEntryClickHouseCheckMarks(int argc, char ** argv) { boost::program_options::options_description desc("Allowed options"); desc.add_options() diff --git a/programs/check-marks/check-marks.cpp b/programs/check-marks/check-marks.cpp new file mode 100644 index 00000000000..52b5bc8e85c --- /dev/null +++ b/programs/check-marks/check-marks.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseCheckMarks(int argc, char ** argv); diff --git a/programs/checksum-for-compressed-block/CMakeLists.txt b/programs/checksum-for-compressed-block/CMakeLists.txt new file mode 100644 index 00000000000..7ab5a4e6e38 --- /dev/null +++ b/programs/checksum-for-compressed-block/CMakeLists.txt @@ -0,0 +1,9 @@ +set (CLICKHOUSE_CHECKSUM_FOR_COMPRESSED_BLOCK_SOURCES ChecksumForCompressedBlock.cpp) + +set (CLICKHOUSE_CHECKSUM_FOR_COMPRESSED_BLOCK_LINK + PRIVATE + clickhouse_functions + dbms +) + +clickhouse_program_add(checksum-for-compressed-block) diff --git a/utils/checksum-for-compressed-block/main.cpp b/programs/checksum-for-compressed-block/ChecksumForCompressedBlock.cpp similarity index 97% rename from utils/checksum-for-compressed-block/main.cpp rename to programs/checksum-for-compressed-block/ChecksumForCompressedBlock.cpp index 4ae06a78ab4..adf74569b6b 100644 --- a/utils/checksum-for-compressed-block/main.cpp +++ b/programs/checksum-for-compressed-block/ChecksumForCompressedBlock.cpp @@ -34,7 +34,7 @@ std::string flipBit(std::string s, size_t pos) } -int main(int, char **) +int mainEntryClickHouseChecksumForCompressedBlock(int, char **) { using namespace DB; ReadBufferFromFileDescriptor in(STDIN_FILENO); diff --git a/programs/checksum-for-compressed-block/checksum-for-compressed-block.cpp b/programs/checksum-for-compressed-block/checksum-for-compressed-block.cpp new file mode 100644 index 00000000000..a3b72c42ca8 --- /dev/null +++ b/programs/checksum-for-compressed-block/checksum-for-compressed-block.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseChecksumForCompressedBlock(int, char **); diff --git a/programs/keeper-bench/CMakeLists.txt b/programs/keeper-bench/CMakeLists.txt new file mode 100644 index 00000000000..2cec36c1d03 --- /dev/null +++ b/programs/keeper-bench/CMakeLists.txt @@ -0,0 +1,20 @@ +if (NOT TARGET ch_contrib::rapidjson) + message (${RECONFIGURE_MESSAGE_LEVEL} "Not building keeper-bench due to rapidjson is disabled") + return() +endif () + +set (CLICKHOUSE_KEEPER_BENCH_SOURCES + Generator.cpp + Runner.cpp + Stats.cpp + KeeperBench.cpp +) + +set (CLICKHOUSE_KEEPER_BENCH_LINK + PRIVATE + dbms + clickhouse_functions + ch_contrib::rapidjson +) + +clickhouse_program_add(keeper-bench) diff --git a/utils/keeper-bench/Generator.cpp b/programs/keeper-bench/Generator.cpp similarity index 100% rename from utils/keeper-bench/Generator.cpp rename to programs/keeper-bench/Generator.cpp diff --git a/utils/keeper-bench/Generator.h b/programs/keeper-bench/Generator.h similarity index 100% rename from utils/keeper-bench/Generator.h rename to programs/keeper-bench/Generator.h diff --git a/utils/keeper-bench/main.cpp b/programs/keeper-bench/KeeperBench.cpp similarity index 98% rename from utils/keeper-bench/main.cpp rename to programs/keeper-bench/KeeperBench.cpp index 0b963abf406..62f961fdc04 100644 --- a/utils/keeper-bench/main.cpp +++ b/programs/keeper-bench/KeeperBench.cpp @@ -20,7 +20,7 @@ std::optional valueToOptional(const boost::program_options::variable_value & } -int main(int argc, char *argv[]) +int mainEntryClickHouseKeeperBench(int argc, char ** argv) { bool print_stacktrace = true; diff --git a/utils/keeper-bench/README.md b/programs/keeper-bench/README.md similarity index 100% rename from utils/keeper-bench/README.md rename to programs/keeper-bench/README.md diff --git a/utils/keeper-bench/Runner.cpp b/programs/keeper-bench/Runner.cpp similarity index 100% rename from utils/keeper-bench/Runner.cpp rename to programs/keeper-bench/Runner.cpp diff --git a/utils/keeper-bench/Runner.h b/programs/keeper-bench/Runner.h similarity index 100% rename from utils/keeper-bench/Runner.h rename to programs/keeper-bench/Runner.h diff --git a/utils/keeper-bench/Stats.cpp b/programs/keeper-bench/Stats.cpp similarity index 100% rename from utils/keeper-bench/Stats.cpp rename to programs/keeper-bench/Stats.cpp diff --git a/utils/keeper-bench/Stats.h b/programs/keeper-bench/Stats.h similarity index 100% rename from utils/keeper-bench/Stats.h rename to programs/keeper-bench/Stats.h diff --git a/utils/keeper-bench/example.yaml b/programs/keeper-bench/example.yaml similarity index 100% rename from utils/keeper-bench/example.yaml rename to programs/keeper-bench/example.yaml diff --git a/programs/keeper-bench/keeper-bench.cpp b/programs/keeper-bench/keeper-bench.cpp new file mode 100644 index 00000000000..b9d9c12a2a5 --- /dev/null +++ b/programs/keeper-bench/keeper-bench.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseKeeperBench(int argc, char ** argv); diff --git a/programs/keeper-data-dumper/CMakeLists.txt b/programs/keeper-data-dumper/CMakeLists.txt new file mode 100644 index 00000000000..024645a1543 --- /dev/null +++ b/programs/keeper-data-dumper/CMakeLists.txt @@ -0,0 +1,14 @@ +if (NOT TARGET ch_contrib::nuraft) + message (WARNING "Not building keeper-data-dumper due to nuraft is disabled") + return () +endif () + +set (CLICKHOUSE_KEEPER_DATA_DUMPER_SOURCES KeeperDataDumper.cpp) + +set (CLICKHOUSE_KEEPER_DATA_DUMPER_LINK + PRIVATE + clickhouse_functions + dbms +) + +clickhouse_program_add(keeper-data-dumper) diff --git a/utils/keeper-data-dumper/main.cpp b/programs/keeper-data-dumper/KeeperDataDumper.cpp similarity index 98% rename from utils/keeper-data-dumper/main.cpp rename to programs/keeper-data-dumper/KeeperDataDumper.cpp index 161115c44f6..71614a4972f 100644 --- a/utils/keeper-data-dumper/main.cpp +++ b/programs/keeper-data-dumper/KeeperDataDumper.cpp @@ -54,7 +54,7 @@ void dumpMachine(std::shared_ptr> ma std::cout << std::flush; } -int main(int argc, char *argv[]) +int mainEntryClickHouseKeeperDataDumper(int argc, char ** argv) { if (argc != 3) { diff --git a/programs/keeper-data-dumper/keeper-data-dumper.cpp b/programs/keeper-data-dumper/keeper-data-dumper.cpp new file mode 100644 index 00000000000..6c839764e14 --- /dev/null +++ b/programs/keeper-data-dumper/keeper-data-dumper.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseKeeperDataDumper(int argc, char ** argv); diff --git a/programs/main.cpp b/programs/main.cpp index 77edad28160..4d2b37b14b3 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -20,18 +20,24 @@ #include /// Universal executable for various clickhouse applications -int mainEntryClickHouseServer(int argc, char ** argv); -int mainEntryClickHouseClient(int argc, char ** argv); -int mainEntryClickHouseLocal(int argc, char ** argv); int mainEntryClickHouseBenchmark(int argc, char ** argv); -int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); +int mainEntryClickHouseCheckMarks(int argc, char ** argv); +int mainEntryClickHouseChecksumForCompressedBlock(int, char **); +int mainEntryClickHouseClient(int argc, char ** argv); int mainEntryClickHouseCompressor(int argc, char ** argv); -int mainEntryClickHouseFormat(int argc, char ** argv); -int mainEntryClickHouseObfuscator(int argc, char ** argv); -int mainEntryClickHouseGitImport(int argc, char ** argv); -int mainEntryClickHouseStaticFilesDiskUploader(int argc, char ** argv); -int mainEntryClickHouseSU(int argc, char ** argv); int mainEntryClickHouseDisks(int argc, char ** argv); +int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); +int mainEntryClickHouseFormat(int argc, char ** argv); +int mainEntryClickHouseGitImport(int argc, char ** argv); +int mainEntryClickHouseKeeperBench(int argc, char ** argv); +int mainEntryClickHouseKeeperDataDumper(int argc, char ** argv); +int mainEntryClickHouseLocal(int argc, char ** argv); +int mainEntryClickHouseObfuscator(int argc, char ** argv); +int mainEntryClickHouseSU(int argc, char ** argv); +int mainEntryClickHouseServer(int argc, char ** argv); +int mainEntryClickHouseStaticFilesDiskUploader(int argc, char ** argv); +int mainEntryClickHouseZooKeeperDumpTree(int argc, char ** argv); +int mainEntryClickHouseZooKeeperRemoveByList(int argc, char ** argv); int mainEntryClickHouseHashBinary(int, char **) { @@ -66,19 +72,25 @@ using MainFunc = int (*)(int, char**); /// Add an item here to register new application std::pair clickhouse_applications[] = { - {"local", mainEntryClickHouseLocal}, - {"client", mainEntryClickHouseClient}, {"benchmark", mainEntryClickHouseBenchmark}, - {"server", mainEntryClickHouseServer}, - {"extract-from-config", mainEntryClickHouseExtractFromConfig}, + {"check-marks", mainEntryClickHouseCheckMarks}, + {"checksum-for-compressed-block", mainEntryClickHouseChecksumForCompressedBlock}, + {"client", mainEntryClickHouseClient}, {"compressor", mainEntryClickHouseCompressor}, + {"disks", mainEntryClickHouseDisks}, + {"extract-from-config", mainEntryClickHouseExtractFromConfig}, {"format", mainEntryClickHouseFormat}, - {"obfuscator", mainEntryClickHouseObfuscator}, {"git-import", mainEntryClickHouseGitImport}, + {"hash-binary", mainEntryClickHouseHashBinary}, + {"keeper-bench", mainEntryClickHouseKeeperBench}, + {"keeper-data-dumper", mainEntryClickHouseKeeperDataDumper}, + {"local", mainEntryClickHouseLocal}, + {"obfuscator", mainEntryClickHouseObfuscator}, + {"server", mainEntryClickHouseServer}, {"static-files-disk-uploader", mainEntryClickHouseStaticFilesDiskUploader}, {"su", mainEntryClickHouseSU}, - {"hash-binary", mainEntryClickHouseHashBinary}, - {"disks", mainEntryClickHouseDisks}, + {"zookeeper-dump-tree", mainEntryClickHouseZooKeeperDumpTree}, + {"zookeeper-remove-by-list", mainEntryClickHouseZooKeeperRemoveByList}, // keeper #if ENABLE_CLICKHOUSE_KEEPER diff --git a/programs/zookeeper-dump-tree/CMakeLists.txt b/programs/zookeeper-dump-tree/CMakeLists.txt new file mode 100644 index 00000000000..b29094da2b5 --- /dev/null +++ b/programs/zookeeper-dump-tree/CMakeLists.txt @@ -0,0 +1,12 @@ +set (CLICKHOUSE_ZOOKEEPER_DUMP_TREE_SOURCES ZooKeeperDumpTree.cpp) + +set (CLICKHOUSE_ZOOKEEPER_DUMP_TREE_LINK + PRIVATE + clickhouse_common_zookeeper_no_log + clickhouse_common_io + dbms + clickhouse_functions + boost::program_options +) + +clickhouse_program_add(zookeeper-dump-tree) diff --git a/utils/zookeeper-dump-tree/main.cpp b/programs/zookeeper-dump-tree/ZooKeeperDumpTree.cpp similarity index 98% rename from utils/zookeeper-dump-tree/main.cpp rename to programs/zookeeper-dump-tree/ZooKeeperDumpTree.cpp index 2d997f877b2..eb659b5abb6 100644 --- a/utils/zookeeper-dump-tree/main.cpp +++ b/programs/zookeeper-dump-tree/ZooKeeperDumpTree.cpp @@ -6,7 +6,7 @@ #include -int main(int argc, char ** argv) +int mainEntryClickHouseZooKeeperDumpTree(int argc, char ** argv) { try { diff --git a/programs/zookeeper-dump-tree/zookeeper-dump-tree.cpp b/programs/zookeeper-dump-tree/zookeeper-dump-tree.cpp new file mode 100644 index 00000000000..5de0ace40c4 --- /dev/null +++ b/programs/zookeeper-dump-tree/zookeeper-dump-tree.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseZooKeeperDumpTree(int argc, char ** argv); diff --git a/programs/zookeeper-remove-by-list/CMakeLists.txt b/programs/zookeeper-remove-by-list/CMakeLists.txt new file mode 100644 index 00000000000..50cdb24a0cc --- /dev/null +++ b/programs/zookeeper-remove-by-list/CMakeLists.txt @@ -0,0 +1,12 @@ +set (CLICKHOUSE_ZOOKEEPER_REMOVE_BY_LIST_SOURCES ZooKeeperRemoveByList.cpp) + +set (CLICKHOUSE_ZOOKEEPER_REMOVE_BY_LIST_LINK + PRIVATE + clickhouse_common_zookeeper_no_log + clickhouse_common_io + dbms + clickhouse_functions + boost::program_options +) + +clickhouse_program_add(zookeeper-remove-by-list) diff --git a/utils/zookeeper-remove-by-list/main.cpp b/programs/zookeeper-remove-by-list/ZooKeeperRemoveByList.cpp similarity index 96% rename from utils/zookeeper-remove-by-list/main.cpp rename to programs/zookeeper-remove-by-list/ZooKeeperRemoveByList.cpp index 4de300ca98a..05e55022bd0 100644 --- a/utils/zookeeper-remove-by-list/main.cpp +++ b/programs/zookeeper-remove-by-list/ZooKeeperRemoveByList.cpp @@ -5,7 +5,7 @@ #include -int main(int argc, char ** argv) +int mainEntryClickHouseZooKeeperRemoveByList(int argc, char ** argv) try { boost::program_options::options_description desc("Allowed options"); diff --git a/programs/zookeeper-remove-by-list/zookeeper-remove-by-list.cpp b/programs/zookeeper-remove-by-list/zookeeper-remove-by-list.cpp new file mode 100644 index 00000000000..f1f62d24098 --- /dev/null +++ b/programs/zookeeper-remove-by-list/zookeeper-remove-by-list.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseZooKeeperRemoveByList(int argc, char ** argv); diff --git a/src/Compression/examples/CMakeLists.txt b/src/Compression/examples/CMakeLists.txt index 039eb203ad5..0d522ffd892 100644 --- a/src/Compression/examples/CMakeLists.txt +++ b/src/Compression/examples/CMakeLists.txt @@ -1,2 +1,5 @@ clickhouse_add_executable (compressed_buffer compressed_buffer.cpp) target_link_libraries (compressed_buffer PRIVATE dbms clickhouse_common_io clickhouse_common_config clickhouse_compression) + +clickhouse_add_executable (decompress_perf decompress_perf.cpp) +target_link_libraries(decompress_perf PRIVATE dbms clickhouse_functions ch_contrib::lz4) diff --git a/utils/compressor/decompress_perf.cpp b/src/Compression/examples/decompress_perf.cpp similarity index 97% rename from utils/compressor/decompress_perf.cpp rename to src/Compression/examples/decompress_perf.cpp index cb98121c024..b9a33105224 100644 --- a/utils/compressor/decompress_perf.cpp +++ b/src/Compression/examples/decompress_perf.cpp @@ -164,6 +164,17 @@ public: } +/* Usage example: + +#!/bin/sh + +./clickhouse-compressor < clickhouse-compressor > compressed +./clickhouse-compressor -d < compressed > clickhouse-compressor2 +cmp clickhouse-compressor clickhouse-compressor2 && echo "Ok." || echo "Fail." + +*/ + + int main(int argc, char ** argv) try { diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 2373a98239a..cb558114e1e 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -11,7 +11,6 @@ endif () # Not used in packages if (ENABLE_UTILS) - add_subdirectory (compressor) add_subdirectory (corrector_utf8) add_subdirectory (zookeeper-cli) add_subdirectory (zookeeper-dump-tree) @@ -23,7 +22,3 @@ if (ENABLE_UTILS) add_subdirectory (keeper-data-dumper) add_subdirectory (memcpy-bench) endif () - -if (ENABLE_FUZZING AND ENABLE_FUZZER_TEST) - add_subdirectory (libfuzzer-test) -endif () diff --git a/utils/check-marks/CMakeLists.txt b/utils/check-marks/CMakeLists.txt deleted file mode 100644 index 857dc51d40a..00000000000 --- a/utils/check-marks/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -clickhouse_add_executable (check-marks main.cpp) -target_link_libraries(check-marks PRIVATE dbms clickhouse_functions boost::program_options) diff --git a/utils/check-mysql-binlog/CMakeLists.txt b/utils/check-mysql-binlog/CMakeLists.txt deleted file mode 100644 index 0ece1684e6b..00000000000 --- a/utils/check-mysql-binlog/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -clickhouse_add_executable(check-mysql-binlog main.cpp) -target_link_libraries(check-mysql-binlog PRIVATE dbms clickhouse_functions boost::program_options) diff --git a/utils/check-mysql-binlog/main.cpp b/utils/check-mysql-binlog/main.cpp deleted file mode 100644 index 484dd46a90c..00000000000 --- a/utils/check-mysql-binlog/main.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include -#include -#include -#include - -bool quit = false; -void signal_handler(int) -{ - quit = true; -} - -static void processBinlogFromFile(const std::string & bin_path, bool disable_checksum) -{ - DB::MySQLReplication::BinlogFromFile binlog; - binlog.open(bin_path); - binlog.setChecksum(disable_checksum ? DB::MySQLReplication::IBinlog::NONE : DB::MySQLReplication::IBinlog::CRC32); - - DB::MySQLReplication::BinlogEventPtr event; - while (binlog.tryReadEvent(event, /*timeout*/ 0) && !quit) - { - DB::WriteBufferFromOStream cout(std::cout); - event->dump(cout); - binlog.getPosition().dump(cout); - cout.finalize(); - } -} - -static void processBinlogFromSocket(const std::string & host, int port, const std::string & user, const std::string & password, const std::string & executed_gtid_set, bool disable_checksum) -{ - DB::MySQLReplication::BinlogFromSocket binlog; - binlog.setChecksum(disable_checksum ? DB::MySQLReplication::IBinlog::NONE : DB::MySQLReplication::IBinlog::CRC32); - - binlog.connect(host, port, user, password); - binlog.start(/*unique number*/ 42, executed_gtid_set); - DB::MySQLReplication::BinlogEventPtr event; - - while (!quit) - { - if (binlog.tryReadEvent(event, /*timeout*/ 100)) - { - if (event->header.type != DB::MySQLReplication::HEARTBEAT_EVENT) - { - DB::WriteBufferFromOStream cout(std::cout); - event->dump(cout); - binlog.getPosition().dump(cout); - cout.finalize(); - } - } - } -} - -int main(int argc, char ** argv) -{ - (void)signal(SIGINT, signal_handler); - boost::program_options::options_description desc("Allowed options"); - - std::string host = "127.0.0.1"; - int port = 3306; - std::string user = "root"; - std::string password; - std::string gtid; - - desc.add_options() - ("help", "Produce help message") - ("disable_checksum", "Disable checksums in binlog files.") - ("binlog", boost::program_options::value(), "Binlog file") - ("host", boost::program_options::value(&host)->default_value(host), "Host to connect") - ("port", boost::program_options::value(&port)->default_value(port), "Port number to connect") - ("user", boost::program_options::value(&user)->default_value(user), "User") - ("password", boost::program_options::value(&password), "Password") - ("gtid", boost::program_options::value(>id), "Executed gtid set"); - - try - { - boost::program_options::variables_map options; - boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); - boost::program_options::notify(options); - - if (options.count("help") || (!options.count("binlog") && !options.count("gtid"))) - { - std::cout << "Usage: " << argv[0] << std::endl; - std::cout << desc << std::endl; - return EXIT_FAILURE; - } - - if (options.count("binlog")) - processBinlogFromFile(options["binlog"].as(), options.count("disable_checksum")); - else - processBinlogFromSocket(host, port, user, password, gtid, options.count("disable_checksum")); - } - catch (std::exception & ex) - { - std::cerr << ex.what() << std::endl; - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; -} diff --git a/utils/checksum-for-compressed-block/CMakeLists.txt b/utils/checksum-for-compressed-block/CMakeLists.txt deleted file mode 100644 index 84378f6c1b7..00000000000 --- a/utils/checksum-for-compressed-block/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -clickhouse_add_executable (checksum-for-compressed-block-find-bit-flips main.cpp) -target_link_libraries(checksum-for-compressed-block-find-bit-flips PRIVATE dbms clickhouse_functions) diff --git a/utils/compressor/CMakeLists.txt b/utils/compressor/CMakeLists.txt deleted file mode 100644 index 81f981f8126..00000000000 --- a/utils/compressor/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -clickhouse_add_executable (decompress_perf decompress_perf.cpp) -target_link_libraries(decompress_perf PRIVATE dbms clickhouse_functions ch_contrib::lz4) diff --git a/utils/compressor/test.sh b/utils/compressor/test.sh deleted file mode 100755 index fec9102f072..00000000000 --- a/utils/compressor/test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -./clickhouse-compressor < clickhouse-compressor > compressed -./clickhouse-compressor -d < compressed > clickhouse-compressor2 -cmp clickhouse-compressor clickhouse-compressor2 && echo "Ok." || echo "Fail." diff --git a/utils/keeper-bench/CMakeLists.txt b/utils/keeper-bench/CMakeLists.txt deleted file mode 100644 index 4fe0d852fd2..00000000000 --- a/utils/keeper-bench/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -if (NOT TARGET ch_contrib::rapidjson) - message (${RECONFIGURE_MESSAGE_LEVEL} "Not building keeper-bench due to rapidjson is disabled") - return() -endif () - -clickhouse_add_executable(keeper-bench Generator.cpp Runner.cpp Stats.cpp main.cpp) -target_link_libraries(keeper-bench PRIVATE dbms clickhouse_functions ch_contrib::rapidjson) diff --git a/utils/keeper-data-dumper/CMakeLists.txt b/utils/keeper-data-dumper/CMakeLists.txt deleted file mode 100644 index 71f29781b73..00000000000 --- a/utils/keeper-data-dumper/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -if (NOT TARGET ch_contrib::nuraft) - message (WARNING "Not building keeper-data-dumper due to nuraft is disabled") - return () -endif () - -clickhouse_add_executable(keeper-data-dumper main.cpp) -target_link_libraries(keeper-data-dumper PRIVATE dbms clickhouse_functions) diff --git a/utils/libfuzzer-test/CMakeLists.txt b/utils/libfuzzer-test/CMakeLists.txt deleted file mode 100644 index 8765787ff8a..00000000000 --- a/utils/libfuzzer-test/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory (test_basic_fuzzer) diff --git a/utils/libfuzzer-test/README.md b/utils/libfuzzer-test/README.md deleted file mode 100644 index 5598cbdb961..00000000000 --- a/utils/libfuzzer-test/README.md +++ /dev/null @@ -1 +0,0 @@ -This folder contains various stuff intended to test libfuzzer functionality. diff --git a/utils/libfuzzer-test/test_basic_fuzzer/CMakeLists.txt b/utils/libfuzzer-test/test_basic_fuzzer/CMakeLists.txt deleted file mode 100644 index dc927f35a4b..00000000000 --- a/utils/libfuzzer-test/test_basic_fuzzer/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_executable (test_basic_fuzzer main.cpp) diff --git a/utils/libfuzzer-test/test_basic_fuzzer/main.cpp b/utils/libfuzzer-test/test_basic_fuzzer/main.cpp deleted file mode 100644 index 7ccad63273d..00000000000 --- a/utils/libfuzzer-test/test_basic_fuzzer/main.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include -#include - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) -{ - if (size > 0 && data[0] == 'H') - if (size > 1 && data[1] == 'I') - if (size > 2 && data[2] == '!') - __builtin_trap(); - return 0; -} diff --git a/utils/zookeeper-cli/CMakeLists.txt b/utils/zookeeper-cli/CMakeLists.txt deleted file mode 100644 index 2d0769b7bf2..00000000000 --- a/utils/zookeeper-cli/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -clickhouse_add_executable(clickhouse-zookeeper-cli - zookeeper-cli.cpp - ${ClickHouse_SOURCE_DIR}/src/Client/LineReader.cpp) -target_link_libraries(clickhouse-zookeeper-cli PRIVATE - clickhouse_common_zookeeper_base - clickhouse_common_zookeeper_no_log - dbms - clickhouse_functions -) diff --git a/utils/zookeeper-cli/zookeeper-cli.cpp b/utils/zookeeper-cli/zookeeper-cli.cpp deleted file mode 100644 index 4aef29847ce..00000000000 --- a/utils/zookeeper-cli/zookeeper-cli.cpp +++ /dev/null @@ -1,223 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include -#include - - -void printStat(const Coordination::Stat & s) -{ - std::cout << "Stat:\n"; - std::cout << " czxid: " << s.czxid << '\n'; - std::cout << " mzxid: " << s.mzxid << '\n'; - std::cout << " ctime: " << s.ctime << '\n'; - std::cout << " mtime: " << s.mtime << '\n'; - std::cout << " version: " << s.version << '\n'; - std::cout << " cversion: " << s.cversion << '\n'; - std::cout << " aversion: " << s.aversion << '\n'; - std::cout << " ephemeralOwner: " << s.ephemeralOwner << '\n'; - std::cout << " dataLength: " << s.dataLength << '\n'; - std::cout << " numChildren: " << s.numChildren << '\n'; - std::cout << " pzxid: " << s.pzxid << std::endl; -} - -void waitForWatch(const zkutil::EventPtr & event) -{ - std::cout << "waiting for watch" << std::endl; - event->wait(); - std::cout << "watch event was signalled" << std::endl; -} - - -void readUntilSpace(std::string & s, DB::ReadBuffer & buf) -{ - s = ""; - while (!buf.eof()) - { - if (isspace(*buf.position())) - return; - s.push_back(*buf.position()); - ++buf.position(); - } -} - -void readMaybeQuoted(std::string & s, DB::ReadBuffer & buf) -{ - if (!buf.eof() && *buf.position() == '\'') - DB::readQuotedString(s, buf); - else - readUntilSpace(s, buf); -} - - -int main(int argc, char ** argv) -{ - try - { - if (argc != 2) - { - std::cerr << "usage: " << argv[0] << " hosts" << std::endl; - return 2; - } - - Poco::AutoPtr channel = new Poco::ConsoleChannel(std::cerr); - Poco::Logger::root().setChannel(channel); - Poco::Logger::root().setLevel("trace"); - - auto zk = zkutil::ZooKeeper::createWithoutKillingPreviousSessions(zkutil::ZooKeeperArgs(argv[1])); - DB::LineReader lr({}, false, {"\\"}, {}); - - do - { - const auto & line = lr.readLine(":3 ", ":3 "); - if (line.empty()) - break; - - try - { - std::stringstream ss(line); // STYLE_CHECK_ALLOW_STD_STRING_STREAM - - std::string cmd; - ss >> cmd; - - if (cmd == "q" || cmd == "quit" || cmd == "exit" || cmd == ":q") - break; - - std::string path; - ss >> path; - if (cmd == "ls") - { - std::string w; - ss >> w; - bool watch = w == "w"; - zkutil::EventPtr event = watch ? std::make_shared() : nullptr; - std::vector v = zk->getChildren(path, nullptr, event); - for (const auto & child : v) - std::cout << child << std::endl; - if (watch) - waitForWatch(event); - } - else if (cmd == "create") - { - DB::ReadBufferFromString in(line); - - std::string path_ignored; - std::string data; - std::string mode; - - DB::assertString("create", in); - DB::skipWhitespaceIfAny(in); - readMaybeQuoted(path_ignored, in); - DB::skipWhitespaceIfAny(in); - readMaybeQuoted(data, in); - DB::skipWhitespaceIfAny(in); - readUntilSpace(mode, in); - - int32_t m; - if (mode == "p") - m = zkutil::CreateMode::Persistent; - else if (mode == "ps") - m = zkutil::CreateMode::PersistentSequential; - else if (mode == "e") - m = zkutil::CreateMode::Ephemeral; - else if (mode == "es") - m = zkutil::CreateMode::EphemeralSequential; - else - { - std::cout << "Bad create mode" << std::endl; - continue; - } - std::cout << zk->create(path, data, m) << std::endl; - } - else if (cmd == "remove") - { - zk->remove(path); - } - else if (cmd == "rmr") - { - zk->removeRecursive(path); - } - else if (cmd == "exists") - { - std::string w; - ss >> w; - bool watch = w == "w"; - zkutil::EventPtr event = watch ? std::make_shared() : nullptr; - Coordination::Stat stat; - bool e = zk->exists(path, &stat, event); - if (e) - printStat(stat); - else - std::cout << path << " does not exist" << std::endl; - if (watch) - waitForWatch(event); - } - else if (cmd == "get") - { - std::string w; - ss >> w; - bool watch = w == "w"; - zkutil::EventPtr event = watch ? std::make_shared() : nullptr; - Coordination::Stat stat; - std::string data = zk->get(path, &stat, event); - std::cout << "Data: " << data << std::endl; - printStat(stat); - if (watch) - waitForWatch(event); - } - else if (cmd == "set") - { - DB::ReadBufferFromString in(line); - - std::string data; - int version = -1; - - DB::assertString("set", in); - DB::skipWhitespaceIfAny(in); - DB::assertString(path, in); - DB::skipWhitespaceIfAny(in); - readMaybeQuoted(data, in); - DB::skipWhitespaceIfAny(in); - - if (!in.eof()) - DB::readText(version, in); - - Coordination::Stat stat; - zk->set(path, data, version, &stat); - printStat(stat); - } - else if (!cmd.empty()) - { - std::cout << "commands:\n"; - std::cout << " q\n"; - std::cout << " ls path [w]\n"; - std::cout << " create path data (p|ps|e|es)\n"; - std::cout << " remove path\n"; - std::cout << " rmr path\n"; - std::cout << " exists path [w]\n"; - std::cout << " get path [w]\n"; - std::cout << " set path data [version]" << std::endl; - continue; - } - - } - catch (const Coordination::Exception & e) - { - std::cerr << "KeeperException: " << e.displayText() << std::endl; - } - } - while (true); - } - catch (const Coordination::Exception & e) - { - std::cerr << "KeeperException: " << e.displayText() << std::endl; - return 1; - } - - return 0; -} diff --git a/utils/zookeeper-dump-tree/CMakeLists.txt b/utils/zookeeper-dump-tree/CMakeLists.txt deleted file mode 100644 index 835d37bd1cd..00000000000 --- a/utils/zookeeper-dump-tree/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -clickhouse_add_executable (zookeeper-dump-tree main.cpp ${SRCS}) -target_link_libraries(zookeeper-dump-tree PRIVATE - clickhouse_common_zookeeper_base - clickhouse_common_zookeeper_no_log - clickhouse_common_io - dbms - clickhouse_functions - boost::program_options) diff --git a/utils/zookeeper-remove-by-list/CMakeLists.txt b/utils/zookeeper-remove-by-list/CMakeLists.txt deleted file mode 100644 index 4365c716596..00000000000 --- a/utils/zookeeper-remove-by-list/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -clickhouse_add_executable (zookeeper-remove-by-list main.cpp ${SRCS}) -target_link_libraries(zookeeper-remove-by-list PRIVATE - clickhouse_common_zookeeper_base - clickhouse_common_zookeeper_no_log - dbms - clickhouse_functions - boost::program_options) From 0c253e68944d46ffd4ca63e36fcaffa6f850f7c0 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 26 Nov 2024 17:03:55 +0000 Subject: [PATCH 454/502] Fix build --- programs/zookeeper-dump-tree/CMakeLists.txt | 1 + programs/zookeeper-remove-by-list/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/programs/zookeeper-dump-tree/CMakeLists.txt b/programs/zookeeper-dump-tree/CMakeLists.txt index b29094da2b5..aec8c8dd26c 100644 --- a/programs/zookeeper-dump-tree/CMakeLists.txt +++ b/programs/zookeeper-dump-tree/CMakeLists.txt @@ -2,6 +2,7 @@ set (CLICKHOUSE_ZOOKEEPER_DUMP_TREE_SOURCES ZooKeeperDumpTree.cpp) set (CLICKHOUSE_ZOOKEEPER_DUMP_TREE_LINK PRIVATE + clickhouse_common_zookeeper_base clickhouse_common_zookeeper_no_log clickhouse_common_io dbms diff --git a/programs/zookeeper-remove-by-list/CMakeLists.txt b/programs/zookeeper-remove-by-list/CMakeLists.txt index 50cdb24a0cc..a0463e43893 100644 --- a/programs/zookeeper-remove-by-list/CMakeLists.txt +++ b/programs/zookeeper-remove-by-list/CMakeLists.txt @@ -2,6 +2,7 @@ set (CLICKHOUSE_ZOOKEEPER_REMOVE_BY_LIST_SOURCES ZooKeeperRemoveByList.cpp) set (CLICKHOUSE_ZOOKEEPER_REMOVE_BY_LIST_LINK PRIVATE + clickhouse_common_zookeeper_base clickhouse_common_zookeeper_no_log clickhouse_common_io dbms From 786bff76b6ab76c97c58eb18dad225d81d576649 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 26 Nov 2024 17:57:50 +0000 Subject: [PATCH 455/502] Fix style check --- utils/check-style/check-style | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index d86ab8e029d..901330e0015 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -14,7 +14,7 @@ LC_ALL="en_US.UTF-8" ROOT_PATH=$(git rev-parse --show-toplevel) -EXCLUDE='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/.*.cpp|utils/keeper-bench/example.yaml' +EXCLUDE='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/.*.cpp|programs/keeper-bench/example.yaml' EXCLUDE_DOCS='Settings\.cpp|FormatFactorySettings\.h' # From [1]: From 83573e799b7b407e046617e88cf0873b2ed85f49 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 26 Nov 2024 22:12:48 +0000 Subject: [PATCH 456/502] Better CMake --- programs/keeper-bench/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/keeper-bench/CMakeLists.txt b/programs/keeper-bench/CMakeLists.txt index 2cec36c1d03..f9e3cdb82fc 100644 --- a/programs/keeper-bench/CMakeLists.txt +++ b/programs/keeper-bench/CMakeLists.txt @@ -1,5 +1,5 @@ if (NOT TARGET ch_contrib::rapidjson) - message (${RECONFIGURE_MESSAGE_LEVEL} "Not building keeper-bench due to rapidjson is disabled") + message (WARNING "Not building keeper-bench due to rapidjson is disabled") return() endif () From ec5830c857fd3ea49b8864eb74e43a9638e4e855 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 26 Nov 2024 23:45:36 +0000 Subject: [PATCH 457/502] Fix fasttest --- programs/CMakeLists.txt | 10 ++++++++-- programs/main.cpp | 17 ++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 1a90334c992..48417c6be30 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -213,8 +213,6 @@ clickhouse_program_install(clickhouse-disks disks) clickhouse_program_install(clickhouse-extract-from-config extract-from-config) clickhouse_program_install(clickhouse-format format) clickhouse_program_install(clickhouse-git-import git-import) -clickhouse_program_install(clickhouse-keeper-bench keeper-bench) -clickhouse_program_install(clickhouse-keeper-data-dumper keeper-data-dumper) clickhouse_program_install(clickhouse-local local chl ch) clickhouse_program_install(clickhouse-obfuscator obfuscator) clickhouse_program_install(clickhouse-server server) @@ -223,6 +221,14 @@ clickhouse_program_install(clickhouse-su su) clickhouse_program_install(clickhouse-zookeeper-dump-tree zookeeper-dump-tree) clickhouse_program_install(clickhouse-zookeeper-remove-by-list zookeeper-remove-by-list) +if (TARGET ch_contrib::nuraft) + clickhouse_program_install(clickhouse-keeper-data-dumper keeper-data-dumper) +endif () + +if (TARGET ch_contrib::rapidjson) + clickhouse_program_install(clickhouse-keeper-bench keeper-bench) +endif () + if (ENABLE_CLICKHOUSE_KEEPER) if (NOT BUILD_STANDALONE_KEEPER AND CREATE_KEEPER_SYMLINK) add_custom_target (clickhouse-keeper ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse) diff --git a/programs/main.cpp b/programs/main.cpp index 4d2b37b14b3..1bdc0cb89c8 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -29,8 +29,6 @@ int mainEntryClickHouseDisks(int argc, char ** argv); int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); int mainEntryClickHouseFormat(int argc, char ** argv); int mainEntryClickHouseGitImport(int argc, char ** argv); -int mainEntryClickHouseKeeperBench(int argc, char ** argv); -int mainEntryClickHouseKeeperDataDumper(int argc, char ** argv); int mainEntryClickHouseLocal(int argc, char ** argv); int mainEntryClickHouseObfuscator(int argc, char ** argv); int mainEntryClickHouseSU(int argc, char ** argv); @@ -56,6 +54,12 @@ int mainEntryClickHouseKeeperConverter(int argc, char ** argv); #if ENABLE_CLICKHOUSE_KEEPER_CLIENT int mainEntryClickHouseKeeperClient(int argc, char ** argv); #endif +#if USE_RAPIDJSON +int mainEntryClickHouseKeeperBench(int argc, char ** argv); +#endif +#if USE_NURAFT +int mainEntryClickHouseKeeperDataDumper(int argc, char ** argv); +#endif // install int mainEntryClickHouseInstall(int argc, char ** argv); @@ -82,8 +86,6 @@ std::pair clickhouse_applications[] = {"format", mainEntryClickHouseFormat}, {"git-import", mainEntryClickHouseGitImport}, {"hash-binary", mainEntryClickHouseHashBinary}, - {"keeper-bench", mainEntryClickHouseKeeperBench}, - {"keeper-data-dumper", mainEntryClickHouseKeeperDataDumper}, {"local", mainEntryClickHouseLocal}, {"obfuscator", mainEntryClickHouseObfuscator}, {"server", mainEntryClickHouseServer}, @@ -102,7 +104,12 @@ std::pair clickhouse_applications[] = #if ENABLE_CLICKHOUSE_KEEPER_CLIENT {"keeper-client", mainEntryClickHouseKeeperClient}, #endif - +#if USE_RAPIDJSON + {"keeper-bench", mainEntryClickHouseKeeperBench}, +#endif +#if USE_NURAFT + {"keeper-data-dumper", mainEntryClickHouseKeeperDataDumper}, +#endif // install {"install", mainEntryClickHouseInstall}, {"start", mainEntryClickHouseStart}, From 7e4ed1d720f5e0afc170307159cd8011d9ca8ca3 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 27 Nov 2024 17:36:09 +0000 Subject: [PATCH 458/502] Remove getlogin from harmful --- base/harmful/harmful.c | 1 - 1 file changed, 1 deletion(-) diff --git a/base/harmful/harmful.c b/base/harmful/harmful.c index 19bb962999f..0b7ee8da1f4 100644 --- a/base/harmful/harmful.c +++ b/base/harmful/harmful.c @@ -65,7 +65,6 @@ TRAP(gethostbyaddr) TRAP(gethostbyname) TRAP(gethostbyname2) TRAP(gethostent) -TRAP(getlogin) TRAP(getnetbyaddr) TRAP(getnetbyname) TRAP(getnetent) From 6024e34bef2a25b6fc3183f4dc2b5d8c43b2ec69 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 27 Nov 2024 18:52:33 +0000 Subject: [PATCH 459/502] Return order to clickhouse-applications --- programs/main.cpp | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/programs/main.cpp b/programs/main.cpp index 1bdc0cb89c8..1b04e25b58b 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -73,24 +73,27 @@ namespace using MainFunc = int (*)(int, char**); -/// Add an item here to register new application +/// Add an item here to register new application. +/// This list has a "priority" - e.g. we need to disambiguate clickhouse --format being +/// either clickouse-format or clickhouse-{local, client} --format. +/// Currently we will prefer the latter option. std::pair clickhouse_applications[] = { - {"benchmark", mainEntryClickHouseBenchmark}, - {"check-marks", mainEntryClickHouseCheckMarks}, - {"checksum-for-compressed-block", mainEntryClickHouseChecksumForCompressedBlock}, - {"client", mainEntryClickHouseClient}, - {"compressor", mainEntryClickHouseCompressor}, - {"disks", mainEntryClickHouseDisks}, - {"extract-from-config", mainEntryClickHouseExtractFromConfig}, - {"format", mainEntryClickHouseFormat}, - {"git-import", mainEntryClickHouseGitImport}, - {"hash-binary", mainEntryClickHouseHashBinary}, {"local", mainEntryClickHouseLocal}, - {"obfuscator", mainEntryClickHouseObfuscator}, + {"client", mainEntryClickHouseClient}, + {"benchmark", mainEntryClickHouseBenchmark}, {"server", mainEntryClickHouseServer}, + {"extract-from-config", mainEntryClickHouseExtractFromConfig}, + {"compressor", mainEntryClickHouseCompressor}, + {"format", mainEntryClickHouseFormat}, + {"obfuscator", mainEntryClickHouseObfuscator}, + {"git-import", mainEntryClickHouseGitImport}, {"static-files-disk-uploader", mainEntryClickHouseStaticFilesDiskUploader}, {"su", mainEntryClickHouseSU}, + {"hash-binary", mainEntryClickHouseHashBinary}, + {"disks", mainEntryClickHouseDisks}, + {"check-marks", mainEntryClickHouseCheckMarks}, + {"checksum-for-compressed-block", mainEntryClickHouseChecksumForCompressedBlock}, {"zookeeper-dump-tree", mainEntryClickHouseZooKeeperDumpTree}, {"zookeeper-remove-by-list", mainEntryClickHouseZooKeeperRemoveByList}, From 6b8efc66c27331f3c6fcecdd237e055fb765e163 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 9 Dec 2024 12:49:47 +0000 Subject: [PATCH 460/502] Fix build --- programs/CMakeLists.txt | 2 +- programs/keeper-bench/CMakeLists.txt | 4 ++-- utils/CMakeLists.txt | 8 -------- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 48417c6be30..d6d92674a30 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -225,7 +225,7 @@ if (TARGET ch_contrib::nuraft) clickhouse_program_install(clickhouse-keeper-data-dumper keeper-data-dumper) endif () -if (TARGET ch_contrib::rapidjson) +if (TARGET ch_contrib::rapidjson AND TARGET ch_contrib::nuraft) clickhouse_program_install(clickhouse-keeper-bench keeper-bench) endif () diff --git a/programs/keeper-bench/CMakeLists.txt b/programs/keeper-bench/CMakeLists.txt index f9e3cdb82fc..76aff16b667 100644 --- a/programs/keeper-bench/CMakeLists.txt +++ b/programs/keeper-bench/CMakeLists.txt @@ -1,5 +1,5 @@ -if (NOT TARGET ch_contrib::rapidjson) - message (WARNING "Not building keeper-bench due to rapidjson is disabled") +if (NOT TARGET ch_contrib::rapidjson OR NOT TARGET ch_contrib::nuraft) + message (WARNING "Not building keeper-bench due to rapidjson or nuraft is disabled") return() endif () diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index cb558114e1e..52fd77b271e 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -12,13 +12,5 @@ endif () # Not used in packages if (ENABLE_UTILS) add_subdirectory (corrector_utf8) - add_subdirectory (zookeeper-cli) - add_subdirectory (zookeeper-dump-tree) - add_subdirectory (zookeeper-remove-by-list) - add_subdirectory (check-marks) - add_subdirectory (checksum-for-compressed-block) - add_subdirectory (check-mysql-binlog) - add_subdirectory (keeper-bench) - add_subdirectory (keeper-data-dumper) add_subdirectory (memcpy-bench) endif () From 551440358470b8346294b52b5adc7e9821b06447 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 9 Dec 2024 17:36:59 +0000 Subject: [PATCH 461/502] Better CMake --- programs/main.cpp | 2 +- programs/zookeeper-dump-tree/CMakeLists.txt | 3 --- programs/zookeeper-remove-by-list/CMakeLists.txt | 3 --- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/programs/main.cpp b/programs/main.cpp index 1b04e25b58b..ab92e4fc7cb 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -54,7 +54,7 @@ int mainEntryClickHouseKeeperConverter(int argc, char ** argv); #if ENABLE_CLICKHOUSE_KEEPER_CLIENT int mainEntryClickHouseKeeperClient(int argc, char ** argv); #endif -#if USE_RAPIDJSON +#if USE_RAPIDJSON && USE_NURAFT int mainEntryClickHouseKeeperBench(int argc, char ** argv); #endif #if USE_NURAFT diff --git a/programs/zookeeper-dump-tree/CMakeLists.txt b/programs/zookeeper-dump-tree/CMakeLists.txt index aec8c8dd26c..7f4164fca25 100644 --- a/programs/zookeeper-dump-tree/CMakeLists.txt +++ b/programs/zookeeper-dump-tree/CMakeLists.txt @@ -2,9 +2,6 @@ set (CLICKHOUSE_ZOOKEEPER_DUMP_TREE_SOURCES ZooKeeperDumpTree.cpp) set (CLICKHOUSE_ZOOKEEPER_DUMP_TREE_LINK PRIVATE - clickhouse_common_zookeeper_base - clickhouse_common_zookeeper_no_log - clickhouse_common_io dbms clickhouse_functions boost::program_options diff --git a/programs/zookeeper-remove-by-list/CMakeLists.txt b/programs/zookeeper-remove-by-list/CMakeLists.txt index a0463e43893..5cc3488182c 100644 --- a/programs/zookeeper-remove-by-list/CMakeLists.txt +++ b/programs/zookeeper-remove-by-list/CMakeLists.txt @@ -2,9 +2,6 @@ set (CLICKHOUSE_ZOOKEEPER_REMOVE_BY_LIST_SOURCES ZooKeeperRemoveByList.cpp) set (CLICKHOUSE_ZOOKEEPER_REMOVE_BY_LIST_LINK PRIVATE - clickhouse_common_zookeeper_base - clickhouse_common_zookeeper_no_log - clickhouse_common_io dbms clickhouse_functions boost::program_options From 9c094a995922eed15ca851f6e7a98d632f15dc62 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 9 Dec 2024 21:35:06 +0000 Subject: [PATCH 462/502] Fixed FreeBSD build --- programs/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/main.cpp b/programs/main.cpp index ab92e4fc7cb..2a7a2fde3cc 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -107,7 +107,7 @@ std::pair clickhouse_applications[] = #if ENABLE_CLICKHOUSE_KEEPER_CLIENT {"keeper-client", mainEntryClickHouseKeeperClient}, #endif -#if USE_RAPIDJSON +#if USE_RAPIDJSON && USE_NURAFT {"keeper-bench", mainEntryClickHouseKeeperBench}, #endif #if USE_NURAFT From 05cfa49c063bb72178b6c907421eea4e34f9ba68 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 9 Dec 2024 22:42:58 +0100 Subject: [PATCH 463/502] stash --- src/Interpreters/HashJoin/HashJoinMethods.h | 3 +- .../HashJoin/HashJoinMethodsImpl.h | 44 +++++++++++-------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/Interpreters/HashJoin/HashJoinMethods.h b/src/Interpreters/HashJoin/HashJoinMethods.h index 10fb50a6b83..af07da53019 100644 --- a/src/Interpreters/HashJoin/HashJoinMethods.h +++ b/src/Interpreters/HashJoin/HashJoinMethods.h @@ -157,12 +157,13 @@ private: AddedColumns & added_columns); /// First to collect all matched rows refs by join keys, then filter out rows which are not true in additional filter expression. - template + template static size_t joinRightColumnsWithAddtitionalFilter( std::vector && key_getter_vector, const std::vector & mapv, AddedColumns & added_columns, JoinStuff::JoinUsedFlags & used_flags [[maybe_unused]], + const Selector & selector, bool need_filter [[maybe_unused]], bool flag_per_row [[maybe_unused]]); diff --git a/src/Interpreters/HashJoin/HashJoinMethodsImpl.h b/src/Interpreters/HashJoin/HashJoinMethodsImpl.h index 1e52278f020..2b5e92b6841 100644 --- a/src/Interpreters/HashJoin/HashJoinMethodsImpl.h +++ b/src/Interpreters/HashJoin/HashJoinMethodsImpl.h @@ -357,9 +357,15 @@ size_t HashJoinMethods::joinRightColumnsSwitchMu { if (added_columns.additional_filter_expression) { - bool mark_per_row_used = join_features.right || join_features.full || mapv.size() > 1; + const bool mark_per_row_used = join_features.right || join_features.full || mapv.size() > 1; return joinRightColumnsWithAddtitionalFilter( - std::forward>(key_getter_vector), mapv, added_columns, used_flags, need_filter, mark_per_row_used); + std::forward>(key_getter_vector), + mapv, + added_columns, + used_flags, + added_columns.src_block.getSelector(), + need_filter, + mark_per_row_used); } } @@ -664,17 +670,18 @@ ColumnPtr HashJoinMethods::buildAdditionalFilter } template -template +template size_t HashJoinMethods::joinRightColumnsWithAddtitionalFilter( std::vector && key_getter_vector, const std::vector & mapv, AddedColumns & added_columns, JoinStuff::JoinUsedFlags & used_flags [[maybe_unused]], + const Selector & selector, bool need_filter [[maybe_unused]], bool flag_per_row [[maybe_unused]]) { constexpr JoinFeatures join_features; - size_t left_block_rows = added_columns.rows_to_add; + const size_t left_block_rows = added_columns.src_block.rows(); if (need_filter) added_columns.filter = IColumn::Filter(left_block_rows, 0); @@ -688,7 +695,7 @@ size_t HashJoinMethods::joinRightColumnsWithAddt using FindResult = typename KeyGetter::FindResult; size_t max_joined_block_rows = added_columns.max_joined_block_rows; - size_t left_row_iter = 0; + size_t it = 0; PreSelectedRows selected_rows; selected_rows.reserve(left_block_rows); std::vector find_results; @@ -705,8 +712,10 @@ size_t HashJoinMethods::joinRightColumnsWithAddt row_replicate_offset.push_back(0); current_added_rows = 0; selected_rows.clear(); - for (; left_row_iter < left_block_rows; ++left_row_iter) + for (; it < left_block_rows; ++it) { + size_t ind = selector[it]; + if constexpr (join_features.need_replication) { if (unlikely(total_added_rows + current_added_rows >= max_joined_block_rows)) @@ -719,13 +728,12 @@ size_t HashJoinMethods::joinRightColumnsWithAddt for (size_t join_clause_idx = 0; join_clause_idx < added_columns.join_on_keys.size(); ++join_clause_idx) { const auto & join_keys = added_columns.join_on_keys[join_clause_idx]; - if (join_keys.null_map && (*join_keys.null_map)[left_row_iter]) + if (join_keys.null_map && (*join_keys.null_map)[ind]) continue; - bool row_acceptable = !join_keys.isRowFiltered(left_row_iter); - auto find_result = row_acceptable - ? key_getter_vector[join_clause_idx].findKey(*(mapv[join_clause_idx]), left_row_iter, *pool) - : FindResult(); + bool row_acceptable = !join_keys.isRowFiltered(ind); + auto find_result + = row_acceptable ? key_getter_vector[join_clause_idx].findKey(*(mapv[join_clause_idx]), ind, *pool) : FindResult(); if (find_result.isFound()) { @@ -878,11 +886,11 @@ size_t HashJoinMethods::joinRightColumnsWithAddt } }; - while (left_row_iter < left_block_rows && !exceeded_max_block_rows) + while (it < left_block_rows && !exceeded_max_block_rows) { - auto left_start_row = left_row_iter; + auto left_start_row = it; collect_keys_matched_rows_refs(); - if (selected_rows.size() != current_added_rows || row_replicate_offset.size() != left_row_iter - left_start_row + 1) + if (selected_rows.size() != current_added_rows || row_replicate_offset.size() != it - left_start_row + 1) { throw Exception( ErrorCodes::LOGICAL_ERROR, @@ -891,7 +899,7 @@ size_t HashJoinMethods::joinRightColumnsWithAddt selected_rows.size(), current_added_rows, row_replicate_offset.size(), - left_row_iter, + it, left_start_row); } auto filter_col = buildAdditionalFilter(left_start_row, selected_rows, row_replicate_offset, added_columns); @@ -907,11 +915,11 @@ size_t HashJoinMethods::joinRightColumnsWithAddt if constexpr (join_features.need_replication) { - added_columns.offsets_to_replicate->resize_assume_reserved(left_row_iter); - added_columns.filter.resize_assume_reserved(left_row_iter); + added_columns.offsets_to_replicate->resize_assume_reserved(it); + added_columns.filter.resize_assume_reserved(it); } added_columns.applyLazyDefaults(); - return left_row_iter; + return it; } template From 11269cf3545e5f6b9fb67d25f09aa5f0069322b9 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 9 Dec 2024 22:51:13 +0100 Subject: [PATCH 464/502] add test --- ...llel_join_with_additional_filter.reference | 8 ++++++ ...2_parallel_join_with_additional_filter.sql | 26 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/queries/0_stateless/03282_parallel_join_with_additional_filter.reference create mode 100644 tests/queries/0_stateless/03282_parallel_join_with_additional_filter.sql diff --git a/tests/queries/0_stateless/03282_parallel_join_with_additional_filter.reference b/tests/queries/0_stateless/03282_parallel_join_with_additional_filter.reference new file mode 100644 index 00000000000..2f5cded0ed5 --- /dev/null +++ b/tests/queries/0_stateless/03282_parallel_join_with_additional_filter.reference @@ -0,0 +1,8 @@ +---- HASH +1 10 alpha 1 5 ALPHA +2 15 beta 2 10 beta +3 20 gamma 0 0 +---- PARALLEL HASH +1 10 alpha 1 5 ALPHA +2 15 beta 2 10 beta +3 20 gamma 0 0 diff --git a/tests/queries/0_stateless/03282_parallel_join_with_additional_filter.sql b/tests/queries/0_stateless/03282_parallel_join_with_additional_filter.sql new file mode 100644 index 00000000000..b04b466eb2a --- /dev/null +++ b/tests/queries/0_stateless/03282_parallel_join_with_additional_filter.sql @@ -0,0 +1,26 @@ +CREATE TABLE t1 ( + key UInt32, + a UInt32, + attr String +) ENGINE = MergeTree ORDER BY key; + +CREATE TABLE t2 ( + key UInt32, + a UInt32, + attr String +) ENGINE = MergeTree ORDER BY key; + +INSERT INTO t1 (key, a, attr) VALUES (1, 10, 'alpha'), (2, 15, 'beta'), (3, 20, 'gamma'); +INSERT INTO t2 (key, a, attr) VALUES (1, 5, 'ALPHA'), (2, 10, 'beta'), (4, 25, 'delta'); + +SET allow_experimental_join_condition = 1; +SET enable_analyzer = 1; +SET max_threads = 16; + +SELECT '---- HASH'; +SELECT t1.*, t2.* FROM t1 LEFT JOIN t2 ON t1.key = t2.key AND (t1.key < t2.a OR t1.a % 2 = 0) ORDER BY ALL SETTINGS join_algorithm = 'hash'; + +SELECT '---- PARALLEL HASH'; +SELECT t1.*, t2.* FROM t1 LEFT JOIN t2 ON t1.key = t2.key AND (t1.key < t2.a OR t1.a % 2 = 0) ORDER BY ALL SETTINGS join_algorithm = 'parallel_hash'; -- { serverError NOT_IMPLEMENTED} + +SELECT t1.*, t2.* FROM t1 LEFT JOIN t2 ON t1.key = t2.key AND (t1.key < t2.a OR t1.a % 2 = 0) ORDER BY ALL; From c56ece0780f0febc27b80a0456e7c81feb7e943e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 9 Dec 2024 22:55:34 +0100 Subject: [PATCH 465/502] Revert "Add total_bytes_with_inactive to system.tables" --- src/Storages/IStorage.h | 9 --------- src/Storages/StorageMergeTree.cpp | 10 ---------- src/Storages/StorageMergeTree.h | 1 - src/Storages/StorageReplicatedMergeTree.cpp | 9 --------- src/Storages/StorageReplicatedMergeTree.h | 1 - src/Storages/System/StorageSystemTables.cpp | 13 ------------- .../integration/test_analyzer_compatibility/test.py | 6 +++--- ..._system_columns_and_system_tables_long.reference | 9 ++++----- .../00753_system_columns_and_system_tables_long.sql | 9 ++------- .../02117_show_create_table_system.reference | 1 - 10 files changed, 9 insertions(+), 59 deletions(-) diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index f842e12ae88..622b2203877 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -727,15 +727,6 @@ public: /// Does not take underlying Storage (if any) into account virtual std::optional totalBytesUncompressed(const Settings &) const { return {}; } - /// If it is possible to quickly determine exact number of bytes for the table on storage: - /// - disk (compressed) - /// - /// Used for: - /// - For total_bytes_with_inactive column in system.tables - // - /// Does not takes underlying Storage (if any) into account - virtual std::optional totalBytesWithInactive(const Settings &) const { return {}; } - /// Number of rows INSERTed since server start. /// /// Does not take the underlying Storage (if any) into account. diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index baa2fd9256e..a99be13ae24 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -319,16 +319,6 @@ std::optional StorageMergeTree::totalBytesUncompressed(const Settings &) return res; } -std::optional StorageMergeTree::totalBytesWithInactive(const Settings &) const -{ - UInt64 res = 0; - auto outdated_parts = getDataPartsVectorForInternalUsage({DataPartState::Outdated}); - for (const auto & part : outdated_parts) - res += part->getBytesOnDisk(); - return res; -} - - SinkToStoragePtr StorageMergeTree::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context, bool /*async_insert*/) { diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index f61d5cb1e9a..7bc070b12b4 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -68,7 +68,6 @@ public: std::optional totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr) const override; std::optional totalBytes(const Settings &) const override; std::optional totalBytesUncompressed(const Settings &) const override; - std::optional totalBytesWithInactive(const Settings &) const override; SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context, bool async_insert) override; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 00e16d7ad29..0941158289d 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5757,15 +5757,6 @@ std::optional StorageReplicatedMergeTree::totalBytesUncompressed(const S return res; } -std::optional StorageReplicatedMergeTree::totalBytesWithInactive(const Settings &) const -{ - UInt64 res = 0; - auto outdated_parts = getDataPartsStateRange(DataPartState::Outdated); - for (const auto & part : outdated_parts) - res += part->getBytesOnDisk(); - return res; -} - void StorageReplicatedMergeTree::assertNotReadonly() const { if (is_readonly) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 22320611635..814ba1e1a4e 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -162,7 +162,6 @@ public: std::optional totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr context) const override; std::optional totalBytes(const Settings & settings) const override; std::optional totalBytesUncompressed(const Settings & settings) const override; - std::optional totalBytesWithInactive(const Settings & settings) const override; SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context, bool async_insert) override; diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index c414891ddb7..9d5c68c261f 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -179,10 +179,6 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_) "Total number of uncompressed bytes, if it's possible to quickly determine the exact number " "of bytes from the part checksums for the table on storage, otherwise NULL (does not take underlying storage (if any) into account)." }, - {"total_bytes_with_inactive", std::make_shared(std::make_shared()), - "Total number of bytes with inactive parts, if it is possible to quickly determine exact number " - "of bytes for the table on storage, otherwise NULL (does not includes any underlying storage). " - }, {"parts", std::make_shared(std::make_shared()), "The total number of parts in this table."}, {"active_parts", std::make_shared(std::make_shared()), "The number of active parts in this table."}, {"total_marks", std::make_shared(std::make_shared()), "The total number of marks in all parts in this table."}, @@ -601,15 +597,6 @@ protected: res_columns[res_index++]->insertDefault(); } - if (columns_mask[src_index++]) - { - auto total_bytes_with_inactive = table->totalBytesWithInactive(settings); - if (total_bytes_with_inactive) - res_columns[res_index++]->insert(*total_bytes_with_inactive); - else - res_columns[res_index++]->insertDefault(); - } - auto table_merge_tree = std::dynamic_pointer_cast(table); if (columns_mask[src_index++]) { diff --git a/tests/integration/test_analyzer_compatibility/test.py b/tests/integration/test_analyzer_compatibility/test.py index 78f3428e02e..42bd140280b 100644 --- a/tests/integration/test_analyzer_compatibility/test.py +++ b/tests/integration/test_analyzer_compatibility/test.py @@ -43,7 +43,7 @@ def test_two_new_versions(start_cluster): query_id = str(uuid.uuid4()) current.query( - "SELECT name FROM clusterAllReplicas('test_cluster_mixed', system.tables);", + "SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables);", query_id=query_id, ) @@ -73,7 +73,7 @@ WHERE initial_query_id = '{query_id}';""" query_id = str(uuid.uuid4()) backward.query( - "SELECT name FROM clusterAllReplicas('test_cluster_mixed', system.tables)", + "SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables)", query_id=query_id, ) @@ -108,7 +108,7 @@ WHERE initial_query_id = '{query_id}';""" # to the remote server. query_id = str(uuid.uuid4()) current.query( - "SELECT name FROM clusterAllReplicas('test_cluster_mixed', system.tables) SETTINGS enable_analyzer = 1;", + "SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables) SETTINGS enable_analyzer = 1;", query_id=query_id, ) diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference index 26946cd55b4..dd5860ae491 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference @@ -1,13 +1,12 @@ -┌─name────────────────┬─partition_key─┬─sorting_key─┬─primary_key─┬─sampling_key─┬─storage_policy─┬─total_rows─┬─total_bytes_with_inactive─┐ -│ check_system_tables │ name2 │ name1 │ name1 │ name1 │ default │ 0 │ 0 │ -└─────────────────────┴───────────────┴─────────────┴─────────────┴──────────────┴────────────────┴────────────┴───────────────────────────┘ +┌─name────────────────┬─partition_key─┬─sorting_key─┬─primary_key─┬─sampling_key─┬─storage_policy─┬─total_rows─┐ +│ check_system_tables │ name2 │ name1 │ name1 │ name1 │ default │ 0 │ +└─────────────────────┴───────────────┴─────────────┴─────────────┴──────────────┴────────────────┴────────────┘ ┌─name──┬─is_in_partition_key─┬─is_in_sorting_key─┬─is_in_primary_key─┬─is_in_sampling_key─┐ │ name1 │ 0 │ 1 │ 1 │ 1 │ │ name2 │ 1 │ 0 │ 0 │ 0 │ │ name3 │ 0 │ 0 │ 0 │ 0 │ └───────┴─────────────────────┴───────────────────┴───────────────────┴────────────────────┘ -1 1 3 231 1 0 -3 1 6 234 2 462 +3 231 1 ┌─name────────────────┬─partition_key─┬─sorting_key───┬─primary_key─┬─sampling_key─┐ │ check_system_tables │ date │ date, version │ date │ │ └─────────────────────┴───────────────┴───────────────┴─────────────┴──────────────┘ diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql index 4c6a7b63a31..009fc0bbb9f 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql @@ -15,7 +15,7 @@ CREATE TABLE check_system_tables SAMPLE BY name1 SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key = false, ratio_of_defaults_for_sparse_serialization = 1; -SELECT name, partition_key, sorting_key, primary_key, sampling_key, storage_policy, total_rows, total_bytes_with_inactive +SELECT name, partition_key, sorting_key, primary_key, sampling_key, storage_policy, total_rows FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase() FORMAT PrettyCompactNoEscapes; @@ -24,12 +24,7 @@ FROM system.columns WHERE table = 'check_system_tables' AND database = currentDa FORMAT PrettyCompactNoEscapes; INSERT INTO check_system_tables VALUES (1, 1, 1); -SELECT parts, active_parts, total_bytes_uncompressed, total_bytes, total_rows, total_bytes_with_inactive FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase(); - -INSERT INTO check_system_tables VALUES (2, 1, 3); -OPTIMIZE TABLE check_system_tables; -SELECT parts, active_parts, total_bytes_uncompressed, total_bytes, total_rows, total_bytes_with_inactive FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase(); - +SELECT total_bytes_uncompressed, total_bytes, total_rows FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase(); DROP TABLE IF EXISTS check_system_tables; diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 6880deed59e..ef5a2c6665f 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -1115,7 +1115,6 @@ CREATE TABLE system.tables `total_rows` Nullable(UInt64), `total_bytes` Nullable(UInt64), `total_bytes_uncompressed` Nullable(UInt64), - `total_bytes_with_inactive` Nullable(UInt64), `parts` Nullable(UInt64), `active_parts` Nullable(UInt64), `total_marks` Nullable(UInt64), From 201f021db1e35d29ad3eed10c067e645deced66b Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 9 Dec 2024 23:02:26 +0100 Subject: [PATCH 466/502] allow with parallel_hash --- src/Planner/PlannerJoins.cpp | 1 + .../03282_parallel_join_with_additional_filter.sql | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index 74ff72f73fe..28be45f699d 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -876,6 +876,7 @@ std::shared_ptr chooseJoinAlgorithm( { if (table_join->getMixedJoinExpression() && !table_join->isEnabledAlgorithm(JoinAlgorithm::HASH) + && !table_join->isEnabledAlgorithm(JoinAlgorithm::PARALLEL_HASH) && !table_join->isEnabledAlgorithm(JoinAlgorithm::GRACE_HASH)) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, diff --git a/tests/queries/0_stateless/03282_parallel_join_with_additional_filter.sql b/tests/queries/0_stateless/03282_parallel_join_with_additional_filter.sql index b04b466eb2a..4dadc777adf 100644 --- a/tests/queries/0_stateless/03282_parallel_join_with_additional_filter.sql +++ b/tests/queries/0_stateless/03282_parallel_join_with_additional_filter.sql @@ -21,6 +21,4 @@ SELECT '---- HASH'; SELECT t1.*, t2.* FROM t1 LEFT JOIN t2 ON t1.key = t2.key AND (t1.key < t2.a OR t1.a % 2 = 0) ORDER BY ALL SETTINGS join_algorithm = 'hash'; SELECT '---- PARALLEL HASH'; -SELECT t1.*, t2.* FROM t1 LEFT JOIN t2 ON t1.key = t2.key AND (t1.key < t2.a OR t1.a % 2 = 0) ORDER BY ALL SETTINGS join_algorithm = 'parallel_hash'; -- { serverError NOT_IMPLEMENTED} - -SELECT t1.*, t2.* FROM t1 LEFT JOIN t2 ON t1.key = t2.key AND (t1.key < t2.a OR t1.a % 2 = 0) ORDER BY ALL; +SELECT t1.*, t2.* FROM t1 LEFT JOIN t2 ON t1.key = t2.key AND (t1.key < t2.a OR t1.a % 2 = 0) ORDER BY ALL SETTINGS join_algorithm = 'parallel_hash'; From 05d8584966b05ed2196fc1ffb3bf0ecbdf9ac910 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 10 Dec 2024 10:34:36 +0000 Subject: [PATCH 467/502] Fix chassert --- .../ObjectStorage/DataLakes/IcebergMetadata.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 980d2f479cb..5352fc641fa 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -128,6 +128,14 @@ bool operator!=(const Poco::JSON::Object & first, const Poco::JSON::Object & sec { return !(first == second); } + +bool schemasAreIdentical(const Poco::JSON::Object & first, const Poco::JSON::Object & second) +{ + static String fields_key = "fields"; + if (!first.has(fields_key) || !second.has(fields_key)) + return false; + return first.getObject(fields_key) == second.getObject(fields_key); +} } @@ -481,7 +489,7 @@ void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schem if (iceberg_table_schemas_by_ids.contains(schema_id)) { chassert(clickhouse_table_schemas_by_ids.contains(schema_id)); - chassert(*iceberg_table_schemas_by_ids.at(schema_id) == *schema_ptr); + chassert(schemasAreIdentical(*iceberg_table_schemas_by_ids.at(schema_id), *schema_ptr)); } else { From 57d446ebfdb7c0aaed5c30292dcb2467a965d8d0 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 10 Dec 2024 10:44:22 +0000 Subject: [PATCH 468/502] Fix bug --- src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 5352fc641fa..8b83f6434db 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -134,7 +134,7 @@ bool schemasAreIdentical(const Poco::JSON::Object & first, const Poco::JSON::Obj static String fields_key = "fields"; if (!first.has(fields_key) || !second.has(fields_key)) return false; - return first.getObject(fields_key) == second.getObject(fields_key); + return (*first.getObject(fields_key)) == (*second.getObject(fields_key)); } } From 72cc0645472136effec57e22e26344cefc0e0c0d Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 10 Dec 2024 11:07:48 +0000 Subject: [PATCH 469/502] Fix bug --- .../ObjectStorage/DataLakes/IcebergMetadata.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 8b83f6434db..9cf4f64cdb3 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -107,7 +107,8 @@ std::pair parseDecimal(const String & type_name) return {precision, scale}; } -bool operator==(const Poco::JSON::Object & first, const Poco::JSON::Object & second) +template +bool equals(const T & first, const T & second) { std::stringstream first_string_stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM std::stringstream second_string_stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM @@ -124,17 +125,23 @@ bool operator==(const Poco::JSON::Object & first, const Poco::JSON::Object & sec return first_string_stream.str() == second_string_stream.str(); } -bool operator!=(const Poco::JSON::Object & first, const Poco::JSON::Object & second) + +bool operator==(const Poco::JSON::Object & first, const Poco::JSON::Object & second) { - return !(first == second); + return equals(first, second); +} + +bool operator==(const Poco::JSON::Array & first, const Poco::JSON::Array & second) +{ + return equals(first, second); } bool schemasAreIdentical(const Poco::JSON::Object & first, const Poco::JSON::Object & second) { static String fields_key = "fields"; - if (!first.has(fields_key) || !second.has(fields_key)) + if (!first.isArray(fields_key) || !second.isArray(fields_key)) return false; - return (*first.getObject(fields_key)) == (*second.getObject(fields_key)); + return *(first.getArray(fields_key)) == *(second.getArray(fields_key)); } } From fb02bec3dc07c15450d265d4ae93b964ab62f7c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 10 Dec 2024 12:22:51 +0100 Subject: [PATCH 470/502] Revert "Enable JIT compilation for more expressions" --- src/Core/AccurateComparison.h | 6 - src/DataTypes/IDataType.h | 4 - src/DataTypes/NumberTraits.h | 33 ---- src/Functions/DivisionUtils.h | 99 +--------- src/Functions/FunctionBinaryArithmetic.h | 121 ++++-------- src/Functions/FunctionUnaryArithmetic.h | 16 +- src/Functions/FunctionsComparison.h | 198 ++------------------ src/Functions/FunctionsConversion.h | 141 -------------- src/Functions/FunctionsLogical.cpp | 39 ---- src/Functions/FunctionsLogical.h | 101 +++------- src/Functions/abs.cpp | 57 +----- src/Functions/assumeNotNull.cpp | 21 --- src/Functions/bitCount.cpp | 25 +-- src/Functions/identity.h | 26 +-- src/Functions/isNotNull.cpp | 22 --- src/Functions/isNull.cpp | 20 -- src/Functions/sign.cpp | 45 +---- src/Interpreters/ActionsDAG.cpp | 12 +- src/Interpreters/ActionsDAG.h | 4 +- src/Interpreters/ExpressionActions.cpp | 9 - src/Interpreters/JIT/CHJIT.cpp | 4 - src/Interpreters/JIT/CompileDAG.cpp | 8 +- src/Processors/QueryPlan/ExpressionStep.cpp | 39 +--- 23 files changed, 112 insertions(+), 938 deletions(-) diff --git a/src/Core/AccurateComparison.h b/src/Core/AccurateComparison.h index 25937a61168..87ff14e40e7 100644 --- a/src/Core/AccurateComparison.h +++ b/src/Core/AccurateComparison.h @@ -209,14 +209,12 @@ template struct EqualsOp using SymmetricOp = EqualsOp; static UInt8 apply(A a, B b) { return accurate::equalsOp(a, b); } - static constexpr bool compilable = true; }; template struct NotEqualsOp { using SymmetricOp = NotEqualsOp; static UInt8 apply(A a, B b) { return accurate::notEqualsOp(a, b); } - static constexpr bool compilable = true; }; template struct GreaterOp; @@ -225,14 +223,12 @@ template struct LessOp { using SymmetricOp = GreaterOp; static UInt8 apply(A a, B b) { return accurate::lessOp(a, b); } - static constexpr bool compilable = true; }; template struct GreaterOp { using SymmetricOp = LessOp; static UInt8 apply(A a, B b) { return accurate::greaterOp(a, b); } - static constexpr bool compilable = true; }; template struct GreaterOrEqualsOp; @@ -241,14 +237,12 @@ template struct LessOrEqualsOp { using SymmetricOp = GreaterOrEqualsOp; static UInt8 apply(A a, B b) { return accurate::lessOrEqualsOp(a, b); } - static constexpr bool compilable = true; }; template struct GreaterOrEqualsOp { using SymmetricOp = LessOrEqualsOp; static UInt8 apply(A a, B b) { return accurate::greaterOrEqualsOp(a, b); } - static constexpr bool compilable = true; }; } diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 2771d056e96..8f06526ddbb 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -555,7 +555,6 @@ inline bool isNullableOrLowCardinalityNullable(const DataTypePtr & data_type) template constexpr bool IsDataTypeDecimal = false; template constexpr bool IsDataTypeNumber = false; -template constexpr bool IsDataTypeNativeNumber = false; template constexpr bool IsDataTypeDateOrDateTime = false; template constexpr bool IsDataTypeDate = false; template constexpr bool IsDataTypeEnum = false; @@ -582,9 +581,6 @@ template constexpr bool IsDataTypeDecimal> = t template <> inline constexpr bool IsDataTypeDecimal = true; template constexpr bool IsDataTypeNumber> = true; -template -requires std::is_arithmetic_v -constexpr bool IsDataTypeNativeNumber> = true; template <> inline constexpr bool IsDataTypeDate = true; template <> inline constexpr bool IsDataTypeDate = true; diff --git a/src/DataTypes/NumberTraits.h b/src/DataTypes/NumberTraits.h index b4a322e1083..5f5962da5ee 100644 --- a/src/DataTypes/NumberTraits.h +++ b/src/DataTypes/NumberTraits.h @@ -205,39 +205,6 @@ struct ResultOfIf ConstructedType, Error>>>; }; -/** Type casting for `modulo` function: - * UInt, UInt -> UInt - * Int, Int -> Int - * UInt, Int -> Int - * UInt64, Int -> Error - * Float, Float -> Float64 - * Float, [U]Int -> Float64 - */ -template -struct ResultOfModuloNativePromotion -{ - static_assert(is_arithmetic_v && is_arithmetic_v); - - static constexpr bool has_float = std::is_floating_point_v || std::is_floating_point_v; - static constexpr bool has_integer = is_integer || is_integer; - static constexpr bool has_signed = is_signed_v || is_signed_v; - static constexpr bool has_unsigned = !is_signed_v || !is_signed_v; - - static constexpr size_t max_size_of_unsigned_integer = max(is_signed_v ? 0 : sizeof(A), is_signed_v ? 0 : sizeof(B)); - static constexpr size_t max_size_of_signed_integer = max(is_signed_v ? sizeof(A) : 0, is_signed_v ? sizeof(B) : 0); - static constexpr size_t max_size_of_integer = max(is_integer ? sizeof(A) : 0, is_integer ? sizeof(B) : 0); - - using ConstructedType = typename Construct< - has_signed, - false, - (has_signed ^ has_unsigned) ? max(max_size_of_unsigned_integer * 2, max_size_of_signed_integer) : max(sizeof(A), sizeof(B))>::Type; - - using Type = std::conditional_t< - std::is_same_v, - A, - std::conditional_t>>; -}; - /** Before applying operator `%` and bitwise operations, operands are cast to whole numbers. */ template struct ToInteger { diff --git a/src/Functions/DivisionUtils.h b/src/Functions/DivisionUtils.h index 69fa1437487..1d9c2ad7ccb 100644 --- a/src/Functions/DivisionUtils.h +++ b/src/Functions/DivisionUtils.h @@ -6,14 +6,8 @@ #include #include -#include "DataTypes/Native.h" #include "config.h" -#if USE_EMBEDDED_COMPILER -# include -# include -#endif - namespace DB { @@ -21,42 +15,8 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_DIVISION; - extern const int LOGICAL_ERROR; } -#if USE_EMBEDDED_COMPILER - -template -static llvm::Value * compileWithNullableValues(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed, F && compile_func) -{ - auto * left_type = left->getType(); - auto * right_type = right->getType(); - - if (!left_type->isStructTy() && !right_type->isStructTy()) - { - // Both arguments are not nullable. - return compile_func(b, left, right, is_signed); - } - - auto * denull_left = left_type->isStructTy() ? b.CreateExtractValue(left, {1}) : left; - auto * denull_right = right_type->isStructTy() ? b.CreateExtractValue(right, {1}) : right; - auto * denull_result = compile_func(b, denull_left, denull_right, is_signed); - - auto * nullable_result_type = toNullableType(b, denull_result->getType()); - llvm::Value * nullable_result = llvm::Constant::getNullValue(nullable_result_type); - nullable_result = b.CreateInsertValue(nullable_result, denull_result, {0}); - - auto * result_is_null = b.CreateExtractValue(nullable_result, {1}); - if (left_type->isStructTy()) - result_is_null = b.CreateOr(result_is_null, b.CreateExtractValue(left, {1})); - if (right_type->isStructTy()) - result_is_null = b.CreateOr(result_is_null, b.CreateExtractValue(right, {1})); - - return b.CreateInsertValue(nullable_result, result_is_null, {1}); -} - -#endif - template inline void throwIfDivisionLeadsToFPE(A a, B b) { @@ -198,39 +158,14 @@ struct ModuloImpl } #if USE_EMBEDDED_COMPILER - static constexpr bool compilable = true; /// Ignore exceptions in LLVM IR - - static llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) - { - return compileWithNullableValues( - b, - left, - right, - is_signed, - [](auto & b_, auto * left_, auto * right_, auto is_signed_) { return compileImpl(b_, left_, right_, is_signed_); }); - } - - static llvm::Value * compileImpl(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) - { - if (left->getType()->isFloatingPointTy()) - return b.CreateFRem(left, right); - else if (left->getType()->isIntegerTy()) - return is_signed ? b.CreateSRem(left, right) : b.CreateURem(left, right); - else - throw Exception(ErrorCodes::LOGICAL_ERROR, "ModuloImpl compilation expected native integer or floating point type"); - } - - #endif + static constexpr bool compilable = false; /// don't know how to throw from LLVM IR +#endif }; template struct ModuloLegacyImpl : ModuloImpl { using ResultType = typename NumberTraits::ResultOfModuloLegacy::Type; - -#if USE_EMBEDDED_COMPILER - static constexpr bool compilable = false; /// moduloLegacy is only used in partition key expression -#endif }; template @@ -259,36 +194,6 @@ struct PositiveModuloImpl : ModuloImpl } return static_cast(res); } - -#if USE_EMBEDDED_COMPILER - static constexpr bool compilable = true; /// Ignore exceptions in LLVM IR - - static llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) - { - return compileWithNullableValues( - b, - left, - right, - is_signed, - [](auto & b_, auto * left_, auto * right_, auto is_signed_) { return compileImpl(b_, left_, right_, is_signed_); }); - } - - static llvm::Value * compileImpl(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) - { - auto * result = ModuloImpl::compileImpl(b, left, right, is_signed); - if (is_signed) - { - /// If result is negative, result += abs(right). - auto * zero = llvm::Constant::getNullValue(result->getType()); - auto * is_negative = b.CreateICmpSLT(result, zero); - auto * abs_right = b.CreateSelect(b.CreateICmpSLT(right, zero), b.CreateNeg(right), right); - return b.CreateSelect(is_negative, b.CreateAdd(result, abs_right), result); - } - else - return result; - } -#endif - }; } diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 528908b6f0f..91adf73b679 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -810,7 +810,6 @@ class FunctionBinaryArithmetic : public IFunction static constexpr bool is_division = IsOperation::division; static constexpr bool is_bit_hamming_distance = IsOperation::bit_hamming_distance; static constexpr bool is_modulo = IsOperation::modulo; - static constexpr bool is_positive_modulo = IsOperation::positive_modulo; static constexpr bool is_int_div = IsOperation::int_div; static constexpr bool is_int_div_or_zero = IsOperation::int_div_or_zero; @@ -2388,105 +2387,59 @@ ColumnPtr executeStringInteger(const ColumnsWithTypeAndName & arguments, const A if (!canBeNativeType(*arguments[0]) || !canBeNativeType(*arguments[1]) || !canBeNativeType(*result_type)) return false; - auto denull_left_type = removeNullable(arguments[0]); - auto denull_right_type = removeNullable(arguments[1]); - WhichDataType data_type_lhs(denull_left_type); - WhichDataType data_type_rhs(denull_right_type); + WhichDataType data_type_lhs(arguments[0]); + WhichDataType data_type_rhs(arguments[1]); if ((data_type_lhs.isDateOrDate32() || data_type_lhs.isDateTime()) || (data_type_rhs.isDateOrDate32() || data_type_rhs.isDateTime())) return false; - return castBothTypes( - denull_left_type.get(), - denull_right_type.get(), - [&](const auto & left, const auto & right) + return castBothTypes(arguments[0].get(), arguments[1].get(), [&](const auto & left, const auto & right) + { + using LeftDataType = std::decay_t; + using RightDataType = std::decay_t; + if constexpr (!std::is_same_v && + !std::is_same_v && + !std::is_same_v && + !std::is_same_v) { - using LeftDataType = std::decay_t; - using RightDataType = std::decay_t; - if constexpr ( - !std::is_same_v && !std::is_same_v - && !std::is_same_v && !std::is_same_v) - { - using ResultDataType = typename BinaryOperationTraits::ResultDataType; - using OpSpec = Op; - - if constexpr ( - !std::is_same_v && !IsDataTypeDecimal - && !IsDataTypeDecimal && !IsDataTypeDecimal && OpSpec::compilable) - { - if constexpr (is_modulo || is_positive_modulo) - { - using LeftType = typename LeftDataType::FieldType; - using RightType = typename RightDataType::FieldType; - using PromotedType = typename NumberTraits::ResultOfModuloNativePromotion::Type; - if constexpr (std::is_arithmetic_v) - { - return true; - } - } - else - return true; - } - } - return false; - }); + using ResultDataType = typename BinaryOperationTraits::ResultDataType; + using OpSpec = Op; + if constexpr (!std::is_same_v && !IsDataTypeDecimal && OpSpec::compilable) + return true; + } + return false; + }); } llvm::Value * compileImpl(llvm::IRBuilderBase & builder, const ValuesWithType & arguments, const DataTypePtr & result_type) const override { assert(2 == arguments.size()); - auto denull_left_type = removeNullable(arguments[0].type); - auto denull_right_type = removeNullable(arguments[1].type); llvm::Value * result = nullptr; - - castBothTypes( - denull_left_type.get(), - denull_right_type.get(), - [&](const auto & left, const auto & right) + castBothTypes(arguments[0].type.get(), arguments[1].type.get(), [&](const auto & left, const auto & right) + { + using LeftDataType = std::decay_t; + using RightDataType = std::decay_t; + if constexpr (!std::is_same_v && + !std::is_same_v && + !std::is_same_v && + !std::is_same_v) { - using LeftDataType = std::decay_t; - using RightDataType = std::decay_t; - if constexpr ( - !std::is_same_v && !std::is_same_v - && !std::is_same_v && !std::is_same_v) + using ResultDataType = typename BinaryOperationTraits::ResultDataType; + using OpSpec = Op; + if constexpr (!std::is_same_v && !IsDataTypeDecimal && OpSpec::compilable) { - using ResultDataType = typename BinaryOperationTraits::ResultDataType; - using OpSpec = Op; - if constexpr ( - !std::is_same_v && !IsDataTypeDecimal - && !IsDataTypeDecimal && !IsDataTypeDecimal && OpSpec::compilable) - { - auto & b = static_cast &>(builder); - if constexpr (is_modulo || is_positive_modulo) - { - using LeftType = typename LeftDataType::FieldType; - using RightType = typename RightDataType::FieldType; - using PromotedType = typename NumberTraits::ResultOfModuloNativePromotion::Type; - if constexpr (std::is_arithmetic_v) - { - DataTypePtr promoted_type = std::make_shared>(); - if (result_type->isNullable()) - promoted_type = std::make_shared(promoted_type); + auto & b = static_cast &>(builder); + auto * lval = nativeCast(b, arguments[0], result_type); + auto * rval = nativeCast(b, arguments[1], result_type); + result = OpSpec::compile(b, lval, rval, std::is_signed_v); - auto * lval = nativeCast(b, arguments[0], promoted_type); - auto * rval = nativeCast(b, arguments[1], promoted_type); - result = nativeCast( - b, promoted_type, OpSpec::compile(b, lval, rval, std::is_signed_v), result_type); - return true; - } - } - else - { - auto * lval = nativeCast(b, arguments[0], result_type); - auto * rval = nativeCast(b, arguments[1], result_type); - result = OpSpec::compile(b, lval, rval, std::is_signed_v); - return true; - } - } + return true; } - return false; - }); + } + + return false; + }); return result; } diff --git a/src/Functions/FunctionUnaryArithmetic.h b/src/Functions/FunctionUnaryArithmetic.h index 72f5e37e99e..cf79bb06430 100644 --- a/src/Functions/FunctionUnaryArithmetic.h +++ b/src/Functions/FunctionUnaryArithmetic.h @@ -489,7 +489,9 @@ public: { using DataType = std::decay_t; if constexpr (std::is_same_v || std::is_same_v) + { return false; + } else { using T0 = typename DataType::FieldType; @@ -511,7 +513,9 @@ public: { using DataType = std::decay_t; if constexpr (std::is_same_v || std::is_same_v) + { return false; + } else { using T0 = typename DataType::FieldType; @@ -519,16 +523,8 @@ public: if constexpr (!std::is_same_v && !IsDataTypeDecimal && Op::compilable) { auto & b = static_cast &>(builder); - if constexpr (std::is_same_v, AbsImpl> || std::is_same_v, BitCountImpl>) - { - /// We don't need to cast the argument to the result type if it's abs/bitcount function. - result = Op::compile(b, arguments[0].value, is_signed_v); - } - else - { - auto * v = nativeCast(b, arguments[0], result_type); - result = Op::compile(b, v, is_signed_v); - } + auto * v = nativeCast(b, arguments[0], result_type); + result = Op::compile(b, v, is_signed_v); return true; } diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 5a4276e0d75..d8ff9b1699e 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -1,21 +1,17 @@ #pragma once -// Include this first, because `#define _asan_poison_address` from -// llvm/Support/Compiler.h conflicts with its forward declaration in -// sanitizer/asan_interface.h -#include -#include -#include +#include +#include +#include -#include +#include #include #include -#include #include +#include #include -#include -#include -#include +#include + #include #include #include @@ -28,23 +24,22 @@ #include #include #include -#include + +#include +#include + #include +#include #include + +#include +#include + #include #include -#include -#include -#include -#include -#include -#include "DataTypes/NumberTraits.h" -#if USE_EMBEDDED_COMPILER -# include -# include -# include -#endif +#include +#include namespace DB { @@ -635,61 +630,6 @@ struct GenericComparisonImpl } }; - -#if USE_EMBEDDED_COMPILER - -template