From 856a2f59562ed73e91293b0b816eefc6577d7ed5 Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Thu, 25 Aug 2022 17:24:24 +0200 Subject: [PATCH 01/86] Allow to modify constrained settings in readonly mode --- .../settings/constraints-on-settings.md | 28 +++- .../settings/permissions-for-queries.md | 3 +- src/Access/SettingsConstraints.cpp | 122 +++++++++++++++--- src/Access/SettingsConstraints.h | 23 +++- src/Access/SettingsProfileElement.cpp | 4 + src/Access/SettingsProfileElement.h | 4 +- src/Access/UsersConfigAccessStorage.cpp | 4 + src/Core/Settings.h | 2 +- src/Storages/System/StorageSystemSettings.cpp | 15 ++- .../StorageSystemSettingsProfileElements.cpp | 26 +++- 10 files changed, 198 insertions(+), 33 deletions(-) diff --git a/docs/en/operations/settings/constraints-on-settings.md b/docs/en/operations/settings/constraints-on-settings.md index d240fde8ff3..37424ea20cb 100644 --- a/docs/en/operations/settings/constraints-on-settings.md +++ b/docs/en/operations/settings/constraints-on-settings.md @@ -31,7 +31,7 @@ The constraints are defined as the following: ``` If the user tries to violate the constraints an exception is thrown and the setting isn’t changed. -There are supported three types of constraints: `min`, `max`, `readonly`. The `min` and `max` constraints specify upper and lower boundaries for a numeric setting and can be used in combination. The `readonly` constraint specifies that the user cannot change the corresponding setting at all. +There are supported three types of constraints: `min`, `max`, `readonly`. The `min` and `max` constraints specify upper and lower boundaries for a numeric setting and can be used in combination. The `readonly` constraint specifies that the user cannot change the corresponding setting at all. Also for `readonly = 1` mode there are two additional types of constraints: `min_in_readonly` and `max_in_readonly`, see details below. **Example:** Let `users.xml` includes lines: @@ -68,6 +68,32 @@ Code: 452, e.displayText() = DB::Exception: Setting max_memory_usage should not Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should not be changed. ``` +## Constraints in read-only mode {#settings-readonly-constraints} +Read-only mode is enabled by `readonly` setting: +- `readonly=0`: No read-only restrictions. +- `readonly=1`: Only read queries are allowed and settings cannot be changes unless explicitly allowed. +- `readonly=2`: Only read queries are allowed, but settings can be changed, except for `readonly` setting itself. + +In both read-only modes settings constraints are applied and additionally in `readonly=1` mode some settings changes can be explicitly allowed in the following way: +``` xml + + + + + lower_boundary + + + upper_boundary + + + lower_boundary + upper_boundary + + + + +``` + **Note:** the `default` profile has special handling: all the constraints defined for the `default` profile become the default constraints, so they restrict all the users until they’re overridden explicitly for these users. [Original article](https://clickhouse.com/docs/en/operations/settings/constraints_on_settings/) diff --git a/docs/en/operations/settings/permissions-for-queries.md b/docs/en/operations/settings/permissions-for-queries.md index 668cb9993eb..a5bce92b49f 100644 --- a/docs/en/operations/settings/permissions-for-queries.md +++ b/docs/en/operations/settings/permissions-for-queries.md @@ -36,8 +36,7 @@ After setting `readonly = 1`, the user can’t change `readonly` and `allow_ddl` When using the `GET` method in the [HTTP interface](../../interfaces/http.md), `readonly = 1` is set automatically. To modify data, use the `POST` method. -Setting `readonly = 1` prohibit the user from changing all the settings. There is a way to prohibit the user -from changing only specific settings, for details see [constraints on settings](../../operations/settings/constraints-on-settings.md). +Setting `readonly = 1` prohibit the user from changing all the settings. There is a way to prohibit the user from changing only specific settings. Also there is a way to allow changing only specific settings under `readonly = 1` restrictions. For details see [constraints on settings](../../operations/settings/constraints-on-settings.md). Default value: 0 diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 34f2e10dc83..fc8e5e91653 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -81,6 +81,36 @@ bool SettingsConstraints::isReadOnly(std::string_view setting_name) const } +void SettingsConstraints::setMinValueInReadOnly(std::string_view setting_name, const Field & min_value_in_readonly) +{ + getConstraintRef(setting_name).min_value_in_readonly = Settings::castValueUtil(setting_name, min_value_in_readonly); +} + +Field SettingsConstraints::getMinValueInReadOnly(std::string_view setting_name) const +{ + const auto * ptr = tryGetConstraint(setting_name); + if (ptr) + return ptr->min_value_in_readonly; + else + return {}; +} + + +void SettingsConstraints::setMaxValueInReadOnly(std::string_view setting_name, const Field & max_value_in_readonly) +{ + getConstraintRef(setting_name).max_value_in_readonly = Settings::castValueUtil(setting_name, max_value_in_readonly); +} + +Field SettingsConstraints::getMaxValueInReadOnly(std::string_view setting_name) const +{ + const auto * ptr = tryGetConstraint(setting_name); + if (ptr) + return ptr->max_value_in_readonly; + else + return {}; +} + + void SettingsConstraints::set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only) { auto & ref = getConstraintRef(setting_name); @@ -89,7 +119,7 @@ void SettingsConstraints::set(std::string_view setting_name, const Field & min_v ref.read_only = read_only; } -void SettingsConstraints::get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const +void SettingsConstraints::get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only, Field & min_value_in_readonly, Field & max_value_in_readonly) const { const auto * ptr = tryGetConstraint(setting_name); if (ptr) @@ -97,12 +127,16 @@ void SettingsConstraints::get(std::string_view setting_name, Field & min_value, min_value = ptr->min_value; max_value = ptr->max_value; read_only = ptr->read_only; + min_value_in_readonly = ptr->min_value_in_readonly; + max_value_in_readonly = ptr->max_value_in_readonly; } else { min_value = Field{}; max_value = Field{}; read_only = false; + min_value_in_readonly = Field{}; + max_value_in_readonly = Field{}; } } @@ -117,6 +151,10 @@ void SettingsConstraints::merge(const SettingsConstraints & other) constraint.max_value = other_constraint.max_value; if (other_constraint.read_only) constraint.read_only = true; + if (!other_constraint.min_value_in_readonly.isNull()) + constraint.min_value_in_readonly = other_constraint.min_value_in_readonly; + if (!other_constraint.max_value_in_readonly.isNull()) + constraint.max_value_in_readonly = other_constraint.max_value_in_readonly; } } @@ -180,10 +218,8 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh } }; - bool cannot_compare = false; - auto less = [&](const Field & left, const Field & right) + auto less_or_cannot_compare = [=](const Field & left, const Field & right) { - cannot_compare = false; if (reaction == THROW_ON_VIOLATION) return applyVisitor(FieldVisitorAccurateLess{}, left, right); else @@ -194,8 +230,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh } catch (...) { - cannot_compare = true; - return false; + return true; } } }; @@ -248,17 +283,10 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh } /** The `readonly` value is understood as follows: - * 0 - everything allowed. - * 1 - only read queries can be made; you can not change the settings. - * 2 - You can only do read queries and you can change the settings, except for the `readonly` setting. + * 0 - no read-only restrictions. + * 1 - only read requests, as well as changing explicitly allowed settings. + * 2 - only read requests, as well as changing settings, except for the `readonly` setting. */ - if (current_settings.readonly == 1) - { - if (reaction == THROW_ON_VIOLATION) - throw Exception("Cannot modify '" + setting_name + "' setting in readonly mode", ErrorCodes::READONLY); - else - return false; - } if (current_settings.readonly > 1 && setting_name == "readonly") { @@ -269,6 +297,58 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh } const Constraint * constraint = tryGetConstraint(setting_name); + if (current_settings.readonly == 1) + { + if (!constraint || (constraint->min_value_in_readonly.isNull() && constraint->max_value_in_readonly.isNull())) { + if (reaction == THROW_ON_VIOLATION) + throw Exception("Cannot modify '" + setting_name + "' setting in readonly mode", ErrorCodes::READONLY); + else + return false; + } + + const Field & min_value = constraint->min_value_in_readonly; + const Field & max_value = constraint->max_value_in_readonly; + if (!min_value.isNull() && !max_value.isNull() && less_or_cannot_compare(max_value, min_value)) + { + if (reaction == THROW_ON_VIOLATION) + throw Exception("Cannot modify '" + setting_name + "' setting in readonly mode", ErrorCodes::READONLY); + else + return false; + } + + // Check left bound + if (!min_value.isNull() && less_or_cannot_compare(new_value, min_value)) + { + if (reaction == THROW_ON_VIOLATION) + { + throw Exception( + "Setting " + setting_name + " in readonly mode shouldn't be less than " + applyVisitor(FieldVisitorToString(), constraint->min_value_in_readonly), + ErrorCodes::READONLY); + } + else + { + new_value = min_value; // to ensure `min` clamping will not cancel `min_in_readonly` clamp + change.value = min_value; + } + } + + // Check right bound + if (!max_value.isNull() && less_or_cannot_compare(max_value, new_value)) + { + if (reaction == THROW_ON_VIOLATION) + { + throw Exception( + "Setting " + setting_name + " in readonly mode shouldn't be greater than " + applyVisitor(FieldVisitorToString(), constraint->max_value_in_readonly), + ErrorCodes::READONLY); + } + else + { + new_value = max_value; // to ensure `max` clamping will not cancel `max_in_readonly` clamp + change.value = max_value; + } + } + } + if (constraint) { if (constraint->read_only) @@ -281,7 +361,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh const Field & min_value = constraint->min_value; const Field & max_value = constraint->max_value; - if (!min_value.isNull() && !max_value.isNull() && (less(max_value, min_value) || cannot_compare)) + if (!min_value.isNull() && !max_value.isNull() && less_or_cannot_compare(max_value, min_value)) { if (reaction == THROW_ON_VIOLATION) throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); @@ -289,7 +369,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh return false; } - if (!min_value.isNull() && (less(new_value, min_value) || cannot_compare)) + if (!min_value.isNull() && less_or_cannot_compare(new_value, min_value)) { if (reaction == THROW_ON_VIOLATION) { @@ -301,7 +381,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh change.value = min_value; } - if (!max_value.isNull() && (less(max_value, new_value) || cannot_compare)) + if (!max_value.isNull() && less_or_cannot_compare(max_value, new_value)) { if (reaction == THROW_ON_VIOLATION) { @@ -342,7 +422,9 @@ const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(st bool SettingsConstraints::Constraint::operator==(const Constraint & other) const { - return (read_only == other.read_only) && (min_value == other.min_value) && (max_value == other.max_value) + return (read_only == other.read_only) + && (min_value == other.min_value) && (max_value == other.max_value) + && (min_value_in_readonly == other.min_value_in_readonly) && (max_value_in_readonly == other.max_value_in_readonly) && (*setting_name == *other.setting_name); } diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 645a690e051..5566578b101 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -33,6 +33,7 @@ class AccessControl; * * 200000 * 20000000000 + * 10000000000 * * * @@ -43,10 +44,11 @@ class AccessControl; * * This class also checks that we are not in the read-only mode. * If a setting cannot be change due to the read-only mode this class throws an exception. - * The value of `readonly` value is understood as follows: - * 0 - everything allowed. - * 1 - only read queries can be made; you can not change the settings. - * 2 - you can only do read queries and you can change the settings, except for the `readonly` setting. + * The value of `readonly` is understood as follows: + * 0 - not read-only mode, no additional checks. + * 1 - only read queries, as well as changing explicitly allowed settings. + * 2 - only read queries and you can change the settings, except for the `readonly` setting. + * */ class SettingsConstraints { @@ -70,8 +72,14 @@ public: void setReadOnly(std::string_view setting_name, bool read_only); bool isReadOnly(std::string_view setting_name) const; + void setMinValueInReadOnly(std::string_view setting_name, const Field & min_value_in_readonly); + Field getMinValueInReadOnly(std::string_view setting_name) const; + + void setMaxValueInReadOnly(std::string_view setting_name, const Field & max_value_in_readonly); + Field getMaxValueInReadOnly(std::string_view setting_name) const; + void set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only); - void get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const; + void get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only, Field & min_value_in_readonly, Field & max_value_in_readonly) const; void merge(const SettingsConstraints & other); @@ -94,6 +102,11 @@ private: Field min_value; Field max_value; + // Allowed range in `readonly=1` mode. + // NOTE: only reasonable to be a subset of [min_value; max_value] range, although there is no enforcement. + Field min_value_in_readonly; + Field max_value_in_readonly; + bool operator ==(const Constraint & other) const; bool operator !=(const Constraint & other) const { return !(*this == other); } }; diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 465f26f37d9..aff41e3553e 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -214,6 +214,10 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints(const AccessC res.setMaxValue(elem.setting_name, elem.max_value); if (elem.readonly) res.setReadOnly(elem.setting_name, *elem.readonly); + if (!elem.min_value_in_readonly.isNull()) + res.setMinValueInReadOnly(elem.setting_name, elem.min_value_in_readonly); + if (!elem.max_value_in_readonly.isNull()) + res.setMaxValueInReadOnly(elem.setting_name, elem.max_value_in_readonly); } } return res; diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index a4124826b40..1f39cd1c788 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -26,8 +26,10 @@ struct SettingsProfileElement Field min_value; Field max_value; std::optional readonly; + Field min_value_in_readonly; + Field max_value_in_readonly; - auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, readonly); } + auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, readonly, min_value_in_readonly, max_value_in_readonly); } friend bool operator==(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() == rhs.toTuple(); } friend bool operator!=(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return !(lhs == rhs); } friend bool operator <(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() < rhs.toTuple(); } diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 1d755fdf1da..ceacdcdb546 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -449,6 +449,10 @@ namespace profile_element.max_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); else if (constraint_type == "readonly") profile_element.readonly = true; + else if (constraint_type == "min_in_readonly") + profile_element.min_value_in_readonly = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); + else if (constraint_type == "max_in_readonly") + profile_element.max_value_in_readonly = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); else throw Exception("Setting " + constraint_type + " value for " + setting_name + " isn't supported", ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 2b808a1ada7..6ab6ad8bb5c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -350,7 +350,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(UInt64, max_ast_elements, 50000, "Maximum size of query syntax tree in number of nodes. Checked after parsing.", 0) \ M(UInt64, max_expanded_ast_elements, 500000, "Maximum size of query syntax tree in number of nodes after expansion of aliases and the asterisk.", 0) \ \ - M(UInt64, readonly, 0, "0 - everything is allowed. 1 - only read requests. 2 - only read requests, as well as changing settings, except for the 'readonly' setting.", 0) \ + M(UInt64, readonly, 0, "0 - no read-only restrictions. 1 - only read requests, as well as changing explicitly allowed settings. 2 - only read requests, as well as changing settings, except for the 'readonly' setting.", 0) \ \ M(UInt64, max_rows_in_set, 0, "Maximum size of the set (in number of elements) resulting from the execution of the IN section.", 0) \ M(UInt64, max_bytes_in_set, 0, "Maximum size of the set (in bytes in memory) resulting from the execution of the IN section.", 0) \ diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index e1f1e4985b4..cac90e93bb6 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -18,6 +18,8 @@ NamesAndTypesList StorageSystemSettings::getNamesAndTypes() {"min", std::make_shared(std::make_shared())}, {"max", std::make_shared(std::make_shared())}, {"readonly", std::make_shared()}, + {"min_in_readonly", std::make_shared(std::make_shared())}, + {"max_in_readonly", std::make_shared(std::make_shared())}, {"type", std::make_shared()}, }; } @@ -40,8 +42,9 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[3]->insert(setting.getDescription()); Field min, max; + Field min_in_readonly, max_in_readonly; bool read_only = false; - constraints.get(setting_name, min, max, read_only); + constraints.get(setting_name, min, max, read_only, min_in_readonly, max_in_readonly); /// These two columns can accept strings only. if (!min.isNull()) @@ -57,10 +60,18 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co read_only = true; } + /// These two columns can accept strings only. + if (!min_in_readonly.isNull()) + min_in_readonly = Settings::valueToStringUtil(setting_name, min_in_readonly); + if (!max_in_readonly.isNull()) + max_in_readonly = Settings::valueToStringUtil(setting_name, max_in_readonly); + res_columns[4]->insert(min); res_columns[5]->insert(max); res_columns[6]->insert(read_only); - res_columns[7]->insert(setting.getTypeName()); + res_columns[7]->insert(min_in_readonly); + res_columns[8]->insert(max_in_readonly); + res_columns[9]->insert(setting.getTypeName()); } } diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp index 565ff5e471e..7dcec16204e 100644 --- a/src/Storages/System/StorageSystemSettingsProfileElements.cpp +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -29,6 +29,8 @@ NamesAndTypesList StorageSystemSettingsProfileElements::getNamesAndTypes() {"min", std::make_shared(std::make_shared())}, {"max", std::make_shared(std::make_shared())}, {"readonly", std::make_shared(std::make_shared())}, + {"min_in_readonly", std::make_shared(std::make_shared())}, + {"max_in_readonly", std::make_shared(std::make_shared())}, {"inherit_profile", std::make_shared(std::make_shared())}, }; return names_and_types; @@ -64,6 +66,10 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns auto & column_max_null_map = assert_cast(*res_columns[i++]).getNullMapData(); auto & column_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()).getData(); auto & column_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_min_in_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_min_in_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_max_in_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_max_in_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); auto & column_inherit_profile = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); auto & column_inherit_profile_null_map = assert_cast(*res_columns[i++]).getNullMapData(); @@ -108,8 +114,26 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns inserted_readonly = true; } + bool inserted_min_in_readonly = false; + if (!element.min_value_in_readonly.isNull() && !element.setting_name.empty()) + { + String str = Settings::valueToStringUtil(element.setting_name, element.min_value_in_readonly); + column_min_in_readonly.insertData(str.data(), str.length()); + column_min_in_readonly_null_map.push_back(false); + inserted_min_in_readonly = true; + } + + bool inserted_max_in_readonly = false; + if (!element.max_value_in_readonly.isNull() && !element.setting_name.empty()) + { + String str = Settings::valueToStringUtil(element.setting_name, element.max_value_in_readonly); + column_max_in_readonly.insertData(str.data(), str.length()); + column_max_in_readonly_null_map.push_back(false); + inserted_max_in_readonly = true; + } + bool inserted_setting_name = false; - if (inserted_value || inserted_min || inserted_max || inserted_readonly) + if (inserted_value || inserted_min || inserted_max || inserted_readonly || inserted_min_in_readonly || inserted_max_in_readonly) { const auto & setting_name = element.setting_name; column_setting_name.insertData(setting_name.data(), setting_name.size()); From c5d1bbf680fe49c209d841d8c8b712146479c112 Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Fri, 26 Aug 2022 21:20:00 +0200 Subject: [PATCH 02/86] reimplement with tag --- .../settings/constraints-on-settings.md | 26 +- src/Access/SettingsConstraints.cpp | 351 ++++++------------ src/Access/SettingsConstraints.h | 95 +++-- src/Access/SettingsProfileElement.cpp | 29 +- src/Access/SettingsProfileElement.h | 10 +- src/Access/UsersConfigAccessStorage.cpp | 16 +- src/Storages/System/StorageSystemSettings.cpp | 23 +- .../StorageSystemSettingsProfileElements.cpp | 31 +- tests/config/users.d/readonly.xml | 1 + 9 files changed, 239 insertions(+), 343 deletions(-) diff --git a/docs/en/operations/settings/constraints-on-settings.md b/docs/en/operations/settings/constraints-on-settings.md index 37424ea20cb..32b7e5c9951 100644 --- a/docs/en/operations/settings/constraints-on-settings.md +++ b/docs/en/operations/settings/constraints-on-settings.md @@ -31,7 +31,9 @@ The constraints are defined as the following: ``` If the user tries to violate the constraints an exception is thrown and the setting isn’t changed. -There are supported three types of constraints: `min`, `max`, `readonly`. The `min` and `max` constraints specify upper and lower boundaries for a numeric setting and can be used in combination. The `readonly` constraint specifies that the user cannot change the corresponding setting at all. Also for `readonly = 1` mode there are two additional types of constraints: `min_in_readonly` and `max_in_readonly`, see details below. +There are supported three types of constraints: `min`, `max`, `readonly`. The `min` and `max` constraints specify upper and lower boundaries for a numeric setting and can be used in combination. The `readonly` constraint specifies that the user cannot change the corresponding setting at all. + +If there are multiple profiles active for a user, then constraints are merged. Constraints for the same setting are replaced during merge, such that the last constraint is used and all previous are ignored. **Example:** Let `users.xml` includes lines: @@ -68,32 +70,38 @@ Code: 452, e.displayText() = DB::Exception: Setting max_memory_usage should not Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should not be changed. ``` -## Constraints in read-only mode {#settings-readonly-constraints} +## Allowances in read-only mode {#settings-readonly-allowance} Read-only mode is enabled by `readonly` setting: - `readonly=0`: No read-only restrictions. - `readonly=1`: Only read queries are allowed and settings cannot be changes unless explicitly allowed. - `readonly=2`: Only read queries are allowed, but settings can be changed, except for `readonly` setting itself. -In both read-only modes settings constraints are applied and additionally in `readonly=1` mode some settings changes can be explicitly allowed in the following way: +In `readonly=0` and `readonly=2` modes settings constraints are applied as usual. But in `readonly=1` by default all settings changes are forbidden and constraints are ignored. But special allowances can be set in the following way: ``` xml - + - lower_boundary + lower_boundary - upper_boundary + upper_boundary - lower_boundary - upper_boundary + lower_boundary + upper_boundary - + + + + + ``` +Constraints and allowances are merged from multiple profiles independently. Previous constraint is not canceled by new allowance, as well as previous allowance is not canceled by new constraint. Instead profile can have one allowance and one constraint for every setting. Allowances are used in `readonly=1` mode, otherwise constraints are used. + **Note:** the `default` profile has special handling: all the constraints defined for the `default` profile become the default constraints, so they restrict all the users until they’re overridden explicitly for these users. [Original article](https://clickhouse.com/docs/en/operations/settings/constraints_on_settings/) diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index fc8e5e91653..a9264cb4ccd 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -36,125 +36,65 @@ void SettingsConstraints::clear() } -void SettingsConstraints::setMinValue(std::string_view setting_name, const Field & min_value) +void SettingsConstraints::constrainMinValue(const String & setting_name, const Field & min_value) { - getConstraintRef(setting_name).min_value = Settings::castValueUtil(setting_name, min_value); + constraints[setting_name].min_value = Settings::castValueUtil(setting_name, min_value); } -Field SettingsConstraints::getMinValue(std::string_view setting_name) const +void SettingsConstraints::constrainMaxValue(const String & setting_name, const Field & max_value) { - const auto * ptr = tryGetConstraint(setting_name); - if (ptr) - return ptr->min_value; - else - return {}; + constraints[setting_name].max_value = Settings::castValueUtil(setting_name, max_value); } - -void SettingsConstraints::setMaxValue(std::string_view setting_name, const Field & max_value) +void SettingsConstraints::constrainReadOnly(const String & setting_name, bool read_only) { - getConstraintRef(setting_name).max_value = Settings::castValueUtil(setting_name, max_value); + constraints[setting_name].read_only = read_only; } -Field SettingsConstraints::getMaxValue(std::string_view setting_name) const +void SettingsConstraints::allowMinValue(const String & setting_name, const Field & min_value) { - const auto * ptr = tryGetConstraint(setting_name); - if (ptr) - return ptr->max_value; - else - return {}; + allowances[setting_name].min_value = Settings::castValueUtil(setting_name, min_value); } - -void SettingsConstraints::setReadOnly(std::string_view setting_name, bool read_only) +void SettingsConstraints::allowMaxValue(const String & setting_name, const Field & max_value) { - getConstraintRef(setting_name).read_only = read_only; + allowances[setting_name].max_value = Settings::castValueUtil(setting_name, max_value); } -bool SettingsConstraints::isReadOnly(std::string_view setting_name) const +void SettingsConstraints::allowReadOnly(const String & setting_name, bool read_only) { - const auto * ptr = tryGetConstraint(setting_name); - if (ptr) - return ptr->read_only; - else - return false; + allowances[setting_name].read_only = read_only; } - -void SettingsConstraints::setMinValueInReadOnly(std::string_view setting_name, const Field & min_value_in_readonly) +void SettingsConstraints::get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const { - getConstraintRef(setting_name).min_value_in_readonly = Settings::castValueUtil(setting_name, min_value_in_readonly); -} - -Field SettingsConstraints::getMinValueInReadOnly(std::string_view setting_name) const -{ - const auto * ptr = tryGetConstraint(setting_name); - if (ptr) - return ptr->min_value_in_readonly; - else - return {}; -} - - -void SettingsConstraints::setMaxValueInReadOnly(std::string_view setting_name, const Field & max_value_in_readonly) -{ - getConstraintRef(setting_name).max_value_in_readonly = Settings::castValueUtil(setting_name, max_value_in_readonly); -} - -Field SettingsConstraints::getMaxValueInReadOnly(std::string_view setting_name) const -{ - const auto * ptr = tryGetConstraint(setting_name); - if (ptr) - return ptr->max_value_in_readonly; - else - return {}; -} - - -void SettingsConstraints::set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only) -{ - auto & ref = getConstraintRef(setting_name); - ref.min_value = Settings::castValueUtil(setting_name, min_value); - ref.max_value = Settings::castValueUtil(setting_name, max_value); - ref.read_only = read_only; -} - -void SettingsConstraints::get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only, Field & min_value_in_readonly, Field & max_value_in_readonly) const -{ - const auto * ptr = tryGetConstraint(setting_name); - if (ptr) - { - min_value = ptr->min_value; - max_value = ptr->max_value; - read_only = ptr->read_only; - min_value_in_readonly = ptr->min_value_in_readonly; - max_value_in_readonly = ptr->max_value_in_readonly; - } - else - { - min_value = Field{}; - max_value = Field{}; - read_only = false; - min_value_in_readonly = Field{}; - max_value_in_readonly = Field{}; - } + auto range = getRange(current_settings, setting_name); + min_value = range.min_value; + max_value = range.max_value; + read_only = range.read_only; } void SettingsConstraints::merge(const SettingsConstraints & other) { for (const auto & [other_name, other_constraint] : other.constraints) { - auto & constraint = getConstraintRef(other_name); + auto & constraint = constraints[other_name]; if (!other_constraint.min_value.isNull()) constraint.min_value = other_constraint.min_value; if (!other_constraint.max_value.isNull()) constraint.max_value = other_constraint.max_value; if (other_constraint.read_only) constraint.read_only = true; - if (!other_constraint.min_value_in_readonly.isNull()) - constraint.min_value_in_readonly = other_constraint.min_value_in_readonly; - if (!other_constraint.max_value_in_readonly.isNull()) - constraint.max_value_in_readonly = other_constraint.max_value_in_readonly; + } + for (const auto & [other_name, other_allowance] : other.allowances) + { + auto & allowance = allowances[other_name]; + if (!other_allowance.min_value.isNull()) + allowance.min_value = other_allowance.min_value; + if (!other_allowance.max_value.isNull()) + allowance.max_value = other_allowance.max_value; + if (other_allowance.read_only) + allowance.read_only = true; } } @@ -218,23 +158,6 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh } }; - auto less_or_cannot_compare = [=](const Field & left, const Field & right) - { - if (reaction == THROW_ON_VIOLATION) - return applyVisitor(FieldVisitorAccurateLess{}, left, right); - else - { - try - { - return applyVisitor(FieldVisitorAccurateLess{}, left, right); - } - catch (...) - { - return true; - } - } - }; - if (reaction == THROW_ON_VIOLATION) { try @@ -274,14 +197,86 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh return false; } - if (!current_settings.allow_ddl && setting_name == "allow_ddl") + return getRange(current_settings, setting_name).check(change, new_value, reaction); +} + +bool SettingsConstraints::Range::check(SettingChange & change, const Field & new_value, ReactionOnViolation reaction) const +{ + const String & setting_name = change.name; + + auto less_or_cannot_compare = [=](const Field & left, const Field & right) { if (reaction == THROW_ON_VIOLATION) - throw Exception("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED); + return applyVisitor(FieldVisitorAccurateLess{}, left, right); + else + { + try + { + return applyVisitor(FieldVisitorAccurateLess{}, left, right); + } + catch (...) + { + return true; + } + } + }; + + if (!explain.empty()) + { + if (reaction == THROW_ON_VIOLATION) + throw Exception(explain, code); else return false; } + if (read_only) + { + if (reaction == THROW_ON_VIOLATION) + throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); + else + return false; + } + + if (!min_value.isNull() && !max_value.isNull() && less_or_cannot_compare(max_value, min_value)) + { + if (reaction == THROW_ON_VIOLATION) + throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); + else + return false; + } + + if (!min_value.isNull() && less_or_cannot_compare(new_value, min_value)) + { + if (reaction == THROW_ON_VIOLATION) + { + throw Exception( + "Setting " + setting_name + " shouldn't be less than " + applyVisitor(FieldVisitorToString(), min_value), + ErrorCodes::SETTING_CONSTRAINT_VIOLATION); + } + else + change.value = min_value; + } + + if (!max_value.isNull() && less_or_cannot_compare(max_value, new_value)) + { + if (reaction == THROW_ON_VIOLATION) + { + throw Exception( + "Setting " + setting_name + " shouldn't be greater than " + applyVisitor(FieldVisitorToString(), max_value), + ErrorCodes::SETTING_CONSTRAINT_VIOLATION); + } + else + change.value = max_value; + } + + return true; +} + +SettingsConstraints::Range SettingsConstraints::getRange(const Settings & current_settings, std::string_view setting_name) const +{ + if (!current_settings.allow_ddl && setting_name == "allow_ddl") + return Range::forbidden("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED); + /** The `readonly` value is understood as follows: * 0 - no read-only restrictions. * 1 - only read requests, as well as changing explicitly allowed settings. @@ -289,147 +284,31 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh */ if (current_settings.readonly > 1 && setting_name == "readonly") - { - if (reaction == THROW_ON_VIOLATION) - throw Exception("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY); - else - return false; - } + return Range::forbidden("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY); - const Constraint * constraint = tryGetConstraint(setting_name); if (current_settings.readonly == 1) { - if (!constraint || (constraint->min_value_in_readonly.isNull() && constraint->max_value_in_readonly.isNull())) { - if (reaction == THROW_ON_VIOLATION) - throw Exception("Cannot modify '" + setting_name + "' setting in readonly mode", ErrorCodes::READONLY); - else - return false; - } - - const Field & min_value = constraint->min_value_in_readonly; - const Field & max_value = constraint->max_value_in_readonly; - if (!min_value.isNull() && !max_value.isNull() && less_or_cannot_compare(max_value, min_value)) - { - if (reaction == THROW_ON_VIOLATION) - throw Exception("Cannot modify '" + setting_name + "' setting in readonly mode", ErrorCodes::READONLY); - else - return false; - } - - // Check left bound - if (!min_value.isNull() && less_or_cannot_compare(new_value, min_value)) - { - if (reaction == THROW_ON_VIOLATION) - { - throw Exception( - "Setting " + setting_name + " in readonly mode shouldn't be less than " + applyVisitor(FieldVisitorToString(), constraint->min_value_in_readonly), - ErrorCodes::READONLY); - } - else - { - new_value = min_value; // to ensure `min` clamping will not cancel `min_in_readonly` clamp - change.value = min_value; - } - } - - // Check right bound - if (!max_value.isNull() && less_or_cannot_compare(max_value, new_value)) - { - if (reaction == THROW_ON_VIOLATION) - { - throw Exception( - "Setting " + setting_name + " in readonly mode shouldn't be greater than " + applyVisitor(FieldVisitorToString(), constraint->max_value_in_readonly), - ErrorCodes::READONLY); - } - else - { - new_value = max_value; // to ensure `max` clamping will not cancel `max_in_readonly` clamp - change.value = max_value; - } - } + auto it = allowances.find(setting_name); + if (it == allowances.end()) + return Range::forbidden("Cannot modify '" + String(setting_name) + "' setting in readonly mode", ErrorCodes::READONLY); + return it->second; } - - if (constraint) + else // For both readonly=0 and readonly=2 { - if (constraint->read_only) - { - if (reaction == THROW_ON_VIOLATION) - throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); - else - return false; - } - - const Field & min_value = constraint->min_value; - const Field & max_value = constraint->max_value; - if (!min_value.isNull() && !max_value.isNull() && less_or_cannot_compare(max_value, min_value)) - { - if (reaction == THROW_ON_VIOLATION) - throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); - else - return false; - } - - if (!min_value.isNull() && less_or_cannot_compare(new_value, min_value)) - { - if (reaction == THROW_ON_VIOLATION) - { - throw Exception( - "Setting " + setting_name + " shouldn't be less than " + applyVisitor(FieldVisitorToString(), constraint->min_value), - ErrorCodes::SETTING_CONSTRAINT_VIOLATION); - } - else - change.value = min_value; - } - - if (!max_value.isNull() && less_or_cannot_compare(max_value, new_value)) - { - if (reaction == THROW_ON_VIOLATION) - { - throw Exception( - "Setting " + setting_name + " shouldn't be greater than " + applyVisitor(FieldVisitorToString(), constraint->max_value), - ErrorCodes::SETTING_CONSTRAINT_VIOLATION); - } - else - change.value = max_value; - } + auto it = constraints.find(setting_name); + if (it == constraints.end()) + return Range::allowed(); + return it->second; } - - return true; } - -SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(std::string_view setting_name) +bool SettingsConstraints::Range::operator==(const Range & other) const { - auto it = constraints.find(setting_name); - if (it == constraints.end()) - { - auto setting_name_ptr = std::make_shared(setting_name); - Constraint new_constraint; - new_constraint.setting_name = setting_name_ptr; - it = constraints.emplace(*setting_name_ptr, std::move(new_constraint)).first; - } - return it->second; -} - -const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(std::string_view setting_name) const -{ - auto it = constraints.find(setting_name); - if (it == constraints.end()) - return nullptr; - return &it->second; -} - - -bool SettingsConstraints::Constraint::operator==(const Constraint & other) const -{ - return (read_only == other.read_only) - && (min_value == other.min_value) && (max_value == other.max_value) - && (min_value_in_readonly == other.min_value_in_readonly) && (max_value_in_readonly == other.max_value_in_readonly) - && (*setting_name == *other.setting_name); + return read_only == other.read_only && min_value == other.min_value && max_value == other.max_value; } bool operator ==(const SettingsConstraints & left, const SettingsConstraints & right) { - return (left.constraints == right.constraints); + return left.constraints == right.constraints && left.allowances == right.allowances; } } diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 5566578b101..eaca894d15e 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -33,12 +33,17 @@ class AccessControl; * * 200000 * 20000000000 - * 10000000000 * * * * * + * + * + * 200000 + * 10000000000 + * + * * * * @@ -63,23 +68,14 @@ public: void clear(); bool empty() const { return constraints.empty(); } - void setMinValue(std::string_view setting_name, const Field & min_value); - Field getMinValue(std::string_view setting_name) const; + void constrainMinValue(const String & setting_name, const Field & min_value); + void constrainMaxValue(const String & setting_name, const Field & max_value); + void constrainReadOnly(const String & setting_name, bool read_only); + void allowMinValue(const String & setting_name, const Field & min_value); + void allowMaxValue(const String & setting_name, const Field & max_value); + void allowReadOnly(const String & setting_name, bool read_only); - void setMaxValue(std::string_view setting_name, const Field & max_value); - Field getMaxValue(std::string_view setting_name) const; - - void setReadOnly(std::string_view setting_name, bool read_only); - bool isReadOnly(std::string_view setting_name) const; - - void setMinValueInReadOnly(std::string_view setting_name, const Field & min_value_in_readonly); - Field getMinValueInReadOnly(std::string_view setting_name) const; - - void setMaxValueInReadOnly(std::string_view setting_name, const Field & max_value_in_readonly); - Field getMaxValueInReadOnly(std::string_view setting_name) const; - - void set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only); - void get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only, Field & min_value_in_readonly, Field & max_value_in_readonly) const; + void get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const; void merge(const SettingsConstraints & other); @@ -95,33 +91,60 @@ public: friend bool operator !=(const SettingsConstraints & left, const SettingsConstraints & right) { return !(left == right); } private: - struct Constraint - { - std::shared_ptr setting_name; - bool read_only = false; - Field min_value; - Field max_value; - - // Allowed range in `readonly=1` mode. - // NOTE: only reasonable to be a subset of [min_value; max_value] range, although there is no enforcement. - Field min_value_in_readonly; - Field max_value_in_readonly; - - bool operator ==(const Constraint & other) const; - bool operator !=(const Constraint & other) const { return !(*this == other); } - }; - enum ReactionOnViolation { THROW_ON_VIOLATION, CLAMP_ON_VIOLATION, }; + + struct Range + { + bool read_only = false; + Field min_value; + Field max_value; + + String explain; + int code; + + bool operator ==(const Range & other) const; + bool operator !=(const Range & other) const { return !(*this == other); } + + bool check(SettingChange & change, const Field & new_value, ReactionOnViolation reaction) const; + + static const Range & allowed() + { + static Range res; + return res; + } + + static Range forbidden(const String & explain, int code) + { + return Range{.read_only = true, .explain = explain, .code = code}; + } + }; + + struct StringHash + { + using is_transparent = void; + size_t operator()(std::string_view txt) const + { + return std::hash{}(txt); + } + size_t operator()(const String & txt) const + { + return std::hash{}(txt); + } + }; + bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const; - Constraint & getConstraintRef(std::string_view setting_name); - const Constraint * tryGetConstraint(std::string_view setting_name) const; + Range getRange(const Settings & current_settings, std::string_view setting_name) const; + + // Special containter for heterogenous lookups: to avoid `String` construction during `find(std::string_view)` + using RangeMap = std::unordered_map>; + RangeMap constraints; + RangeMap allowances; - std::unordered_map constraints; const AccessControl * access_control = nullptr; }; diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index aff41e3553e..7ff182fd139 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -208,16 +208,25 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints(const AccessC { if (!elem.setting_name.empty() && (elem.setting_name != ALLOW_BACKUP_SETTING_NAME)) { - if (!elem.min_value.isNull()) - res.setMinValue(elem.setting_name, elem.min_value); - if (!elem.max_value.isNull()) - res.setMaxValue(elem.setting_name, elem.max_value); - if (elem.readonly) - res.setReadOnly(elem.setting_name, *elem.readonly); - if (!elem.min_value_in_readonly.isNull()) - res.setMinValueInReadOnly(elem.setting_name, elem.min_value_in_readonly); - if (!elem.max_value_in_readonly.isNull()) - res.setMaxValueInReadOnly(elem.setting_name, elem.max_value_in_readonly); + switch (elem.kind) + { + case SettingsProfileElement::RangeKind::Constrain: + if (!elem.min_value.isNull()) + res.constrainMinValue(elem.setting_name, elem.min_value); + if (!elem.max_value.isNull()) + res.constrainMaxValue(elem.setting_name, elem.max_value); + if (elem.readonly) + res.constrainReadOnly(elem.setting_name, *elem.readonly); + break; + case SettingsProfileElement::RangeKind::Allow: + if (!elem.min_value.isNull()) + res.allowMinValue(elem.setting_name, elem.min_value); + if (!elem.max_value.isNull()) + res.allowMaxValue(elem.setting_name, elem.max_value); + if (elem.readonly) + res.allowReadOnly(elem.setting_name, *elem.readonly); + break; + } } } return res; diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index 1f39cd1c788..bcc9ea7f0c4 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -21,15 +21,19 @@ struct SettingsProfileElement { std::optional parent_profile; + enum class RangeKind { + Constrain = 0, + Allow + }; + String setting_name; Field value; + RangeKind kind = RangeKind::Constrain; Field min_value; Field max_value; std::optional readonly; - Field min_value_in_readonly; - Field max_value_in_readonly; - auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, readonly, min_value_in_readonly, max_value_in_readonly); } + auto toTuple() const { return std::tie(parent_profile, setting_name, value, kind, min_value, max_value, readonly); } friend bool operator==(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() == rhs.toTuple(); } friend bool operator!=(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return !(lhs == rhs); } friend bool operator <(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() < rhs.toTuple(); } diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index ceacdcdb546..6ffd9bcb681 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -425,7 +425,8 @@ namespace SettingsProfileElements parseSettingsConstraints(const Poco::Util::AbstractConfiguration & config, const String & path_to_constraints, - const AccessControl & access_control) + const AccessControl & access_control, + SettingsProfileElement::RangeKind kind) { SettingsProfileElements profile_elements; Poco::Util::AbstractConfiguration::Keys keys; @@ -437,6 +438,7 @@ namespace SettingsProfileElement profile_element; profile_element.setting_name = setting_name; + profile_element.kind = kind; Poco::Util::AbstractConfiguration::Keys constraint_types; String path_to_name = path_to_constraints + "." + setting_name; config.keys(path_to_name, constraint_types); @@ -449,10 +451,6 @@ namespace profile_element.max_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); else if (constraint_type == "readonly") profile_element.readonly = true; - else if (constraint_type == "min_in_readonly") - profile_element.min_value_in_readonly = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); - else if (constraint_type == "max_in_readonly") - profile_element.max_value_in_readonly = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); else throw Exception("Setting " + constraint_type + " value for " + setting_name + " isn't supported", ErrorCodes::NOT_IMPLEMENTED); } @@ -491,7 +489,13 @@ namespace if (key == "constraints" || key.starts_with("constraints[")) { - profile->elements.merge(parseSettingsConstraints(config, profile_config + "." + key, access_control)); + profile->elements.merge(parseSettingsConstraints(config, profile_config + "." + key, access_control, SettingsProfileElement::RangeKind::Constrain)); + continue; + } + + if (key == "allow" || key.starts_with("allow[")) + { + profile->elements.merge(parseSettingsConstraints(config, profile_config + "." + key, access_control, SettingsProfileElement::RangeKind::Allow)); continue; } diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index cac90e93bb6..a0a8803169b 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -18,8 +18,6 @@ NamesAndTypesList StorageSystemSettings::getNamesAndTypes() {"min", std::make_shared(std::make_shared())}, {"max", std::make_shared(std::make_shared())}, {"readonly", std::make_shared()}, - {"min_in_readonly", std::make_shared(std::make_shared())}, - {"max_in_readonly", std::make_shared(std::make_shared())}, {"type", std::make_shared()}, }; } @@ -42,9 +40,8 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[3]->insert(setting.getDescription()); Field min, max; - Field min_in_readonly, max_in_readonly; bool read_only = false; - constraints.get(setting_name, min, max, read_only, min_in_readonly, max_in_readonly); + constraints.get(settings, setting_name, min, max, read_only); /// These two columns can accept strings only. if (!min.isNull()) @@ -52,26 +49,10 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co if (!max.isNull()) max = Settings::valueToStringUtil(setting_name, max); - if (!read_only) - { - if ((settings.readonly == 1) - || ((settings.readonly > 1) && (setting_name == "readonly")) - || ((!settings.allow_ddl) && (setting_name == "allow_ddl"))) - read_only = true; - } - - /// These two columns can accept strings only. - if (!min_in_readonly.isNull()) - min_in_readonly = Settings::valueToStringUtil(setting_name, min_in_readonly); - if (!max_in_readonly.isNull()) - max_in_readonly = Settings::valueToStringUtil(setting_name, max_in_readonly); - res_columns[4]->insert(min); res_columns[5]->insert(max); res_columns[6]->insert(read_only); - res_columns[7]->insert(min_in_readonly); - res_columns[8]->insert(max_in_readonly); - res_columns[9]->insert(setting.getTypeName()); + res_columns[7]->insert(setting.getTypeName()); } } diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp index 7dcec16204e..b1f2ad36999 100644 --- a/src/Storages/System/StorageSystemSettingsProfileElements.cpp +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -29,8 +29,7 @@ NamesAndTypesList StorageSystemSettingsProfileElements::getNamesAndTypes() {"min", std::make_shared(std::make_shared())}, {"max", std::make_shared(std::make_shared())}, {"readonly", std::make_shared(std::make_shared())}, - {"min_in_readonly", std::make_shared(std::make_shared())}, - {"max_in_readonly", std::make_shared(std::make_shared())}, + {"allowance", std::make_shared(std::make_shared())}, {"inherit_profile", std::make_shared(std::make_shared())}, }; return names_and_types; @@ -66,10 +65,8 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns auto & column_max_null_map = assert_cast(*res_columns[i++]).getNullMapData(); auto & column_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()).getData(); auto & column_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); - auto & column_min_in_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); - auto & column_min_in_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); - auto & column_max_in_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); - auto & column_max_in_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_allowance = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()).getData(); + auto & column_allowance_null_map = assert_cast(*res_columns[i++]).getNullMapData(); auto & column_inherit_profile = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); auto & column_inherit_profile_null_map = assert_cast(*res_columns[i++]).getNullMapData(); @@ -114,26 +111,16 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns inserted_readonly = true; } - bool inserted_min_in_readonly = false; - if (!element.min_value_in_readonly.isNull() && !element.setting_name.empty()) + bool inserted_allowance = false; + if (element.kind == SettingsProfileElement::RangeKind::Allow && !element.setting_name.empty()) { - String str = Settings::valueToStringUtil(element.setting_name, element.min_value_in_readonly); - column_min_in_readonly.insertData(str.data(), str.length()); - column_min_in_readonly_null_map.push_back(false); - inserted_min_in_readonly = true; - } - - bool inserted_max_in_readonly = false; - if (!element.max_value_in_readonly.isNull() && !element.setting_name.empty()) - { - String str = Settings::valueToStringUtil(element.setting_name, element.max_value_in_readonly); - column_max_in_readonly.insertData(str.data(), str.length()); - column_max_in_readonly_null_map.push_back(false); - inserted_max_in_readonly = true; + column_allowance.push_back(true); + column_allowance_null_map.push_back(false); + inserted_allowance = true; } bool inserted_setting_name = false; - if (inserted_value || inserted_min || inserted_max || inserted_readonly || inserted_min_in_readonly || inserted_max_in_readonly) + if (inserted_value || inserted_min || inserted_max || inserted_readonly || inserted_allowance) { const auto & setting_name = element.setting_name; column_setting_name.insertData(setting_name.data(), setting_name.size()); diff --git a/tests/config/users.d/readonly.xml b/tests/config/users.d/readonly.xml index 8e463205348..69cdba8b8f6 100644 --- a/tests/config/users.d/readonly.xml +++ b/tests/config/users.d/readonly.xml @@ -4,6 +4,7 @@ 1 + 8 From 136308314fe938fb3718c3e5097c3d9269bd985e Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Fri, 26 Aug 2022 21:20:52 +0200 Subject: [PATCH 03/86] cleanup --- tests/config/users.d/readonly.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/config/users.d/readonly.xml b/tests/config/users.d/readonly.xml index 69cdba8b8f6..8e463205348 100644 --- a/tests/config/users.d/readonly.xml +++ b/tests/config/users.d/readonly.xml @@ -4,7 +4,6 @@ 1 - 8 From f8cfb2e53ae24bea5e1f9ddf98b58794d959c798 Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Mon, 29 Aug 2022 14:33:48 +0200 Subject: [PATCH 04/86] fix style, typos and tests --- src/Access/SettingsConstraints.h | 2 +- src/Access/SettingsProfileElement.h | 3 ++- .../queries/0_stateless/01293_create_role.reference | 12 ++++++------ .../01294_create_settings_profile.reference | 12 ++++++------ ...05_drop_settings_profile_while_assigned.reference | 2 +- .../02117_show_create_table_system.reference | 1 + 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index eaca894d15e..c6cd54a9e68 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -140,7 +140,7 @@ private: Range getRange(const Settings & current_settings, std::string_view setting_name) const; - // Special containter for heterogenous lookups: to avoid `String` construction during `find(std::string_view)` + // Special container for heterogeneous lookups: to avoid `String` construction during `find(std::string_view)` using RangeMap = std::unordered_map>; RangeMap constraints; RangeMap allowances; diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index bcc9ea7f0c4..d8cd6a3ca81 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -21,7 +21,8 @@ struct SettingsProfileElement { std::optional parent_profile; - enum class RangeKind { + enum class RangeKind + { Constrain = 0, Allow }; diff --git a/tests/queries/0_stateless/01293_create_role.reference b/tests/queries/0_stateless/01293_create_role.reference index 8d9a259ecf5..4c287d6b9f8 100644 --- a/tests/queries/0_stateless/01293_create_role.reference +++ b/tests/queries/0_stateless/01293_create_role.reference @@ -30,9 +30,9 @@ CREATE ROLE r2_01293 SETTINGS readonly = 1 -- system.roles r1_01293 local directory -- system.settings_profile_elements -\N \N r1_01293 0 readonly 1 \N \N \N \N -\N \N r2_01293 0 \N \N \N \N \N default -\N \N r3_01293 0 max_memory_usage 5000000 4000000 6000000 0 \N -\N \N r4_01293 0 \N \N \N \N \N default -\N \N r4_01293 1 max_memory_usage 5000000 \N \N \N \N -\N \N r4_01293 2 readonly 1 \N \N \N \N +\N \N r1_01293 0 readonly 1 \N \N \N \N \N +\N \N r2_01293 0 \N \N \N \N \N \N default +\N \N r3_01293 0 max_memory_usage 5000000 4000000 6000000 0 \N \N +\N \N r4_01293 0 \N \N \N \N \N \N default +\N \N r4_01293 1 max_memory_usage 5000000 \N \N \N \N \N +\N \N r4_01293 2 readonly 1 \N \N \N \N \N diff --git a/tests/queries/0_stateless/01294_create_settings_profile.reference b/tests/queries/0_stateless/01294_create_settings_profile.reference index da47b084070..90b440e7746 100644 --- a/tests/queries/0_stateless/01294_create_settings_profile.reference +++ b/tests/queries/0_stateless/01294_create_settings_profile.reference @@ -60,9 +60,9 @@ s4_01294 local directory 1 0 ['r1_01294'] [] s5_01294 local directory 3 0 ['u1_01294'] [] s6_01294 local directory 0 1 [] ['r1_01294','u1_01294'] -- system.settings_profile_elements -s2_01294 \N \N 0 readonly 0 \N \N \N \N -s3_01294 \N \N 0 max_memory_usage 5000000 4000000 6000000 1 \N -s4_01294 \N \N 0 max_memory_usage 5000000 \N \N \N \N -s5_01294 \N \N 0 \N \N \N \N \N default -s5_01294 \N \N 1 readonly 0 \N \N \N \N -s5_01294 \N \N 2 max_memory_usage \N \N 6000000 0 \N +s2_01294 \N \N 0 readonly 0 \N \N \N \N \N +s3_01294 \N \N 0 max_memory_usage 5000000 4000000 6000000 1 \N \N +s4_01294 \N \N 0 max_memory_usage 5000000 \N \N \N \N \N +s5_01294 \N \N 0 \N \N \N \N \N \N default +s5_01294 \N \N 1 readonly 0 \N \N \N \N \N +s5_01294 \N \N 2 max_memory_usage \N \N 6000000 0 \N \N diff --git a/tests/queries/0_stateless/01605_drop_settings_profile_while_assigned.reference b/tests/queries/0_stateless/01605_drop_settings_profile_while_assigned.reference index 47942812a11..ac580c02596 100644 --- a/tests/queries/0_stateless/01605_drop_settings_profile_while_assigned.reference +++ b/tests/queries/0_stateless/01605_drop_settings_profile_while_assigned.reference @@ -1,2 +1,2 @@ -\N test_01605 \N 0 \N \N \N \N \N test_01605 +\N test_01605 \N 0 \N \N \N \N \N \N test_01605 PROFILE DROPPED diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 3ca3f856b95..65ab6557437 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -968,6 +968,7 @@ CREATE TABLE system.settings_profile_elements `min` Nullable(String), `max` Nullable(String), `readonly` Nullable(UInt8), + `allowance` Nullable(UInt8), `inherit_profile` Nullable(String) ) ENGINE = SystemSettingsProfileElements From 7fec55eea4520871d4d05c0a80646ec925d63535 Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Fri, 2 Sep 2022 04:12:05 +0200 Subject: [PATCH 05/86] work in progress --- programs/server/config.xml | 6 ++ src/Access/AccessControl.cpp | 1 + src/Access/AccessControl.h | 4 ++ src/Access/SettingsConstraints.cpp | 72 ++++++++----------- src/Access/SettingsConstraints.h | 31 ++++---- src/Access/SettingsProfileElement.cpp | 45 ++++++------ src/Access/SettingsProfileElement.h | 12 +--- .../Access/ASTSettingsProfileElement.cpp | 18 ++++- .../Access/ASTSettingsProfileElement.h | 12 +++- src/Storages/System/StorageSystemSettings.cpp | 5 +- .../StorageSystemSettingsProfileElements.cpp | 20 +++--- .../enable_access_control_improvements.xml | 1 + .../helpers/0_common_instance_config.xml | 1 + .../disable_access_control_improvements.xml | 1 + 14 files changed, 121 insertions(+), 108 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index 2ce3fe7754f..00a0608f607 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -640,6 +640,12 @@ executed by any user. You can change this behaviour by setting this to true. If it's set to true then this query requires "GRANT SELECT ON information_schema." just like as for ordinary tables. --> false + + + false diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index c6729459988..81a887dbaa4 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -171,6 +171,7 @@ void AccessControl::setUpFromMainConfig(const Poco::Util::AbstractConfiguration setOnClusterQueriesRequireClusterGrant(config_.getBool("access_control_improvements.on_cluster_queries_require_cluster_grant", false)); setSelectFromSystemDatabaseRequiresGrant(config_.getBool("access_control_improvements.select_from_system_db_requires_grant", false)); setSelectFromInformationSchemaRequiresGrant(config_.getBool("access_control_improvements.select_from_information_schema_requires_grant", false)); + setSettingsConstraintsReplacePrevious(config_.getBool("access_control_improvements.settings_constraints_replace_previous", false)); addStoragesFromMainConfig(config_, config_path_, get_zookeeper_function_); } diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index ab9cdba9ad1..e8a787ada0b 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -158,6 +158,9 @@ public: void setSelectFromInformationSchemaRequiresGrant(bool enable) { select_from_information_schema_requires_grant = enable; } bool doesSelectFromInformationSchemaRequireGrant() const { return select_from_information_schema_requires_grant; } + void setSettingsConstraintsReplacePrevious(bool enable) { settings_constraints_replace_previous = enable; } + bool doesSettingsConstraintsReplacePrevious() const { return settings_constraints_replace_previous; } + std::shared_ptr getContextAccess( const UUID & user_id, const std::vector & current_roles, @@ -223,6 +226,7 @@ private: std::atomic_bool on_cluster_queries_require_cluster_grant = false; std::atomic_bool select_from_system_db_requires_grant = false; std::atomic_bool select_from_information_schema_requires_grant = false; + std::atomic_bool settings_constraints_replace_previous = false; }; } diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index a9264cb4ccd..f49847ad701 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -36,65 +36,53 @@ void SettingsConstraints::clear() } -void SettingsConstraints::constrainMinValue(const String & setting_name, const Field & min_value) +void SettingsConstraints::setMinValue(const String & setting_name, const Field & min_value) { constraints[setting_name].min_value = Settings::castValueUtil(setting_name, min_value); } -void SettingsConstraints::constrainMaxValue(const String & setting_name, const Field & max_value) +void SettingsConstraints::setMaxValue(const String & setting_name, const Field & max_value) { constraints[setting_name].max_value = Settings::castValueUtil(setting_name, max_value); } -void SettingsConstraints::constrainReadOnly(const String & setting_name, bool read_only) +void SettingsConstraints::setIsConst(const String & setting_name, bool is_const) { - constraints[setting_name].read_only = read_only; + constraints[setting_name].is_const = is_const; } -void SettingsConstraints::allowMinValue(const String & setting_name, const Field & min_value) +void SettingsConstraints::setChangableInReadonly(const String & setting_name, bool changeable_in_readonly) { - allowances[setting_name].min_value = Settings::castValueUtil(setting_name, min_value); + constraints[setting_name].changeable_in_readonly = changeable_in_readonly; } -void SettingsConstraints::allowMaxValue(const String & setting_name, const Field & max_value) -{ - allowances[setting_name].max_value = Settings::castValueUtil(setting_name, max_value); -} - -void SettingsConstraints::allowReadOnly(const String & setting_name, bool read_only) -{ - allowances[setting_name].read_only = read_only; -} - -void SettingsConstraints::get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const +void SettingsConstraints::get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, bool & is_const) const { auto range = getRange(current_settings, setting_name); min_value = range.min_value; max_value = range.max_value; - read_only = range.read_only; + is_const = range.is_const; } void SettingsConstraints::merge(const SettingsConstraints & other) { - for (const auto & [other_name, other_constraint] : other.constraints) + if (access_control.doesSettingsConstraintsReplacePrevious()) { - auto & constraint = constraints[other_name]; - if (!other_constraint.min_value.isNull()) - constraint.min_value = other_constraint.min_value; - if (!other_constraint.max_value.isNull()) - constraint.max_value = other_constraint.max_value; - if (other_constraint.read_only) - constraint.read_only = true; + for (const auto & [other_name, other_constraint] : other.constraints) + constraints[other_name] = other_constraint; } - for (const auto & [other_name, other_allowance] : other.allowances) + else { - auto & allowance = allowances[other_name]; - if (!other_allowance.min_value.isNull()) - allowance.min_value = other_allowance.min_value; - if (!other_allowance.max_value.isNull()) - allowance.max_value = other_allowance.max_value; - if (other_allowance.read_only) - allowance.read_only = true; + for (const auto & [other_name, other_constraint] : other.constraints) + { + auto & constraint = constraints[other_name]; + if (!other_constraint.min_value.isNull()) + constraint.min_value = other_constraint.min_value; + if (!other_constraint.max_value.isNull()) + constraint.max_value = other_constraint.max_value; + if (other_constraint.is_const) + constraint.is_const = true; // NOTE: In this mode flag cannot be overriden to be false + } } } @@ -229,7 +217,7 @@ bool SettingsConstraints::Range::check(SettingChange & change, const Field & new return false; } - if (read_only) + if (is_const) { if (reaction == THROW_ON_VIOLATION) throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); @@ -279,36 +267,34 @@ SettingsConstraints::Range SettingsConstraints::getRange(const Settings & curren /** The `readonly` value is understood as follows: * 0 - no read-only restrictions. - * 1 - only read requests, as well as changing explicitly allowed settings. + * 1 - only read requests, as well as changing settings with `changable_in_readonly` flag. * 2 - only read requests, as well as changing settings, except for the `readonly` setting. */ if (current_settings.readonly > 1 && setting_name == "readonly") return Range::forbidden("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY); + auto it = constraints.find(setting_name); if (current_settings.readonly == 1) { - auto it = allowances.find(setting_name); - if (it == allowances.end()) + if (it == constraints.end() || !it->second.changeable_in_readonly) return Range::forbidden("Cannot modify '" + String(setting_name) + "' setting in readonly mode", ErrorCodes::READONLY); - return it->second; } else // For both readonly=0 and readonly=2 { - auto it = constraints.find(setting_name); if (it == constraints.end()) return Range::allowed(); - return it->second; } + return it->second; } bool SettingsConstraints::Range::operator==(const Range & other) const { - return read_only == other.read_only && min_value == other.min_value && max_value == other.max_value; + return is_const == other.is_const && changeable_in_readonly == other.changeable_in_readonly && min_value == other.min_value && max_value == other.max_value; } bool operator ==(const SettingsConstraints & left, const SettingsConstraints & right) { - return left.constraints == right.constraints && left.allowances == right.allowances; + return left.constraints == right.constraints; } } diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index c6cd54a9e68..f9dedb49b68 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -35,15 +35,12 @@ class AccessControl; * 20000000000 * * - * + * * + * + * + * * - * - * - * 200000 - * 10000000000 - * - * * * * @@ -51,7 +48,7 @@ class AccessControl; * If a setting cannot be change due to the read-only mode this class throws an exception. * The value of `readonly` is understood as follows: * 0 - not read-only mode, no additional checks. - * 1 - only read queries, as well as changing explicitly allowed settings. + * 1 - only read queries, as well as changing settings with flag. * 2 - only read queries and you can change the settings, except for the `readonly` setting. * */ @@ -68,14 +65,12 @@ public: void clear(); bool empty() const { return constraints.empty(); } - void constrainMinValue(const String & setting_name, const Field & min_value); - void constrainMaxValue(const String & setting_name, const Field & max_value); - void constrainReadOnly(const String & setting_name, bool read_only); - void allowMinValue(const String & setting_name, const Field & min_value); - void allowMaxValue(const String & setting_name, const Field & max_value); - void allowReadOnly(const String & setting_name, bool read_only); + void setMinValue(const String & setting_name, const Field & min_value); + void setMaxValue(const String & setting_name, const Field & max_value); + void setIsConst(const String & setting_name, bool is_const); + void setChangableInReadonly(const String & setting_name, bool is_const); - void get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const; + void get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, bool & is_const) const; void merge(const SettingsConstraints & other); @@ -99,7 +94,8 @@ private: struct Range { - bool read_only = false; + bool is_const = false; + bool changeable_in_readonly = false; Field min_value; Field max_value; @@ -119,7 +115,7 @@ private: static Range forbidden(const String & explain, int code) { - return Range{.read_only = true, .explain = explain, .code = code}; + return Range{.is_const = true, .explain = explain, .code = code}; } }; @@ -143,7 +139,6 @@ private: // Special container for heterogeneous lookups: to avoid `String` construction during `find(std::string_view)` using RangeMap = std::unordered_map>; RangeMap constraints; - RangeMap allowances; const AccessControl * access_control = nullptr; }; diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 7ff182fd139..7cd5d127815 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -56,7 +56,23 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A value = ast.value; min_value = ast.min_value; max_value = ast.max_value; - readonly = ast.readonly; + changeable_in_readonly = false; + switch (ast.type) + { + case ASTSettingsProfileElement::ConstraintType::NONE: + is_const.reset(); + break; + case ASTSettingsProfileElement::ConstraintType::WRITABLE: + is_const.emplace(false); + break; + case ASTSettingsProfileElement::ConstraintType::CONST: + is_const.emplace(true); + break; + case ASTSettingsProfileElement::ConstraintType::CHANGEABLE_IN_READONLY: + is_const.reset(); + changeable_in_readonly = true; + break; + } if (!value.isNull()) value = Settings::castValueUtil(setting_name, value); @@ -208,25 +224,14 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints(const AccessC { if (!elem.setting_name.empty() && (elem.setting_name != ALLOW_BACKUP_SETTING_NAME)) { - switch (elem.kind) - { - case SettingsProfileElement::RangeKind::Constrain: - if (!elem.min_value.isNull()) - res.constrainMinValue(elem.setting_name, elem.min_value); - if (!elem.max_value.isNull()) - res.constrainMaxValue(elem.setting_name, elem.max_value); - if (elem.readonly) - res.constrainReadOnly(elem.setting_name, *elem.readonly); - break; - case SettingsProfileElement::RangeKind::Allow: - if (!elem.min_value.isNull()) - res.allowMinValue(elem.setting_name, elem.min_value); - if (!elem.max_value.isNull()) - res.allowMaxValue(elem.setting_name, elem.max_value); - if (elem.readonly) - res.allowReadOnly(elem.setting_name, *elem.readonly); - break; - } + if (!elem.min_value.isNull()) + res.setMinValue(elem.setting_name, elem.min_value); + if (!elem.max_value.isNull()) + res.setMaxValue(elem.setting_name, elem.max_value); + if (elem.is_const) + res.setIsConst(elem.setting_name, *elem.is_const); + if (elem.changeable_in_readonly) + res.setChangableInReadonly(elem.setting_name, true); } } return res; diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index d8cd6a3ca81..2d6e5cf7b2f 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -21,20 +21,14 @@ struct SettingsProfileElement { std::optional parent_profile; - enum class RangeKind - { - Constrain = 0, - Allow - }; - String setting_name; Field value; - RangeKind kind = RangeKind::Constrain; Field min_value; Field max_value; - std::optional readonly; + std::optional is_const; + bool changeable_in_readonly; - auto toTuple() const { return std::tie(parent_profile, setting_name, value, kind, min_value, max_value, readonly); } + auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, is_const, changeable_in_readonly); } friend bool operator==(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() == rhs.toTuple(); } friend bool operator!=(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return !(lhs == rhs); } friend bool operator <(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() < rhs.toTuple(); } diff --git a/src/Parsers/Access/ASTSettingsProfileElement.cpp b/src/Parsers/Access/ASTSettingsProfileElement.cpp index 23dba8a926f..12de74ab417 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.cpp +++ b/src/Parsers/Access/ASTSettingsProfileElement.cpp @@ -52,10 +52,22 @@ void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, Form << applyVisitor(FieldVisitorToString{}, max_value); } - if (readonly) + switch (type) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << (*readonly ? " READONLY" : " WRITABLE") - << (settings.hilite ? IAST::hilite_none : ""); + case ConstraintType::NONE: + break; + case ConstraintType::WRITABLE: + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WRITABLE" + << (settings.hilite ? IAST::hilite_none : ""); + break; + case ConstraintType::CONST: + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " READONLY" + << (settings.hilite ? IAST::hilite_none : ""); + break; + case ConstraintType::CHANGEABLE_IN_READONLY: + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CHANGEABLE_IN_READONLY" + << (settings.hilite ? IAST::hilite_none : ""); + break; } } diff --git a/src/Parsers/Access/ASTSettingsProfileElement.h b/src/Parsers/Access/ASTSettingsProfileElement.h index 6a54bca3213..d31f409cbd8 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.h +++ b/src/Parsers/Access/ASTSettingsProfileElement.h @@ -7,7 +7,7 @@ namespace DB { /** Represents a settings profile's element like the following - * {variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE]} | PROFILE 'profile_name' + * {variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY]} | PROFILE 'profile_name' */ class ASTSettingsProfileElement : public IAST { @@ -17,7 +17,13 @@ public: Field value; Field min_value; Field max_value; - std::optional readonly; + enum class ConstraintType { + NONE, // default + WRITABLE, + CONST, // equals READONLY + CHANGEABLE_IN_READONLY, + }; + ConstraintType type; bool id_mode = false; /// If true then `parent_profile` keeps UUID, not a name. bool use_inherit_keyword = false; /// If true then this element is a part of ASTCreateSettingsProfileQuery. @@ -30,7 +36,7 @@ public: /** Represents settings profile's elements like the following - * {{variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE]} | PROFILE 'profile_name'} [,...] + * {{variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY]} | PROFILE 'profile_name'} [,...] */ class ASTSettingsProfileElements : public IAST { diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index a0a8803169b..35c9ea442d4 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -40,8 +40,9 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[3]->insert(setting.getDescription()); Field min, max; - bool read_only = false; - constraints.get(settings, setting_name, min, max, read_only); + bool is_const = false; + bool changeable_in_readonly = false; + constraints.get(settings, setting_name, min, max, is_const, changeable_in_readonly); /// These two columns can accept strings only. if (!min.isNull()) diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp index b1f2ad36999..ea23b052764 100644 --- a/src/Storages/System/StorageSystemSettingsProfileElements.cpp +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -29,7 +29,7 @@ NamesAndTypesList StorageSystemSettingsProfileElements::getNamesAndTypes() {"min", std::make_shared(std::make_shared())}, {"max", std::make_shared(std::make_shared())}, {"readonly", std::make_shared(std::make_shared())}, - {"allowance", std::make_shared(std::make_shared())}, + {"changeable_in_readonly", std::make_shared(std::make_shared())}, {"inherit_profile", std::make_shared(std::make_shared())}, }; return names_and_types; @@ -65,8 +65,8 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns auto & column_max_null_map = assert_cast(*res_columns[i++]).getNullMapData(); auto & column_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()).getData(); auto & column_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); - auto & column_allowance = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()).getData(); - auto & column_allowance_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_changeable_in_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()).getData(); + auto & column_changeable_in_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); auto & column_inherit_profile = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); auto & column_inherit_profile_null_map = assert_cast(*res_columns[i++]).getNullMapData(); @@ -104,23 +104,23 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns } bool inserted_readonly = false; - if (element.readonly && !element.setting_name.empty()) + if (element.is_const && !element.setting_name.empty()) { column_readonly.push_back(*element.readonly); column_readonly_null_map.push_back(false); inserted_readonly = true; } - bool inserted_allowance = false; - if (element.kind == SettingsProfileElement::RangeKind::Allow && !element.setting_name.empty()) + bool inserted_changeable_in_readonly = false; + if (element.changeable_in_readonly && !element.setting_name.empty()) { - column_allowance.push_back(true); - column_allowance_null_map.push_back(false); - inserted_allowance = true; + column_changeable_in_readonly.push_back(true); + column_changeable_in_readonly_null_map.push_back(false); + inserted_changeable_in_readonly = true; } bool inserted_setting_name = false; - if (inserted_value || inserted_min || inserted_max || inserted_readonly || inserted_allowance) + if (inserted_value || inserted_min || inserted_max || inserted_readonly || inserted_changeable_in_readonly) { const auto & setting_name = element.setting_name; column_setting_name.insertData(setting_name.data(), setting_name.size()); diff --git a/tests/config/config.d/enable_access_control_improvements.xml b/tests/config/config.d/enable_access_control_improvements.xml index 5a186548098..564b656a0ad 100644 --- a/tests/config/config.d/enable_access_control_improvements.xml +++ b/tests/config/config.d/enable_access_control_improvements.xml @@ -4,5 +4,6 @@ true true true + true diff --git a/tests/integration/helpers/0_common_instance_config.xml b/tests/integration/helpers/0_common_instance_config.xml index 64f0ce9e361..27563e47c35 100644 --- a/tests/integration/helpers/0_common_instance_config.xml +++ b/tests/integration/helpers/0_common_instance_config.xml @@ -24,5 +24,6 @@ true true true + true diff --git a/tests/integration/test_disabled_access_control_improvements/configs/config.d/disable_access_control_improvements.xml b/tests/integration/test_disabled_access_control_improvements/configs/config.d/disable_access_control_improvements.xml index 7969c638fd7..a335c7f8a1f 100644 --- a/tests/integration/test_disabled_access_control_improvements/configs/config.d/disable_access_control_improvements.xml +++ b/tests/integration/test_disabled_access_control_improvements/configs/config.d/disable_access_control_improvements.xml @@ -3,5 +3,6 @@ + From 014d109175cfb76ee846ad2bc9bd03a02b54c75f Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Fri, 2 Sep 2022 16:20:09 +0200 Subject: [PATCH 06/86] fix build, fix docs, fix comments, logical fixes, test are still to be fixed and new test are to be added --- .../settings/constraints-on-settings.md | 54 +++++++------------ programs/server/config.xml | 3 +- src/Access/SettingsConstraints.cpp | 2 +- src/Access/SettingsConstraints.h | 2 +- src/Access/SettingsProfileElement.cpp | 16 +++++- src/Access/UsersConfigAccessStorage.cpp | 26 ++++----- .../Access/ParserSettingsProfileElement.cpp | 29 +++++----- src/Storages/System/StorageSystemSettings.cpp | 5 +- .../StorageSystemSettingsProfileElements.cpp | 2 +- 9 files changed, 72 insertions(+), 67 deletions(-) diff --git a/docs/en/operations/settings/constraints-on-settings.md b/docs/en/operations/settings/constraints-on-settings.md index 08966db0ad6..651b6465f7e 100644 --- a/docs/en/operations/settings/constraints-on-settings.md +++ b/docs/en/operations/settings/constraints-on-settings.md @@ -26,15 +26,33 @@ The constraints are defined as the following: + + lower_boundary + upper_boundary + + ``` If the user tries to violate the constraints an exception is thrown and the setting isn’t changed. -There are supported three types of constraints: `min`, `max`, `readonly`. The `min` and `max` constraints specify upper and lower boundaries for a numeric setting and can be used in combination. The `readonly` constraint specifies that the user cannot change the corresponding setting at all. +There are supported few types of constraints: `min`, `max`, `readonly` (with alias `const`) and `changeable_in_readonly`. The `min` and `max` constraints specify upper and lower boundaries for a numeric setting and can be used in combination. The `readonly` or `const` constraint specifies that the user cannot change the corresponding setting at all. The `changeable_in_readonly` constraint type allows user to change the setting within `min`/`max` range even if `readonly` setting is set to 1, otherwise settings are not allow to be changed in `readonly=1` mode. Note that `changeable_in_readonly` is supported only if `settings_constraints_replace_previous` is enabled: +``` xml + + true + +``` + +If there are multiple profiles active for a user, then constraints are merged. Merge process depends on `settings_constraints_replace_previous`: +- **true** (recommended): constraints for the same setting are replaced during merge, such that the last constraint is used and all previous are ignored including fields that are not set in new constraint. +- **false** (default): constraints for the same setting are merged in a way that every not set type of constraint is taken from previous profile and every set type of constraint is replaced by value from new profile. + +Read-only mode is enabled by `readonly` setting (not to confuse with `readonly` constraint type): +- `readonly=0`: No read-only restrictions. +- `readonly=1`: Only read queries are allowed and settings cannot be changes unless `changeable_in_readonly` is set. +- `readonly=2`: Only read queries are allowed, but settings can be changed, except for `readonly` setting itself. -If there are multiple profiles active for a user, then constraints are merged. Constraints for the same setting are replaced during merge, such that the last constraint is used and all previous are ignored. **Example:** Let `users.xml` includes lines: @@ -71,38 +89,6 @@ Code: 452, e.displayText() = DB::Exception: Setting max_memory_usage should not Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should not be changed. ``` -## Allowances in read-only mode {#settings-readonly-allowance} -Read-only mode is enabled by `readonly` setting: -- `readonly=0`: No read-only restrictions. -- `readonly=1`: Only read queries are allowed and settings cannot be changes unless explicitly allowed. -- `readonly=2`: Only read queries are allowed, but settings can be changed, except for `readonly` setting itself. - -In `readonly=0` and `readonly=2` modes settings constraints are applied as usual. But in `readonly=1` by default all settings changes are forbidden and constraints are ignored. But special allowances can be set in the following way: -``` xml - - - - - lower_boundary - - - upper_boundary - - - lower_boundary - upper_boundary - - - - - - - - -``` - -Constraints and allowances are merged from multiple profiles independently. Previous constraint is not canceled by new allowance, as well as previous allowance is not canceled by new constraint. Instead profile can have one allowance and one constraint for every setting. Allowances are used in `readonly=1` mode, otherwise constraints are used. - **Note:** the `default` profile has special handling: all the constraints defined for the `default` profile become the default constraints, so they restrict all the users until they’re overridden explicitly for these users. [Original article](https://clickhouse.com/docs/en/operations/settings/constraints_on_settings/) diff --git a/programs/server/config.xml b/programs/server/config.xml index 00a0608f607..0c58c9777d1 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -644,7 +644,8 @@ + actions of previous constraint (defined in other profiles) for the same specific setting, including fields that are not set by new constraint. + Also it enables 'changeable_in_readonly' constraint type --> false diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index f49847ad701..51065cef062 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -66,7 +66,7 @@ void SettingsConstraints::get(const Settings & current_settings, std::string_vie void SettingsConstraints::merge(const SettingsConstraints & other) { - if (access_control.doesSettingsConstraintsReplacePrevious()) + if (access_control->doesSettingsConstraintsReplacePrevious()) { for (const auto & [other_name, other_constraint] : other.constraints) constraints[other_name] = other_constraint; diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index f9dedb49b68..5a90d7e698c 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -140,7 +140,7 @@ private: using RangeMap = std::unordered_map>; RangeMap constraints; - const AccessControl * access_control = nullptr; + const AccessControl * access_control; }; } diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 7cd5d127815..0eeb0712161 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -69,6 +69,8 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A is_const.emplace(true); break; case ASTSettingsProfileElement::ConstraintType::CHANGEABLE_IN_READONLY: + if (!access_control->doesSettingsConstraintsReplacePrevious()) + throw Exception("CHANGEABLE_IN_READONLY for " + setting_name + " is not allowed unless settings_constraints_replace_previous is enabled", ErrorCodes::NOT_IMPLEMENTED); is_const.reset(); changeable_in_readonly = true; break; @@ -96,7 +98,12 @@ std::shared_ptr SettingsProfileElement::toAST() const ast->value = value; ast->min_value = min_value; ast->max_value = max_value; - ast->readonly = readonly; + if (changeable_in_readonly) + ast->type = ASTSettingsProfileElement::ConstraintType::CHANGEABLE_IN_READONLY; + if (is_const.has_value()) + ast->type = *is_const ? ASTSettingsProfileElement::ConstraintType::CONST : ASTSettingsProfileElement::ConstraintType::WRITABLE; + else + ast->type = ASTSettingsProfileElement::ConstraintType::NONE; return ast; } @@ -117,7 +124,12 @@ std::shared_ptr SettingsProfileElement::toASTWithName ast->value = value; ast->min_value = min_value; ast->max_value = max_value; - ast->readonly = readonly; + if (changeable_in_readonly) + ast->type = ASTSettingsProfileElement::ConstraintType::CHANGEABLE_IN_READONLY; + if (is_const.has_value()) + ast->type = *is_const ? ASTSettingsProfileElement::ConstraintType::CONST : ASTSettingsProfileElement::ConstraintType::WRITABLE; + else + ast->type = ASTSettingsProfileElement::ConstraintType::NONE; return ast; } diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 6ffd9bcb681..b3280ac1fd3 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -425,8 +425,7 @@ namespace SettingsProfileElements parseSettingsConstraints(const Poco::Util::AbstractConfiguration & config, const String & path_to_constraints, - const AccessControl & access_control, - SettingsProfileElement::RangeKind kind) + const AccessControl & access_control) { SettingsProfileElements profile_elements; Poco::Util::AbstractConfiguration::Keys keys; @@ -438,7 +437,6 @@ namespace SettingsProfileElement profile_element; profile_element.setting_name = setting_name; - profile_element.kind = kind; Poco::Util::AbstractConfiguration::Keys constraint_types; String path_to_name = path_to_constraints + "." + setting_name; config.keys(path_to_name, constraint_types); @@ -449,11 +447,21 @@ namespace profile_element.min_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); else if (constraint_type == "max") profile_element.max_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); - else if (constraint_type == "readonly") - profile_element.readonly = true; + else if (constraint_type == "readonly" || constraint_type == "const") + profile_element.is_const = true; + else if (constraint_type == "changeable_in_readonly") + { + if (access_control.doesSettingsConstraintsReplacePrevious()) + profile_element.changeable_in_readonly = true; + else + throw Exception("Setting changeable_in_readonly for " + setting_name + " is not allowed unless settings_constraints_replace_previous is enabled", ErrorCodes::NOT_IMPLEMENTED); + } else throw Exception("Setting " + constraint_type + " value for " + setting_name + " isn't supported", ErrorCodes::NOT_IMPLEMENTED); } + if (profile_element.is_const && profile_element.changeable_in_readonly) + throw Exception("Both settings changeable_in_readonly and const/readonly cannot be used for " + setting_name, ErrorCodes::NOT_IMPLEMENTED); + profile_elements.push_back(std::move(profile_element)); } @@ -489,13 +497,7 @@ namespace if (key == "constraints" || key.starts_with("constraints[")) { - profile->elements.merge(parseSettingsConstraints(config, profile_config + "." + key, access_control, SettingsProfileElement::RangeKind::Constrain)); - continue; - } - - if (key == "allow" || key.starts_with("allow[")) - { - profile->elements.merge(parseSettingsConstraints(config, profile_config + "." + key, access_control, SettingsProfileElement::RangeKind::Allow)); + profile->elements.merge(parseSettingsConstraints(config, profile_config + "." + key, access_control)); continue; } diff --git a/src/Parsers/Access/ParserSettingsProfileElement.cpp b/src/Parsers/Access/ParserSettingsProfileElement.cpp index f4ca52dea2a..7c39c1fbb7b 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.cpp +++ b/src/Parsers/Access/ParserSettingsProfileElement.cpp @@ -95,18 +95,23 @@ namespace } - bool parseReadonlyOrWritableKeyword(IParserBase::Pos & pos, Expected & expected, std::optional & readonly) + bool parseConstraintTypeKeyword(IParserBase::Pos & pos, Expected & expected, ASTSettingsProfileElement::ConstraintType & type) { return IParserBase::wrapParseImpl(pos, [&] { - if (ParserKeyword{"READONLY"}.ignore(pos, expected)) + if (ParserKeyword{"READONLY"}.ignore(pos, expected) || ParserKeyword{"CONST"}.ignore(pos, expected)) { - readonly = true; + type = ASTSettingsProfileElement::ConstraintType::CONST; return true; } else if (ParserKeyword{"WRITABLE"}.ignore(pos, expected)) { - readonly = false; + type = ASTSettingsProfileElement::ConstraintType::WRITABLE; + return true; + } + else if (ParserKeyword{"CHANGEABLE_IN_READONLY"}.ignore(pos, expected)) + { + type = ASTSettingsProfileElement::ConstraintType::CHANGEABLE_IN_READONLY; return true; } else @@ -122,7 +127,7 @@ namespace Field & value, Field & min_value, Field & max_value, - std::optional & readonly) + ASTSettingsProfileElement::ConstraintType & type) { return IParserBase::wrapParseImpl(pos, [&] { @@ -134,11 +139,11 @@ namespace Field res_value; Field res_min_value; Field res_max_value; - std::optional res_readonly; + ASTSettingsProfileElement::ConstraintType res_type; bool has_value_or_constraint = false; while (parseValue(pos, expected, res_value) || parseMinMaxValue(pos, expected, res_min_value, res_max_value) - || parseReadonlyOrWritableKeyword(pos, expected, res_readonly)) + || parseConstraintTypeKeyword(pos, expected, res_type)) { has_value_or_constraint = true; } @@ -147,7 +152,7 @@ namespace return false; if (boost::iequals(res_setting_name, "PROFILE") && res_value.isNull() && res_min_value.isNull() && res_max_value.isNull() - && res_readonly) + && res_type == ASTSettingsProfileElement::ConstraintType::CONST) { /// Ambiguity: "profile readonly" can be treated either as a profile named "readonly" or /// as a setting named 'profile' with the readonly constraint. @@ -159,7 +164,7 @@ namespace value = std::move(res_value); min_value = std::move(res_min_value); max_value = std::move(res_max_value); - readonly = res_readonly; + type = res_type; return true; }); } @@ -179,9 +184,9 @@ namespace Field value; Field min_value; Field max_value; - std::optional readonly; + ASTSettingsProfileElement::ConstraintType type; - bool ok = parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, readonly); + bool ok = parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, type); if (!ok && (parseProfileKeyword(pos, expected, use_inherit_keyword) || previous_element_was_parent_profile)) ok = parseProfileNameOrID(pos, expected, id_mode, parent_profile); @@ -195,7 +200,7 @@ namespace result->value = std::move(value); result->min_value = std::move(min_value); result->max_value = std::move(max_value); - result->readonly = readonly; + result->type = type; result->id_mode = id_mode; result->use_inherit_keyword = use_inherit_keyword; return true; diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index 35c9ea442d4..858d1973e25 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -41,8 +41,7 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co Field min, max; bool is_const = false; - bool changeable_in_readonly = false; - constraints.get(settings, setting_name, min, max, is_const, changeable_in_readonly); + constraints.get(settings, setting_name, min, max, is_const); /// These two columns can accept strings only. if (!min.isNull()) @@ -52,7 +51,7 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[4]->insert(min); res_columns[5]->insert(max); - res_columns[6]->insert(read_only); + res_columns[6]->insert(is_const); res_columns[7]->insert(setting.getTypeName()); } } diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp index ea23b052764..3a79aa632d0 100644 --- a/src/Storages/System/StorageSystemSettingsProfileElements.cpp +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -106,7 +106,7 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns bool inserted_readonly = false; if (element.is_const && !element.setting_name.empty()) { - column_readonly.push_back(*element.readonly); + column_readonly.push_back(*element.is_const); column_readonly_null_map.push_back(false); inserted_readonly = true; } From a6a95e38ca9008f75df748c31094e7fc26dcb975 Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Fri, 2 Sep 2022 18:20:47 +0200 Subject: [PATCH 07/86] fix style --- src/Access/SettingsProfileElement.cpp | 4 ++++ src/Parsers/Access/ASTSettingsProfileElement.h | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 0eeb0712161..a0efe14090a 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -18,6 +18,10 @@ namespace constexpr const char ALLOW_BACKUP_SETTING_NAME[] = "allow_backup"; } +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} SettingsProfileElement::SettingsProfileElement(const ASTSettingsProfileElement & ast) { diff --git a/src/Parsers/Access/ASTSettingsProfileElement.h b/src/Parsers/Access/ASTSettingsProfileElement.h index d31f409cbd8..5ae5316c318 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.h +++ b/src/Parsers/Access/ASTSettingsProfileElement.h @@ -17,7 +17,8 @@ public: Field value; Field min_value; Field max_value; - enum class ConstraintType { + enum class ConstraintType + { NONE, // default WRITABLE, CONST, // equals READONLY From 7a2d750f7f6e9ad0ac07f4fbdc213fbce05da2f2 Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Fri, 2 Sep 2022 18:59:16 +0200 Subject: [PATCH 08/86] fix typo --- src/Access/SettingsConstraints.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 51065cef062..5d99aadb1e3 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -81,7 +81,7 @@ void SettingsConstraints::merge(const SettingsConstraints & other) if (!other_constraint.max_value.isNull()) constraint.max_value = other_constraint.max_value; if (other_constraint.is_const) - constraint.is_const = true; // NOTE: In this mode flag cannot be overriden to be false + constraint.is_const = true; // NOTE: In this mode flag cannot be overridden to be false } } } From 77ee4c04aa017271e7deceb305413bdfceceae03 Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Tue, 6 Sep 2022 20:28:10 +0200 Subject: [PATCH 09/86] fix stateless tests --- programs/server/config.xml | 4 +- src/Access/SettingsConstraints.cpp | 41 ++++++--------- src/Access/SettingsConstraints.h | 15 ++---- src/Access/SettingsProfileElement.cpp | 52 +++---------------- src/Access/SettingsProfileElement.h | 6 +-- src/Access/UsersConfigAccessStorage.cpp | 13 +++-- src/Common/SettingConstraintType.h | 26 ++++++++++ .../Access/ASTSettingsProfileElement.cpp | 8 +-- .../Access/ASTSettingsProfileElement.h | 11 +--- .../Access/ParserSettingsProfileElement.cpp | 16 +++--- .../Access/ParserSettingsProfileElement.h | 2 +- src/Storages/System/StorageSystemSettings.cpp | 6 +-- .../StorageSystemSettingsProfileElements.cpp | 6 +-- .../02117_show_create_table_system.reference | 2 +- 14 files changed, 91 insertions(+), 117 deletions(-) create mode 100644 src/Common/SettingConstraintType.h diff --git a/programs/server/config.xml b/programs/server/config.xml index cdc81d91024..a11dc7ea83f 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -66,7 +66,7 @@ For example, as below: {"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"} To enable JSON logging support, please uncomment the entire tag below. - + a) You can modify key names by changing values under tag values inside tag. For example, to change DATE_TIME to MY_DATE_TIME, you can do like: MY_DATE_TIME @@ -667,7 +667,7 @@ previous profile. You can change this behaviour by setting this to true. If it's set to true then if settings profile has a constraint for a specific setting, then this constraint completely cancels all actions of previous constraint (defined in other profiles) for the same specific setting, including fields that are not set by new constraint. - Also it enables 'changeable_in_readonly' constraint type --> + It also enables 'changeable_in_readonly' constraint type --> false diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 5d99aadb1e3..26e5cc930d7 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -35,33 +35,24 @@ void SettingsConstraints::clear() constraints.clear(); } - -void SettingsConstraints::setMinValue(const String & setting_name, const Field & min_value) +void SettingsConstraints::set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintType type) { - constraints[setting_name].min_value = Settings::castValueUtil(setting_name, min_value); + if (min_value.isNull() && max_value.isNull() && type == SettingConstraintType::NONE) + return; // Do not even create entry to avoid issues during profile inheritance + auto & constraint = constraints[setting_name]; + if (!min_value.isNull()) + constraint.min_value = Settings::castValueUtil(setting_name, min_value); + if (!max_value.isNull()) + constraint.max_value = Settings::castValueUtil(setting_name, max_value); + constraint.type = type; } -void SettingsConstraints::setMaxValue(const String & setting_name, const Field & max_value) -{ - constraints[setting_name].max_value = Settings::castValueUtil(setting_name, max_value); -} - -void SettingsConstraints::setIsConst(const String & setting_name, bool is_const) -{ - constraints[setting_name].is_const = is_const; -} - -void SettingsConstraints::setChangableInReadonly(const String & setting_name, bool changeable_in_readonly) -{ - constraints[setting_name].changeable_in_readonly = changeable_in_readonly; -} - -void SettingsConstraints::get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, bool & is_const) const +void SettingsConstraints::get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintType & type) const { auto range = getRange(current_settings, setting_name); min_value = range.min_value; max_value = range.max_value; - is_const = range.is_const; + type = range.type; } void SettingsConstraints::merge(const SettingsConstraints & other) @@ -80,8 +71,8 @@ void SettingsConstraints::merge(const SettingsConstraints & other) constraint.min_value = other_constraint.min_value; if (!other_constraint.max_value.isNull()) constraint.max_value = other_constraint.max_value; - if (other_constraint.is_const) - constraint.is_const = true; // NOTE: In this mode flag cannot be overridden to be false + if (other_constraint.type == SettingConstraintType::CONST) + constraint.type = SettingConstraintType::CONST; // NOTE: In this mode flag cannot be overridden to be false } } } @@ -217,7 +208,7 @@ bool SettingsConstraints::Range::check(SettingChange & change, const Field & new return false; } - if (is_const) + if (type == SettingConstraintType::CONST) { if (reaction == THROW_ON_VIOLATION) throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); @@ -277,7 +268,7 @@ SettingsConstraints::Range SettingsConstraints::getRange(const Settings & curren auto it = constraints.find(setting_name); if (current_settings.readonly == 1) { - if (it == constraints.end() || !it->second.changeable_in_readonly) + if (it == constraints.end() || it->second.type != SettingConstraintType::CHANGEABLE_IN_READONLY) return Range::forbidden("Cannot modify '" + String(setting_name) + "' setting in readonly mode", ErrorCodes::READONLY); } else // For both readonly=0 and readonly=2 @@ -290,7 +281,7 @@ SettingsConstraints::Range SettingsConstraints::getRange(const Settings & curren bool SettingsConstraints::Range::operator==(const Range & other) const { - return is_const == other.is_const && changeable_in_readonly == other.changeable_in_readonly && min_value == other.min_value && max_value == other.max_value; + return type == other.type && min_value == other.min_value && max_value == other.max_value; } bool operator ==(const SettingsConstraints & left, const SettingsConstraints & right) diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 5a90d7e698c..e94db6ad705 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -1,9 +1,9 @@ #pragma once +#include #include #include - namespace Poco::Util { class AbstractConfiguration; @@ -65,12 +65,8 @@ public: void clear(); bool empty() const { return constraints.empty(); } - void setMinValue(const String & setting_name, const Field & min_value); - void setMaxValue(const String & setting_name, const Field & max_value); - void setIsConst(const String & setting_name, bool is_const); - void setChangableInReadonly(const String & setting_name, bool is_const); - - void get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, bool & is_const) const; + void set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintType type); + void get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintType & type) const; void merge(const SettingsConstraints & other); @@ -94,8 +90,7 @@ private: struct Range { - bool is_const = false; - bool changeable_in_readonly = false; + SettingConstraintType type = SettingConstraintType::NONE; Field min_value; Field max_value; @@ -115,7 +110,7 @@ private: static Range forbidden(const String & explain, int code) { - return Range{.is_const = true, .explain = explain, .code = code}; + return Range{.type = SettingConstraintType::CONST, .explain = explain, .code = code}; } }; diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index a0efe14090a..6851169db5e 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -60,25 +60,10 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A value = ast.value; min_value = ast.min_value; max_value = ast.max_value; - changeable_in_readonly = false; - switch (ast.type) - { - case ASTSettingsProfileElement::ConstraintType::NONE: - is_const.reset(); - break; - case ASTSettingsProfileElement::ConstraintType::WRITABLE: - is_const.emplace(false); - break; - case ASTSettingsProfileElement::ConstraintType::CONST: - is_const.emplace(true); - break; - case ASTSettingsProfileElement::ConstraintType::CHANGEABLE_IN_READONLY: - if (!access_control->doesSettingsConstraintsReplacePrevious()) - throw Exception("CHANGEABLE_IN_READONLY for " + setting_name + " is not allowed unless settings_constraints_replace_previous is enabled", ErrorCodes::NOT_IMPLEMENTED); - is_const.reset(); - changeable_in_readonly = true; - break; - } + type = ast.type; + + if (type == SettingConstraintType::CHANGEABLE_IN_READONLY && !access_control->doesSettingsConstraintsReplacePrevious()) + throw Exception("CHANGEABLE_IN_READONLY for " + setting_name + " is not allowed unless settings_constraints_replace_previous is enabled", ErrorCodes::NOT_IMPLEMENTED); if (!value.isNull()) value = Settings::castValueUtil(setting_name, value); @@ -102,12 +87,7 @@ std::shared_ptr SettingsProfileElement::toAST() const ast->value = value; ast->min_value = min_value; ast->max_value = max_value; - if (changeable_in_readonly) - ast->type = ASTSettingsProfileElement::ConstraintType::CHANGEABLE_IN_READONLY; - if (is_const.has_value()) - ast->type = *is_const ? ASTSettingsProfileElement::ConstraintType::CONST : ASTSettingsProfileElement::ConstraintType::WRITABLE; - else - ast->type = ASTSettingsProfileElement::ConstraintType::NONE; + ast->type = type; return ast; } @@ -128,12 +108,7 @@ std::shared_ptr SettingsProfileElement::toASTWithName ast->value = value; ast->min_value = min_value; ast->max_value = max_value; - if (changeable_in_readonly) - ast->type = ASTSettingsProfileElement::ConstraintType::CHANGEABLE_IN_READONLY; - if (is_const.has_value()) - ast->type = *is_const ? ASTSettingsProfileElement::ConstraintType::CONST : ASTSettingsProfileElement::ConstraintType::WRITABLE; - else - ast->type = ASTSettingsProfileElement::ConstraintType::NONE; + ast->type = type; return ast; } @@ -237,19 +212,8 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints(const AccessC { SettingsConstraints res{access_control}; for (const auto & elem : *this) - { - if (!elem.setting_name.empty() && (elem.setting_name != ALLOW_BACKUP_SETTING_NAME)) - { - if (!elem.min_value.isNull()) - res.setMinValue(elem.setting_name, elem.min_value); - if (!elem.max_value.isNull()) - res.setMaxValue(elem.setting_name, elem.max_value); - if (elem.is_const) - res.setIsConst(elem.setting_name, *elem.is_const); - if (elem.changeable_in_readonly) - res.setChangableInReadonly(elem.setting_name, true); - } - } + if (!elem.setting_name.empty() && elem.setting_name != ALLOW_BACKUP_SETTING_NAME) + res.set(elem.setting_name, elem.min_value, elem.max_value, elem.type); return res; } diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index 2d6e5cf7b2f..2d0444a1589 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -25,10 +26,9 @@ struct SettingsProfileElement Field value; Field min_value; Field max_value; - std::optional is_const; - bool changeable_in_readonly; + SettingConstraintType type = SettingConstraintType::NONE; - auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, is_const, changeable_in_readonly); } + auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, type); } friend bool operator==(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() == rhs.toTuple(); } friend bool operator!=(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return !(lhs == rhs); } friend bool operator <(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() < rhs.toTuple(); } diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index b3280ac1fd3..4adbfeef98b 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -441,6 +441,7 @@ namespace String path_to_name = path_to_constraints + "." + setting_name; config.keys(path_to_name, constraint_types); + size_t type_specifiers_count = 0; for (const String & constraint_type : constraint_types) { if (constraint_type == "min") @@ -448,19 +449,23 @@ namespace else if (constraint_type == "max") profile_element.max_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); else if (constraint_type == "readonly" || constraint_type == "const") - profile_element.is_const = true; + { + type_specifiers_count++; + profile_element.type = SettingConstraintType::CONST; + } else if (constraint_type == "changeable_in_readonly") { + type_specifiers_count++; if (access_control.doesSettingsConstraintsReplacePrevious()) - profile_element.changeable_in_readonly = true; + profile_element.type = SettingConstraintType::CHANGEABLE_IN_READONLY; else throw Exception("Setting changeable_in_readonly for " + setting_name + " is not allowed unless settings_constraints_replace_previous is enabled", ErrorCodes::NOT_IMPLEMENTED); } else throw Exception("Setting " + constraint_type + " value for " + setting_name + " isn't supported", ErrorCodes::NOT_IMPLEMENTED); } - if (profile_element.is_const && profile_element.changeable_in_readonly) - throw Exception("Both settings changeable_in_readonly and const/readonly cannot be used for " + setting_name, ErrorCodes::NOT_IMPLEMENTED); + if (type_specifiers_count > 1) + throw Exception("Not more than one constraint type specifier (const/readonly/changeable_in_readonly) is allowed for " + setting_name, ErrorCodes::NOT_IMPLEMENTED); profile_elements.push_back(std::move(profile_element)); } diff --git a/src/Common/SettingConstraintType.h b/src/Common/SettingConstraintType.h new file mode 100644 index 00000000000..8a515c8dfb8 --- /dev/null +++ b/src/Common/SettingConstraintType.h @@ -0,0 +1,26 @@ +#pragma once + +#include + + +namespace DB +{ + +enum class SettingConstraintType +{ + // Default. Behave in the same way as WRITABLE, but is not inherited unless `settings_constraints_replace_previous` is set. + NONE, + + // Setting can be change within specified range only in `readonly=0` or `readonly=2` mode. + WRITABLE, + + // Setting cannot be changed at all. + // Either READONLY or CONST keyword in SQL syntax can be used ( or in config.xml) to enable this. + // NOTE: name `CONST` is choosen to avoid confusion with `readonly` setting. + CONST, + + // Setting can be changed within specified range, regardless of `readonly` setting value. + CHANGEABLE_IN_READONLY, +}; + +} diff --git a/src/Parsers/Access/ASTSettingsProfileElement.cpp b/src/Parsers/Access/ASTSettingsProfileElement.cpp index 12de74ab417..85ec5bc9233 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.cpp +++ b/src/Parsers/Access/ASTSettingsProfileElement.cpp @@ -54,17 +54,17 @@ void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, Form switch (type) { - case ConstraintType::NONE: + case SettingConstraintType::NONE: break; - case ConstraintType::WRITABLE: + case SettingConstraintType::WRITABLE: settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WRITABLE" << (settings.hilite ? IAST::hilite_none : ""); break; - case ConstraintType::CONST: + case SettingConstraintType::CONST: settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " READONLY" << (settings.hilite ? IAST::hilite_none : ""); break; - case ConstraintType::CHANGEABLE_IN_READONLY: + case SettingConstraintType::CHANGEABLE_IN_READONLY: settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CHANGEABLE_IN_READONLY" << (settings.hilite ? IAST::hilite_none : ""); break; diff --git a/src/Parsers/Access/ASTSettingsProfileElement.h b/src/Parsers/Access/ASTSettingsProfileElement.h index 5ae5316c318..c3816c65958 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.h +++ b/src/Parsers/Access/ASTSettingsProfileElement.h @@ -2,7 +2,7 @@ #include #include - +#include namespace DB { @@ -17,14 +17,7 @@ public: Field value; Field min_value; Field max_value; - enum class ConstraintType - { - NONE, // default - WRITABLE, - CONST, // equals READONLY - CHANGEABLE_IN_READONLY, - }; - ConstraintType type; + SettingConstraintType type = SettingConstraintType::NONE; bool id_mode = false; /// If true then `parent_profile` keeps UUID, not a name. bool use_inherit_keyword = false; /// If true then this element is a part of ASTCreateSettingsProfileQuery. diff --git a/src/Parsers/Access/ParserSettingsProfileElement.cpp b/src/Parsers/Access/ParserSettingsProfileElement.cpp index 7c39c1fbb7b..c7b73a78d0c 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.cpp +++ b/src/Parsers/Access/ParserSettingsProfileElement.cpp @@ -95,23 +95,23 @@ namespace } - bool parseConstraintTypeKeyword(IParserBase::Pos & pos, Expected & expected, ASTSettingsProfileElement::ConstraintType & type) + bool parseConstraintTypeKeyword(IParserBase::Pos & pos, Expected & expected, SettingConstraintType & type) { return IParserBase::wrapParseImpl(pos, [&] { if (ParserKeyword{"READONLY"}.ignore(pos, expected) || ParserKeyword{"CONST"}.ignore(pos, expected)) { - type = ASTSettingsProfileElement::ConstraintType::CONST; + type = SettingConstraintType::CONST; return true; } else if (ParserKeyword{"WRITABLE"}.ignore(pos, expected)) { - type = ASTSettingsProfileElement::ConstraintType::WRITABLE; + type = SettingConstraintType::WRITABLE; return true; } else if (ParserKeyword{"CHANGEABLE_IN_READONLY"}.ignore(pos, expected)) { - type = ASTSettingsProfileElement::ConstraintType::CHANGEABLE_IN_READONLY; + type = SettingConstraintType::CHANGEABLE_IN_READONLY; return true; } else @@ -127,7 +127,7 @@ namespace Field & value, Field & min_value, Field & max_value, - ASTSettingsProfileElement::ConstraintType & type) + SettingConstraintType & type) { return IParserBase::wrapParseImpl(pos, [&] { @@ -139,7 +139,7 @@ namespace Field res_value; Field res_min_value; Field res_max_value; - ASTSettingsProfileElement::ConstraintType res_type; + SettingConstraintType res_type; bool has_value_or_constraint = false; while (parseValue(pos, expected, res_value) || parseMinMaxValue(pos, expected, res_min_value, res_max_value) @@ -152,7 +152,7 @@ namespace return false; if (boost::iequals(res_setting_name, "PROFILE") && res_value.isNull() && res_min_value.isNull() && res_max_value.isNull() - && res_type == ASTSettingsProfileElement::ConstraintType::CONST) + && res_type == SettingConstraintType::CONST) { /// Ambiguity: "profile readonly" can be treated either as a profile named "readonly" or /// as a setting named 'profile' with the readonly constraint. @@ -184,7 +184,7 @@ namespace Field value; Field min_value; Field max_value; - ASTSettingsProfileElement::ConstraintType type; + SettingConstraintType type = SettingConstraintType::NONE; bool ok = parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, type); diff --git a/src/Parsers/Access/ParserSettingsProfileElement.h b/src/Parsers/Access/ParserSettingsProfileElement.h index 8843591a56c..082fc66625f 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.h +++ b/src/Parsers/Access/ParserSettingsProfileElement.h @@ -6,7 +6,7 @@ namespace DB { /** Parses a string like this: - * {variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE]} | PROFILE 'profile_name' + * {variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY]} | PROFILE 'profile_name' */ class ParserSettingsProfileElement : public IParserBase { diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index 858d1973e25..33f1c4cc6b1 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -40,8 +40,8 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[3]->insert(setting.getDescription()); Field min, max; - bool is_const = false; - constraints.get(settings, setting_name, min, max, is_const); + SettingConstraintType type; + constraints.get(settings, setting_name, min, max, type); /// These two columns can accept strings only. if (!min.isNull()) @@ -51,7 +51,7 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[4]->insert(min); res_columns[5]->insert(max); - res_columns[6]->insert(is_const); + res_columns[6]->insert(type == SettingConstraintType::CONST); res_columns[7]->insert(setting.getTypeName()); } } diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp index 3a79aa632d0..7e83b75c468 100644 --- a/src/Storages/System/StorageSystemSettingsProfileElements.cpp +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -104,15 +104,15 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns } bool inserted_readonly = false; - if (element.is_const && !element.setting_name.empty()) + if ((element.type == SettingConstraintType::CONST || element.type == SettingConstraintType::WRITABLE) && !element.setting_name.empty()) { - column_readonly.push_back(*element.is_const); + column_readonly.push_back(element.type == SettingConstraintType::CONST); column_readonly_null_map.push_back(false); inserted_readonly = true; } bool inserted_changeable_in_readonly = false; - if (element.changeable_in_readonly && !element.setting_name.empty()) + if (element.type == SettingConstraintType::CHANGEABLE_IN_READONLY && !element.setting_name.empty()) { column_changeable_in_readonly.push_back(true); column_changeable_in_readonly_null_map.push_back(false); diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index dd2f6651d5c..2cfd99ec9f4 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -969,7 +969,7 @@ CREATE TABLE system.settings_profile_elements `min` Nullable(String), `max` Nullable(String), `readonly` Nullable(UInt8), - `allowance` Nullable(UInt8), + `changeable_in_readonly` Nullable(UInt8), `inherit_profile` Nullable(String) ) ENGINE = SystemSettingsProfileElements From 35524d0175cc861c32cc88f17988e51a9ede2159 Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Tue, 6 Sep 2022 20:51:33 +0200 Subject: [PATCH 10/86] typo --- src/Common/SettingConstraintType.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/SettingConstraintType.h b/src/Common/SettingConstraintType.h index 8a515c8dfb8..9302a598fbf 100644 --- a/src/Common/SettingConstraintType.h +++ b/src/Common/SettingConstraintType.h @@ -16,7 +16,7 @@ enum class SettingConstraintType // Setting cannot be changed at all. // Either READONLY or CONST keyword in SQL syntax can be used ( or in config.xml) to enable this. - // NOTE: name `CONST` is choosen to avoid confusion with `readonly` setting. + // NOTE: name `CONST` is chosen to avoid confusion with `readonly` setting. CONST, // Setting can be changed within specified range, regardless of `readonly` setting value. From 9da22f7b7cf20a7b0472d58438c818256b9ff1e2 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 7 Sep 2022 15:06:56 +0000 Subject: [PATCH 11/86] Fix possible use-heap-after-free in -Map combinator --- src/AggregateFunctions/AggregateFunctionMap.h | 5 ++++ src/Interpreters/AggregationUtils.cpp | 30 ++++++++++++------- .../02418_map_combinator_bug.reference | 0 .../0_stateless/02418_map_combinator_bug.sql | 11 +++++++ 4 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/02418_map_combinator_bug.reference create mode 100644 tests/queries/0_stateless/02418_map_combinator_bug.sql diff --git a/src/AggregateFunctions/AggregateFunctionMap.h b/src/AggregateFunctions/AggregateFunctionMap.h index 9ed4b48c281..f2c56755504 100644 --- a/src/AggregateFunctions/AggregateFunctionMap.h +++ b/src/AggregateFunctions/AggregateFunctionMap.h @@ -84,6 +84,11 @@ private: using Base = IAggregateFunctionDataHelper>; public: + bool isState() const override + { + return nested_func->isState(); + } + AggregateFunctionMap(AggregateFunctionPtr nested, const DataTypes & types) : Base(types, nested->getParameters()), nested_func(nested) { if (types.empty()) diff --git a/src/Interpreters/AggregationUtils.cpp b/src/Interpreters/AggregationUtils.cpp index 43062546450..9b237a24928 100644 --- a/src/Interpreters/AggregationUtils.cpp +++ b/src/Interpreters/AggregationUtils.cpp @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -6,6 +7,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int ILLEGAL_COLUMN; } OutputBlockColumns prepareOutputBlockColumns( @@ -50,19 +52,27 @@ OutputBlockColumns prepareOutputBlockColumns( if (aggregate_functions[i]->isState()) { + IColumn * column_to_check = final_aggregate_columns[i].get(); + /// Aggregate state can be wrapped into array/map if aggregate function ends with -Resample/Map combinator + if (auto * column_map = typeid_cast(final_aggregate_columns[i].get())) + column_to_check = &column_map->getNestedData().getColumn(1); + else if (auto * column_array = typeid_cast(final_aggregate_columns[i].get())) + column_to_check = &column_array->getData(); + /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. - if (auto * column_aggregate_func = typeid_cast(final_aggregate_columns[i].get())) + if (auto * column_aggregate_func = typeid_cast(column_to_check)) + { for (auto & pool : aggregates_pools) column_aggregate_func->addArena(pool); - - /// Aggregate state can be wrapped into array if aggregate function ends with -Resample combinator. - final_aggregate_columns[i]->forEachSubcolumn( - [&aggregates_pools](auto & subcolumn) - { - if (auto * column_aggregate_func = typeid_cast(subcolumn.get())) - for (auto & pool : aggregates_pools) - column_aggregate_func->addArena(pool); - }); + } + else + { + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Aggregate function {} was marked as State, but result column {} doesn't contain AggregateFunction column", + aggregate_functions[i]->getName(), + final_aggregate_columns[i]->getName()); + } } } } diff --git a/tests/queries/0_stateless/02418_map_combinator_bug.reference b/tests/queries/0_stateless/02418_map_combinator_bug.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02418_map_combinator_bug.sql b/tests/queries/0_stateless/02418_map_combinator_bug.sql new file mode 100644 index 00000000000..9fae64cd3f9 --- /dev/null +++ b/tests/queries/0_stateless/02418_map_combinator_bug.sql @@ -0,0 +1,11 @@ +drop table if exists test; +create table test (x Map(UInt8, AggregateFunction(uniq, UInt64))) engine=Memory; +insert into test select uniqStateMap(map(1, number)) from numbers(10); +select * from test format Null; +drop table test; + +create table test (x AggregateFunction(uniq, UInt64), y Int64) engine=Memory; +insert into test select uniqState(number) as x, number as y from numbers(10) group by number; +select uniqStateMap(map(1, x)) OVER (PARTITION BY y) from test; -- {serverError ILLEGAL_COLUMN} +drop table test; + From 5c0c19c533444a90451dbede0dac09f4fbc16d4c Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 7 Sep 2022 18:49:34 +0000 Subject: [PATCH 12/86] Make generic solution --- .../AggregateFunctionArray.h | 5 ++++ .../AggregateFunctionDistinct.h | 5 ++++ .../AggregateFunctionForEach.h | 5 ++++ src/AggregateFunctions/AggregateFunctionIf.h | 5 ++++ src/AggregateFunctions/AggregateFunctionMap.h | 5 ++++ .../AggregateFunctionNull.h | 5 ++++ .../AggregateFunctionOrFill.h | 5 ++++ .../AggregateFunctionResample.h | 5 ++++ .../AggregateFunctionState.h | 5 ++++ src/AggregateFunctions/IAggregateFunction.h | 8 ++++++ src/Interpreters/AggregationUtils.cpp | 25 +++---------------- ... => 02418_aggregate_combinators.reference} | 0 .../02418_aggregate_combinators.sql | 25 +++++++++++++++++++ .../0_stateless/02418_map_combinator_bug.sql | 11 -------- 14 files changed, 82 insertions(+), 32 deletions(-) rename tests/queries/0_stateless/{02418_map_combinator_bug.reference => 02418_aggregate_combinators.reference} (100%) create mode 100644 tests/queries/0_stateless/02418_aggregate_combinators.sql delete mode 100644 tests/queries/0_stateless/02418_map_combinator_bug.sql diff --git a/src/AggregateFunctions/AggregateFunctionArray.h b/src/AggregateFunctions/AggregateFunctionArray.h index 85e0dfc8050..a5564ea2bb3 100644 --- a/src/AggregateFunctions/AggregateFunctionArray.h +++ b/src/AggregateFunctions/AggregateFunctionArray.h @@ -99,6 +99,11 @@ public: return nested_func->isState(); } + IColumn * extractStateColumnFromResultColumn(IColumn * column) const override + { + return nested_func->extractStateColumnFromResultColumn(column); + } + void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { const IColumn * nested[num_arguments]; diff --git a/src/AggregateFunctions/AggregateFunctionDistinct.h b/src/AggregateFunctions/AggregateFunctionDistinct.h index 5afe104bcc0..93d0c58b20a 100644 --- a/src/AggregateFunctions/AggregateFunctionDistinct.h +++ b/src/AggregateFunctions/AggregateFunctionDistinct.h @@ -245,6 +245,11 @@ public: return nested_func->isState(); } + IColumn * extractStateColumnFromResultColumn(IColumn * column) const override + { + return nested_func->extractStateColumnFromResultColumn(column); + } + AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; diff --git a/src/AggregateFunctions/AggregateFunctionForEach.h b/src/AggregateFunctions/AggregateFunctionForEach.h index 064b7b00c86..e12b8e552de 100644 --- a/src/AggregateFunctions/AggregateFunctionForEach.h +++ b/src/AggregateFunctions/AggregateFunctionForEach.h @@ -264,6 +264,11 @@ public: return nested_func->isState(); } + IColumn * extractStateColumnFromResultColumn(IColumn * column) const override + { + return nested_func->extractStateColumnFromResultColumn(&assert_cast(column)->getData()); + } + AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; diff --git a/src/AggregateFunctions/AggregateFunctionIf.h b/src/AggregateFunctions/AggregateFunctionIf.h index 18104f94fad..10d46a54b74 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.h +++ b/src/AggregateFunctions/AggregateFunctionIf.h @@ -183,6 +183,11 @@ public: return nested_func->isState(); } + IColumn * extractStateColumnFromResultColumn(IColumn * column) const override + { + return nested_func->extractStateColumnFromResultColumn(column); + } + AggregateFunctionPtr getOwnNullAdapter( const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params, const AggregateFunctionProperties & properties) const override; diff --git a/src/AggregateFunctions/AggregateFunctionMap.h b/src/AggregateFunctions/AggregateFunctionMap.h index f2c56755504..1a68063e9d6 100644 --- a/src/AggregateFunctions/AggregateFunctionMap.h +++ b/src/AggregateFunctions/AggregateFunctionMap.h @@ -89,6 +89,11 @@ public: return nested_func->isState(); } + IColumn * extractStateColumnFromResultColumn(IColumn * column) const override + { + return nested_func->extractStateColumnFromResultColumn(&assert_cast(column)->getNestedData().getColumn(1)); + } + AggregateFunctionMap(AggregateFunctionPtr nested, const DataTypes & types) : Base(types, nested->getParameters()), nested_func(nested) { if (types.empty()) diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index ca284680800..10a9c38e41c 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -189,6 +189,11 @@ public: return nested_function->isState(); } + IColumn * extractStateColumnFromResultColumn(IColumn * column) const override + { + return nested_function->extractStateColumnFromResultColumn(column); + } + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } #if USE_EMBEDDED_COMPILER diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.h b/src/AggregateFunctions/AggregateFunctionOrFill.h index 4eca1b4df92..6c669b53d3e 100644 --- a/src/AggregateFunctions/AggregateFunctionOrFill.h +++ b/src/AggregateFunctions/AggregateFunctionOrFill.h @@ -67,6 +67,11 @@ public: return nested_function->isState(); } + IColumn * extractStateColumnFromResultColumn(IColumn * column) const override + { + return nested_function->extractStateColumnFromResultColumn(column); + } + bool allocatesMemoryInArena() const override { return nested_function->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/AggregateFunctionResample.h b/src/AggregateFunctions/AggregateFunctionResample.h index 5d7bf95ceee..5f4e173d571 100644 --- a/src/AggregateFunctions/AggregateFunctionResample.h +++ b/src/AggregateFunctions/AggregateFunctionResample.h @@ -91,6 +91,11 @@ public: return nested_function->isState(); } + IColumn * extractStateColumnFromResultColumn(IColumn * column) const override + { + return nested_function->extractStateColumnFromResultColumn(&assert_cast(column)->getData()); + } + bool allocatesMemoryInArena() const override { return nested_function->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/AggregateFunctionState.h b/src/AggregateFunctions/AggregateFunctionState.h index a598e4838cc..8de17580758 100644 --- a/src/AggregateFunctions/AggregateFunctionState.h +++ b/src/AggregateFunctions/AggregateFunctionState.h @@ -112,6 +112,11 @@ public: /// Aggregate function or aggregate function state. bool isState() const override { return true; } + IColumn * extractStateColumnFromResultColumn(IColumn * column) const override + { + return assert_cast(column); + } + bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 87cb1006d36..efea5e9f0f4 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -172,6 +172,14 @@ public: */ virtual bool isState() const { return false; } + virtual IColumn * extractStateColumnFromResultColumn(IColumn *) const + { + if (isState()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Function {} is marked as State but method extractStateColumnFromResultColumn is not implemented"); + + throw Exception("Method extractStateColumnFromResultColumn is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } + /** The inner loop that uses the function pointer is better than using the virtual function. * The reason is that in the case of virtual functions GCC 5.1.2 generates code, * which, at each iteration of the loop, reloads the function address (the offset value in the virtual function table) from memory to the register. diff --git a/src/Interpreters/AggregationUtils.cpp b/src/Interpreters/AggregationUtils.cpp index 9b237a24928..146d950d2c5 100644 --- a/src/Interpreters/AggregationUtils.cpp +++ b/src/Interpreters/AggregationUtils.cpp @@ -52,27 +52,10 @@ OutputBlockColumns prepareOutputBlockColumns( if (aggregate_functions[i]->isState()) { - IColumn * column_to_check = final_aggregate_columns[i].get(); - /// Aggregate state can be wrapped into array/map if aggregate function ends with -Resample/Map combinator - if (auto * column_map = typeid_cast(final_aggregate_columns[i].get())) - column_to_check = &column_map->getNestedData().getColumn(1); - else if (auto * column_array = typeid_cast(final_aggregate_columns[i].get())) - column_to_check = &column_array->getData(); - - /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. - if (auto * column_aggregate_func = typeid_cast(column_to_check)) - { - for (auto & pool : aggregates_pools) - column_aggregate_func->addArena(pool); - } - else - { - throw Exception( - ErrorCodes::ILLEGAL_COLUMN, - "Aggregate function {} was marked as State, but result column {} doesn't contain AggregateFunction column", - aggregate_functions[i]->getName(), - final_aggregate_columns[i]->getName()); - } + auto * column_aggregate_func = assert_cast( + aggregate_functions[i]->extractStateColumnFromResultColumn(final_aggregate_columns[i].get())); + for (auto & pool : aggregates_pools) + column_aggregate_func->addArena(pool); } } } diff --git a/tests/queries/0_stateless/02418_map_combinator_bug.reference b/tests/queries/0_stateless/02418_aggregate_combinators.reference similarity index 100% rename from tests/queries/0_stateless/02418_map_combinator_bug.reference rename to tests/queries/0_stateless/02418_aggregate_combinators.reference diff --git a/tests/queries/0_stateless/02418_aggregate_combinators.sql b/tests/queries/0_stateless/02418_aggregate_combinators.sql new file mode 100644 index 00000000000..227ce71296f --- /dev/null +++ b/tests/queries/0_stateless/02418_aggregate_combinators.sql @@ -0,0 +1,25 @@ +drop table if exists test; +create table test (x Map(UInt8, AggregateFunction(uniq, UInt64))) engine=Memory; +insert into test select uniqStateMap(map(1, number)) from numbers(10); +select * from test format Null; +truncate table test; +drop table test; + +create table test (x Map(UInt8, Array(Map(UInt8, Array(AggregateFunction(uniq, UInt64)))))) engine=Memory; +insert into test select uniqStateForEachMapForEachMap(map(1, [map(1, [number, number])])) from numbers(10); +select * from test format Null; +truncate table test; +drop table test; + +create table test (x Array(Array(AggregateFunction(uniq, UInt64)))) engine=Memory; +insert into test select uniqStateForEachResample(30, 75, 30)([number, number + 1], 30) from numbers(10); +select * from test format Null; +truncate table test; +drop table test; + +create table test (x Array(Array(Map(UInt8, AggregateFunction(uniq, UInt64))))) engine=Memory; +insert into test select uniqStateMapForEachResample(30, 75, 30)([map(1, number)], 30) from numbers(10); +select * from test format Null; +truncate table test; +drop table test; + diff --git a/tests/queries/0_stateless/02418_map_combinator_bug.sql b/tests/queries/0_stateless/02418_map_combinator_bug.sql deleted file mode 100644 index 9fae64cd3f9..00000000000 --- a/tests/queries/0_stateless/02418_map_combinator_bug.sql +++ /dev/null @@ -1,11 +0,0 @@ -drop table if exists test; -create table test (x Map(UInt8, AggregateFunction(uniq, UInt64))) engine=Memory; -insert into test select uniqStateMap(map(1, number)) from numbers(10); -select * from test format Null; -drop table test; - -create table test (x AggregateFunction(uniq, UInt64), y Int64) engine=Memory; -insert into test select uniqState(number) as x, number as y from numbers(10) group by number; -select uniqStateMap(map(1, x)) OVER (PARTITION BY y) from test; -- {serverError ILLEGAL_COLUMN} -drop table test; - From 0475eb0b17050fc5d1f13b4cee114acc3b1b29c0 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 7 Sep 2022 18:53:22 +0000 Subject: [PATCH 13/86] Make better --- src/AggregateFunctions/AggregateFunctionState.h | 2 +- src/Interpreters/AggregationUtils.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionState.h b/src/AggregateFunctions/AggregateFunctionState.h index 8de17580758..24d1a694e0a 100644 --- a/src/AggregateFunctions/AggregateFunctionState.h +++ b/src/AggregateFunctions/AggregateFunctionState.h @@ -114,7 +114,7 @@ public: IColumn * extractStateColumnFromResultColumn(IColumn * column) const override { - return assert_cast(column); + return column; } bool allocatesMemoryInArena() const override diff --git a/src/Interpreters/AggregationUtils.cpp b/src/Interpreters/AggregationUtils.cpp index 146d950d2c5..4d2f7647628 100644 --- a/src/Interpreters/AggregationUtils.cpp +++ b/src/Interpreters/AggregationUtils.cpp @@ -1,5 +1,4 @@ #include -#include namespace DB { From 93d38b94514ef7a29218ee24a1efa4035b92de0b Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Thu, 8 Sep 2022 13:19:18 +0200 Subject: [PATCH 14/86] Fix style --- src/Interpreters/AggregationUtils.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/AggregationUtils.cpp b/src/Interpreters/AggregationUtils.cpp index 4d2f7647628..6c51cc1c9ac 100644 --- a/src/Interpreters/AggregationUtils.cpp +++ b/src/Interpreters/AggregationUtils.cpp @@ -6,7 +6,6 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; - extern const int ILLEGAL_COLUMN; } OutputBlockColumns prepareOutputBlockColumns( From 992ba99da43dfdd9c5420683febce5345e175c7e Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Thu, 8 Sep 2022 15:23:29 +0200 Subject: [PATCH 15/86] Add comment --- src/AggregateFunctions/IAggregateFunction.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index efea5e9f0f4..4ff16974b92 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -172,6 +172,10 @@ public: */ virtual bool isState() const { return false; } + /** Special method for aggregate functions with combinator -State, it takes resulting column and extracts + * column that has type ColumnAggregateFunction. This is needed because by using different combinators + * resulting column can have complex type like nested Arrays/Maps with ColumnAggregateFunction inside it. + */ virtual IColumn * extractStateColumnFromResultColumn(IColumn *) const { if (isState()) From 62541ab7647fb3384e9ce30dfe65960fa92b763e Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Thu, 8 Sep 2022 17:43:09 +0200 Subject: [PATCH 16/86] fix more tests and clang tidy build --- src/Access/SettingsProfileElement.cpp | 8 ++++---- src/Common/SettingConstraintType.h | 2 -- tests/integration/test_settings_profile/test.py | 12 +++++++----- .../queries/0_stateless/01292_create_user.reference | 12 ++++++------ 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 6851169db5e..64bb4d3769a 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -50,11 +50,14 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A { setting_name = ast.setting_name; - /// Optionally check if a setting with that name is allowed. if (access_control) { + /// Check if a setting with that name is allowed. if (setting_name != ALLOW_BACKUP_SETTING_NAME) access_control->checkSettingNameIsAllowed(setting_name); + /// Check if a CHANGEABLE_IN_READONLY is allowed. + if (ast.type == SettingConstraintType::CHANGEABLE_IN_READONLY && !access_control->doesSettingsConstraintsReplacePrevious()) + throw Exception("CHANGEABLE_IN_READONLY for " + setting_name + " is not allowed unless settings_constraints_replace_previous is enabled", ErrorCodes::NOT_IMPLEMENTED); } value = ast.value; @@ -62,9 +65,6 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A max_value = ast.max_value; type = ast.type; - if (type == SettingConstraintType::CHANGEABLE_IN_READONLY && !access_control->doesSettingsConstraintsReplacePrevious()) - throw Exception("CHANGEABLE_IN_READONLY for " + setting_name + " is not allowed unless settings_constraints_replace_previous is enabled", ErrorCodes::NOT_IMPLEMENTED); - if (!value.isNull()) value = Settings::castValueUtil(setting_name, value); if (!min_value.isNull()) diff --git a/src/Common/SettingConstraintType.h b/src/Common/SettingConstraintType.h index 9302a598fbf..eee5e630b9a 100644 --- a/src/Common/SettingConstraintType.h +++ b/src/Common/SettingConstraintType.h @@ -1,7 +1,5 @@ #pragma once -#include - namespace DB { diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index a90273e7a7a..83c504f6d0c 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -102,6 +102,7 @@ def test_smoke(): 110000000, "\\N", "\\N", + "\\N", ] ] @@ -148,7 +149,7 @@ def test_smoke(): ) ) assert system_settings_profile_elements(user_name="robin") == [ - ["\\N", "robin", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] + ["\\N", "robin", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] ] instance.query("ALTER USER robin SETTINGS NONE") @@ -215,11 +216,12 @@ def test_settings_from_granted_role(): 110000000, "\\N", "\\N", + "\\N", ], - ["xyz", "\\N", "\\N", 1, "max_ast_depth", 2000, "\\N", "\\N", "\\N", "\\N"], + ["xyz", "\\N", "\\N", 1, "max_ast_depth", 2000, "\\N", "\\N", "\\N", "\\N", "\\N"], ] assert system_settings_profile_elements(role_name="worker") == [ - ["\\N", "\\N", "worker", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] + ["\\N", "\\N", "worker", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] ] instance.query("REVOKE worker FROM robin") @@ -315,13 +317,13 @@ def test_inheritance(): ["xyz", "local directory", 1, 0, "[]", "[]"] ] assert system_settings_profile_elements(profile_name="xyz") == [ - ["xyz", "\\N", "\\N", 0, "max_memory_usage", 100000002, "\\N", "\\N", 1, "\\N"] + ["xyz", "\\N", "\\N", 0, "max_memory_usage", 100000002, "\\N", "\\N", 1, "\\N", "\\N"] ] assert system_settings_profile("alpha") == [ ["alpha", "local directory", 1, 0, "['robin']", "[]"] ] assert system_settings_profile_elements(profile_name="alpha") == [ - ["alpha", "\\N", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] + ["alpha", "\\N", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] ] assert system_settings_profile_elements(user_name="robin") == [] diff --git a/tests/queries/0_stateless/01292_create_user.reference b/tests/queries/0_stateless/01292_create_user.reference index 997a9504bb5..454df63d506 100644 --- a/tests/queries/0_stateless/01292_create_user.reference +++ b/tests/queries/0_stateless/01292_create_user.reference @@ -109,9 +109,9 @@ u2_01292 local directory no_password {} [] [] [] ['%.%.myhost.com'] 0 [] [] u3_01292 local directory sha256_password {} ['192.169.1.1','192.168.0.0/16'] ['localhost'] [] [] 0 ['r1_01292'] [] u4_01292 local directory double_sha1_password {} ['::/0'] [] [] [] 1 [] ['r1_01292'] -- system.settings_profile_elements -\N u1_01292 \N 0 readonly 1 \N \N \N \N -\N u2_01292 \N 0 \N \N \N \N \N default -\N u3_01292 \N 0 max_memory_usage 5000000 4000000 6000000 0 \N -\N u4_01292 \N 0 \N \N \N \N \N default -\N u4_01292 \N 1 max_memory_usage 5000000 \N \N \N \N -\N u4_01292 \N 2 readonly 1 \N \N \N \N +\N u1_01292 \N 0 readonly 1 \N \N \N \N \N +\N u2_01292 \N 0 \N \N \N \N \N \N default +\N u3_01292 \N 0 max_memory_usage 5000000 4000000 6000000 0 \N \N +\N u4_01292 \N 0 \N \N \N \N \N \N default +\N u4_01292 \N 1 max_memory_usage 5000000 \N \N \N \N \N +\N u4_01292 \N 2 readonly 1 \N \N \N \N \N From 807b09f6b0344c9142ea2e5421f1f0a3398b2e5a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 8 Sep 2022 15:52:00 +0000 Subject: [PATCH 17/86] Automatic style fix --- .../integration/test_settings_profile/test.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 83c504f6d0c..0e6fbb9fdfe 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -218,7 +218,19 @@ def test_settings_from_granted_role(): "\\N", "\\N", ], - ["xyz", "\\N", "\\N", 1, "max_ast_depth", 2000, "\\N", "\\N", "\\N", "\\N", "\\N"], + [ + "xyz", + "\\N", + "\\N", + 1, + "max_ast_depth", + 2000, + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + ], ] assert system_settings_profile_elements(role_name="worker") == [ ["\\N", "\\N", "worker", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] @@ -317,7 +329,19 @@ def test_inheritance(): ["xyz", "local directory", 1, 0, "[]", "[]"] ] assert system_settings_profile_elements(profile_name="xyz") == [ - ["xyz", "\\N", "\\N", 0, "max_memory_usage", 100000002, "\\N", "\\N", 1, "\\N", "\\N"] + [ + "xyz", + "\\N", + "\\N", + 0, + "max_memory_usage", + 100000002, + "\\N", + "\\N", + 1, + "\\N", + "\\N", + ] ] assert system_settings_profile("alpha") == [ ["alpha", "local directory", 1, 0, "['robin']", "[]"] From 76eb001e785fa40885889914d51cfad5e545adfa Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Fri, 9 Sep 2022 14:01:11 +0200 Subject: [PATCH 18/86] add test for changeable_in_readonly + fix SQL in docs --- .../en/sql-reference/statements/alter/role.md | 2 +- .../statements/alter/settings-profile.md | 2 +- .../sql-reference/statements/create/role.md | 2 +- .../statements/create/settings-profile.md | 2 +- .../ru/sql-reference/statements/alter/role.md | 2 +- .../statements/alter/settings-profile.md | 2 +- .../sql-reference/statements/create/role.md | 2 +- .../statements/create/settings-profile.md | 2 +- docs/zh/sql-reference/statements/alter.md | 6 +- .../zh/sql-reference/statements/alter/role.md | 2 +- .../statements/alter/settings-profile.md | 2 +- src/Parsers/Access/ASTCreateRoleQuery.h | 4 +- .../Access/ASTCreateSettingsProfileQuery.h | 4 +- src/Parsers/Access/ASTCreateUserQuery.h | 4 +- src/Parsers/Access/ParserCreateRoleQuery.h | 4 +- .../Access/ParserCreateSettingsProfileQuery.h | 4 +- src/Parsers/Access/ParserCreateUserQuery.h | 4 +- .../integration/test_settings_profile/test.py | 62 +++++++++++++++++++ 18 files changed, 87 insertions(+), 25 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/role.md b/docs/en/sql-reference/statements/alter/role.md index 2bee9fd0dc6..c068d6c4fce 100644 --- a/docs/en/sql-reference/statements/alter/role.md +++ b/docs/en/sql-reference/statements/alter/role.md @@ -13,5 +13,5 @@ Syntax: ``` sql ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` diff --git a/docs/en/sql-reference/statements/alter/settings-profile.md b/docs/en/sql-reference/statements/alter/settings-profile.md index 234bb22ae14..adfe65e3c80 100644 --- a/docs/en/sql-reference/statements/alter/settings-profile.md +++ b/docs/en/sql-reference/statements/alter/settings-profile.md @@ -13,5 +13,5 @@ Syntax: ``` sql ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` diff --git a/docs/en/sql-reference/statements/create/role.md b/docs/en/sql-reference/statements/create/role.md index 84b02042fe9..6c80204688b 100644 --- a/docs/en/sql-reference/statements/create/role.md +++ b/docs/en/sql-reference/statements/create/role.md @@ -11,7 +11,7 @@ Syntax: ``` sql CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [, name2 ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` ## Managing Roles diff --git a/docs/en/sql-reference/statements/create/settings-profile.md b/docs/en/sql-reference/statements/create/settings-profile.md index 4c7e4c30ea0..8883b22896b 100644 --- a/docs/en/sql-reference/statements/create/settings-profile.md +++ b/docs/en/sql-reference/statements/create/settings-profile.md @@ -12,7 +12,7 @@ Syntax: ``` sql CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` `ON CLUSTER` clause allows creating settings profiles on a cluster, see [Distributed DDL](../../../sql-reference/distributed-ddl.md). diff --git a/docs/ru/sql-reference/statements/alter/role.md b/docs/ru/sql-reference/statements/alter/role.md index a86ff780b8d..4e84260fd40 100644 --- a/docs/ru/sql-reference/statements/alter/role.md +++ b/docs/ru/sql-reference/statements/alter/role.md @@ -13,6 +13,6 @@ sidebar_label: ROLE ``` sql ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` diff --git a/docs/ru/sql-reference/statements/alter/settings-profile.md b/docs/ru/sql-reference/statements/alter/settings-profile.md index ec1cd1f72e6..8166f17597c 100644 --- a/docs/ru/sql-reference/statements/alter/settings-profile.md +++ b/docs/ru/sql-reference/statements/alter/settings-profile.md @@ -13,6 +13,6 @@ sidebar_label: SETTINGS PROFILE ``` sql ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` diff --git a/docs/ru/sql-reference/statements/create/role.md b/docs/ru/sql-reference/statements/create/role.md index 9e06ad1914e..4a93de8a74c 100644 --- a/docs/ru/sql-reference/statements/create/role.md +++ b/docs/ru/sql-reference/statements/create/role.md @@ -12,7 +12,7 @@ sidebar_label: "Роль" ```sql CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [, name2 ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` ## Управление ролями {#managing-roles} diff --git a/docs/ru/sql-reference/statements/create/settings-profile.md b/docs/ru/sql-reference/statements/create/settings-profile.md index d85b2aadeda..9aa77e4c241 100644 --- a/docs/ru/sql-reference/statements/create/settings-profile.md +++ b/docs/ru/sql-reference/statements/create/settings-profile.md @@ -13,7 +13,7 @@ sidebar_label: "Профиль настроек" ``` sql CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` Секция `ON CLUSTER` позволяет создавать профили на кластере, см. [Распределенные DDL запросы](../../../sql-reference/distributed-ddl.md). diff --git a/docs/zh/sql-reference/statements/alter.md b/docs/zh/sql-reference/statements/alter.md index 2e143d3b654..23edfd633db 100644 --- a/docs/zh/sql-reference/statements/alter.md +++ b/docs/zh/sql-reference/statements/alter.md @@ -500,7 +500,7 @@ ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name] [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` ### 说明 {#alter-user-dscr} @@ -540,7 +540,7 @@ ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 ``` sql ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name] [RENAME TO new_name] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` ## 修改row policy {#alter-row-policy-statement} @@ -584,7 +584,7 @@ ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name] ``` sql ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name] [RENAME TO new_name] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` [Original article](https://clickhouse.com/docs/en/query_language/alter/) diff --git a/docs/zh/sql-reference/statements/alter/role.md b/docs/zh/sql-reference/statements/alter/role.md index e364571359f..3647f94e46e 100644 --- a/docs/zh/sql-reference/statements/alter/role.md +++ b/docs/zh/sql-reference/statements/alter/role.md @@ -13,5 +13,5 @@ sidebar_label: 角色 ``` sql ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` diff --git a/docs/zh/sql-reference/statements/alter/settings-profile.md b/docs/zh/sql-reference/statements/alter/settings-profile.md index e4365b25c1a..e21f18f920b 100644 --- a/docs/zh/sql-reference/statements/alter/settings-profile.md +++ b/docs/zh/sql-reference/statements/alter/settings-profile.md @@ -13,5 +13,5 @@ sidebar_label: 配置文件设置 ``` sql ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` diff --git a/src/Parsers/Access/ASTCreateRoleQuery.h b/src/Parsers/Access/ASTCreateRoleQuery.h index 0e0773c8972..906ea683e1a 100644 --- a/src/Parsers/Access/ASTCreateRoleQuery.h +++ b/src/Parsers/Access/ASTCreateRoleQuery.h @@ -10,11 +10,11 @@ class ASTSettingsProfileElements; /** CREATE ROLE [IF NOT EXISTS | OR REPLACE] name - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * * ALTER ROLE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] */ class ASTCreateRoleQuery : public IAST, public ASTQueryWithOnCluster { diff --git a/src/Parsers/Access/ASTCreateSettingsProfileQuery.h b/src/Parsers/Access/ASTCreateSettingsProfileQuery.h index 64546bc7230..441ec0f5233 100644 --- a/src/Parsers/Access/ASTCreateSettingsProfileQuery.h +++ b/src/Parsers/Access/ASTCreateSettingsProfileQuery.h @@ -11,12 +11,12 @@ class ASTRolesOrUsersSet; /** CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] * * ALTER SETTINGS PROFILE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] */ class ASTCreateSettingsProfileQuery : public IAST, public ASTQueryWithOnCluster diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index c61f2cdd9fd..b8eb70e2041 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -19,7 +19,7 @@ class ASTSettingsProfileElements; * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [DEFAULT DATABASE database | NONE] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] * * ALTER USER [IF EXISTS] name @@ -28,7 +28,7 @@ class ASTSettingsProfileElements; * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [DEFAULT DATABASE database | NONE] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] */ class ASTCreateUserQuery : public IAST, public ASTQueryWithOnCluster diff --git a/src/Parsers/Access/ParserCreateRoleQuery.h b/src/Parsers/Access/ParserCreateRoleQuery.h index 1fdee67eaab..883ea89854a 100644 --- a/src/Parsers/Access/ParserCreateRoleQuery.h +++ b/src/Parsers/Access/ParserCreateRoleQuery.h @@ -7,11 +7,11 @@ namespace DB { /** Parses queries like * CREATE ROLE [IF NOT EXISTS | OR REPLACE] name - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * * ALTER ROLE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] */ class ParserCreateRoleQuery : public IParserBase { diff --git a/src/Parsers/Access/ParserCreateSettingsProfileQuery.h b/src/Parsers/Access/ParserCreateSettingsProfileQuery.h index ab730fcd8eb..bee5bdcb2d1 100644 --- a/src/Parsers/Access/ParserCreateSettingsProfileQuery.h +++ b/src/Parsers/Access/ParserCreateSettingsProfileQuery.h @@ -7,11 +7,11 @@ namespace DB { /** Parses queries like * CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] * * ALTER SETTINGS PROFILE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] */ class ParserCreateSettingsProfileQuery : public IParserBase { diff --git a/src/Parsers/Access/ParserCreateUserQuery.h b/src/Parsers/Access/ParserCreateUserQuery.h index 215133a777c..0cc8c9b6649 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.h +++ b/src/Parsers/Access/ParserCreateUserQuery.h @@ -10,7 +10,7 @@ namespace DB * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] * * ALTER USER [IF EXISTS] name @@ -18,7 +18,7 @@ namespace DB * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] */ class ParserCreateUserQuery : public IParserBase diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 0e6fbb9fdfe..8acafab257b 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -392,6 +392,68 @@ def test_alter_and_drop(): instance.query("SET max_memory_usage = 120000000", user="robin") +def test_changeable_in_readonly(): + instance.query( + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000003 MIN 90000000 MAX 110000000 CHANGEABLE_IN_READONLY SETTINGS readonly = 1 TO robin" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "100000003\n" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'readonly'", + user="robin", + ) + == "1\n" + ) + assert ( + "Setting max_memory_usage shouldn't be less than 90000000" + in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + ) + assert ( + "Setting max_memory_usage shouldn't be greater than 110000000" + in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin" + ) + ) + + assert system_settings_profile_elements(profile_name="xyz") == [ + [ + "xyz", + "\\N", + "\\N", + 0, + "max_memory_usage", + 100000003, + 90000000, + 110000000, + "\\N", + 1, + "\\N", + ], + [ + "xyz", + "\\N", + "\\N", + 1, + "readonly", + 1, + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + ] + ] + + instance.query("SET max_memory_usage = 90000000", user="robin") + instance.query("SET max_memory_usage = 110000000", user="robin") + + def test_show_profiles(): instance.query("CREATE SETTINGS PROFILE xyz") assert instance.query("SHOW SETTINGS PROFILES") == "default\nreadonly\nxyz\n" From aeac3916efda768b5fea93ba0658fde65f7ccf34 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 9 Sep 2022 12:27:13 +0000 Subject: [PATCH 19/86] Automatic style fix --- tests/integration/test_settings_profile/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 8acafab257b..07472e8114c 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -447,7 +447,7 @@ def test_changeable_in_readonly(): "\\N", "\\N", "\\N", - ] + ], ] instance.query("SET max_memory_usage = 90000000", user="robin") From 4810a59866a3cee37138e6dabf32afef6d6a72ad Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Mon, 12 Sep 2022 13:26:51 +0200 Subject: [PATCH 20/86] fix uninitialized variables --- src/Parsers/Access/ParserSettingsProfileElement.cpp | 2 +- src/Storages/System/StorageSystemSettings.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Parsers/Access/ParserSettingsProfileElement.cpp b/src/Parsers/Access/ParserSettingsProfileElement.cpp index c7b73a78d0c..766bb972355 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.cpp +++ b/src/Parsers/Access/ParserSettingsProfileElement.cpp @@ -139,7 +139,7 @@ namespace Field res_value; Field res_min_value; Field res_max_value; - SettingConstraintType res_type; + SettingConstraintType res_type = SettingConstraintType::NONE; bool has_value_or_constraint = false; while (parseValue(pos, expected, res_value) || parseMinMaxValue(pos, expected, res_min_value, res_max_value) diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index 33f1c4cc6b1..161eb17b280 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -40,7 +40,7 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[3]->insert(setting.getDescription()); Field min, max; - SettingConstraintType type; + SettingConstraintType type = SettingConstraintType::NONE; constraints.get(settings, setting_name, min, max, type); /// These two columns can accept strings only. From c31818260fa49568ef3ca9698945c4475cda5c7c Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Mon, 12 Sep 2022 21:03:06 +0200 Subject: [PATCH 21/86] renames and refactoring --- src/Access/SettingsConstraints.cpp | 45 +++++++++-------- src/Access/SettingsConstraints.h | 50 +++++++++++-------- src/Access/SettingsProfileElement.cpp | 10 ++-- src/Access/SettingsProfileElement.h | 6 +-- src/Access/UsersConfigAccessStorage.cpp | 14 +++--- ...tType.h => SettingConstraintWritability.h} | 4 +- .../Access/ASTSettingsProfileElement.cpp | 12 ++--- .../Access/ASTSettingsProfileElement.h | 4 +- .../Access/ParserSettingsProfileElement.cpp | 24 ++++----- src/Storages/System/StorageSystemSettings.cpp | 6 +-- .../StorageSystemSettingsProfileElements.cpp | 6 +-- 11 files changed, 96 insertions(+), 85 deletions(-) rename src/Common/{SettingConstraintType.h => SettingConstraintWritability.h} (92%) diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 26e5cc930d7..69c7c7e7f64 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -35,24 +35,24 @@ void SettingsConstraints::clear() constraints.clear(); } -void SettingsConstraints::set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintType type) +void SettingsConstraints::set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability) { - if (min_value.isNull() && max_value.isNull() && type == SettingConstraintType::NONE) + if (min_value.isNull() && max_value.isNull() && writability == SettingConstraintWritability::DEFAULT) return; // Do not even create entry to avoid issues during profile inheritance auto & constraint = constraints[setting_name]; if (!min_value.isNull()) constraint.min_value = Settings::castValueUtil(setting_name, min_value); if (!max_value.isNull()) constraint.max_value = Settings::castValueUtil(setting_name, max_value); - constraint.type = type; + constraint.writability = writability; } -void SettingsConstraints::get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintType & type) const +void SettingsConstraints::get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const { - auto range = getRange(current_settings, setting_name); - min_value = range.min_value; - max_value = range.max_value; - type = range.type; + auto checker = getChecker(current_settings, setting_name); + min_value = checker.constraint.min_value; + max_value = checker.constraint.max_value; + writability = checker.constraint.writability; } void SettingsConstraints::merge(const SettingsConstraints & other) @@ -71,8 +71,8 @@ void SettingsConstraints::merge(const SettingsConstraints & other) constraint.min_value = other_constraint.min_value; if (!other_constraint.max_value.isNull()) constraint.max_value = other_constraint.max_value; - if (other_constraint.type == SettingConstraintType::CONST) - constraint.type = SettingConstraintType::CONST; // NOTE: In this mode flag cannot be overridden to be false + if (other_constraint.writability == SettingConstraintWritability::CONST) + constraint.writability = SettingConstraintWritability::CONST; // NOTE: In this mode flag cannot be overridden to be false } } } @@ -176,10 +176,10 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh return false; } - return getRange(current_settings, setting_name).check(change, new_value, reaction); + return getChecker(current_settings, setting_name).check(change, new_value, reaction); } -bool SettingsConstraints::Range::check(SettingChange & change, const Field & new_value, ReactionOnViolation reaction) const +bool SettingsConstraints::Checker::check(SettingChange & change, const Field & new_value, ReactionOnViolation reaction) const { const String & setting_name = change.name; @@ -208,7 +208,7 @@ bool SettingsConstraints::Range::check(SettingChange & change, const Field & new return false; } - if (type == SettingConstraintType::CONST) + if (constraint.writability == SettingConstraintWritability::CONST) { if (reaction == THROW_ON_VIOLATION) throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); @@ -216,6 +216,9 @@ bool SettingsConstraints::Range::check(SettingChange & change, const Field & new return false; } + const auto & min_value = constraint.min_value; + const auto & max_value = constraint.max_value; + if (!min_value.isNull() && !max_value.isNull() && less_or_cannot_compare(max_value, min_value)) { if (reaction == THROW_ON_VIOLATION) @@ -251,10 +254,10 @@ bool SettingsConstraints::Range::check(SettingChange & change, const Field & new return true; } -SettingsConstraints::Range SettingsConstraints::getRange(const Settings & current_settings, std::string_view setting_name) const +SettingsConstraints::Checker SettingsConstraints::getChecker(const Settings & current_settings, std::string_view setting_name) const { if (!current_settings.allow_ddl && setting_name == "allow_ddl") - return Range::forbidden("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED); + return Checker("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED); /** The `readonly` value is understood as follows: * 0 - no read-only restrictions. @@ -263,25 +266,25 @@ SettingsConstraints::Range SettingsConstraints::getRange(const Settings & curren */ if (current_settings.readonly > 1 && setting_name == "readonly") - return Range::forbidden("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY); + return Checker("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY); auto it = constraints.find(setting_name); if (current_settings.readonly == 1) { - if (it == constraints.end() || it->second.type != SettingConstraintType::CHANGEABLE_IN_READONLY) - return Range::forbidden("Cannot modify '" + String(setting_name) + "' setting in readonly mode", ErrorCodes::READONLY); + if (it == constraints.end() || it->second.writability != SettingConstraintWritability::CHANGEABLE_IN_READONLY) + return Checker("Cannot modify '" + String(setting_name) + "' setting in readonly mode", ErrorCodes::READONLY); } else // For both readonly=0 and readonly=2 { if (it == constraints.end()) - return Range::allowed(); + return Checker(); // Allowed } return it->second; } -bool SettingsConstraints::Range::operator==(const Range & other) const +bool SettingsConstraints::Constraint::operator==(const Constraint & other) const { - return type == other.type && min_value == other.min_value && max_value == other.max_value; + return writability == other.writability && min_value == other.min_value && max_value == other.max_value; } bool operator ==(const SettingsConstraints & left, const SettingsConstraints & right) diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index e94db6ad705..5491c52ef3f 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -65,8 +65,8 @@ public: void clear(); bool empty() const { return constraints.empty(); } - void set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintType type); - void get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintType & type) const; + void set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability); + void get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const; void merge(const SettingsConstraints & other); @@ -88,30 +88,38 @@ private: CLAMP_ON_VIOLATION, }; - struct Range + struct Constraint { - SettingConstraintType type = SettingConstraintType::NONE; + SettingConstraintWritability writability = SettingConstraintWritability::DEFAULT; Field min_value; Field max_value; + bool operator ==(const Constraint & other) const; + bool operator !=(const Constraint & other) const { return !(*this == other); } + }; + + struct Checker { + Constraint constraint; String explain; - int code; + int code = 0; - bool operator ==(const Range & other) const; - bool operator !=(const Range & other) const { return !(*this == other); } + // Allows everything + Checker() = default; + // Forbidden with explanation + Checker(const String & explain_, int code_) + : constraint{.writability = SettingConstraintWritability::CONST} + , explain(explain_) + , code(code_) + {} + + // Allow of forbid depending on range defined by constraint, also used to return stored constraint + Checker(const Constraint & constraint_) + : constraint(constraint_) + {} + + // Perform checking bool check(SettingChange & change, const Field & new_value, ReactionOnViolation reaction) const; - - static const Range & allowed() - { - static Range res; - return res; - } - - static Range forbidden(const String & explain, int code) - { - return Range{.type = SettingConstraintType::CONST, .explain = explain, .code = code}; - } }; struct StringHash @@ -129,11 +137,11 @@ private: bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const; - Range getRange(const Settings & current_settings, std::string_view setting_name) const; + Checker getChecker(const Settings & current_settings, std::string_view setting_name) const; // Special container for heterogeneous lookups: to avoid `String` construction during `find(std::string_view)` - using RangeMap = std::unordered_map>; - RangeMap constraints; + using Constraints = std::unordered_map>; + Constraints constraints; const AccessControl * access_control; }; diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 64bb4d3769a..48285ecdba5 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -56,14 +56,14 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A if (setting_name != ALLOW_BACKUP_SETTING_NAME) access_control->checkSettingNameIsAllowed(setting_name); /// Check if a CHANGEABLE_IN_READONLY is allowed. - if (ast.type == SettingConstraintType::CHANGEABLE_IN_READONLY && !access_control->doesSettingsConstraintsReplacePrevious()) + if (ast.writability == SettingConstraintWritability::CHANGEABLE_IN_READONLY && !access_control->doesSettingsConstraintsReplacePrevious()) throw Exception("CHANGEABLE_IN_READONLY for " + setting_name + " is not allowed unless settings_constraints_replace_previous is enabled", ErrorCodes::NOT_IMPLEMENTED); } value = ast.value; min_value = ast.min_value; max_value = ast.max_value; - type = ast.type; + writability = ast.writability; if (!value.isNull()) value = Settings::castValueUtil(setting_name, value); @@ -87,7 +87,7 @@ std::shared_ptr SettingsProfileElement::toAST() const ast->value = value; ast->min_value = min_value; ast->max_value = max_value; - ast->type = type; + ast->writability = writability; return ast; } @@ -108,7 +108,7 @@ std::shared_ptr SettingsProfileElement::toASTWithName ast->value = value; ast->min_value = min_value; ast->max_value = max_value; - ast->type = type; + ast->writability = writability; return ast; } @@ -213,7 +213,7 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints(const AccessC SettingsConstraints res{access_control}; for (const auto & elem : *this) if (!elem.setting_name.empty() && elem.setting_name != ALLOW_BACKUP_SETTING_NAME) - res.set(elem.setting_name, elem.min_value, elem.max_value, elem.type); + res.set(elem.setting_name, elem.min_value, elem.max_value, elem.writability); return res; } diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index 2d0444a1589..d2bf59f6799 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include @@ -26,9 +26,9 @@ struct SettingsProfileElement Field value; Field min_value; Field max_value; - SettingConstraintType type = SettingConstraintType::NONE; + SettingConstraintWritability writability = SettingConstraintWritability::DEFAULT; - auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, type); } + auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, writability); } friend bool operator==(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() == rhs.toTuple(); } friend bool operator!=(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return !(lhs == rhs); } friend bool operator <(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() < rhs.toTuple(); } diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 4adbfeef98b..d5d3b68e6f1 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -441,7 +441,7 @@ namespace String path_to_name = path_to_constraints + "." + setting_name; config.keys(path_to_name, constraint_types); - size_t type_specifiers_count = 0; + size_t writability_count = 0; for (const String & constraint_type : constraint_types) { if (constraint_type == "min") @@ -450,22 +450,22 @@ namespace profile_element.max_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); else if (constraint_type == "readonly" || constraint_type == "const") { - type_specifiers_count++; - profile_element.type = SettingConstraintType::CONST; + writability_count++; + profile_element.writability = SettingConstraintWritability::CONST; } else if (constraint_type == "changeable_in_readonly") { - type_specifiers_count++; + writability_count++; if (access_control.doesSettingsConstraintsReplacePrevious()) - profile_element.type = SettingConstraintType::CHANGEABLE_IN_READONLY; + profile_element.writability = SettingConstraintWritability::CHANGEABLE_IN_READONLY; else throw Exception("Setting changeable_in_readonly for " + setting_name + " is not allowed unless settings_constraints_replace_previous is enabled", ErrorCodes::NOT_IMPLEMENTED); } else throw Exception("Setting " + constraint_type + " value for " + setting_name + " isn't supported", ErrorCodes::NOT_IMPLEMENTED); } - if (type_specifiers_count > 1) - throw Exception("Not more than one constraint type specifier (const/readonly/changeable_in_readonly) is allowed for " + setting_name, ErrorCodes::NOT_IMPLEMENTED); + if (writability_count > 1) + throw Exception("Not more than one constraint writability specifier (const/readonly/changeable_in_readonly) is allowed for " + setting_name, ErrorCodes::NOT_IMPLEMENTED); profile_elements.push_back(std::move(profile_element)); } diff --git a/src/Common/SettingConstraintType.h b/src/Common/SettingConstraintWritability.h similarity index 92% rename from src/Common/SettingConstraintType.h rename to src/Common/SettingConstraintWritability.h index eee5e630b9a..583411d1d31 100644 --- a/src/Common/SettingConstraintType.h +++ b/src/Common/SettingConstraintWritability.h @@ -4,10 +4,10 @@ namespace DB { -enum class SettingConstraintType +enum class SettingConstraintWritability { // Default. Behave in the same way as WRITABLE, but is not inherited unless `settings_constraints_replace_previous` is set. - NONE, + DEFAULT, // Setting can be change within specified range only in `readonly=0` or `readonly=2` mode. WRITABLE, diff --git a/src/Parsers/Access/ASTSettingsProfileElement.cpp b/src/Parsers/Access/ASTSettingsProfileElement.cpp index 85ec5bc9233..db860cb08a6 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.cpp +++ b/src/Parsers/Access/ASTSettingsProfileElement.cpp @@ -52,19 +52,19 @@ void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, Form << applyVisitor(FieldVisitorToString{}, max_value); } - switch (type) + switch (writability) { - case SettingConstraintType::NONE: + case SettingConstraintWritability::DEFAULT: break; - case SettingConstraintType::WRITABLE: + case SettingConstraintWritability::WRITABLE: settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WRITABLE" << (settings.hilite ? IAST::hilite_none : ""); break; - case SettingConstraintType::CONST: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " READONLY" + case SettingConstraintWritability::CONST: + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CONST" << (settings.hilite ? IAST::hilite_none : ""); break; - case SettingConstraintType::CHANGEABLE_IN_READONLY: + case SettingConstraintWritability::CHANGEABLE_IN_READONLY: settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CHANGEABLE_IN_READONLY" << (settings.hilite ? IAST::hilite_none : ""); break; diff --git a/src/Parsers/Access/ASTSettingsProfileElement.h b/src/Parsers/Access/ASTSettingsProfileElement.h index c3816c65958..f67313c1601 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.h +++ b/src/Parsers/Access/ASTSettingsProfileElement.h @@ -2,7 +2,7 @@ #include #include -#include +#include namespace DB { @@ -17,7 +17,7 @@ public: Field value; Field min_value; Field max_value; - SettingConstraintType type = SettingConstraintType::NONE; + SettingConstraintWritability writability = SettingConstraintWritability::DEFAULT; bool id_mode = false; /// If true then `parent_profile` keeps UUID, not a name. bool use_inherit_keyword = false; /// If true then this element is a part of ASTCreateSettingsProfileQuery. diff --git a/src/Parsers/Access/ParserSettingsProfileElement.cpp b/src/Parsers/Access/ParserSettingsProfileElement.cpp index 766bb972355..caee993953b 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.cpp +++ b/src/Parsers/Access/ParserSettingsProfileElement.cpp @@ -95,23 +95,23 @@ namespace } - bool parseConstraintTypeKeyword(IParserBase::Pos & pos, Expected & expected, SettingConstraintType & type) + bool parseConstraintWritabilityKeyword(IParserBase::Pos & pos, Expected & expected, SettingConstraintWritability & writability) { return IParserBase::wrapParseImpl(pos, [&] { if (ParserKeyword{"READONLY"}.ignore(pos, expected) || ParserKeyword{"CONST"}.ignore(pos, expected)) { - type = SettingConstraintType::CONST; + writability = SettingConstraintWritability::CONST; return true; } else if (ParserKeyword{"WRITABLE"}.ignore(pos, expected)) { - type = SettingConstraintType::WRITABLE; + writability = SettingConstraintWritability::WRITABLE; return true; } else if (ParserKeyword{"CHANGEABLE_IN_READONLY"}.ignore(pos, expected)) { - type = SettingConstraintType::CHANGEABLE_IN_READONLY; + writability = SettingConstraintWritability::CHANGEABLE_IN_READONLY; return true; } else @@ -127,7 +127,7 @@ namespace Field & value, Field & min_value, Field & max_value, - SettingConstraintType & type) + SettingConstraintWritability & writability) { return IParserBase::wrapParseImpl(pos, [&] { @@ -139,11 +139,11 @@ namespace Field res_value; Field res_min_value; Field res_max_value; - SettingConstraintType res_type = SettingConstraintType::NONE; + SettingConstraintWritability res_writability = SettingConstraintWritability::DEFAULT; bool has_value_or_constraint = false; while (parseValue(pos, expected, res_value) || parseMinMaxValue(pos, expected, res_min_value, res_max_value) - || parseConstraintTypeKeyword(pos, expected, res_type)) + || parseConstraintWritabilityKeyword(pos, expected, res_writability)) { has_value_or_constraint = true; } @@ -152,7 +152,7 @@ namespace return false; if (boost::iequals(res_setting_name, "PROFILE") && res_value.isNull() && res_min_value.isNull() && res_max_value.isNull() - && res_type == SettingConstraintType::CONST) + && res_writability == SettingConstraintWritability::CONST) { /// Ambiguity: "profile readonly" can be treated either as a profile named "readonly" or /// as a setting named 'profile' with the readonly constraint. @@ -164,7 +164,7 @@ namespace value = std::move(res_value); min_value = std::move(res_min_value); max_value = std::move(res_max_value); - type = res_type; + writability = res_writability; return true; }); } @@ -184,9 +184,9 @@ namespace Field value; Field min_value; Field max_value; - SettingConstraintType type = SettingConstraintType::NONE; + SettingConstraintWritability writability = SettingConstraintWritability::DEFAULT; - bool ok = parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, type); + bool ok = parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, writability); if (!ok && (parseProfileKeyword(pos, expected, use_inherit_keyword) || previous_element_was_parent_profile)) ok = parseProfileNameOrID(pos, expected, id_mode, parent_profile); @@ -200,7 +200,7 @@ namespace result->value = std::move(value); result->min_value = std::move(min_value); result->max_value = std::move(max_value); - result->type = type; + result->writability = writability; result->id_mode = id_mode; result->use_inherit_keyword = use_inherit_keyword; return true; diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index 161eb17b280..4ccf87e4071 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -40,8 +40,8 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[3]->insert(setting.getDescription()); Field min, max; - SettingConstraintType type = SettingConstraintType::NONE; - constraints.get(settings, setting_name, min, max, type); + SettingConstraintWritability writability = SettingConstraintWritability::DEFAULT; + constraints.get(settings, setting_name, min, max, writability); /// These two columns can accept strings only. if (!min.isNull()) @@ -51,7 +51,7 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[4]->insert(min); res_columns[5]->insert(max); - res_columns[6]->insert(type == SettingConstraintType::CONST); + res_columns[6]->insert(writability == SettingConstraintWritability::CONST); res_columns[7]->insert(setting.getTypeName()); } } diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp index 7e83b75c468..62aa327230c 100644 --- a/src/Storages/System/StorageSystemSettingsProfileElements.cpp +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -104,15 +104,15 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns } bool inserted_readonly = false; - if ((element.type == SettingConstraintType::CONST || element.type == SettingConstraintType::WRITABLE) && !element.setting_name.empty()) + if ((element.writability == SettingConstraintWritability::CONST || element.writability == SettingConstraintWritability::WRITABLE) && !element.setting_name.empty()) { - column_readonly.push_back(element.type == SettingConstraintType::CONST); + column_readonly.push_back(element.writability == SettingConstraintWritability::CONST); column_readonly_null_map.push_back(false); inserted_readonly = true; } bool inserted_changeable_in_readonly = false; - if (element.type == SettingConstraintType::CHANGEABLE_IN_READONLY && !element.setting_name.empty()) + if (element.writability == SettingConstraintWritability::CHANGEABLE_IN_READONLY && !element.setting_name.empty()) { column_changeable_in_readonly.push_back(true); column_changeable_in_readonly_null_map.push_back(false); From cad3a6b7641e009b27e134bb11f9174e05189865 Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Mon, 12 Sep 2022 22:01:45 +0200 Subject: [PATCH 22/86] fix style --- src/Access/SettingsConstraints.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 5491c52ef3f..f29959f6601 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -98,7 +98,8 @@ private: bool operator !=(const Constraint & other) const { return !(*this == other); } }; - struct Checker { + struct Checker + { Constraint constraint; String explain; int code = 0; From 9cd78585c3dc2681dc2bc60085bedbc1d25f82fd Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Tue, 13 Sep 2022 00:12:40 +0200 Subject: [PATCH 23/86] fix tests --- tests/integration/test_disk_access_storage/test.py | 4 ++-- tests/integration/test_settings_profile/test.py | 4 ++-- tests/queries/0_stateless/01292_create_user.reference | 4 ++-- tests/queries/0_stateless/01292_create_user.sql | 4 ++-- tests/queries/0_stateless/01293_create_role.reference | 4 ++-- tests/queries/0_stateless/01293_create_role.sql | 4 ++-- .../0_stateless/01294_create_settings_profile.reference | 8 ++++---- .../queries/0_stateless/01294_create_settings_profile.sql | 6 +++--- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index 273a00adffe..bcfc9d718a7 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -93,7 +93,7 @@ def test_alter(): instance.query("GRANT SELECT ON mydb.mytable TO u1") instance.query("GRANT SELECT ON mydb.* TO rx WITH GRANT OPTION") instance.query( - "ALTER SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY" + "ALTER SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 CONST" ) def check(): @@ -124,7 +124,7 @@ def test_alter(): ) assert ( instance.query("SHOW CREATE SETTINGS PROFILE s1") - == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY\n" + == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 CONST\n" ) assert ( instance.query("SHOW CREATE SETTINGS PROFILE s2") diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 29d4da95251..3a19ae0c906 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -302,12 +302,12 @@ def test_settings_from_granted_role(): def test_inheritance(): instance.query( - "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY" + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 CONST" ) instance.query("CREATE SETTINGS PROFILE alpha SETTINGS PROFILE xyz TO robin") assert ( instance.query("SHOW CREATE SETTINGS PROFILE xyz") - == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY\n" + == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 CONST\n" ) assert ( instance.query("SHOW CREATE SETTINGS PROFILE alpha") diff --git a/tests/queries/0_stateless/01292_create_user.reference b/tests/queries/0_stateless/01292_create_user.reference index 454df63d506..732f7e715d8 100644 --- a/tests/queries/0_stateless/01292_create_user.reference +++ b/tests/queries/0_stateless/01292_create_user.reference @@ -58,9 +58,9 @@ CREATE USER u2_01292 SETTINGS PROFILE default CREATE USER u3_01292 SETTINGS max_memory_usage = 5000000 CREATE USER u4_01292 SETTINGS max_memory_usage MIN 5000000 CREATE USER u5_01292 SETTINGS max_memory_usage MAX 5000000 -CREATE USER u6_01292 SETTINGS max_memory_usage READONLY +CREATE USER u6_01292 SETTINGS max_memory_usage CONST CREATE USER u7_01292 SETTINGS max_memory_usage WRITABLE -CREATE USER u8_01292 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 READONLY +CREATE USER u8_01292 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST CREATE USER u9_01292 SETTINGS PROFILE default, max_memory_usage = 5000000 WRITABLE CREATE USER u1_01292 SETTINGS readonly = 1 CREATE USER u2_01292 SETTINGS readonly = 1 diff --git a/tests/queries/0_stateless/01292_create_user.sql b/tests/queries/0_stateless/01292_create_user.sql index a9582376825..d0f157d36b0 100644 --- a/tests/queries/0_stateless/01292_create_user.sql +++ b/tests/queries/0_stateless/01292_create_user.sql @@ -122,9 +122,9 @@ CREATE USER u2_01292 SETTINGS PROFILE 'default'; CREATE USER u3_01292 SETTINGS max_memory_usage=5000000; CREATE USER u4_01292 SETTINGS max_memory_usage MIN=5000000; CREATE USER u5_01292 SETTINGS max_memory_usage MAX=5000000; -CREATE USER u6_01292 SETTINGS max_memory_usage READONLY; +CREATE USER u6_01292 SETTINGS max_memory_usage CONST; CREATE USER u7_01292 SETTINGS max_memory_usage WRITABLE; -CREATE USER u8_01292 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY; +CREATE USER u8_01292 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 CONST; CREATE USER u9_01292 SETTINGS PROFILE 'default', max_memory_usage=5000000 WRITABLE; SHOW CREATE USER u1_01292; SHOW CREATE USER u2_01292; diff --git a/tests/queries/0_stateless/01293_create_role.reference b/tests/queries/0_stateless/01293_create_role.reference index 4c287d6b9f8..d33693e61fa 100644 --- a/tests/queries/0_stateless/01293_create_role.reference +++ b/tests/queries/0_stateless/01293_create_role.reference @@ -15,9 +15,9 @@ CREATE ROLE r2_01293 SETTINGS PROFILE default CREATE ROLE r3_01293 SETTINGS max_memory_usage = 5000000 CREATE ROLE r4_01293 SETTINGS max_memory_usage MIN 5000000 CREATE ROLE r5_01293 SETTINGS max_memory_usage MAX 5000000 -CREATE ROLE r6_01293 SETTINGS max_memory_usage READONLY +CREATE ROLE r6_01293 SETTINGS max_memory_usage CONST CREATE ROLE r7_01293 SETTINGS max_memory_usage WRITABLE -CREATE ROLE r8_01293 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 READONLY +CREATE ROLE r8_01293 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST CREATE ROLE r9_01293 SETTINGS PROFILE default, max_memory_usage = 5000000 WRITABLE CREATE ROLE r1_01293 SETTINGS readonly = 1 CREATE ROLE r2_01293 SETTINGS readonly = 1 diff --git a/tests/queries/0_stateless/01293_create_role.sql b/tests/queries/0_stateless/01293_create_role.sql index 963a1020e3f..f22edfeec3e 100644 --- a/tests/queries/0_stateless/01293_create_role.sql +++ b/tests/queries/0_stateless/01293_create_role.sql @@ -31,9 +31,9 @@ CREATE ROLE r2_01293 SETTINGS PROFILE 'default'; CREATE ROLE r3_01293 SETTINGS max_memory_usage=5000000; CREATE ROLE r4_01293 SETTINGS max_memory_usage MIN=5000000; CREATE ROLE r5_01293 SETTINGS max_memory_usage MAX=5000000; -CREATE ROLE r6_01293 SETTINGS max_memory_usage READONLY; +CREATE ROLE r6_01293 SETTINGS max_memory_usage CONST; CREATE ROLE r7_01293 SETTINGS max_memory_usage WRITABLE; -CREATE ROLE r8_01293 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY; +CREATE ROLE r8_01293 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 CONST; CREATE ROLE r9_01293 SETTINGS PROFILE 'default', max_memory_usage=5000000 WRITABLE; SHOW CREATE ROLE r1_01293; SHOW CREATE ROLE r2_01293; diff --git a/tests/queries/0_stateless/01294_create_settings_profile.reference b/tests/queries/0_stateless/01294_create_settings_profile.reference index 90b440e7746..8b7972176fe 100644 --- a/tests/queries/0_stateless/01294_create_settings_profile.reference +++ b/tests/queries/0_stateless/01294_create_settings_profile.reference @@ -11,9 +11,9 @@ CREATE SETTINGS PROFILE s2_01294 SETTINGS INHERIT default CREATE SETTINGS PROFILE s3_01294 SETTINGS max_memory_usage = 5000000 CREATE SETTINGS PROFILE s4_01294 SETTINGS max_memory_usage MIN 5000000 CREATE SETTINGS PROFILE s5_01294 SETTINGS max_memory_usage MAX 5000000 -CREATE SETTINGS PROFILE s6_01294 SETTINGS max_memory_usage READONLY +CREATE SETTINGS PROFILE s6_01294 SETTINGS max_memory_usage CONST CREATE SETTINGS PROFILE s7_01294 SETTINGS max_memory_usage WRITABLE -CREATE SETTINGS PROFILE s8_01294 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 READONLY +CREATE SETTINGS PROFILE s8_01294 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST CREATE SETTINGS PROFILE s9_01294 SETTINGS INHERIT default, max_memory_usage = 5000000 WRITABLE CREATE SETTINGS PROFILE s10_01294 SETTINGS INHERIT s1_01294, INHERIT s3_01294, INHERIT default, readonly = 0, max_memory_usage MAX 6000000 CREATE SETTINGS PROFILE s1_01294 SETTINGS readonly = 0 @@ -47,11 +47,11 @@ CREATE SETTINGS PROFILE s3_01294 TO r1_01294 CREATE SETTINGS PROFILE s4_01294 TO r1_01294 -- readonly ambiguity CREATE SETTINGS PROFILE s1_01294 SETTINGS readonly = 1 -CREATE SETTINGS PROFILE s2_01294 SETTINGS readonly READONLY +CREATE SETTINGS PROFILE s2_01294 SETTINGS readonly CONST CREATE SETTINGS PROFILE s3_01294 SETTINGS INHERIT readonly CREATE SETTINGS PROFILE s4_01294 SETTINGS INHERIT readonly, INHERIT readonly CREATE SETTINGS PROFILE s5_01294 SETTINGS INHERIT readonly, readonly = 1 -CREATE SETTINGS PROFILE s6_01294 SETTINGS INHERIT readonly, readonly READONLY +CREATE SETTINGS PROFILE s6_01294 SETTINGS INHERIT readonly, readonly CONST -- system.settings_profiles s1_01294 local directory 0 0 [] [] s2_01294 local directory 1 0 ['r1_01294'] [] diff --git a/tests/queries/0_stateless/01294_create_settings_profile.sql b/tests/queries/0_stateless/01294_create_settings_profile.sql index b7dd91ad6ed..565b4e70367 100644 --- a/tests/queries/0_stateless/01294_create_settings_profile.sql +++ b/tests/queries/0_stateless/01294_create_settings_profile.sql @@ -25,9 +25,9 @@ CREATE PROFILE s2_01294 SETTINGS INHERIT 'default'; CREATE PROFILE s3_01294 SETTINGS max_memory_usage=5000000; CREATE PROFILE s4_01294 SETTINGS max_memory_usage MIN=5000000; CREATE PROFILE s5_01294 SETTINGS max_memory_usage MAX=5000000; -CREATE PROFILE s6_01294 SETTINGS max_memory_usage READONLY; +CREATE PROFILE s6_01294 SETTINGS max_memory_usage CONST; CREATE PROFILE s7_01294 SETTINGS max_memory_usage WRITABLE; -CREATE PROFILE s8_01294 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY; +CREATE PROFILE s8_01294 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 CONST; CREATE PROFILE s9_01294 SETTINGS INHERIT 'default', max_memory_usage=5000000 WRITABLE; CREATE PROFILE s10_01294 SETTINGS INHERIT s1_01294, s3_01294, INHERIT default, readonly=0, max_memory_usage MAX 6000000; SHOW CREATE PROFILE s1_01294; @@ -106,7 +106,7 @@ DROP PROFILE s1_01294, s2_01294, s3_01294, s4_01294, s5_01294, s6_01294; SELECT '-- system.settings_profiles'; CREATE PROFILE s1_01294; CREATE PROFILE s2_01294 SETTINGS readonly=0 TO r1_01294;; -CREATE PROFILE s3_01294 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY TO r1_01294; +CREATE PROFILE s3_01294 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 CONST TO r1_01294; CREATE PROFILE s4_01294 SETTINGS max_memory_usage=5000000 TO r1_01294; CREATE PROFILE s5_01294 SETTINGS INHERIT default, readonly=0, max_memory_usage MAX 6000000 WRITABLE TO u1_01294; CREATE PROFILE s6_01294 TO ALL EXCEPT u1_01294, r1_01294; From ebea9f9359dce393e6a0ac6e5b5a267d32a07356 Mon Sep 17 00:00:00 2001 From: "teng.ma" Date: Thu, 11 Aug 2022 14:57:31 +0800 Subject: [PATCH 24/86] Feature: Support Event Tracing when calling Aws S3 API Description: ============ In Computing && Storage Architecture, using S3 as remote / Shared storage, the Method to access S3 is using AWS S3 API There is a gap between ClickHouse DB with Ozone Operation In ClickHouse, operation is SQL and background task In S3 , the operation should be AWS S3 API And one sql maybe can mapped to multiple API Solution: ========= Added Calling API as event into system.events table --- contrib/datasketches-cpp | 2 +- src/Common/ProfileEvents.cpp | 12 + src/Disks/S3/DiskS3.cpp | 1147 ++++++++++++++++++++++++++++++++++ src/IO/ReadBufferFromS3.cpp | 4 +- src/IO/S3Common.cpp | 6 + src/IO/WriteBufferFromS3.cpp | 14 +- 6 files changed, 1182 insertions(+), 3 deletions(-) create mode 100644 src/Disks/S3/DiskS3.cpp diff --git a/contrib/datasketches-cpp b/contrib/datasketches-cpp index 7abd49bb2e7..7d73d7610db 160000 --- a/contrib/datasketches-cpp +++ b/contrib/datasketches-cpp @@ -1 +1 @@ -Subproject commit 7abd49bb2e72bf9a5029993d31dcb1872da88292 +Subproject commit 7d73d7610db31d4e1ecde0fb3a7ee90ef371207f diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 2997a0c0d08..e9632aa050c 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -410,6 +410,18 @@ The server successfully detected this situation and will download merged part fr M(OverflowBreak, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'break' and the result is incomplete.") \ M(OverflowThrow, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'throw' and exception was thrown.") \ M(OverflowAny, "Number of times approximate GROUP BY was in effect: when aggregation was performed only on top of first 'max_rows_to_group_by' unique keys and other keys were ignored due to 'group_by_overflow_mode' = 'any'.") \ + M(ScalarSubqueriesCacheMiss, "Number of times a read from a scalar subquery was not cached and had to be calculated completely") \ + M(DeleteS3Keys, "Number of s3 API Delete Keys be called") \ + M(CopyS3Object, "Number of s3 API CopyObject be called") \ + M(ListS3Objects, "Number of s3 API ListObjects be called") \ + M(HeadS3Object, "Number of s3 API HeadObject be called") \ + M(CreateS3MultipartUpload, "Number of s3 API CreateMultipartUpload be called") \ + M(UploadS3PartCopy, "Number of s3 API UploadPartCopy be called") \ + M(UploadS3Part, "Number of s3 API UploadS3Part be called") \ + M(AbortS3MultipartUpload, "Number of s3 API AbortMultipartUpload be called") \ + M(CompleteS3MultipartUpload, "Number of s3 API CompleteS3MultipartUpload be called") \ + M(PutS3ObjectRequest, "Number of s3 API PutS3ObjectRequest be called") \ + M(GetS3ObjectRequest, "Number of s3 API GetS3ObjectRequest be called") namespace ProfileEvents { diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp new file mode 100644 index 00000000000..fe54e992518 --- /dev/null +++ b/src/Disks/S3/DiskS3.cpp @@ -0,0 +1,1147 @@ +#include "DiskS3.h" + +#if USE_AWS_S3 +#include "Disks/DiskFactory.h" + +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event DeleteS3Keys; + extern const Event CopyS3Object; + extern const Event ListS3Objects; + extern const Event HeadS3Object; + extern const Event CreateS3MultipartUpload; + extern const Event UploadS3PartCopy; + extern const Event AbortS3MultipartUpload; + extern const Event CompleteS3MultipartUpload; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int S3_ERROR; + extern const int FILE_ALREADY_EXISTS; + extern const int UNKNOWN_FORMAT; + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; +} + + + +/// Helper class to collect keys into chunks of maximum size (to prepare batch requests to AWS API) +/// see https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html +class S3PathKeeper : public RemoteFSPathKeeper +{ +public: + using Chunk = Aws::Vector; + using Chunks = std::list; + + explicit S3PathKeeper(size_t chunk_limit_) : RemoteFSPathKeeper(chunk_limit_) {} + + void addPath(const String & path) override + { + if (chunks.empty() || chunks.back().size() >= chunk_limit) + { + /// add one more chunk + chunks.push_back(Chunks::value_type()); + chunks.back().reserve(chunk_limit); + } + Aws::S3::Model::ObjectIdentifier obj; + obj.SetKey(path); + chunks.back().push_back(obj); + } + + void removePaths(Fn auto && remove_chunk_func) + { + for (auto & chunk : chunks) + remove_chunk_func(std::move(chunk)); + } + + static String getChunkKeys(const Chunk & chunk) + { + String res; + for (const auto & obj : chunk) + { + const auto & key = obj.GetKey(); + if (!res.empty()) + res.append(", "); + res.append(key.c_str(), key.size()); + } + return res; + } + +private: + Chunks chunks; +}; + +template +void throwIfError(Aws::Utils::Outcome & response) +{ + if (!response.IsSuccess()) + { + const auto & err = response.GetError(); + throw Exception(std::to_string(static_cast(err.GetErrorType())) + ": " + err.GetMessage(), ErrorCodes::S3_ERROR); + } +} + +template +void throwIfError(const Aws::Utils::Outcome & response) +{ + if (!response.IsSuccess()) + { + const auto & err = response.GetError(); + throw Exception(err.GetMessage(), static_cast(err.GetErrorType())); + } +} +template +void logIfError(Aws::Utils::Outcome & response, Fn auto && msg) +{ + try + { + throwIfError(response); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__, msg()); + } +} + +template +void logIfError(const Aws::Utils::Outcome & response, Fn auto && msg) +{ + try + { + throwIfError(response); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__, msg()); + } +} + +DiskS3::DiskS3( + String name_, + String bucket_, + String s3_root_path_, + DiskPtr metadata_disk_, + FileCachePtr cache_, + ContextPtr context_, + SettingsPtr settings_, + GetDiskSettings settings_getter_) + : IDiskRemote(name_, s3_root_path_, metadata_disk_, std::move(cache_), "DiskS3", settings_->thread_pool_size) + , bucket(std::move(bucket_)) + , current_settings(std::move(settings_)) + , settings_getter(settings_getter_) + , context(context_) +{ +} + +RemoteFSPathKeeperPtr DiskS3::createFSPathKeeper() const +{ + auto settings = current_settings.get(); + return std::make_shared(settings->objects_chunk_size_to_delete); +} + +void DiskS3::removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper) +{ + auto settings = current_settings.get(); + auto * s3_paths_keeper = dynamic_cast(fs_paths_keeper.get()); + + if (s3_paths_keeper) + s3_paths_keeper->removePaths([&](S3PathKeeper::Chunk && chunk) + { + ProfileEvents::increment(ProfileEvents::DeleteS3Keys); + String keys = S3PathKeeper::getChunkKeys(chunk); + LOG_TRACE(log, "Remove AWS keys {}", keys); + Aws::S3::Model::Delete delkeys; + delkeys.SetObjects(chunk); + Aws::S3::Model::DeleteObjectsRequest request; + request.SetBucket(bucket); + request.SetDelete(delkeys); + auto outcome = settings->client->DeleteObjects(request); + // Do not throw here, continue deleting other chunks + logIfError(outcome, [&](){return "Can't remove AWS keys: " + keys;}); + }); +} + +void DiskS3::moveFile(const String & from_path, const String & to_path) +{ + auto settings = current_settings.get(); + + moveFile(from_path, to_path, settings->send_metadata); +} + +void DiskS3::moveFile(const String & from_path, const String & to_path, bool send_metadata) +{ + if (exists(to_path)) + throw Exception("File already exists: " + to_path, ErrorCodes::FILE_ALREADY_EXISTS); + + if (send_metadata) + { + auto revision = ++revision_counter; + const ObjectMetadata object_metadata { + {"from_path", from_path}, + {"to_path", to_path} + }; + createFileOperationObject("rename", revision, object_metadata); + } + metadata_disk->moveFile(from_path, to_path); +} + +std::unique_ptr DiskS3::readFile(const String & path, const ReadSettings & read_settings, std::optional, std::optional) const +{ + auto settings = current_settings.get(); + auto metadata = readMetadata(path); + + LOG_TEST(log, "Read from file by path: {}. Existing S3 objects: {}", + backQuote(metadata_disk->getPath() + path), metadata.remote_fs_objects.size()); + + ReadSettings disk_read_settings{read_settings}; + if (cache) + disk_read_settings.remote_fs_cache = cache; + + auto s3_impl = std::make_unique( + path, settings->client, bucket, metadata, + settings->s3_max_single_read_retries, disk_read_settings); + + if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) + { + auto reader = getThreadPoolReader(); + return std::make_unique(reader, disk_read_settings, std::move(s3_impl)); + } + else + { + auto buf = std::make_unique(std::move(s3_impl)); + return std::make_unique(std::move(buf), settings->min_bytes_for_seek); + } +} + +std::unique_ptr DiskS3::writeFile(const String & path, size_t buf_size, WriteMode mode) +{ + auto settings = current_settings.get(); + + /// Path to store new S3 object. + auto blob_name = getRandomASCIIString(); + + std::optional object_metadata; + if (settings->send_metadata) + { + auto revision = ++revision_counter; + object_metadata = { + {"path", path} + }; + blob_name = "r" + revisionToString(revision) + "-file-" + blob_name; + } + + LOG_TRACE(log, "{} to file by path: {}. S3 path: {}", + mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_disk->getPath() + path), remote_fs_root_path + blob_name); + + ScheduleFunc schedule = [pool = &getThreadPoolWriter(), thread_group = CurrentThread::getGroup()](auto callback) + { + pool->scheduleOrThrow([callback = std::move(callback), thread_group]() + { + if (thread_group) + CurrentThread::attachTo(thread_group); + + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + + /// After we detached from the thread_group, parent for memory_tracker inside ThreadStatus will be reset to it's parent. + /// Typically, it may be changes from Process to User. + /// Usually it could be ok, because thread pool task is executed before user-level memory tracker is destroyed. + /// However, thread could stay alive inside the thread pool, and it's ThreadStatus as well. + /// When, finally, we destroy the thread (and the ThreadStatus), + /// it can use memory tracker in the ~ThreadStatus in order to alloc/free untracked_memory,\ + /// and by this time user-level memory tracker may be already destroyed. + /// + /// As a work-around, reset memory tracker to total, which is always alive. + CurrentThread::get().memory_tracker.setParent(&total_memory_tracker); + ); + callback(); + }); + }; + + auto s3_buffer = std::make_unique( + settings->client, + bucket, + remote_fs_root_path + blob_name, + settings->s3_min_upload_part_size, + settings->s3_upload_part_size_multiply_factor, + settings->s3_upload_part_size_multiply_parts_count_threshold, + settings->s3_max_single_part_upload_size, + std::move(object_metadata), + buf_size, std::move(schedule)); + + auto create_metadata_callback = [this, path, blob_name, mode] (size_t count) + { + readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_name, count] (Metadata & metadata) { metadata.addObject(blob_name, count); return true; }); + }; + + return std::make_unique(std::move(s3_buffer), std::move(create_metadata_callback), path); +} + +void DiskS3::createHardLink(const String & src_path, const String & dst_path) +{ + auto settings = current_settings.get(); + createHardLink(src_path, dst_path, settings->send_metadata); +} + +void DiskS3::createHardLink(const String & src_path, const String & dst_path, bool send_metadata) +{ + /// We don't need to record hardlinks created to shadow folder. + if (send_metadata && !dst_path.starts_with("shadow/")) + { + auto revision = ++revision_counter; + const ObjectMetadata object_metadata { + {"src_path", src_path}, + {"dst_path", dst_path} + }; + createFileOperationObject("hardlink", revision, object_metadata); + } + + IDiskRemote::createHardLink(src_path, dst_path); +} + +void DiskS3::shutdown() +{ + auto settings = current_settings.get(); + /// This call stops any next retry attempts for ongoing S3 requests. + /// If S3 request is failed and the method below is executed S3 client immediately returns the last failed S3 request outcome. + /// If S3 is healthy nothing wrong will be happened and S3 requests will be processed in a regular way without errors. + /// This should significantly speed up shutdown process if S3 is unhealthy. + settings->client->DisableRequestProcessing(); +} + +void DiskS3::createFileOperationObject(const String & operation_name, UInt64 revision, const DiskS3::ObjectMetadata & metadata) +{ + auto settings = current_settings.get(); + const String key = "operations/r" + revisionToString(revision) + "-" + operation_name; + WriteBufferFromS3 buffer( + settings->client, + bucket, + remote_fs_root_path + key, + settings->s3_min_upload_part_size, + settings->s3_upload_part_size_multiply_factor, + settings->s3_upload_part_size_multiply_parts_count_threshold, + settings->s3_max_single_part_upload_size, + metadata); + + buffer.write('0'); + buffer.finalize(); +} + +void DiskS3::startup() +{ + auto settings = current_settings.get(); + + /// Need to be enabled if it was disabled during shutdown() call. + settings->client->EnableRequestProcessing(); + + if (!settings->send_metadata) + return; + + LOG_INFO(log, "Starting up disk {}", name); + + restore(); + + if (readSchemaVersion(bucket, remote_fs_root_path) < RESTORABLE_SCHEMA_VERSION) + migrateToRestorableSchema(); + + findLastRevision(); + + LOG_INFO(log, "Disk {} started up", name); +} + +void DiskS3::findLastRevision() +{ + /// Construct revision number from high to low bits. + String revision; + revision.reserve(64); + for (int bit = 0; bit < 64; ++bit) + { + auto revision_prefix = revision + "1"; + + LOG_TRACE(log, "Check object exists with revision prefix {}", revision_prefix); + + /// Check file or operation with such revision prefix exists. + if (checkObjectExists(bucket, remote_fs_root_path + "r" + revision_prefix) + || checkObjectExists(bucket, remote_fs_root_path + "operations/r" + revision_prefix)) + revision += "1"; + else + revision += "0"; + } + revision_counter = static_cast(std::bitset<64>(revision).to_ullong()); + LOG_INFO(log, "Found last revision number {} for disk {}", revision_counter, name); +} + +int DiskS3::readSchemaVersion(const String & source_bucket, const String & source_path) +{ + int version = 0; + if (!checkObjectExists(source_bucket, source_path + SCHEMA_VERSION_OBJECT)) + return version; + + auto settings = current_settings.get(); + ReadBufferFromS3 buffer( + settings->client, + source_bucket, + source_path + SCHEMA_VERSION_OBJECT, + settings->s3_max_single_read_retries, + context->getReadSettings()); + + readIntText(version, buffer); + + return version; +} + +void DiskS3::saveSchemaVersion(const int & version) +{ + auto settings = current_settings.get(); + + WriteBufferFromS3 buffer( + settings->client, + bucket, + remote_fs_root_path + SCHEMA_VERSION_OBJECT, + settings->s3_min_upload_part_size, + settings->s3_upload_part_size_multiply_factor, + settings->s3_upload_part_size_multiply_parts_count_threshold, + settings->s3_max_single_part_upload_size); + + writeIntText(version, buffer); + buffer.finalize(); +} + +void DiskS3::updateObjectMetadata(const String & key, const ObjectMetadata & metadata) +{ + copyObjectImpl(bucket, key, bucket, key, std::nullopt, metadata); +} + +void DiskS3::migrateFileToRestorableSchema(const String & path) +{ + LOG_TRACE(log, "Migrate file {} to restorable schema", metadata_disk->getPath() + path); + + auto meta = readMetadata(path); + + for (const auto & [key, _] : meta.remote_fs_objects) + { + ObjectMetadata metadata { + {"path", path} + }; + updateObjectMetadata(remote_fs_root_path + key, metadata); + } +} + +void DiskS3::migrateToRestorableSchemaRecursive(const String & path, Futures & results) +{ + checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. + + LOG_TRACE(log, "Migrate directory {} to restorable schema", metadata_disk->getPath() + path); + + bool dir_contains_only_files = true; + for (auto it = iterateDirectory(path); it->isValid(); it->next()) + if (isDirectory(it->path())) + { + dir_contains_only_files = false; + break; + } + + /// The whole directory can be migrated asynchronously. + if (dir_contains_only_files) + { + auto result = getExecutor().execute([this, path] + { + for (auto it = iterateDirectory(path); it->isValid(); it->next()) + migrateFileToRestorableSchema(it->path()); + }); + + results.push_back(std::move(result)); + } + else + { + for (auto it = iterateDirectory(path); it->isValid(); it->next()) + if (!isDirectory(it->path())) + { + auto source_path = it->path(); + auto result = getExecutor().execute([this, source_path] + { + migrateFileToRestorableSchema(source_path); + }); + + results.push_back(std::move(result)); + } + else + migrateToRestorableSchemaRecursive(it->path(), results); + } +} + +void DiskS3::migrateToRestorableSchema() +{ + try + { + LOG_INFO(log, "Start migration to restorable schema for disk {}", name); + + Futures results; + + for (const auto & root : data_roots) + if (exists(root)) + migrateToRestorableSchemaRecursive(root + '/', results); + + for (auto & result : results) + result.wait(); + for (auto & result : results) + result.get(); + + saveSchemaVersion(RESTORABLE_SCHEMA_VERSION); + } + catch (const Exception &) + { + tryLogCurrentException(log, fmt::format("Failed to migrate to restorable schema for disk {}", name)); + + throw; + } +} + +bool DiskS3::checkObjectExists(const String & source_bucket, const String & prefix) const +{ + auto settings = current_settings.get(); + ProfileEvents::increment(ProfileEvents::ListS3Objects); + Aws::S3::Model::ListObjectsV2Request request; + request.SetBucket(source_bucket); + request.SetPrefix(prefix); + request.SetMaxKeys(1); + + auto outcome = settings->client->ListObjectsV2(request); + throwIfError(outcome); + + return !outcome.GetResult().GetContents().empty(); +} + +bool DiskS3::checkUniqueId(const String & id) const +{ + auto settings = current_settings.get(); + /// Check that we have right s3 and have access rights + /// Actually interprets id as s3 object name and checks if it exists + ProfileEvents::increment(ProfileEvents::ListS3Objects); + Aws::S3::Model::ListObjectsV2Request request; + request.SetBucket(bucket); + request.SetPrefix(id); + + auto outcome = settings->client->ListObjectsV2(request); + throwIfError(outcome); + + Aws::Vector object_list = outcome.GetResult().GetContents(); + + for (const auto & object : object_list) + if (object.GetKey() == id) + return true; + return false; +} + +Aws::S3::Model::HeadObjectResult DiskS3::headObject(const String & source_bucket, const String & key) const +{ + auto settings = current_settings.get(); + ProfileEvents::increment(ProfileEvents::HeadS3Object); + Aws::S3::Model::HeadObjectRequest request; + request.SetBucket(source_bucket); + request.SetKey(key); + + auto outcome = settings->client->HeadObject(request); + throwIfError(outcome); + + return outcome.GetResultWithOwnership(); +} + +void DiskS3::listObjects(const String & source_bucket, const String & source_path, std::function callback) const +{ + auto settings = current_settings.get(); + ProfileEvents::increment(ProfileEvents::ListS3Objects); + Aws::S3::Model::ListObjectsV2Request request; + request.SetBucket(source_bucket); + request.SetPrefix(source_path); + request.SetMaxKeys(settings->list_object_keys_size); + + Aws::S3::Model::ListObjectsV2Outcome outcome; + do + { + outcome = settings->client->ListObjectsV2(request); + throwIfError(outcome); + + bool should_continue = callback(outcome.GetResult()); + + if (!should_continue) + break; + + request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); + } while (outcome.GetResult().GetIsTruncated()); +} + +void DiskS3::copyObject(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, + std::optional head) const +{ + if (head && (head->GetContentLength() >= static_cast(5_GiB))) + copyObjectMultipartImpl(src_bucket, src_key, dst_bucket, dst_key, head); + else + copyObjectImpl(src_bucket, src_key, dst_bucket, dst_key); +} + +void DiskS3::copyObjectImpl(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, + std::optional head, + std::optional> metadata) const +{ + auto settings = current_settings.get(); + ProfileEvents::increment(ProfileEvents::CopyS3Object); + Aws::S3::Model::CopyObjectRequest request; + request.SetCopySource(src_bucket + "/" + src_key); + request.SetBucket(dst_bucket); + request.SetKey(dst_key); + if (metadata) + { + request.SetMetadata(*metadata); + request.SetMetadataDirective(Aws::S3::Model::MetadataDirective::REPLACE); + } + + auto outcome = settings->client->CopyObject(request); + + if (!outcome.IsSuccess() && outcome.GetError().GetExceptionName() == "EntityTooLarge") + { // Can't come here with MinIO, MinIO allows single part upload for large objects. + copyObjectMultipartImpl(src_bucket, src_key, dst_bucket, dst_key, head, metadata); + return; + } + + throwIfError(outcome); +} + +void DiskS3::copyObjectMultipartImpl(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, + std::optional head, + std::optional> metadata) const +{ + LOG_TRACE(log, "Multipart copy upload has created. Src Bucket: {}, Src Key: {}, Dst Bucket: {}, Dst Key: {}, Metadata: {}", + src_bucket, src_key, dst_bucket, dst_key, metadata ? "REPLACE" : "NOT_SET"); + + auto settings = current_settings.get(); + + if (!head) + head = headObject(src_bucket, src_key); + + size_t size = head->GetContentLength(); + + String multipart_upload_id; + + { + ProfileEvents::increment(ProfileEvents::CreateS3MultipartUpload); + Aws::S3::Model::CreateMultipartUploadRequest request; + request.SetBucket(dst_bucket); + request.SetKey(dst_key); + if (metadata) + request.SetMetadata(*metadata); + + auto outcome = settings->client->CreateMultipartUpload(request); + + throwIfError(outcome); + + multipart_upload_id = outcome.GetResult().GetUploadId(); + } + + std::vector part_tags; + + size_t upload_part_size = settings->s3_min_upload_part_size; + for (size_t position = 0, part_number = 1; position < size; ++part_number, position += upload_part_size) + { + ProfileEvents::increment(ProfileEvents::UploadS3PartCopy); + Aws::S3::Model::UploadPartCopyRequest part_request; + part_request.SetCopySource(src_bucket + "/" + src_key); + part_request.SetBucket(dst_bucket); + part_request.SetKey(dst_key); + part_request.SetUploadId(multipart_upload_id); + part_request.SetPartNumber(part_number); + part_request.SetCopySourceRange(fmt::format("bytes={}-{}", position, std::min(size, position + upload_part_size) - 1)); + + auto outcome = settings->client->UploadPartCopy(part_request); + if (!outcome.IsSuccess()) + { + ProfileEvents::increment(ProfileEvents::AbortS3MultipartUpload); + Aws::S3::Model::AbortMultipartUploadRequest abort_request; + abort_request.SetBucket(dst_bucket); + abort_request.SetKey(dst_key); + abort_request.SetUploadId(multipart_upload_id); + settings->client->AbortMultipartUpload(abort_request); + // In error case we throw exception later with first error from UploadPartCopy + } + throwIfError(outcome); + + auto etag = outcome.GetResult().GetCopyPartResult().GetETag(); + part_tags.push_back(etag); + } + + { + ProfileEvents::increment(ProfileEvents::CompleteS3MultipartUpload); + Aws::S3::Model::CompleteMultipartUploadRequest req; + req.SetBucket(dst_bucket); + req.SetKey(dst_key); + req.SetUploadId(multipart_upload_id); + + Aws::S3::Model::CompletedMultipartUpload multipart_upload; + for (size_t i = 0; i < part_tags.size(); ++i) + { + Aws::S3::Model::CompletedPart part; + multipart_upload.AddParts(part.WithETag(part_tags[i]).WithPartNumber(i + 1)); + } + + req.SetMultipartUpload(multipart_upload); + + auto outcome = settings->client->CompleteMultipartUpload(req); + + throwIfError(outcome); + + LOG_TRACE(log, "Multipart copy upload has completed. Src Bucket: {}, Src Key: {}, Dst Bucket: {}, Dst Key: {}, " + "Upload_id: {}, Parts: {}", src_bucket, src_key, dst_bucket, dst_key, multipart_upload_id, part_tags.size()); + } +} + +struct DiskS3::RestoreInformation +{ + UInt64 revision = LATEST_REVISION; + String source_bucket; + String source_path; + bool detached = false; +}; + +void DiskS3::readRestoreInformation(DiskS3::RestoreInformation & restore_information) +{ + const ReadSettings read_settings; + auto buffer = metadata_disk->readFile(RESTORE_FILE_NAME, read_settings, 512); + buffer->next(); + + try + { + std::map properties; + + while (buffer->hasPendingData()) + { + String property; + readText(property, *buffer); + assertChar('\n', *buffer); + + auto pos = property.find('='); + if (pos == String::npos || pos == 0 || pos == property.length()) + throw Exception(fmt::format("Invalid property {} in restore file", property), ErrorCodes::UNKNOWN_FORMAT); + + auto key = property.substr(0, pos); + auto value = property.substr(pos + 1); + + auto it = properties.find(key); + if (it != properties.end()) + throw Exception(fmt::format("Property key duplication {} in restore file", key), ErrorCodes::UNKNOWN_FORMAT); + + properties[key] = value; + } + + for (const auto & [key, value] : properties) + { + ReadBufferFromString value_buffer (value); + + if (key == "revision") + readIntText(restore_information.revision, value_buffer); + else if (key == "source_bucket") + readText(restore_information.source_bucket, value_buffer); + else if (key == "source_path") + readText(restore_information.source_path, value_buffer); + else if (key == "detached") + readBoolTextWord(restore_information.detached, value_buffer); + else + throw Exception(fmt::format("Unknown key {} in restore file", key), ErrorCodes::UNKNOWN_FORMAT); + } + } + catch (const Exception &) + { + tryLogCurrentException(log, "Failed to read restore information"); + throw; + } +} + +void DiskS3::restore() +{ + if (!exists(RESTORE_FILE_NAME)) + return; + + try + { + RestoreInformation information; + information.source_bucket = bucket; + information.source_path = remote_fs_root_path; + + readRestoreInformation(information); + if (information.revision == 0) + information.revision = LATEST_REVISION; + if (!information.source_path.ends_with('/')) + information.source_path += '/'; + + if (information.source_bucket == bucket) + { + /// In this case we need to additionally cleanup S3 from objects with later revision. + /// Will be simply just restore to different path. + if (information.source_path == remote_fs_root_path && information.revision != LATEST_REVISION) + throw Exception("Restoring to the same bucket and path is allowed if revision is latest (0)", ErrorCodes::BAD_ARGUMENTS); + + /// This case complicates S3 cleanup in case of unsuccessful restore. + if (information.source_path != remote_fs_root_path && remote_fs_root_path.starts_with(information.source_path)) + throw Exception("Restoring to the same bucket is allowed only if source path is not a sub-path of configured path in S3 disk", ErrorCodes::BAD_ARGUMENTS); + } + + LOG_INFO(log, "Starting to restore disk {}. Revision: {}, Source bucket: {}, Source path: {}", + name, information.revision, information.source_bucket, information.source_path); + + if (readSchemaVersion(information.source_bucket, information.source_path) < RESTORABLE_SCHEMA_VERSION) + throw Exception("Source bucket doesn't have restorable schema.", ErrorCodes::BAD_ARGUMENTS); + + LOG_INFO(log, "Removing old metadata..."); + + bool cleanup_s3 = information.source_bucket != bucket || information.source_path != remote_fs_root_path; + for (const auto & root : data_roots) + if (exists(root)) + removeSharedRecursive(root + '/', !cleanup_s3); + + restoreFiles(information); + restoreFileOperations(information); + + metadata_disk->removeFile(RESTORE_FILE_NAME); + + saveSchemaVersion(RESTORABLE_SCHEMA_VERSION); + + LOG_INFO(log, "Restore disk {} finished", name); + } + catch (const Exception &) + { + tryLogCurrentException(log, fmt::format("Failed to restore disk {}", name)); + + throw; + } +} + +void DiskS3::restoreFiles(const RestoreInformation & restore_information) +{ + LOG_INFO(log, "Starting restore files for disk {}", name); + + std::vector> results; + auto restore_files = [this, &restore_information, &results](auto list_result) + { + std::vector keys; + for (const auto & row : list_result.GetContents()) + { + const String & key = row.GetKey(); + + /// Skip file operations objects. They will be processed separately. + if (key.find("/operations/") != String::npos) + continue; + + const auto [revision, _] = extractRevisionAndOperationFromKey(key); + /// Filter early if it's possible to get revision from key. + if (revision > restore_information.revision) + continue; + + keys.push_back(key); + } + + if (!keys.empty()) + { + auto result = getExecutor().execute([this, &restore_information, keys]() + { + processRestoreFiles(restore_information.source_bucket, restore_information.source_path, keys); + }); + + results.push_back(std::move(result)); + } + + return true; + }; + + /// Execute. + listObjects(restore_information.source_bucket, restore_information.source_path, restore_files); + + for (auto & result : results) + result.wait(); + for (auto & result : results) + result.get(); + + LOG_INFO(log, "Files are restored for disk {}", name); +} + +void DiskS3::processRestoreFiles(const String & source_bucket, const String & source_path, Strings keys) +{ + for (const auto & key : keys) + { + auto head_result = headObject(source_bucket, key); + auto object_metadata = head_result.GetMetadata(); + + /// Restore file if object has 'path' in metadata. + auto path_entry = object_metadata.find("path"); + if (path_entry == object_metadata.end()) + { + /// Such keys can remain after migration, we can skip them. + LOG_WARNING(log, "Skip key {} because it doesn't have 'path' in metadata", key); + continue; + } + + const auto & path = path_entry->second; + + createDirectories(directoryPath(path)); + auto relative_key = shrinkKey(source_path, key); + + /// Copy object if we restore to different bucket / path. + if (bucket != source_bucket || remote_fs_root_path != source_path) + copyObject(source_bucket, key, bucket, remote_fs_root_path + relative_key, head_result); + + auto updater = [relative_key, head_result] (Metadata & metadata) + { + metadata.addObject(relative_key, head_result.GetContentLength()); + return true; + }; + + createUpdateAndStoreMetadata(path, false, updater); + + LOG_TRACE(log, "Restored file {}", path); + } +} + +void DiskS3::restoreFileOperations(const RestoreInformation & restore_information) +{ + auto settings = current_settings.get(); + + LOG_INFO(log, "Starting restore file operations for disk {}", name); + + /// Enable recording file operations if we restore to different bucket / path. + bool send_metadata = bucket != restore_information.source_bucket || remote_fs_root_path != restore_information.source_path; + + std::set renames; + auto restore_file_operations = [this, &restore_information, &renames, &send_metadata](auto list_result) + { + const String rename = "rename"; + const String hardlink = "hardlink"; + + for (const auto & row : list_result.GetContents()) + { + const String & key = row.GetKey(); + + const auto [revision, operation] = extractRevisionAndOperationFromKey(key); + if (revision == UNKNOWN_REVISION) + { + LOG_WARNING(log, "Skip key {} with unknown revision", key); + continue; + } + + /// S3 ensures that keys will be listed in ascending UTF-8 bytes order (revision order). + /// We can stop processing if revision of the object is already more than required. + if (revision > restore_information.revision) + return false; + + /// Keep original revision if restore to different bucket / path. + if (send_metadata) + revision_counter = revision - 1; + + auto object_metadata = headObject(restore_information.source_bucket, key).GetMetadata(); + if (operation == rename) + { + auto from_path = object_metadata["from_path"]; + auto to_path = object_metadata["to_path"]; + if (exists(from_path)) + { + moveFile(from_path, to_path, send_metadata); + LOG_TRACE(log, "Revision {}. Restored rename {} -> {}", revision, from_path, to_path); + + if (restore_information.detached && isDirectory(to_path)) + { + /// Sometimes directory paths are passed without trailing '/'. We should keep them in one consistent way. + if (!from_path.ends_with('/')) + from_path += '/'; + if (!to_path.ends_with('/')) + to_path += '/'; + + /// Always keep latest actual directory path to avoid 'detaching' not existing paths. + auto it = renames.find(from_path); + if (it != renames.end()) + renames.erase(it); + + renames.insert(to_path); + } + } + } + else if (operation == hardlink) + { + auto src_path = object_metadata["src_path"]; + auto dst_path = object_metadata["dst_path"]; + if (exists(src_path)) + { + createDirectories(directoryPath(dst_path)); + createHardLink(src_path, dst_path, send_metadata); + LOG_TRACE(log, "Revision {}. Restored hardlink {} -> {}", revision, src_path, dst_path); + } + } + } + + return true; + }; + + /// Execute. + listObjects(restore_information.source_bucket, restore_information.source_path + "operations/", restore_file_operations); + + if (restore_information.detached) + { + Strings not_finished_prefixes{"tmp_", "delete_tmp_", "attaching_", "deleting_"}; + + for (const auto & path : renames) + { + /// Skip already detached parts. + if (path.find("/detached/") != std::string::npos) + continue; + + /// Skip not finished parts. They shouldn't be in 'detached' directory, because CH wouldn't be able to finish processing them. + fs::path directory_path(path); + auto directory_name = directory_path.parent_path().filename().string(); + + auto predicate = [&directory_name](String & prefix) { return directory_name.starts_with(prefix); }; + if (std::any_of(not_finished_prefixes.begin(), not_finished_prefixes.end(), predicate)) + continue; + + auto detached_path = pathToDetached(path); + + LOG_TRACE(log, "Move directory to 'detached' {} -> {}", path, detached_path); + + fs::path from_path = fs::path(path); + fs::path to_path = fs::path(detached_path); + if (path.ends_with('/')) + to_path /= from_path.parent_path().filename(); + else + to_path /= from_path.filename(); + + /// to_path may exist and non-empty in case for example abrupt restart, so remove it before rename + if (metadata_disk->exists(to_path)) + metadata_disk->removeRecursive(to_path); + + createDirectories(directoryPath(to_path)); + metadata_disk->moveDirectory(from_path, to_path); + } + } + + LOG_INFO(log, "File operations restored for disk {}", name); +} + +std::tuple DiskS3::extractRevisionAndOperationFromKey(const String & key) +{ + String revision_str; + String operation; + + re2::RE2::FullMatch(key, key_regexp, &revision_str, &operation); + + return {(revision_str.empty() ? UNKNOWN_REVISION : static_cast(std::bitset<64>(revision_str).to_ullong())), operation}; +} + +String DiskS3::shrinkKey(const String & path, const String & key) +{ + if (!key.starts_with(path)) + throw Exception("The key " + key + " prefix mismatch with given " + path, ErrorCodes::LOGICAL_ERROR); + + return key.substr(path.length()); +} + +String DiskS3::revisionToString(UInt64 revision) +{ + return std::bitset<64>(revision).to_string(); +} + +String DiskS3::pathToDetached(const String & source_path) +{ + if (source_path.ends_with('/')) + return fs::path(source_path).parent_path().parent_path() / "detached/"; + return fs::path(source_path).parent_path() / "detached/"; +} + +void DiskS3::onFreeze(const String & path) +{ + createDirectories(path); + auto revision_file_buf = metadata_disk->writeFile(path + "revision.txt", 32); + writeIntText(revision_counter.load(), *revision_file_buf); + revision_file_buf->finalize(); +} + +void DiskS3::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context_, const String &, const DisksMap &) +{ + auto new_settings = settings_getter(config, "storage_configuration.disks." + name, context_); + + current_settings.set(std::move(new_settings)); + + if (AsyncExecutor * exec = dynamic_cast(&getExecutor())) + exec->setMaxThreads(current_settings.get()->thread_pool_size); +} + +DiskS3Settings::DiskS3Settings( + const std::shared_ptr & client_, + size_t s3_max_single_read_retries_, + size_t s3_min_upload_part_size_, + size_t s3_upload_part_size_multiply_factor_, + size_t s3_upload_part_size_multiply_parts_count_threshold_, + size_t s3_max_single_part_upload_size_, + size_t min_bytes_for_seek_, + bool send_metadata_, + int thread_pool_size_, + int list_object_keys_size_, + int objects_chunk_size_to_delete_) + : client(client_) + , s3_max_single_read_retries(s3_max_single_read_retries_) + , s3_min_upload_part_size(s3_min_upload_part_size_) + , s3_upload_part_size_multiply_factor(s3_upload_part_size_multiply_factor_) + , s3_upload_part_size_multiply_parts_count_threshold(s3_upload_part_size_multiply_parts_count_threshold_) + , s3_max_single_part_upload_size(s3_max_single_part_upload_size_) + , min_bytes_for_seek(min_bytes_for_seek_) + , send_metadata(send_metadata_) + , thread_pool_size(thread_pool_size_) + , list_object_keys_size(list_object_keys_size_) + , objects_chunk_size_to_delete(objects_chunk_size_to_delete_) +{ +} + +} + +#endif diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 380365f9b95..7b6c5a796e2 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -24,6 +24,8 @@ namespace ProfileEvents extern const Event ReadBufferFromS3Bytes; extern const Event ReadBufferFromS3RequestsErrors; extern const Event ReadBufferSeekCancelConnection; + extern const Event HeadS3Object; + extern const Event GetS3ObjectRequest; } namespace DB @@ -249,7 +251,6 @@ size_t ReadBufferFromS3::getFileSize() return *file_size; auto object_size = S3::getObjectSize(client_ptr, bucket, key, version_id); - file_size = object_size; return *file_size; } @@ -275,6 +276,7 @@ SeekableReadBuffer::Range ReadBufferFromS3::getRemainingReadRange() const std::unique_ptr ReadBufferFromS3::initialize() { + ProfileEvents::increment(ProfileEvents::GetS3ObjectRequest); Aws::S3::Model::GetObjectRequest req; req.SetBucket(bucket); req.SetKey(key); diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index e97fa707c13..7bf4795161a 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -63,6 +63,11 @@ const char * S3_LOGGER_TAG_NAMES[][2] = { {"AWSAuthV4Signer", "AWSClient (AWSAuthV4Signer)"}, }; +namespace ProfileEvents +{ + extern const Event HeadS3Object; +} + const std::pair & convertLogLevel(Aws::Utils::Logging::LogLevel log_level) { static const std::unordered_map> mapping = @@ -814,6 +819,7 @@ namespace S3 S3::ObjectInfo getObjectInfo(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error) { + ProfileEvents::increment(ProfileEvents::HeadS3Object); Aws::S3::Model::HeadObjectRequest req; req.SetBucket(bucket); req.SetKey(key); diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index b09abda85db..c538f8c8f26 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -1,4 +1,5 @@ #include +#include #if USE_AWS_S3 @@ -24,6 +25,11 @@ namespace ProfileEvents { extern const Event WriteBufferFromS3Bytes; + extern const Event S3WriteBytes; + extern const Event CreateS3MultipartUpload; + extern const Event CompleteS3MultipartUpload; + extern const Event UploadS3Part; + extern const Event PutS3ObjectRequest; } namespace DB @@ -182,6 +188,7 @@ void WriteBufferFromS3::finalizeImpl() void WriteBufferFromS3::createMultipartUpload() { + ProfileEvents::increment(ProfileEvents::CreateS3MultipartUpload); Aws::S3::Model::CreateMultipartUploadRequest req; req.SetBucket(bucket); req.SetKey(key); @@ -255,6 +262,7 @@ void WriteBufferFromS3::writePart() try { + ProfileEvents::increment(ProfileEvents::UploadS3Part); fillUploadRequest(task->req, part_number); schedule([this, task, task_finish_notify]() @@ -282,6 +290,7 @@ void WriteBufferFromS3::writePart() UploadPartTask task; auto & tags = TSA_SUPPRESS_WARNING_FOR_WRITE(part_tags); /// Suppress warning because schedule == false. + ProfileEvents::increment(ProfileEvents::UploadS3Part); fillUploadRequest(task.req, tags.size() + 1); processUploadRequest(task); tags.push_back(task.tag); @@ -326,6 +335,7 @@ void WriteBufferFromS3::completeMultipartUpload() if (tags.empty()) throw Exception("Failed to complete multipart upload. No parts have uploaded", ErrorCodes::S3_ERROR); + ProfileEvents::increment(ProfileEvents::CompleteS3MultipartUpload); Aws::S3::Model::CompleteMultipartUploadRequest req; req.SetBucket(bucket); req.SetKey(key); @@ -372,7 +382,7 @@ void WriteBufferFromS3::makeSinglepartUpload() /// Notify waiting thread when put object task finished auto task_notify_finish = [&]() - { + { std::lock_guard lock(bg_tasks_mutex); put_object_task->is_finished = true; @@ -384,6 +394,7 @@ void WriteBufferFromS3::makeSinglepartUpload() try { + ProfileEvents::increment(ProfileEvents::PutS3ObjectRequest); fillPutRequest(put_object_task->req); schedule([this, task_notify_finish]() @@ -409,6 +420,7 @@ void WriteBufferFromS3::makeSinglepartUpload() else { PutObjectTask task; + ProfileEvents::increment(ProfileEvents::PutS3ObjectRequest); fillPutRequest(task.req); processPutRequest(task); } From a85c91748827a8002a903c33a9e06c1a5e40b2c5 Mon Sep 17 00:00:00 2001 From: "teng.ma" Date: Tue, 6 Sep 2022 14:57:49 +0800 Subject: [PATCH 25/86] fix master conflict --- src/Common/ProfileEvents.cpp | 1 - src/IO/ReadBufferFromS3.cpp | 1 + src/IO/S3Common.cpp | 5 ----- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index e9632aa050c..2ad868ff6dc 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -410,7 +410,6 @@ The server successfully detected this situation and will download merged part fr M(OverflowBreak, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'break' and the result is incomplete.") \ M(OverflowThrow, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'throw' and exception was thrown.") \ M(OverflowAny, "Number of times approximate GROUP BY was in effect: when aggregation was performed only on top of first 'max_rows_to_group_by' unique keys and other keys were ignored due to 'group_by_overflow_mode' = 'any'.") \ - M(ScalarSubqueriesCacheMiss, "Number of times a read from a scalar subquery was not cached and had to be calculated completely") \ M(DeleteS3Keys, "Number of s3 API Delete Keys be called") \ M(CopyS3Object, "Number of s3 API CopyObject be called") \ M(ListS3Objects, "Number of s3 API ListObjects be called") \ diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 7b6c5a796e2..b1884bd53bb 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -250,6 +250,7 @@ size_t ReadBufferFromS3::getFileSize() if (file_size) return *file_size; + ProfileEvents::increment(ProfileEvents::HeadS3Object); auto object_size = S3::getObjectSize(client_ptr, bucket, key, version_id); file_size = object_size; return *file_size; diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 7bf4795161a..9fafc304a87 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -63,10 +63,6 @@ const char * S3_LOGGER_TAG_NAMES[][2] = { {"AWSAuthV4Signer", "AWSClient (AWSAuthV4Signer)"}, }; -namespace ProfileEvents -{ - extern const Event HeadS3Object; -} const std::pair & convertLogLevel(Aws::Utils::Logging::LogLevel log_level) { @@ -819,7 +815,6 @@ namespace S3 S3::ObjectInfo getObjectInfo(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error) { - ProfileEvents::increment(ProfileEvents::HeadS3Object); Aws::S3::Model::HeadObjectRequest req; req.SetBucket(bucket); req.SetKey(key); From 5badb1b186d0475d271e23c0a61cfff256db46af Mon Sep 17 00:00:00 2001 From: mateng0915 Date: Mon, 12 Sep 2022 16:01:17 +0800 Subject: [PATCH 26/86] resolve the review comments --- src/Common/ProfileEvents.cpp | 2 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 25 +++++++++++++++++++ src/IO/ReadBufferFromS3.cpp | 2 -- src/IO/WriteBufferFromS3.cpp | 7 ++---- src/Storages/StorageS3.cpp | 12 +++++++++ 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 2ad868ff6dc..8f37fcde50d 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -410,7 +410,7 @@ The server successfully detected this situation and will download merged part fr M(OverflowBreak, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'break' and the result is incomplete.") \ M(OverflowThrow, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'throw' and exception was thrown.") \ M(OverflowAny, "Number of times approximate GROUP BY was in effect: when aggregation was performed only on top of first 'max_rows_to_group_by' unique keys and other keys were ignored due to 'group_by_overflow_mode' = 'any'.") \ - M(DeleteS3Keys, "Number of s3 API Delete Keys be called") \ + M(DeleteS3Objects, "Number of s3 API DeleteObjects be called") \ M(CopyS3Object, "Number of s3 API CopyObject be called") \ M(ListS3Objects, "Number of s3 API ListObjects be called") \ M(HeadS3Object, "Number of s3 API HeadObject be called") \ diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 998b521cc56..faa97c4848f 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -1,4 +1,5 @@ #include +#include #if USE_AWS_S3 @@ -31,6 +32,18 @@ #include #include +namespace ProfileEvents +{ + extern const Event DeleteS3Objects; + extern const Event HeadS3Object; + extern const Event ListS3Objects; + extern const Event CopyS3Object; + extern const Event CreateS3MultipartUpload; + extern const Event UploadS3PartCopy; + extern const Event AbortS3MultipartUpload; + extern const Event CompleteS3MultipartUpload; +} + namespace DB { @@ -96,6 +109,8 @@ std::string S3ObjectStorage::generateBlobNameForPath(const std::string & /* path Aws::S3::Model::HeadObjectOutcome S3ObjectStorage::requestObjectHeadData(const std::string & bucket_from, const std::string & key) const { auto client_ptr = client.get(); + + ProfileEvents::increment(ProfileEvents::HeadS3Object); Aws::S3::Model::HeadObjectRequest request; request.SetBucket(bucket_from); request.SetKey(key); @@ -211,6 +226,7 @@ void S3ObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize auto settings_ptr = s3_settings.get(); auto client_ptr = client.get(); + ProfileEvents::increment(ProfileEvents::ListS3Objects); Aws::S3::Model::ListObjectsV2Request request; request.SetBucket(bucket); request.SetPrefix(path); @@ -239,6 +255,7 @@ void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exis { auto client_ptr = client.get(); + ProfileEvents::increment(ProfileEvents::DeleteS3Objects); Aws::S3::Model::DeleteObjectRequest request; request.SetBucket(bucket); request.SetKey(object.absolute_path); @@ -284,6 +301,8 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e Aws::S3::Model::Delete delkeys; delkeys.SetObjects(current_chunk); + + ProfileEvents::increment(ProfileEvents::DeleteS3Objects); Aws::S3::Model::DeleteObjectsRequest request; request.SetBucket(bucket); request.SetDelete(delkeys); @@ -357,6 +376,8 @@ void S3ObjectStorage::copyObjectImpl( std::optional metadata) const { auto client_ptr = client.get(); + + ProfileEvents::increment(ProfileEvents::CopyS3Object); Aws::S3::Model::CopyObjectRequest request; request.SetCopySource(src_bucket + "/" + src_key); request.SetBucket(dst_bucket); @@ -405,6 +426,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( String multipart_upload_id; { + ProfileEvents::increment(ProfileEvents::CreateS3MultipartUpload); Aws::S3::Model::CreateMultipartUploadRequest request; request.SetBucket(dst_bucket); request.SetKey(dst_key); @@ -423,6 +445,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( size_t upload_part_size = settings_ptr->s3_settings.min_upload_part_size; for (size_t position = 0, part_number = 1; position < size; ++part_number, position += upload_part_size) { + ProfileEvents::increment(ProfileEvents::UploadS3PartCopy); Aws::S3::Model::UploadPartCopyRequest part_request; part_request.SetCopySource(src_bucket + "/" + src_key); part_request.SetBucket(dst_bucket); @@ -434,6 +457,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( auto outcome = client_ptr->UploadPartCopy(part_request); if (!outcome.IsSuccess()) { + ProfileEvents::increment(ProfileEvents::AbortS3MultipartUpload); Aws::S3::Model::AbortMultipartUploadRequest abort_request; abort_request.SetBucket(dst_bucket); abort_request.SetKey(dst_key); @@ -448,6 +472,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( } { + ProfileEvents::increment(ProfileEvents::CompleteS3MultipartUpload); Aws::S3::Model::CompleteMultipartUploadRequest req; req.SetBucket(dst_bucket); req.SetKey(dst_key); diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index b1884bd53bb..f2cc3f72250 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -24,7 +24,6 @@ namespace ProfileEvents extern const Event ReadBufferFromS3Bytes; extern const Event ReadBufferFromS3RequestsErrors; extern const Event ReadBufferSeekCancelConnection; - extern const Event HeadS3Object; extern const Event GetS3ObjectRequest; } @@ -250,7 +249,6 @@ size_t ReadBufferFromS3::getFileSize() if (file_size) return *file_size; - ProfileEvents::increment(ProfileEvents::HeadS3Object); auto object_size = S3::getObjectSize(client_ptr, bucket, key, version_id); file_size = object_size; return *file_size; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index c538f8c8f26..6a92337d21d 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -26,7 +26,6 @@ namespace ProfileEvents { extern const Event WriteBufferFromS3Bytes; extern const Event S3WriteBytes; - extern const Event CreateS3MultipartUpload; extern const Event CompleteS3MultipartUpload; extern const Event UploadS3Part; extern const Event PutS3ObjectRequest; @@ -188,7 +187,6 @@ void WriteBufferFromS3::finalizeImpl() void WriteBufferFromS3::createMultipartUpload() { - ProfileEvents::increment(ProfileEvents::CreateS3MultipartUpload); Aws::S3::Model::CreateMultipartUploadRequest req; req.SetBucket(bucket); req.SetKey(key); @@ -262,7 +260,6 @@ void WriteBufferFromS3::writePart() try { - ProfileEvents::increment(ProfileEvents::UploadS3Part); fillUploadRequest(task->req, part_number); schedule([this, task, task_finish_notify]() @@ -290,7 +287,6 @@ void WriteBufferFromS3::writePart() UploadPartTask task; auto & tags = TSA_SUPPRESS_WARNING_FOR_WRITE(part_tags); /// Suppress warning because schedule == false. - ProfileEvents::increment(ProfileEvents::UploadS3Part); fillUploadRequest(task.req, tags.size() + 1); processUploadRequest(task); tags.push_back(task.tag); @@ -312,6 +308,7 @@ void WriteBufferFromS3::fillUploadRequest(Aws::S3::Model::UploadPartRequest & re void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) { + ProfileEvents::increment(ProfileEvents::UploadS3Part); auto outcome = client_ptr->UploadPart(task.req); if (outcome.IsSuccess()) @@ -420,7 +417,6 @@ void WriteBufferFromS3::makeSinglepartUpload() else { PutObjectTask task; - ProfileEvents::increment(ProfileEvents::PutS3ObjectRequest); fillPutRequest(task.req); processPutRequest(task); } @@ -441,6 +437,7 @@ void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req) void WriteBufferFromS3::processPutRequest(const PutObjectTask & task) { + ProfileEvents::increment(ProfileEvents::PutS3ObjectRequest); auto outcome = client_ptr->PutObject(task.req); bool with_pool = static_cast(schedule); if (outcome.IsSuccess()) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 627679d6779..12a34946b45 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -1,4 +1,5 @@ #include +#include #include "IO/ParallelReadBuffer.h" #include "IO/IOThreadPool.h" #include "Parsers/ASTCreateQuery.h" @@ -63,6 +64,12 @@ namespace fs = std::filesystem; static const String PARTITION_ID_WILDCARD = "{_partition_id}"; +namespace ProfileEvents +{ + extern const Event DeleteS3Objects; + extern const Event ListS3Objects; +} + namespace DB { @@ -164,6 +171,7 @@ private: { buffer.clear(); + ProfileEvents::increment(ProfileEvents::ListS3Objects); outcome = client.ListObjectsV2(request); if (!outcome.IsSuccess()) throw Exception(ErrorCodes::S3_ERROR, "Could not list objects in bucket {} with prefix {}, S3 exception: {}, message: {}", @@ -552,6 +560,7 @@ Chunk StorageS3Source::generate() static bool checkIfObjectExists(const std::shared_ptr & client, const String & bucket, const String & key) { bool is_finished = false; + Aws::S3::Model::ListObjectsV2Request request; Aws::S3::Model::ListObjectsV2Outcome outcome; @@ -559,6 +568,8 @@ static bool checkIfObjectExists(const std::shared_ptr & request.SetPrefix(key); while (!is_finished) { + + ProfileEvents::increment(ProfileEvents::ListS3Objects); outcome = client->ListObjectsV2(request); if (!outcome.IsSuccess()) throw Exception( @@ -1036,6 +1047,7 @@ void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, delkeys.AddObjects(std::move(obj)); } + ProfileEvents::increment(ProfileEvents::DeleteS3Objects); Aws::S3::Model::DeleteObjectsRequest request; request.SetBucket(s3_configuration.uri.bucket); request.SetDelete(delkeys); From 9a91815f86f90c04c295874350fb1d33f5defa70 Mon Sep 17 00:00:00 2001 From: mateng0915 Date: Mon, 12 Sep 2022 16:22:12 +0800 Subject: [PATCH 27/86] revert submodel --- contrib/datasketches-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/datasketches-cpp b/contrib/datasketches-cpp index 7d73d7610db..7abd49bb2e7 160000 --- a/contrib/datasketches-cpp +++ b/contrib/datasketches-cpp @@ -1 +1 @@ -Subproject commit 7d73d7610db31d4e1ecde0fb3a7ee90ef371207f +Subproject commit 7abd49bb2e72bf9a5029993d31dcb1872da88292 From f38e262706c52a246ffb1faedb2edcab55411911 Mon Sep 17 00:00:00 2001 From: mateng0915 Date: Mon, 12 Sep 2022 16:24:47 +0800 Subject: [PATCH 28/86] fixed the unnecessory changes --- src/IO/WriteBufferFromS3.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 6a92337d21d..34fa8bb2ae2 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -379,7 +379,7 @@ void WriteBufferFromS3::makeSinglepartUpload() /// Notify waiting thread when put object task finished auto task_notify_finish = [&]() - { + { std::lock_guard lock(bg_tasks_mutex); put_object_task->is_finished = true; From f7f976e94ebd407937b9a4822d96f79cf7bccc0a Mon Sep 17 00:00:00 2001 From: mateng0915 Date: Mon, 12 Sep 2022 16:29:42 +0800 Subject: [PATCH 29/86] remove extra space --- src/IO/ReadBufferFromS3.cpp | 1 + src/IO/S3Common.cpp | 1 - src/IO/WriteBufferFromS3.cpp | 1 - src/Storages/StorageS3.cpp | 2 -- 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index f2cc3f72250..323d909764a 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -250,6 +250,7 @@ size_t ReadBufferFromS3::getFileSize() return *file_size; auto object_size = S3::getObjectSize(client_ptr, bucket, key, version_id); + file_size = object_size; return *file_size; } diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 9fafc304a87..e97fa707c13 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -63,7 +63,6 @@ const char * S3_LOGGER_TAG_NAMES[][2] = { {"AWSAuthV4Signer", "AWSClient (AWSAuthV4Signer)"}, }; - const std::pair & convertLogLevel(Aws::Utils::Logging::LogLevel log_level) { static const std::unordered_map> mapping = diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 34fa8bb2ae2..0917b5e03b1 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -391,7 +391,6 @@ void WriteBufferFromS3::makeSinglepartUpload() try { - ProfileEvents::increment(ProfileEvents::PutS3ObjectRequest); fillPutRequest(put_object_task->req); schedule([this, task_notify_finish]() diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 12a34946b45..7a24d63f186 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -560,7 +560,6 @@ Chunk StorageS3Source::generate() static bool checkIfObjectExists(const std::shared_ptr & client, const String & bucket, const String & key) { bool is_finished = false; - Aws::S3::Model::ListObjectsV2Request request; Aws::S3::Model::ListObjectsV2Outcome outcome; @@ -568,7 +567,6 @@ static bool checkIfObjectExists(const std::shared_ptr & request.SetPrefix(key); while (!is_finished) { - ProfileEvents::increment(ProfileEvents::ListS3Objects); outcome = client->ListObjectsV2(request); if (!outcome.IsSuccess()) From 372a0b779465b0a1cfc73baffeb41cf1b710d733 Mon Sep 17 00:00:00 2001 From: mateng0915 Date: Mon, 12 Sep 2022 16:44:28 +0800 Subject: [PATCH 30/86] added events into s3 integration test --- tests/integration/test_profile_events_s3/test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/integration/test_profile_events_s3/test.py b/tests/integration/test_profile_events_s3/test.py index a0f664df000..34dc25e5232 100644 --- a/tests/integration/test_profile_events_s3/test.py +++ b/tests/integration/test_profile_events_s3/test.py @@ -56,6 +56,17 @@ init_list = { "DiskS3WriteRequestsErrorsTotal": 0, "DiskS3WriteRequestsErrors503": 0, "DiskS3WriteRequestsRedirects": 0, + "DeleteS3Objects": 0, + "CopyS3Object": 0, + "ListS3Objects": 0, + "HeadS3Object": 0, + "CreateS3MultipartUpload": 0, + "UploadS3PartCopy": 0, + "UploadS3Part": 0, + "AbortS3MultipartUpload": 0, + "CompleteS3MultipartUpload": 0, + "PutS3ObjectRequest": 0, + "GetS3ObjectRequest" 0, } From 31a090075634cf3f25a008b72b7913591f6fc0a8 Mon Sep 17 00:00:00 2001 From: mateng0915 Date: Mon, 12 Sep 2022 20:32:21 +0800 Subject: [PATCH 31/86] deleted local file --- src/Disks/S3/DiskS3.cpp | 1147 --------------------------------------- 1 file changed, 1147 deletions(-) delete mode 100644 src/Disks/S3/DiskS3.cpp diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp deleted file mode 100644 index fe54e992518..00000000000 --- a/src/Disks/S3/DiskS3.cpp +++ /dev/null @@ -1,1147 +0,0 @@ -#include "DiskS3.h" - -#if USE_AWS_S3 -#include "Disks/DiskFactory.h" - -#include -#include -#include - -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ProfileEvents -{ - extern const Event DeleteS3Keys; - extern const Event CopyS3Object; - extern const Event ListS3Objects; - extern const Event HeadS3Object; - extern const Event CreateS3MultipartUpload; - extern const Event UploadS3PartCopy; - extern const Event AbortS3MultipartUpload; - extern const Event CompleteS3MultipartUpload; -} - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int S3_ERROR; - extern const int FILE_ALREADY_EXISTS; - extern const int UNKNOWN_FORMAT; - extern const int BAD_ARGUMENTS; - extern const int LOGICAL_ERROR; -} - - - -/// Helper class to collect keys into chunks of maximum size (to prepare batch requests to AWS API) -/// see https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html -class S3PathKeeper : public RemoteFSPathKeeper -{ -public: - using Chunk = Aws::Vector; - using Chunks = std::list; - - explicit S3PathKeeper(size_t chunk_limit_) : RemoteFSPathKeeper(chunk_limit_) {} - - void addPath(const String & path) override - { - if (chunks.empty() || chunks.back().size() >= chunk_limit) - { - /// add one more chunk - chunks.push_back(Chunks::value_type()); - chunks.back().reserve(chunk_limit); - } - Aws::S3::Model::ObjectIdentifier obj; - obj.SetKey(path); - chunks.back().push_back(obj); - } - - void removePaths(Fn auto && remove_chunk_func) - { - for (auto & chunk : chunks) - remove_chunk_func(std::move(chunk)); - } - - static String getChunkKeys(const Chunk & chunk) - { - String res; - for (const auto & obj : chunk) - { - const auto & key = obj.GetKey(); - if (!res.empty()) - res.append(", "); - res.append(key.c_str(), key.size()); - } - return res; - } - -private: - Chunks chunks; -}; - -template -void throwIfError(Aws::Utils::Outcome & response) -{ - if (!response.IsSuccess()) - { - const auto & err = response.GetError(); - throw Exception(std::to_string(static_cast(err.GetErrorType())) + ": " + err.GetMessage(), ErrorCodes::S3_ERROR); - } -} - -template -void throwIfError(const Aws::Utils::Outcome & response) -{ - if (!response.IsSuccess()) - { - const auto & err = response.GetError(); - throw Exception(err.GetMessage(), static_cast(err.GetErrorType())); - } -} -template -void logIfError(Aws::Utils::Outcome & response, Fn auto && msg) -{ - try - { - throwIfError(response); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__, msg()); - } -} - -template -void logIfError(const Aws::Utils::Outcome & response, Fn auto && msg) -{ - try - { - throwIfError(response); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__, msg()); - } -} - -DiskS3::DiskS3( - String name_, - String bucket_, - String s3_root_path_, - DiskPtr metadata_disk_, - FileCachePtr cache_, - ContextPtr context_, - SettingsPtr settings_, - GetDiskSettings settings_getter_) - : IDiskRemote(name_, s3_root_path_, metadata_disk_, std::move(cache_), "DiskS3", settings_->thread_pool_size) - , bucket(std::move(bucket_)) - , current_settings(std::move(settings_)) - , settings_getter(settings_getter_) - , context(context_) -{ -} - -RemoteFSPathKeeperPtr DiskS3::createFSPathKeeper() const -{ - auto settings = current_settings.get(); - return std::make_shared(settings->objects_chunk_size_to_delete); -} - -void DiskS3::removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper) -{ - auto settings = current_settings.get(); - auto * s3_paths_keeper = dynamic_cast(fs_paths_keeper.get()); - - if (s3_paths_keeper) - s3_paths_keeper->removePaths([&](S3PathKeeper::Chunk && chunk) - { - ProfileEvents::increment(ProfileEvents::DeleteS3Keys); - String keys = S3PathKeeper::getChunkKeys(chunk); - LOG_TRACE(log, "Remove AWS keys {}", keys); - Aws::S3::Model::Delete delkeys; - delkeys.SetObjects(chunk); - Aws::S3::Model::DeleteObjectsRequest request; - request.SetBucket(bucket); - request.SetDelete(delkeys); - auto outcome = settings->client->DeleteObjects(request); - // Do not throw here, continue deleting other chunks - logIfError(outcome, [&](){return "Can't remove AWS keys: " + keys;}); - }); -} - -void DiskS3::moveFile(const String & from_path, const String & to_path) -{ - auto settings = current_settings.get(); - - moveFile(from_path, to_path, settings->send_metadata); -} - -void DiskS3::moveFile(const String & from_path, const String & to_path, bool send_metadata) -{ - if (exists(to_path)) - throw Exception("File already exists: " + to_path, ErrorCodes::FILE_ALREADY_EXISTS); - - if (send_metadata) - { - auto revision = ++revision_counter; - const ObjectMetadata object_metadata { - {"from_path", from_path}, - {"to_path", to_path} - }; - createFileOperationObject("rename", revision, object_metadata); - } - metadata_disk->moveFile(from_path, to_path); -} - -std::unique_ptr DiskS3::readFile(const String & path, const ReadSettings & read_settings, std::optional, std::optional) const -{ - auto settings = current_settings.get(); - auto metadata = readMetadata(path); - - LOG_TEST(log, "Read from file by path: {}. Existing S3 objects: {}", - backQuote(metadata_disk->getPath() + path), metadata.remote_fs_objects.size()); - - ReadSettings disk_read_settings{read_settings}; - if (cache) - disk_read_settings.remote_fs_cache = cache; - - auto s3_impl = std::make_unique( - path, settings->client, bucket, metadata, - settings->s3_max_single_read_retries, disk_read_settings); - - if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) - { - auto reader = getThreadPoolReader(); - return std::make_unique(reader, disk_read_settings, std::move(s3_impl)); - } - else - { - auto buf = std::make_unique(std::move(s3_impl)); - return std::make_unique(std::move(buf), settings->min_bytes_for_seek); - } -} - -std::unique_ptr DiskS3::writeFile(const String & path, size_t buf_size, WriteMode mode) -{ - auto settings = current_settings.get(); - - /// Path to store new S3 object. - auto blob_name = getRandomASCIIString(); - - std::optional object_metadata; - if (settings->send_metadata) - { - auto revision = ++revision_counter; - object_metadata = { - {"path", path} - }; - blob_name = "r" + revisionToString(revision) + "-file-" + blob_name; - } - - LOG_TRACE(log, "{} to file by path: {}. S3 path: {}", - mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_disk->getPath() + path), remote_fs_root_path + blob_name); - - ScheduleFunc schedule = [pool = &getThreadPoolWriter(), thread_group = CurrentThread::getGroup()](auto callback) - { - pool->scheduleOrThrow([callback = std::move(callback), thread_group]() - { - if (thread_group) - CurrentThread::attachTo(thread_group); - - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - - /// After we detached from the thread_group, parent for memory_tracker inside ThreadStatus will be reset to it's parent. - /// Typically, it may be changes from Process to User. - /// Usually it could be ok, because thread pool task is executed before user-level memory tracker is destroyed. - /// However, thread could stay alive inside the thread pool, and it's ThreadStatus as well. - /// When, finally, we destroy the thread (and the ThreadStatus), - /// it can use memory tracker in the ~ThreadStatus in order to alloc/free untracked_memory,\ - /// and by this time user-level memory tracker may be already destroyed. - /// - /// As a work-around, reset memory tracker to total, which is always alive. - CurrentThread::get().memory_tracker.setParent(&total_memory_tracker); - ); - callback(); - }); - }; - - auto s3_buffer = std::make_unique( - settings->client, - bucket, - remote_fs_root_path + blob_name, - settings->s3_min_upload_part_size, - settings->s3_upload_part_size_multiply_factor, - settings->s3_upload_part_size_multiply_parts_count_threshold, - settings->s3_max_single_part_upload_size, - std::move(object_metadata), - buf_size, std::move(schedule)); - - auto create_metadata_callback = [this, path, blob_name, mode] (size_t count) - { - readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_name, count] (Metadata & metadata) { metadata.addObject(blob_name, count); return true; }); - }; - - return std::make_unique(std::move(s3_buffer), std::move(create_metadata_callback), path); -} - -void DiskS3::createHardLink(const String & src_path, const String & dst_path) -{ - auto settings = current_settings.get(); - createHardLink(src_path, dst_path, settings->send_metadata); -} - -void DiskS3::createHardLink(const String & src_path, const String & dst_path, bool send_metadata) -{ - /// We don't need to record hardlinks created to shadow folder. - if (send_metadata && !dst_path.starts_with("shadow/")) - { - auto revision = ++revision_counter; - const ObjectMetadata object_metadata { - {"src_path", src_path}, - {"dst_path", dst_path} - }; - createFileOperationObject("hardlink", revision, object_metadata); - } - - IDiskRemote::createHardLink(src_path, dst_path); -} - -void DiskS3::shutdown() -{ - auto settings = current_settings.get(); - /// This call stops any next retry attempts for ongoing S3 requests. - /// If S3 request is failed and the method below is executed S3 client immediately returns the last failed S3 request outcome. - /// If S3 is healthy nothing wrong will be happened and S3 requests will be processed in a regular way without errors. - /// This should significantly speed up shutdown process if S3 is unhealthy. - settings->client->DisableRequestProcessing(); -} - -void DiskS3::createFileOperationObject(const String & operation_name, UInt64 revision, const DiskS3::ObjectMetadata & metadata) -{ - auto settings = current_settings.get(); - const String key = "operations/r" + revisionToString(revision) + "-" + operation_name; - WriteBufferFromS3 buffer( - settings->client, - bucket, - remote_fs_root_path + key, - settings->s3_min_upload_part_size, - settings->s3_upload_part_size_multiply_factor, - settings->s3_upload_part_size_multiply_parts_count_threshold, - settings->s3_max_single_part_upload_size, - metadata); - - buffer.write('0'); - buffer.finalize(); -} - -void DiskS3::startup() -{ - auto settings = current_settings.get(); - - /// Need to be enabled if it was disabled during shutdown() call. - settings->client->EnableRequestProcessing(); - - if (!settings->send_metadata) - return; - - LOG_INFO(log, "Starting up disk {}", name); - - restore(); - - if (readSchemaVersion(bucket, remote_fs_root_path) < RESTORABLE_SCHEMA_VERSION) - migrateToRestorableSchema(); - - findLastRevision(); - - LOG_INFO(log, "Disk {} started up", name); -} - -void DiskS3::findLastRevision() -{ - /// Construct revision number from high to low bits. - String revision; - revision.reserve(64); - for (int bit = 0; bit < 64; ++bit) - { - auto revision_prefix = revision + "1"; - - LOG_TRACE(log, "Check object exists with revision prefix {}", revision_prefix); - - /// Check file or operation with such revision prefix exists. - if (checkObjectExists(bucket, remote_fs_root_path + "r" + revision_prefix) - || checkObjectExists(bucket, remote_fs_root_path + "operations/r" + revision_prefix)) - revision += "1"; - else - revision += "0"; - } - revision_counter = static_cast(std::bitset<64>(revision).to_ullong()); - LOG_INFO(log, "Found last revision number {} for disk {}", revision_counter, name); -} - -int DiskS3::readSchemaVersion(const String & source_bucket, const String & source_path) -{ - int version = 0; - if (!checkObjectExists(source_bucket, source_path + SCHEMA_VERSION_OBJECT)) - return version; - - auto settings = current_settings.get(); - ReadBufferFromS3 buffer( - settings->client, - source_bucket, - source_path + SCHEMA_VERSION_OBJECT, - settings->s3_max_single_read_retries, - context->getReadSettings()); - - readIntText(version, buffer); - - return version; -} - -void DiskS3::saveSchemaVersion(const int & version) -{ - auto settings = current_settings.get(); - - WriteBufferFromS3 buffer( - settings->client, - bucket, - remote_fs_root_path + SCHEMA_VERSION_OBJECT, - settings->s3_min_upload_part_size, - settings->s3_upload_part_size_multiply_factor, - settings->s3_upload_part_size_multiply_parts_count_threshold, - settings->s3_max_single_part_upload_size); - - writeIntText(version, buffer); - buffer.finalize(); -} - -void DiskS3::updateObjectMetadata(const String & key, const ObjectMetadata & metadata) -{ - copyObjectImpl(bucket, key, bucket, key, std::nullopt, metadata); -} - -void DiskS3::migrateFileToRestorableSchema(const String & path) -{ - LOG_TRACE(log, "Migrate file {} to restorable schema", metadata_disk->getPath() + path); - - auto meta = readMetadata(path); - - for (const auto & [key, _] : meta.remote_fs_objects) - { - ObjectMetadata metadata { - {"path", path} - }; - updateObjectMetadata(remote_fs_root_path + key, metadata); - } -} - -void DiskS3::migrateToRestorableSchemaRecursive(const String & path, Futures & results) -{ - checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. - - LOG_TRACE(log, "Migrate directory {} to restorable schema", metadata_disk->getPath() + path); - - bool dir_contains_only_files = true; - for (auto it = iterateDirectory(path); it->isValid(); it->next()) - if (isDirectory(it->path())) - { - dir_contains_only_files = false; - break; - } - - /// The whole directory can be migrated asynchronously. - if (dir_contains_only_files) - { - auto result = getExecutor().execute([this, path] - { - for (auto it = iterateDirectory(path); it->isValid(); it->next()) - migrateFileToRestorableSchema(it->path()); - }); - - results.push_back(std::move(result)); - } - else - { - for (auto it = iterateDirectory(path); it->isValid(); it->next()) - if (!isDirectory(it->path())) - { - auto source_path = it->path(); - auto result = getExecutor().execute([this, source_path] - { - migrateFileToRestorableSchema(source_path); - }); - - results.push_back(std::move(result)); - } - else - migrateToRestorableSchemaRecursive(it->path(), results); - } -} - -void DiskS3::migrateToRestorableSchema() -{ - try - { - LOG_INFO(log, "Start migration to restorable schema for disk {}", name); - - Futures results; - - for (const auto & root : data_roots) - if (exists(root)) - migrateToRestorableSchemaRecursive(root + '/', results); - - for (auto & result : results) - result.wait(); - for (auto & result : results) - result.get(); - - saveSchemaVersion(RESTORABLE_SCHEMA_VERSION); - } - catch (const Exception &) - { - tryLogCurrentException(log, fmt::format("Failed to migrate to restorable schema for disk {}", name)); - - throw; - } -} - -bool DiskS3::checkObjectExists(const String & source_bucket, const String & prefix) const -{ - auto settings = current_settings.get(); - ProfileEvents::increment(ProfileEvents::ListS3Objects); - Aws::S3::Model::ListObjectsV2Request request; - request.SetBucket(source_bucket); - request.SetPrefix(prefix); - request.SetMaxKeys(1); - - auto outcome = settings->client->ListObjectsV2(request); - throwIfError(outcome); - - return !outcome.GetResult().GetContents().empty(); -} - -bool DiskS3::checkUniqueId(const String & id) const -{ - auto settings = current_settings.get(); - /// Check that we have right s3 and have access rights - /// Actually interprets id as s3 object name and checks if it exists - ProfileEvents::increment(ProfileEvents::ListS3Objects); - Aws::S3::Model::ListObjectsV2Request request; - request.SetBucket(bucket); - request.SetPrefix(id); - - auto outcome = settings->client->ListObjectsV2(request); - throwIfError(outcome); - - Aws::Vector object_list = outcome.GetResult().GetContents(); - - for (const auto & object : object_list) - if (object.GetKey() == id) - return true; - return false; -} - -Aws::S3::Model::HeadObjectResult DiskS3::headObject(const String & source_bucket, const String & key) const -{ - auto settings = current_settings.get(); - ProfileEvents::increment(ProfileEvents::HeadS3Object); - Aws::S3::Model::HeadObjectRequest request; - request.SetBucket(source_bucket); - request.SetKey(key); - - auto outcome = settings->client->HeadObject(request); - throwIfError(outcome); - - return outcome.GetResultWithOwnership(); -} - -void DiskS3::listObjects(const String & source_bucket, const String & source_path, std::function callback) const -{ - auto settings = current_settings.get(); - ProfileEvents::increment(ProfileEvents::ListS3Objects); - Aws::S3::Model::ListObjectsV2Request request; - request.SetBucket(source_bucket); - request.SetPrefix(source_path); - request.SetMaxKeys(settings->list_object_keys_size); - - Aws::S3::Model::ListObjectsV2Outcome outcome; - do - { - outcome = settings->client->ListObjectsV2(request); - throwIfError(outcome); - - bool should_continue = callback(outcome.GetResult()); - - if (!should_continue) - break; - - request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); - } while (outcome.GetResult().GetIsTruncated()); -} - -void DiskS3::copyObject(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, - std::optional head) const -{ - if (head && (head->GetContentLength() >= static_cast(5_GiB))) - copyObjectMultipartImpl(src_bucket, src_key, dst_bucket, dst_key, head); - else - copyObjectImpl(src_bucket, src_key, dst_bucket, dst_key); -} - -void DiskS3::copyObjectImpl(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, - std::optional head, - std::optional> metadata) const -{ - auto settings = current_settings.get(); - ProfileEvents::increment(ProfileEvents::CopyS3Object); - Aws::S3::Model::CopyObjectRequest request; - request.SetCopySource(src_bucket + "/" + src_key); - request.SetBucket(dst_bucket); - request.SetKey(dst_key); - if (metadata) - { - request.SetMetadata(*metadata); - request.SetMetadataDirective(Aws::S3::Model::MetadataDirective::REPLACE); - } - - auto outcome = settings->client->CopyObject(request); - - if (!outcome.IsSuccess() && outcome.GetError().GetExceptionName() == "EntityTooLarge") - { // Can't come here with MinIO, MinIO allows single part upload for large objects. - copyObjectMultipartImpl(src_bucket, src_key, dst_bucket, dst_key, head, metadata); - return; - } - - throwIfError(outcome); -} - -void DiskS3::copyObjectMultipartImpl(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, - std::optional head, - std::optional> metadata) const -{ - LOG_TRACE(log, "Multipart copy upload has created. Src Bucket: {}, Src Key: {}, Dst Bucket: {}, Dst Key: {}, Metadata: {}", - src_bucket, src_key, dst_bucket, dst_key, metadata ? "REPLACE" : "NOT_SET"); - - auto settings = current_settings.get(); - - if (!head) - head = headObject(src_bucket, src_key); - - size_t size = head->GetContentLength(); - - String multipart_upload_id; - - { - ProfileEvents::increment(ProfileEvents::CreateS3MultipartUpload); - Aws::S3::Model::CreateMultipartUploadRequest request; - request.SetBucket(dst_bucket); - request.SetKey(dst_key); - if (metadata) - request.SetMetadata(*metadata); - - auto outcome = settings->client->CreateMultipartUpload(request); - - throwIfError(outcome); - - multipart_upload_id = outcome.GetResult().GetUploadId(); - } - - std::vector part_tags; - - size_t upload_part_size = settings->s3_min_upload_part_size; - for (size_t position = 0, part_number = 1; position < size; ++part_number, position += upload_part_size) - { - ProfileEvents::increment(ProfileEvents::UploadS3PartCopy); - Aws::S3::Model::UploadPartCopyRequest part_request; - part_request.SetCopySource(src_bucket + "/" + src_key); - part_request.SetBucket(dst_bucket); - part_request.SetKey(dst_key); - part_request.SetUploadId(multipart_upload_id); - part_request.SetPartNumber(part_number); - part_request.SetCopySourceRange(fmt::format("bytes={}-{}", position, std::min(size, position + upload_part_size) - 1)); - - auto outcome = settings->client->UploadPartCopy(part_request); - if (!outcome.IsSuccess()) - { - ProfileEvents::increment(ProfileEvents::AbortS3MultipartUpload); - Aws::S3::Model::AbortMultipartUploadRequest abort_request; - abort_request.SetBucket(dst_bucket); - abort_request.SetKey(dst_key); - abort_request.SetUploadId(multipart_upload_id); - settings->client->AbortMultipartUpload(abort_request); - // In error case we throw exception later with first error from UploadPartCopy - } - throwIfError(outcome); - - auto etag = outcome.GetResult().GetCopyPartResult().GetETag(); - part_tags.push_back(etag); - } - - { - ProfileEvents::increment(ProfileEvents::CompleteS3MultipartUpload); - Aws::S3::Model::CompleteMultipartUploadRequest req; - req.SetBucket(dst_bucket); - req.SetKey(dst_key); - req.SetUploadId(multipart_upload_id); - - Aws::S3::Model::CompletedMultipartUpload multipart_upload; - for (size_t i = 0; i < part_tags.size(); ++i) - { - Aws::S3::Model::CompletedPart part; - multipart_upload.AddParts(part.WithETag(part_tags[i]).WithPartNumber(i + 1)); - } - - req.SetMultipartUpload(multipart_upload); - - auto outcome = settings->client->CompleteMultipartUpload(req); - - throwIfError(outcome); - - LOG_TRACE(log, "Multipart copy upload has completed. Src Bucket: {}, Src Key: {}, Dst Bucket: {}, Dst Key: {}, " - "Upload_id: {}, Parts: {}", src_bucket, src_key, dst_bucket, dst_key, multipart_upload_id, part_tags.size()); - } -} - -struct DiskS3::RestoreInformation -{ - UInt64 revision = LATEST_REVISION; - String source_bucket; - String source_path; - bool detached = false; -}; - -void DiskS3::readRestoreInformation(DiskS3::RestoreInformation & restore_information) -{ - const ReadSettings read_settings; - auto buffer = metadata_disk->readFile(RESTORE_FILE_NAME, read_settings, 512); - buffer->next(); - - try - { - std::map properties; - - while (buffer->hasPendingData()) - { - String property; - readText(property, *buffer); - assertChar('\n', *buffer); - - auto pos = property.find('='); - if (pos == String::npos || pos == 0 || pos == property.length()) - throw Exception(fmt::format("Invalid property {} in restore file", property), ErrorCodes::UNKNOWN_FORMAT); - - auto key = property.substr(0, pos); - auto value = property.substr(pos + 1); - - auto it = properties.find(key); - if (it != properties.end()) - throw Exception(fmt::format("Property key duplication {} in restore file", key), ErrorCodes::UNKNOWN_FORMAT); - - properties[key] = value; - } - - for (const auto & [key, value] : properties) - { - ReadBufferFromString value_buffer (value); - - if (key == "revision") - readIntText(restore_information.revision, value_buffer); - else if (key == "source_bucket") - readText(restore_information.source_bucket, value_buffer); - else if (key == "source_path") - readText(restore_information.source_path, value_buffer); - else if (key == "detached") - readBoolTextWord(restore_information.detached, value_buffer); - else - throw Exception(fmt::format("Unknown key {} in restore file", key), ErrorCodes::UNKNOWN_FORMAT); - } - } - catch (const Exception &) - { - tryLogCurrentException(log, "Failed to read restore information"); - throw; - } -} - -void DiskS3::restore() -{ - if (!exists(RESTORE_FILE_NAME)) - return; - - try - { - RestoreInformation information; - information.source_bucket = bucket; - information.source_path = remote_fs_root_path; - - readRestoreInformation(information); - if (information.revision == 0) - information.revision = LATEST_REVISION; - if (!information.source_path.ends_with('/')) - information.source_path += '/'; - - if (information.source_bucket == bucket) - { - /// In this case we need to additionally cleanup S3 from objects with later revision. - /// Will be simply just restore to different path. - if (information.source_path == remote_fs_root_path && information.revision != LATEST_REVISION) - throw Exception("Restoring to the same bucket and path is allowed if revision is latest (0)", ErrorCodes::BAD_ARGUMENTS); - - /// This case complicates S3 cleanup in case of unsuccessful restore. - if (information.source_path != remote_fs_root_path && remote_fs_root_path.starts_with(information.source_path)) - throw Exception("Restoring to the same bucket is allowed only if source path is not a sub-path of configured path in S3 disk", ErrorCodes::BAD_ARGUMENTS); - } - - LOG_INFO(log, "Starting to restore disk {}. Revision: {}, Source bucket: {}, Source path: {}", - name, information.revision, information.source_bucket, information.source_path); - - if (readSchemaVersion(information.source_bucket, information.source_path) < RESTORABLE_SCHEMA_VERSION) - throw Exception("Source bucket doesn't have restorable schema.", ErrorCodes::BAD_ARGUMENTS); - - LOG_INFO(log, "Removing old metadata..."); - - bool cleanup_s3 = information.source_bucket != bucket || information.source_path != remote_fs_root_path; - for (const auto & root : data_roots) - if (exists(root)) - removeSharedRecursive(root + '/', !cleanup_s3); - - restoreFiles(information); - restoreFileOperations(information); - - metadata_disk->removeFile(RESTORE_FILE_NAME); - - saveSchemaVersion(RESTORABLE_SCHEMA_VERSION); - - LOG_INFO(log, "Restore disk {} finished", name); - } - catch (const Exception &) - { - tryLogCurrentException(log, fmt::format("Failed to restore disk {}", name)); - - throw; - } -} - -void DiskS3::restoreFiles(const RestoreInformation & restore_information) -{ - LOG_INFO(log, "Starting restore files for disk {}", name); - - std::vector> results; - auto restore_files = [this, &restore_information, &results](auto list_result) - { - std::vector keys; - for (const auto & row : list_result.GetContents()) - { - const String & key = row.GetKey(); - - /// Skip file operations objects. They will be processed separately. - if (key.find("/operations/") != String::npos) - continue; - - const auto [revision, _] = extractRevisionAndOperationFromKey(key); - /// Filter early if it's possible to get revision from key. - if (revision > restore_information.revision) - continue; - - keys.push_back(key); - } - - if (!keys.empty()) - { - auto result = getExecutor().execute([this, &restore_information, keys]() - { - processRestoreFiles(restore_information.source_bucket, restore_information.source_path, keys); - }); - - results.push_back(std::move(result)); - } - - return true; - }; - - /// Execute. - listObjects(restore_information.source_bucket, restore_information.source_path, restore_files); - - for (auto & result : results) - result.wait(); - for (auto & result : results) - result.get(); - - LOG_INFO(log, "Files are restored for disk {}", name); -} - -void DiskS3::processRestoreFiles(const String & source_bucket, const String & source_path, Strings keys) -{ - for (const auto & key : keys) - { - auto head_result = headObject(source_bucket, key); - auto object_metadata = head_result.GetMetadata(); - - /// Restore file if object has 'path' in metadata. - auto path_entry = object_metadata.find("path"); - if (path_entry == object_metadata.end()) - { - /// Such keys can remain after migration, we can skip them. - LOG_WARNING(log, "Skip key {} because it doesn't have 'path' in metadata", key); - continue; - } - - const auto & path = path_entry->second; - - createDirectories(directoryPath(path)); - auto relative_key = shrinkKey(source_path, key); - - /// Copy object if we restore to different bucket / path. - if (bucket != source_bucket || remote_fs_root_path != source_path) - copyObject(source_bucket, key, bucket, remote_fs_root_path + relative_key, head_result); - - auto updater = [relative_key, head_result] (Metadata & metadata) - { - metadata.addObject(relative_key, head_result.GetContentLength()); - return true; - }; - - createUpdateAndStoreMetadata(path, false, updater); - - LOG_TRACE(log, "Restored file {}", path); - } -} - -void DiskS3::restoreFileOperations(const RestoreInformation & restore_information) -{ - auto settings = current_settings.get(); - - LOG_INFO(log, "Starting restore file operations for disk {}", name); - - /// Enable recording file operations if we restore to different bucket / path. - bool send_metadata = bucket != restore_information.source_bucket || remote_fs_root_path != restore_information.source_path; - - std::set renames; - auto restore_file_operations = [this, &restore_information, &renames, &send_metadata](auto list_result) - { - const String rename = "rename"; - const String hardlink = "hardlink"; - - for (const auto & row : list_result.GetContents()) - { - const String & key = row.GetKey(); - - const auto [revision, operation] = extractRevisionAndOperationFromKey(key); - if (revision == UNKNOWN_REVISION) - { - LOG_WARNING(log, "Skip key {} with unknown revision", key); - continue; - } - - /// S3 ensures that keys will be listed in ascending UTF-8 bytes order (revision order). - /// We can stop processing if revision of the object is already more than required. - if (revision > restore_information.revision) - return false; - - /// Keep original revision if restore to different bucket / path. - if (send_metadata) - revision_counter = revision - 1; - - auto object_metadata = headObject(restore_information.source_bucket, key).GetMetadata(); - if (operation == rename) - { - auto from_path = object_metadata["from_path"]; - auto to_path = object_metadata["to_path"]; - if (exists(from_path)) - { - moveFile(from_path, to_path, send_metadata); - LOG_TRACE(log, "Revision {}. Restored rename {} -> {}", revision, from_path, to_path); - - if (restore_information.detached && isDirectory(to_path)) - { - /// Sometimes directory paths are passed without trailing '/'. We should keep them in one consistent way. - if (!from_path.ends_with('/')) - from_path += '/'; - if (!to_path.ends_with('/')) - to_path += '/'; - - /// Always keep latest actual directory path to avoid 'detaching' not existing paths. - auto it = renames.find(from_path); - if (it != renames.end()) - renames.erase(it); - - renames.insert(to_path); - } - } - } - else if (operation == hardlink) - { - auto src_path = object_metadata["src_path"]; - auto dst_path = object_metadata["dst_path"]; - if (exists(src_path)) - { - createDirectories(directoryPath(dst_path)); - createHardLink(src_path, dst_path, send_metadata); - LOG_TRACE(log, "Revision {}. Restored hardlink {} -> {}", revision, src_path, dst_path); - } - } - } - - return true; - }; - - /// Execute. - listObjects(restore_information.source_bucket, restore_information.source_path + "operations/", restore_file_operations); - - if (restore_information.detached) - { - Strings not_finished_prefixes{"tmp_", "delete_tmp_", "attaching_", "deleting_"}; - - for (const auto & path : renames) - { - /// Skip already detached parts. - if (path.find("/detached/") != std::string::npos) - continue; - - /// Skip not finished parts. They shouldn't be in 'detached' directory, because CH wouldn't be able to finish processing them. - fs::path directory_path(path); - auto directory_name = directory_path.parent_path().filename().string(); - - auto predicate = [&directory_name](String & prefix) { return directory_name.starts_with(prefix); }; - if (std::any_of(not_finished_prefixes.begin(), not_finished_prefixes.end(), predicate)) - continue; - - auto detached_path = pathToDetached(path); - - LOG_TRACE(log, "Move directory to 'detached' {} -> {}", path, detached_path); - - fs::path from_path = fs::path(path); - fs::path to_path = fs::path(detached_path); - if (path.ends_with('/')) - to_path /= from_path.parent_path().filename(); - else - to_path /= from_path.filename(); - - /// to_path may exist and non-empty in case for example abrupt restart, so remove it before rename - if (metadata_disk->exists(to_path)) - metadata_disk->removeRecursive(to_path); - - createDirectories(directoryPath(to_path)); - metadata_disk->moveDirectory(from_path, to_path); - } - } - - LOG_INFO(log, "File operations restored for disk {}", name); -} - -std::tuple DiskS3::extractRevisionAndOperationFromKey(const String & key) -{ - String revision_str; - String operation; - - re2::RE2::FullMatch(key, key_regexp, &revision_str, &operation); - - return {(revision_str.empty() ? UNKNOWN_REVISION : static_cast(std::bitset<64>(revision_str).to_ullong())), operation}; -} - -String DiskS3::shrinkKey(const String & path, const String & key) -{ - if (!key.starts_with(path)) - throw Exception("The key " + key + " prefix mismatch with given " + path, ErrorCodes::LOGICAL_ERROR); - - return key.substr(path.length()); -} - -String DiskS3::revisionToString(UInt64 revision) -{ - return std::bitset<64>(revision).to_string(); -} - -String DiskS3::pathToDetached(const String & source_path) -{ - if (source_path.ends_with('/')) - return fs::path(source_path).parent_path().parent_path() / "detached/"; - return fs::path(source_path).parent_path() / "detached/"; -} - -void DiskS3::onFreeze(const String & path) -{ - createDirectories(path); - auto revision_file_buf = metadata_disk->writeFile(path + "revision.txt", 32); - writeIntText(revision_counter.load(), *revision_file_buf); - revision_file_buf->finalize(); -} - -void DiskS3::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context_, const String &, const DisksMap &) -{ - auto new_settings = settings_getter(config, "storage_configuration.disks." + name, context_); - - current_settings.set(std::move(new_settings)); - - if (AsyncExecutor * exec = dynamic_cast(&getExecutor())) - exec->setMaxThreads(current_settings.get()->thread_pool_size); -} - -DiskS3Settings::DiskS3Settings( - const std::shared_ptr & client_, - size_t s3_max_single_read_retries_, - size_t s3_min_upload_part_size_, - size_t s3_upload_part_size_multiply_factor_, - size_t s3_upload_part_size_multiply_parts_count_threshold_, - size_t s3_max_single_part_upload_size_, - size_t min_bytes_for_seek_, - bool send_metadata_, - int thread_pool_size_, - int list_object_keys_size_, - int objects_chunk_size_to_delete_) - : client(client_) - , s3_max_single_read_retries(s3_max_single_read_retries_) - , s3_min_upload_part_size(s3_min_upload_part_size_) - , s3_upload_part_size_multiply_factor(s3_upload_part_size_multiply_factor_) - , s3_upload_part_size_multiply_parts_count_threshold(s3_upload_part_size_multiply_parts_count_threshold_) - , s3_max_single_part_upload_size(s3_max_single_part_upload_size_) - , min_bytes_for_seek(min_bytes_for_seek_) - , send_metadata(send_metadata_) - , thread_pool_size(thread_pool_size_) - , list_object_keys_size(list_object_keys_size_) - , objects_chunk_size_to_delete(objects_chunk_size_to_delete_) -{ -} - -} - -#endif From fc731341457eefe222c4a2489d4f3675da4c4a1f Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 14 Sep 2022 18:03:36 +0200 Subject: [PATCH 32/86] fix race between DROP and short form of ATTACH --- src/Databases/DatabaseOnDisk.cpp | 1 + src/Interpreters/InterpreterCreateQuery.cpp | 32 +++++++++++++++------ src/Interpreters/InterpreterCreateQuery.h | 4 ++- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index b5bb6c7c759..796142884a3 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -185,6 +185,7 @@ void DatabaseOnDisk::createTable( if (create.attach_short_syntax) { /// Metadata already exists, table was detached + assert(fs::exists(getObjectMetadataPath(table_name))); removeDetachedPermanentlyFlag(local_context, table_name, table_metadata_path, true); attachTable(local_context, table_name, table, getTableDataPath(create)); return; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index e66fe543ab0..b086adb9e73 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1001,6 +1001,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) String current_database = getContext()->getCurrentDatabase(); auto database_name = create.database ? create.getDatabase() : current_database; + DDLGuardPtr ddl_guard; + // If this is a stub ATTACH query, read the query definition from the database if (create.attach && !create.storage && !create.columns_list) { @@ -1022,6 +1024,10 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (!create.cluster.empty()) return executeQueryOnCluster(create); + /// For short syntax of ATTACH query we have to lock table name here, before reading metadata + /// and hold it until table is attached + ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); + bool if_not_exists = create.if_not_exists; // Table SQL definition is available even if the table is detached (even permanently) @@ -1053,6 +1059,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.attach_from_path) { + chassert(!ddl_guard); fs::path user_files = fs::path(getContext()->getUserFilesPath()).lexically_normal(); fs::path root_path = fs::path(getContext()->getPath()).lexically_normal(); @@ -1147,6 +1154,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (need_add_to_database && database->getEngineName() == "Replicated") { + chassert(!ddl_guard); auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); if (auto * ptr = typeid_cast(database.get()); @@ -1159,13 +1167,20 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) } if (!create.cluster.empty()) + { + chassert(!ddl_guard); return executeQueryOnCluster(create); + } if (create.replace_table) + { + chassert(!ddl_guard); return doCreateOrReplaceTable(create, properties); + } /// Actually creates table - bool created = doCreateTable(create, properties); + bool created = doCreateTable(create, properties, ddl_guard); + ddl_guard.reset(); if (!created) /// Table already exists return {}; @@ -1180,7 +1195,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) } bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, - const InterpreterCreateQuery::TableProperties & properties) + const InterpreterCreateQuery::TableProperties & properties, + DDLGuardPtr & ddl_guard) { if (create.temporary) { @@ -1193,16 +1209,12 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, return true; } - std::unique_ptr guard; + if (!ddl_guard) + ddl_guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); String data_path; DatabasePtr database; - /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). - * If table doesn't exist, one thread is creating table, while others wait in DDLGuard. - */ - guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); - database = DatabaseCatalog::instance().getDatabase(create.getDatabase()); assertOrSetUUID(create, database); @@ -1411,7 +1423,9 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, try { /// Create temporary table (random name will be generated) - [[maybe_unused]] bool done = InterpreterCreateQuery(query_ptr, create_context).doCreateTable(create, properties); + DDLGuardPtr ddl_guard; + [[maybe_unused]] bool done = InterpreterCreateQuery(query_ptr, create_context).doCreateTable(create, properties, ddl_guard); + ddl_guard.reset(); assert(done); created = true; diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 984310b2952..4d11387f44c 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -18,7 +18,9 @@ class ASTExpressionList; class ASTConstraintDeclaration; class ASTStorage; class IDatabase; +class DDLGuard; using DatabasePtr = std::shared_ptr; +using DDLGuardPtr = std::unique_ptr; /** Allows to create new table or database, @@ -89,7 +91,7 @@ private: AccessRightsElements getRequiredAccess() const; /// Create IStorage and add it to database. If table already exists and IF NOT EXISTS specified, do nothing and return false. - bool doCreateTable(ASTCreateQuery & create, const TableProperties & properties); + bool doCreateTable(ASTCreateQuery & create, const TableProperties & properties, DDLGuardPtr & ddl_guard); BlockIO doCreateOrReplaceTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties); /// Inserts data in created table if it's CREATE ... SELECT BlockIO fillTableIfNeeded(const ASTCreateQuery & create); From f1a0501eb2ab2d9b6de5a5d2f13ad86b26623085 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 14 Sep 2022 18:01:49 +0000 Subject: [PATCH 33/86] Fix memory leaks and segfaults in combinators --- .../AggregateFunctionArray.h | 10 ++--- .../AggregateFunctionDistinct.h | 11 ++--- .../AggregateFunctionForEach.h | 25 +++++++---- src/AggregateFunctions/AggregateFunctionIf.h | 10 ++--- src/AggregateFunctions/AggregateFunctionMap.h | 41 ++++++++++++++++++- .../AggregateFunctionMerge.h | 10 +++++ .../AggregateFunctionNull.h | 10 ++--- .../AggregateFunctionOrFill.h | 10 ++--- .../AggregateFunctionResample.h | 11 ++--- .../AggregateFunctionSimpleState.h | 7 ++++ .../AggregateFunctionState.h | 7 +--- src/AggregateFunctions/IAggregateFunction.h | 39 +++++++++--------- src/Columns/ColumnAggregateFunction.cpp | 2 +- src/Columns/ColumnArray.h | 8 ++++ src/Columns/ColumnConst.h | 6 +++ src/Columns/ColumnLowCardinality.h | 13 ++++++ src/Columns/ColumnMap.cpp | 6 +++ src/Columns/ColumnMap.h | 1 + src/Columns/ColumnNullable.h | 8 ++++ src/Columns/ColumnObject.cpp | 12 ++++++ src/Columns/ColumnObject.h | 1 + src/Columns/ColumnSparse.cpp | 8 ++++ src/Columns/ColumnSparse.h | 1 + src/Columns/ColumnTuple.cpp | 9 ++++ src/Columns/ColumnTuple.h | 1 + src/Columns/ColumnUnique.h | 9 ++++ src/Columns/IColumn.h | 3 ++ src/Interpreters/AggregationUtils.cpp | 13 ++++-- src/Interpreters/Aggregator.cpp | 23 ++++------- .../02418_aggregate_combinators.reference | 10 +++++ .../02418_aggregate_combinators.sql | 11 +++++ 31 files changed, 253 insertions(+), 83 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionArray.h b/src/AggregateFunctions/AggregateFunctionArray.h index a5564ea2bb3..642f98eb029 100644 --- a/src/AggregateFunctions/AggregateFunctionArray.h +++ b/src/AggregateFunctions/AggregateFunctionArray.h @@ -79,6 +79,11 @@ public: nested_func->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + nested_func->destroyUpToState(place); + } + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); @@ -99,11 +104,6 @@ public: return nested_func->isState(); } - IColumn * extractStateColumnFromResultColumn(IColumn * column) const override - { - return nested_func->extractStateColumnFromResultColumn(column); - } - void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { const IColumn * nested[num_arguments]; diff --git a/src/AggregateFunctions/AggregateFunctionDistinct.h b/src/AggregateFunctions/AggregateFunctionDistinct.h index 93d0c58b20a..622015ccc57 100644 --- a/src/AggregateFunctions/AggregateFunctionDistinct.h +++ b/src/AggregateFunctions/AggregateFunctionDistinct.h @@ -225,6 +225,12 @@ public: nested_func->destroy(getNestedPlace(place)); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + this->data(place).~Data(); + nested_func->destroyUpToState(getNestedPlace(place)); + } + String getName() const override { return nested_func->getName() + "Distinct"; @@ -245,11 +251,6 @@ public: return nested_func->isState(); } - IColumn * extractStateColumnFromResultColumn(IColumn * column) const override - { - return nested_func->extractStateColumnFromResultColumn(column); - } - AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; diff --git a/src/AggregateFunctions/AggregateFunctionForEach.h b/src/AggregateFunctions/AggregateFunctionForEach.h index e12b8e552de..d25c752d42a 100644 --- a/src/AggregateFunctions/AggregateFunctionForEach.h +++ b/src/AggregateFunctions/AggregateFunctionForEach.h @@ -66,6 +66,7 @@ private: if (old_size < new_size) { char * old_state = state.array_of_aggregate_datas; + char * new_state = arena.alignedAlloc( new_size * nested_size_of_data, nested_func->alignOfData()); @@ -139,18 +140,33 @@ public: return nested_func->getDefaultVersion(); } - void destroy(AggregateDataPtr __restrict place) const noexcept override + template + void destroyImpl(AggregateDataPtr __restrict place) const noexcept { AggregateFunctionForEachData & state = data(place); char * nested_state = state.array_of_aggregate_datas; for (size_t i = 0; i < state.dynamic_array_size; ++i) { - nested_func->destroy(nested_state); + if constexpr (up_tp_state) + nested_func->destroyUpToState(nested_state); + else + nested_func->destroy(nested_state); + nested_state += nested_size_of_data; } } + void destroy(AggregateDataPtr __restrict place) const noexcept override + { + destroyImpl(place); + } + + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + destroyImpl(place); + } + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); @@ -264,11 +280,6 @@ public: return nested_func->isState(); } - IColumn * extractStateColumnFromResultColumn(IColumn * column) const override - { - return nested_func->extractStateColumnFromResultColumn(&assert_cast(column)->getData()); - } - AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; diff --git a/src/AggregateFunctions/AggregateFunctionIf.h b/src/AggregateFunctions/AggregateFunctionIf.h index 10d46a54b74..58ecadfa167 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.h +++ b/src/AggregateFunctions/AggregateFunctionIf.h @@ -86,6 +86,11 @@ public: nested_func->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + nested_func->destroyUpToState(place); + } + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); @@ -183,11 +188,6 @@ public: return nested_func->isState(); } - IColumn * extractStateColumnFromResultColumn(IColumn * column) const override - { - return nested_func->extractStateColumnFromResultColumn(column); - } - AggregateFunctionPtr getOwnNullAdapter( const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params, const AggregateFunctionProperties & properties) const override; diff --git a/src/AggregateFunctions/AggregateFunctionMap.h b/src/AggregateFunctions/AggregateFunctionMap.h index 1a68063e9d6..d19b83b4cd4 100644 --- a/src/AggregateFunctions/AggregateFunctionMap.h +++ b/src/AggregateFunctions/AggregateFunctionMap.h @@ -89,9 +89,19 @@ public: return nested_func->isState(); } - IColumn * extractStateColumnFromResultColumn(IColumn * column) const override + bool isVersioned() const override { - return nested_func->extractStateColumnFromResultColumn(&assert_cast(column)->getNestedData().getColumn(1)); + return nested_func->isVersioned(); + } + + size_t getDefaultVersion() const override + { + return nested_func->getDefaultVersion(); + } + + bool hasTrivialDestructor() const override + { + return false; } AggregateFunctionMap(AggregateFunctionPtr nested, const DataTypes & types) : Base(types, nested->getParameters()), nested_func(nested) @@ -197,6 +207,33 @@ public: } } + template + void destroyImpl(AggregateDataPtr __restrict place) const noexcept + { + AggregateFunctionMapCombinatorData & state = Base::data(place); + + for (const auto & [key, nested_place] : state.merged_maps) + { + if constexpr (up_to_state) + nested_func->destroyUpToState(nested_place); + else + nested_func->destroy(nested_place); + } + + auto to_destroy = std::move(state.merged_maps); + this->data(place).~Data(); + } + + void destroy(AggregateDataPtr __restrict place) const noexcept override + { + destroyImpl(place); + } + + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + destroyImpl(place); + } + void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { auto & merged_maps = this->data(place).merged_maps; diff --git a/src/AggregateFunctions/AggregateFunctionMerge.h b/src/AggregateFunctions/AggregateFunctionMerge.h index 7bf0f5ea00f..bb2d36eeed1 100644 --- a/src/AggregateFunctions/AggregateFunctionMerge.h +++ b/src/AggregateFunctions/AggregateFunctionMerge.h @@ -80,6 +80,11 @@ public: nested_func->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + nested_func->destroyUpToState(place); + } + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); @@ -126,6 +131,11 @@ public: } AggregateFunctionPtr getNestedFunction() const override { return nested_func; } + + bool isState() const override + { + return nested_func->isState(); + } }; } diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index 10a9c38e41c..80fd70e59f3 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -114,6 +114,11 @@ public: nested_function->destroy(nestedPlace(place)); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + nested_function->destroyUpToState(nestedPlace(place)); + } + bool hasTrivialDestructor() const override { return nested_function->hasTrivialDestructor(); @@ -189,11 +194,6 @@ public: return nested_function->isState(); } - IColumn * extractStateColumnFromResultColumn(IColumn * column) const override - { - return nested_function->extractStateColumnFromResultColumn(column); - } - AggregateFunctionPtr getNestedFunction() const override { return nested_function; } #if USE_EMBEDDED_COMPILER diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.h b/src/AggregateFunctions/AggregateFunctionOrFill.h index 6c669b53d3e..c5a0d60224a 100644 --- a/src/AggregateFunctions/AggregateFunctionOrFill.h +++ b/src/AggregateFunctions/AggregateFunctionOrFill.h @@ -67,11 +67,6 @@ public: return nested_function->isState(); } - IColumn * extractStateColumnFromResultColumn(IColumn * column) const override - { - return nested_function->extractStateColumnFromResultColumn(column); - } - bool allocatesMemoryInArena() const override { return nested_function->allocatesMemoryInArena(); @@ -103,6 +98,11 @@ public: nested_function->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + nested_function->destroyUpToState(place); + } + void add( AggregateDataPtr __restrict place, const IColumn ** columns, diff --git a/src/AggregateFunctions/AggregateFunctionResample.h b/src/AggregateFunctions/AggregateFunctionResample.h index 5f4e173d571..b29259f9c84 100644 --- a/src/AggregateFunctions/AggregateFunctionResample.h +++ b/src/AggregateFunctions/AggregateFunctionResample.h @@ -91,11 +91,6 @@ public: return nested_function->isState(); } - IColumn * extractStateColumnFromResultColumn(IColumn * column) const override - { - return nested_function->extractStateColumnFromResultColumn(&assert_cast(column)->getData()); - } - bool allocatesMemoryInArena() const override { return nested_function->allocatesMemoryInArena(); @@ -139,6 +134,12 @@ public: nested_function->destroy(place + i * size_of_data); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + for (size_t i = 0; i < total; ++i) + nested_function->destroyUpToState(place + i * size_of_data); + } + void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { Key key; diff --git a/src/AggregateFunctions/AggregateFunctionSimpleState.h b/src/AggregateFunctions/AggregateFunctionSimpleState.h index d63d8b71b8c..c9a4dafc752 100644 --- a/src/AggregateFunctions/AggregateFunctionSimpleState.h +++ b/src/AggregateFunctions/AggregateFunctionSimpleState.h @@ -56,10 +56,17 @@ public: return nested_func->getDefaultVersion(); } + bool isState() const override + { + return nested_func->isState(); + } + void create(AggregateDataPtr __restrict place) const override { nested_func->create(place); } void destroy(AggregateDataPtr __restrict place) const noexcept override { nested_func->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override { nested_func->destroyUpToState(place); } + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); } size_t sizeOfData() const override { return nested_func->sizeOfData(); } diff --git a/src/AggregateFunctions/AggregateFunctionState.h b/src/AggregateFunctions/AggregateFunctionState.h index 24d1a694e0a..6ab3dbab625 100644 --- a/src/AggregateFunctions/AggregateFunctionState.h +++ b/src/AggregateFunctions/AggregateFunctionState.h @@ -69,6 +69,8 @@ public: nested_func->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict) const noexcept override {} + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); @@ -112,11 +114,6 @@ public: /// Aggregate function or aggregate function state. bool isState() const override { return true; } - IColumn * extractStateColumnFromResultColumn(IColumn * column) const override - { - return column; - } - bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 4ff16974b92..69d4433f5b3 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -113,6 +113,17 @@ public: /// Delete data for aggregation. virtual void destroy(AggregateDataPtr __restrict place) const noexcept = 0; + /// Delete all combinator states that were used after combinator -State. + /// For example for uniqArrayStateForEachMap(...) it will destroy + /// states that were created by combinators Map and ForEach. + /// It's needed because ColumnAggregateFunction in the result will be + /// responsible only for destruction of states that were created + /// by aggregate function and all combinators before -State combinator. + virtual void destroyUpToState(AggregateDataPtr __restrict place) const noexcept + { + destroy(place); + } + /// It is not necessary to delete data. virtual bool hasTrivialDestructor() const = 0; @@ -168,21 +179,12 @@ public: /** Returns true for aggregate functions of type -State * They are executed as other aggregate functions, but not finalized (return an aggregation state that can be combined with another). - * Also returns true when the final value of this aggregate function contains State of other aggregate function inside. */ virtual bool isState() const { return false; } - /** Special method for aggregate functions with combinator -State, it takes resulting column and extracts - * column that has type ColumnAggregateFunction. This is needed because by using different combinators - * resulting column can have complex type like nested Arrays/Maps with ColumnAggregateFunction inside it. + /** Same as isState but also returns true when the final value of this aggregate function contains + * State of other aggregate function inside (when some other combinators are used after -State) */ - virtual IColumn * extractStateColumnFromResultColumn(IColumn *) const - { - if (isState()) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Function {} is marked as State but method extractStateColumnFromResultColumn is not implemented"); - - throw Exception("Method extractStateColumnFromResultColumn is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED); - } /** The inner loop that uses the function pointer is better than using the virtual function. * The reason is that in the case of virtual functions GCC 5.1.2 generates code, @@ -289,8 +291,7 @@ public: Arena * arena) const = 0; /** Insert result of aggregate function into result column with batch size. - * If destroy_place_after_insert is true. Then implementation of this method - * must destroy aggregate place if insert state into result column was successful. + * The implementation of this method will destroy aggregate place up to -State if insert state into result column was successful. * All places that were not inserted must be destroyed if there was exception during insert into result column. */ virtual void insertResultIntoBatch( @@ -299,8 +300,7 @@ public: AggregateDataPtr * places, size_t place_offset, IColumn & to, - Arena * arena, - bool destroy_place_after_insert) const = 0; + Arena * arena) const = 0; /** Destroy batch of aggregate places. */ @@ -624,8 +624,7 @@ public: AggregateDataPtr * places, size_t place_offset, IColumn & to, - Arena * arena, - bool destroy_place_after_insert) const override + Arena * arena) const override { size_t batch_index = row_begin; @@ -634,9 +633,9 @@ public: for (; batch_index < row_end; ++batch_index) { static_cast(this)->insertResultInto(places[batch_index] + place_offset, to, arena); - - if (destroy_place_after_insert) - static_cast(this)->destroy(places[batch_index] + place_offset); + /// For State AggregateFunction ownership of aggregate place is passed to result column after insert, + /// so we need to destroy all states up to state of -State combinator. + static_cast(this)->destroyUpToState(places[batch_index] + place_offset); } } catch (...) diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index 55f851cb7c6..f9288ef8393 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -162,7 +162,7 @@ MutableColumnPtr ColumnAggregateFunction::convertToValues(MutableColumnPtr colum }; callback(res); - res->forEachSubcolumn(callback); + res->forEachSubcolumnRecursively(callback); for (auto * val : data) func->insertResultInto(val, *res, &column_aggregate_func.createOrGetArena()); diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 1d88b2e6a26..4e951ec28b8 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -157,6 +157,14 @@ public: callback(data); } + void forEachSubcolumnRecursively(ColumnCallback callback) override + { + callback(offsets); + offsets->forEachSubcolumnRecursively(callback); + callback(data); + data->forEachSubcolumnRecursively(callback); + } + bool structureEquals(const IColumn & rhs) const override { if (const auto * rhs_concrete = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h index 99a230720a4..4ae02b3689c 100644 --- a/src/Columns/ColumnConst.h +++ b/src/Columns/ColumnConst.h @@ -240,6 +240,12 @@ public: callback(data); } + void forEachSubcolumnRecursively(ColumnCallback callback) override + { + callback(data); + data->forEachSubcolumnRecursively(callback); + } + bool structureEquals(const IColumn & rhs) const override { if (const auto * rhs_concrete = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index 4fa6a0be4ed..9e9df0b5366 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -177,6 +177,19 @@ public: callback(dictionary.getColumnUniquePtr()); } + void forEachSubcolumnRecursively(ColumnCallback callback) override + { + callback(idx.getPositionsPtr()); + idx.getPositionsPtr()->forEachSubcolumnRecursively(callback); + + /// Column doesn't own dictionary if it's shared. + if (!dictionary.isShared()) + { + callback(dictionary.getColumnUniquePtr()); + dictionary.getColumnUniquePtr()->forEachSubcolumnRecursively(callback); + } + } + bool structureEquals(const IColumn & rhs) const override { if (const auto * rhs_low_cardinality = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp index 06851f992ac..34ade945aef 100644 --- a/src/Columns/ColumnMap.cpp +++ b/src/Columns/ColumnMap.cpp @@ -278,6 +278,12 @@ void ColumnMap::forEachSubcolumn(ColumnCallback callback) callback(nested); } +void ColumnMap::forEachSubcolumnRecursively(ColumnCallback callback) +{ + callback(nested); + nested->forEachSubcolumnRecursively(callback); +} + bool ColumnMap::structureEquals(const IColumn & rhs) const { if (const auto * rhs_map = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h index 95838e70d10..a3e171008ff 100644 --- a/src/Columns/ColumnMap.h +++ b/src/Columns/ColumnMap.h @@ -89,6 +89,7 @@ public: size_t allocatedBytes() const override; void protect() override; void forEachSubcolumn(ColumnCallback callback) override; + void forEachSubcolumnRecursively(ColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; double getRatioOfDefaultRows(double sample_ratio) const override; void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override; diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index e832f6d20e5..90b514f6115 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -148,6 +148,14 @@ public: callback(null_map); } + void forEachSubcolumnRecursively(ColumnCallback callback) override + { + callback(nested_column); + nested_column->forEachSubcolumnRecursively(callback); + callback(null_map); + null_map->forEachSubcolumnRecursively(callback); + } + bool structureEquals(const IColumn & rhs) const override { if (const auto * rhs_nullable = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 5b72b838b99..86586559ff7 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -671,6 +671,18 @@ void ColumnObject::forEachSubcolumn(ColumnCallback callback) callback(part); } +void ColumnObject::forEachSubcolumnRecursively(ColumnCallback callback) +{ + for (auto & entry : subcolumns) + { + for (auto & part : entry->data.data) + { + callback(part); + part->forEachSubcolumnRecursively(callback); + } + } +} + void ColumnObject::insert(const Field & field) { const auto & object = field.get(); diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index 07099307258..f32356fed6e 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -211,6 +211,7 @@ public: size_t byteSize() const override; size_t allocatedBytes() const override; void forEachSubcolumn(ColumnCallback callback) override; + void forEachSubcolumnRecursively(ColumnCallback callback) override; void insert(const Field & field) override; void insertDefault() override; void insertFrom(const IColumn & src, size_t n) override; diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp index c6f0c753d64..0c3f8d11adc 100644 --- a/src/Columns/ColumnSparse.cpp +++ b/src/Columns/ColumnSparse.cpp @@ -750,6 +750,14 @@ void ColumnSparse::forEachSubcolumn(ColumnCallback callback) callback(offsets); } +void ColumnSparse::forEachSubcolumnRecursively(ColumnCallback callback) +{ + callback(values); + values->forEachSubcolumnRecursively(callback); + callback(offsets); + offsets->forEachSubcolumnRecursively(callback); +} + const IColumn::Offsets & ColumnSparse::getOffsetsData() const { return assert_cast(*offsets).getData(); diff --git a/src/Columns/ColumnSparse.h b/src/Columns/ColumnSparse.h index 04d8f6ae9b8..5814cd77637 100644 --- a/src/Columns/ColumnSparse.h +++ b/src/Columns/ColumnSparse.h @@ -140,6 +140,7 @@ public: ColumnPtr compress() const override; void forEachSubcolumn(ColumnCallback callback) override; + void forEachSubcolumnRecursively(ColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 6763c11bb9a..d0c54466bc9 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -501,6 +501,15 @@ void ColumnTuple::forEachSubcolumn(ColumnCallback callback) callback(column); } +void ColumnTuple::forEachSubcolumnRecursively(ColumnCallback callback) +{ + for (auto & column : columns) + { + callback(column); + column->forEachSubcolumnRecursively(callback); + } +} + bool ColumnTuple::structureEquals(const IColumn & rhs) const { if (const auto * rhs_tuple = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index b1de8df74a9..385de7db1e7 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -97,6 +97,7 @@ public: size_t allocatedBytes() const override; void protect() override; void forEachSubcolumn(ColumnCallback callback) override; + void forEachSubcolumnRecursively(ColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; bool isCollationSupported() const override; ColumnPtr compress() const override; diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index 58891e30e12..f1a8af9ea90 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -115,6 +115,15 @@ public: nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask); } + void forEachSubcolumnRecursively(IColumn::ColumnCallback callback) override + { + callback(column_holder); + column_holder->forEachSubcolumnRecursively(callback); + reverse_index.setColumn(getRawColumnPtr()); + if (is_nullable) + nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask); + } + bool structureEquals(const IColumn & rhs) const override { if (auto rhs_concrete = typeid_cast(&rhs)) diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 974925d247e..254abc8416b 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -421,6 +421,9 @@ public: using ColumnCallback = std::function; virtual void forEachSubcolumn(ColumnCallback) {} + /// Similar to forEachSubcolumn but it also do recursive calls. + virtual void forEachSubcolumnRecursively(ColumnCallback) {} + /// Columns have equal structure. /// If true - you can use "compareAt", "insertFrom", etc. methods. [[nodiscard]] virtual bool structureEquals(const IColumn &) const diff --git a/src/Interpreters/AggregationUtils.cpp b/src/Interpreters/AggregationUtils.cpp index 6c51cc1c9ac..ed5e1512a1f 100644 --- a/src/Interpreters/AggregationUtils.cpp +++ b/src/Interpreters/AggregationUtils.cpp @@ -50,10 +50,15 @@ OutputBlockColumns prepareOutputBlockColumns( if (aggregate_functions[i]->isState()) { - auto * column_aggregate_func = assert_cast( - aggregate_functions[i]->extractStateColumnFromResultColumn(final_aggregate_columns[i].get())); - for (auto & pool : aggregates_pools) - column_aggregate_func->addArena(pool); + auto callback = [&](auto & subcolumn) + { + /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. + if (auto * column_aggregate_func = typeid_cast(subcolumn.get())) + for (auto & pool : aggregates_pools) + column_aggregate_func->addArena(pool); + }; + callback(final_aggregate_columns[i]); + final_aggregate_columns[i]->forEachSubcolumnRecursively(callback); } } } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index ef55f92f63a..0eb6ce7afad 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1754,8 +1754,8 @@ inline void Aggregator::insertAggregatesIntoColumns(Mapped & mapped, MutableColu * It is also tricky, because there are aggregate functions with "-State" modifier. * When we call "insertResultInto" for them, they insert a pointer to the state to ColumnAggregateFunction * and ColumnAggregateFunction will take ownership of this state. - * So, for aggregate functions with "-State" modifier, the state must not be destroyed - * after it has been transferred to ColumnAggregateFunction. + * So, for aggregate functions with "-State" modifier, only states of all combinators that are used + * after -State will be destroyed after result has been transferred to ColumnAggregateFunction. * But we should mark that the data no longer owns these states. */ @@ -1778,8 +1778,8 @@ inline void Aggregator::insertAggregatesIntoColumns(Mapped & mapped, MutableColu /** Destroy states that are no longer needed. This loop does not throw. * - * Don't destroy states for "-State" aggregate functions, - * because the ownership of this state is transferred to ColumnAggregateFunction + * For functions with -State combinator we destroy only states of all combinators that are used + * after -State, because the ownership of the rest states is transferred to ColumnAggregateFunction * and ColumnAggregateFunction will take care. * * But it's only for states that has been transferred to ColumnAggregateFunction @@ -1787,10 +1787,10 @@ inline void Aggregator::insertAggregatesIntoColumns(Mapped & mapped, MutableColu */ for (size_t destroy_i = 0; destroy_i < params.aggregates_size; ++destroy_i) { - /// If ownership was not transferred to ColumnAggregateFunction. - if (!(destroy_i < insert_i && aggregate_functions[destroy_i]->isState())) - aggregate_functions[destroy_i]->destroy( - mapped + offsets_of_aggregate_states[destroy_i]); + if (destroy_i < insert_i) + aggregate_functions[destroy_i]->destroyUpToState(mapped + offsets_of_aggregate_states[destroy_i]); + else + aggregate_functions[destroy_i]->destroy(mapped + offsets_of_aggregate_states[destroy_i]); } /// Mark the cell as destroyed so it will not be destroyed in destructor. @@ -1855,12 +1855,7 @@ Block Aggregator::insertResultsIntoColumns(PaddedPODArray & pl size_t destroy_index = aggregate_functions_destroy_index; ++aggregate_functions_destroy_index; - /// For State AggregateFunction ownership of aggregate place is passed to result column after insert - bool is_state = aggregate_functions[destroy_index]->isState(); - bool destroy_place_after_insert = !is_state; - - aggregate_functions[destroy_index]->insertResultIntoBatch( - 0, places.size(), places.data(), offset, *final_aggregate_column, arena, destroy_place_after_insert); + aggregate_functions[destroy_index]->insertResultIntoBatch(0, places.size(), places.data(), offset, *final_aggregate_column, arena); } } catch (...) diff --git a/tests/queries/0_stateless/02418_aggregate_combinators.reference b/tests/queries/0_stateless/02418_aggregate_combinators.reference index e69de29bb2d..499b9862147 100644 --- a/tests/queries/0_stateless/02418_aggregate_combinators.reference +++ b/tests/queries/0_stateless/02418_aggregate_combinators.reference @@ -0,0 +1,10 @@ +{1:'\0\n\0\0\0\0u,4ƣe\thuULsE|'} +{1:[{1:['\0\n\0\0\0\0u,4ƣe\thuULsE|','\0\n\0\0\0\0u,4ƣe\thuULsE|']}]} +[['\0\n\0\0\0\0u,4ƣe\thuULsE|','\0\nu,4ƣe\th\rH%uULsE|'],[]] +[[{1:'\0\n\0\0\0\0u,4ƣe\thuULsE|'}],[]] +['\0\n\0\0\0\0u,4ƣe\thuULsE|'] +10 +{1:10} +{1:[{1:[10,10]}]} +[[10,10],[]] +[[{1:10}],[]] diff --git a/tests/queries/0_stateless/02418_aggregate_combinators.sql b/tests/queries/0_stateless/02418_aggregate_combinators.sql index 227ce71296f..029660456fd 100644 --- a/tests/queries/0_stateless/02418_aggregate_combinators.sql +++ b/tests/queries/0_stateless/02418_aggregate_combinators.sql @@ -1,24 +1,35 @@ +select uniqStateMap(map(1, number)) from numbers(10); +select uniqStateForEachMapForEachMap(map(1, [map(1, [number, number])])) from numbers(10); +select uniqStateForEachResample(30, 75, 30)([number, number + 1], 30) from numbers(10); +select uniqStateMapForEachResample(30, 75, 30)([map(1, number)], 30) from numbers(10); +select uniqStateForEachMerge(x) as y from (select uniqStateForEachState([number]) as x from numbers(10)); +select uniqMerge(y[1]) from (select uniqStateForEachMerge(x) as y from (select uniqStateForEachState([number]) as x from numbers(10))); + drop table if exists test; create table test (x Map(UInt8, AggregateFunction(uniq, UInt64))) engine=Memory; insert into test select uniqStateMap(map(1, number)) from numbers(10); select * from test format Null; +select mapApply(k, v -> (k, finalizeAggregation(v)), x) from test; truncate table test; drop table test; create table test (x Map(UInt8, Array(Map(UInt8, Array(AggregateFunction(uniq, UInt64)))))) engine=Memory; insert into test select uniqStateForEachMapForEachMap(map(1, [map(1, [number, number])])) from numbers(10); +select mapApply(k, v -> (k, arrayMap(x -> mapApply(k, v -> (k, arrayMap(x -> finalizeAggregation(x), v)), x), v)), x) from test; select * from test format Null; truncate table test; drop table test; create table test (x Array(Array(AggregateFunction(uniq, UInt64)))) engine=Memory; insert into test select uniqStateForEachResample(30, 75, 30)([number, number + 1], 30) from numbers(10); +select arrayMap(x -> arrayMap(x -> finalizeAggregation(x), x), x) from test; select * from test format Null; truncate table test; drop table test; create table test (x Array(Array(Map(UInt8, AggregateFunction(uniq, UInt64))))) engine=Memory; insert into test select uniqStateMapForEachResample(30, 75, 30)([map(1, number)], 30) from numbers(10); +select arrayMap(x -> arrayMap(x -> mapApply(k, v -> (k, finalizeAggregation(v)), x), x), x) from test; select * from test format Null; truncate table test; drop table test; From 4971c1f04d995ebdb790d87f6ee7d09a8100a36b Mon Sep 17 00:00:00 2001 From: serxa Date: Thu, 15 Sep 2022 11:34:14 +0000 Subject: [PATCH 34/86] review fixes --- src/Access/SettingsConstraints.cpp | 2 - src/Access/SettingsConstraints.h | 4 +- src/Access/SettingsProfileElement.cpp | 7 +++- src/Access/SettingsProfileElement.h | 2 +- src/Common/SettingConstraintWritability.h | 7 ++-- .../Access/ASTSettingsProfileElement.cpp | 33 ++++++++------- .../Access/ASTSettingsProfileElement.h | 2 +- .../Access/ParserSettingsProfileElement.cpp | 8 ++-- src/Storages/System/StorageSystemSettings.cpp | 2 +- .../StorageSystemSettingsProfileElements.cpp | 42 ++++++++++--------- .../integration/test_settings_profile/test.py | 20 ++++----- .../0_stateless/01292_create_user.reference | 12 +++--- .../01294_create_settings_profile.reference | 12 +++--- ..._settings_profile_while_assigned.reference | 2 +- .../02117_show_create_table_system.reference | 3 +- 15 files changed, 78 insertions(+), 80 deletions(-) diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 69c7c7e7f64..e8a532ff48b 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -37,8 +37,6 @@ void SettingsConstraints::clear() void SettingsConstraints::set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability) { - if (min_value.isNull() && max_value.isNull() && writability == SettingConstraintWritability::DEFAULT) - return; // Do not even create entry to avoid issues during profile inheritance auto & constraint = constraints[setting_name]; if (!min_value.isNull()) constraint.min_value = Settings::castValueUtil(setting_name, min_value); diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index f29959f6601..3c4038d92c4 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -90,7 +90,7 @@ private: struct Constraint { - SettingConstraintWritability writability = SettingConstraintWritability::DEFAULT; + SettingConstraintWritability writability = SettingConstraintWritability::WRITABLE; Field min_value; Field max_value; @@ -114,7 +114,7 @@ private: , code(code_) {} - // Allow of forbid depending on range defined by constraint, also used to return stored constraint + // Allow or forbid depending on range defined by constraint, also used to return stored constraint Checker(const Constraint & constraint_) : constraint(constraint_) {} diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 48285ecdba5..474ffec0d21 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -213,7 +214,11 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints(const AccessC SettingsConstraints res{access_control}; for (const auto & elem : *this) if (!elem.setting_name.empty() && elem.setting_name != ALLOW_BACKUP_SETTING_NAME) - res.set(elem.setting_name, elem.min_value, elem.max_value, elem.writability); + res.set( + elem.setting_name, + elem.min_value, + elem.max_value, + elem.writability ? *elem.writability : SettingConstraintWritability::WRITABLE); return res; } diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index d2bf59f6799..c02e9947d61 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -26,7 +26,7 @@ struct SettingsProfileElement Field value; Field min_value; Field max_value; - SettingConstraintWritability writability = SettingConstraintWritability::DEFAULT; + std::optional writability; auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, writability); } friend bool operator==(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() == rhs.toTuple(); } diff --git a/src/Common/SettingConstraintWritability.h b/src/Common/SettingConstraintWritability.h index 583411d1d31..4a179dad286 100644 --- a/src/Common/SettingConstraintWritability.h +++ b/src/Common/SettingConstraintWritability.h @@ -6,10 +6,7 @@ namespace DB enum class SettingConstraintWritability { - // Default. Behave in the same way as WRITABLE, but is not inherited unless `settings_constraints_replace_previous` is set. - DEFAULT, - - // Setting can be change within specified range only in `readonly=0` or `readonly=2` mode. + // Default. Setting can be change within specified range only in `readonly=0` or `readonly=2` mode. WRITABLE, // Setting cannot be changed at all. @@ -19,6 +16,8 @@ enum class SettingConstraintWritability // Setting can be changed within specified range, regardless of `readonly` setting value. CHANGEABLE_IN_READONLY, + + MAX }; } diff --git a/src/Parsers/Access/ASTSettingsProfileElement.cpp b/src/Parsers/Access/ASTSettingsProfileElement.cpp index db860cb08a6..b224092e411 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.cpp +++ b/src/Parsers/Access/ASTSettingsProfileElement.cpp @@ -52,22 +52,23 @@ void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, Form << applyVisitor(FieldVisitorToString{}, max_value); } - switch (writability) - { - case SettingConstraintWritability::DEFAULT: - break; - case SettingConstraintWritability::WRITABLE: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WRITABLE" - << (settings.hilite ? IAST::hilite_none : ""); - break; - case SettingConstraintWritability::CONST: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CONST" - << (settings.hilite ? IAST::hilite_none : ""); - break; - case SettingConstraintWritability::CHANGEABLE_IN_READONLY: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CHANGEABLE_IN_READONLY" - << (settings.hilite ? IAST::hilite_none : ""); - break; + if (writability) { + switch (*writability) + { + case SettingConstraintWritability::WRITABLE: + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WRITABLE" + << (settings.hilite ? IAST::hilite_none : ""); + break; + case SettingConstraintWritability::CONST: + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CONST" + << (settings.hilite ? IAST::hilite_none : ""); + break; + case SettingConstraintWritability::CHANGEABLE_IN_READONLY: + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CHANGEABLE_IN_READONLY" + << (settings.hilite ? IAST::hilite_none : ""); + break; + case SettingConstraintWritability::MAX: break; + } } } diff --git a/src/Parsers/Access/ASTSettingsProfileElement.h b/src/Parsers/Access/ASTSettingsProfileElement.h index f67313c1601..275257e4f8e 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.h +++ b/src/Parsers/Access/ASTSettingsProfileElement.h @@ -17,7 +17,7 @@ public: Field value; Field min_value; Field max_value; - SettingConstraintWritability writability = SettingConstraintWritability::DEFAULT; + std::optional writability; bool id_mode = false; /// If true then `parent_profile` keeps UUID, not a name. bool use_inherit_keyword = false; /// If true then this element is a part of ASTCreateSettingsProfileQuery. diff --git a/src/Parsers/Access/ParserSettingsProfileElement.cpp b/src/Parsers/Access/ParserSettingsProfileElement.cpp index caee993953b..db23a806a12 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.cpp +++ b/src/Parsers/Access/ParserSettingsProfileElement.cpp @@ -95,7 +95,7 @@ namespace } - bool parseConstraintWritabilityKeyword(IParserBase::Pos & pos, Expected & expected, SettingConstraintWritability & writability) + bool parseConstraintWritabilityKeyword(IParserBase::Pos & pos, Expected & expected, std::optional & writability) { return IParserBase::wrapParseImpl(pos, [&] { @@ -127,7 +127,7 @@ namespace Field & value, Field & min_value, Field & max_value, - SettingConstraintWritability & writability) + std::optional & writability) { return IParserBase::wrapParseImpl(pos, [&] { @@ -139,7 +139,7 @@ namespace Field res_value; Field res_min_value; Field res_max_value; - SettingConstraintWritability res_writability = SettingConstraintWritability::DEFAULT; + std::optional res_writability; bool has_value_or_constraint = false; while (parseValue(pos, expected, res_value) || parseMinMaxValue(pos, expected, res_min_value, res_max_value) @@ -184,7 +184,7 @@ namespace Field value; Field min_value; Field max_value; - SettingConstraintWritability writability = SettingConstraintWritability::DEFAULT; + std::optional writability; bool ok = parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, writability); diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index 4ccf87e4071..f0ee90531c9 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -40,7 +40,7 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[3]->insert(setting.getDescription()); Field min, max; - SettingConstraintWritability writability = SettingConstraintWritability::DEFAULT; + SettingConstraintWritability writability = SettingConstraintWritability::WRITABLE; constraints.get(settings, setting_name, min, max, writability); /// These two columns can accept strings only. diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp index 62aa327230c..6785a4392e1 100644 --- a/src/Storages/System/StorageSystemSettingsProfileElements.cpp +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -12,11 +12,24 @@ #include #include #include +#include namespace DB { +const std::vector> & getSettingConstraintWritabilityEnumValues() +{ + static const std::vector> values = [] + { + std::vector> res; + for (auto value : collections::range(SettingConstraintWritability::MAX)) + res.emplace_back(toString(value), static_cast(value)); + return res; + }(); + return values; +} + NamesAndTypesList StorageSystemSettingsProfileElements::getNamesAndTypes() { NamesAndTypesList names_and_types{ @@ -28,8 +41,7 @@ NamesAndTypesList StorageSystemSettingsProfileElements::getNamesAndTypes() {"value", std::make_shared(std::make_shared())}, {"min", std::make_shared(std::make_shared())}, {"max", std::make_shared(std::make_shared())}, - {"readonly", std::make_shared(std::make_shared())}, - {"changeable_in_readonly", std::make_shared(std::make_shared())}, + {"writability", std::make_shared(std::make_shared(getSettingConstraintWritabilityEnumValues()))}, {"inherit_profile", std::make_shared(std::make_shared())}, }; return names_and_types; @@ -63,10 +75,8 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns auto & column_min_null_map = assert_cast(*res_columns[i++]).getNullMapData(); auto & column_max = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); auto & column_max_null_map = assert_cast(*res_columns[i++]).getNullMapData(); - auto & column_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()).getData(); - auto & column_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); - auto & column_changeable_in_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()).getData(); - auto & column_changeable_in_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_writability = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_writability_null_map = assert_cast(*res_columns[i++]).getNullMapData(); auto & column_inherit_profile = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); auto & column_inherit_profile_null_map = assert_cast(*res_columns[i++]).getNullMapData(); @@ -103,24 +113,16 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns inserted_max = true; } - bool inserted_readonly = false; - if ((element.writability == SettingConstraintWritability::CONST || element.writability == SettingConstraintWritability::WRITABLE) && !element.setting_name.empty()) + bool inserted_writability = false; + if (element.writability && !element.setting_name.empty()) { - column_readonly.push_back(element.writability == SettingConstraintWritability::CONST); - column_readonly_null_map.push_back(false); - inserted_readonly = true; - } - - bool inserted_changeable_in_readonly = false; - if (element.writability == SettingConstraintWritability::CHANGEABLE_IN_READONLY && !element.setting_name.empty()) - { - column_changeable_in_readonly.push_back(true); - column_changeable_in_readonly_null_map.push_back(false); - inserted_changeable_in_readonly = true; + column_writability.insertValue(static_cast(*element.writability)); + column_writability_null_map.push_back(false); + inserted_writability = true; } bool inserted_setting_name = false; - if (inserted_value || inserted_min || inserted_max || inserted_readonly || inserted_changeable_in_readonly) + if (inserted_value || inserted_min || inserted_max || inserted_writability) { const auto & setting_name = element.setting_name; column_setting_name.insertData(setting_name.data(), setting_name.size()); diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 29d4da95251..3358315cca7 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -102,7 +102,6 @@ def test_smoke(): 110000000, "\\N", "\\N", - "\\N", ] ] @@ -149,7 +148,7 @@ def test_smoke(): ) ) assert system_settings_profile_elements(user_name="robin") == [ - ["\\N", "robin", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] + ["\\N", "robin", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] ] instance.query("ALTER USER robin SETTINGS NONE") @@ -216,7 +215,6 @@ def test_settings_from_granted_role(): 110000000, "\\N", "\\N", - "\\N", ], [ "xyz", @@ -229,11 +227,10 @@ def test_settings_from_granted_role(): "\\N", "\\N", "\\N", - "\\N", ], ] assert system_settings_profile_elements(role_name="worker") == [ - ["\\N", "\\N", "worker", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] + ["\\N", "\\N", "worker", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] ] instance.query("REVOKE worker FROM robin") @@ -302,12 +299,12 @@ def test_settings_from_granted_role(): def test_inheritance(): instance.query( - "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY" + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 CONST" ) instance.query("CREATE SETTINGS PROFILE alpha SETTINGS PROFILE xyz TO robin") assert ( instance.query("SHOW CREATE SETTINGS PROFILE xyz") - == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY\n" + == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 CONST\n" ) assert ( instance.query("SHOW CREATE SETTINGS PROFILE alpha") @@ -338,8 +335,7 @@ def test_inheritance(): 100000002, "\\N", "\\N", - 1, - "\\N", + "CONST", "\\N", ] ] @@ -347,7 +343,7 @@ def test_inheritance(): ["alpha", "local directory", 1, 0, "['robin']", "[]"] ] assert system_settings_profile_elements(profile_name="alpha") == [ - ["alpha", "\\N", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] + ["alpha", "\\N", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] ] assert system_settings_profile_elements(user_name="robin") == [] @@ -431,8 +427,7 @@ def test_changeable_in_readonly(): 100000003, 90000000, 110000000, - "\\N", - 1, + "CHANGEABLE_IN_READONLY", "\\N", ], [ @@ -446,7 +441,6 @@ def test_changeable_in_readonly(): "\\N", "\\N", "\\N", - "\\N", ], ] diff --git a/tests/queries/0_stateless/01292_create_user.reference b/tests/queries/0_stateless/01292_create_user.reference index 454df63d506..997a9504bb5 100644 --- a/tests/queries/0_stateless/01292_create_user.reference +++ b/tests/queries/0_stateless/01292_create_user.reference @@ -109,9 +109,9 @@ u2_01292 local directory no_password {} [] [] [] ['%.%.myhost.com'] 0 [] [] u3_01292 local directory sha256_password {} ['192.169.1.1','192.168.0.0/16'] ['localhost'] [] [] 0 ['r1_01292'] [] u4_01292 local directory double_sha1_password {} ['::/0'] [] [] [] 1 [] ['r1_01292'] -- system.settings_profile_elements -\N u1_01292 \N 0 readonly 1 \N \N \N \N \N -\N u2_01292 \N 0 \N \N \N \N \N \N default -\N u3_01292 \N 0 max_memory_usage 5000000 4000000 6000000 0 \N \N -\N u4_01292 \N 0 \N \N \N \N \N \N default -\N u4_01292 \N 1 max_memory_usage 5000000 \N \N \N \N \N -\N u4_01292 \N 2 readonly 1 \N \N \N \N \N +\N u1_01292 \N 0 readonly 1 \N \N \N \N +\N u2_01292 \N 0 \N \N \N \N \N default +\N u3_01292 \N 0 max_memory_usage 5000000 4000000 6000000 0 \N +\N u4_01292 \N 0 \N \N \N \N \N default +\N u4_01292 \N 1 max_memory_usage 5000000 \N \N \N \N +\N u4_01292 \N 2 readonly 1 \N \N \N \N diff --git a/tests/queries/0_stateless/01294_create_settings_profile.reference b/tests/queries/0_stateless/01294_create_settings_profile.reference index 90b440e7746..68a84c1e927 100644 --- a/tests/queries/0_stateless/01294_create_settings_profile.reference +++ b/tests/queries/0_stateless/01294_create_settings_profile.reference @@ -60,9 +60,9 @@ s4_01294 local directory 1 0 ['r1_01294'] [] s5_01294 local directory 3 0 ['u1_01294'] [] s6_01294 local directory 0 1 [] ['r1_01294','u1_01294'] -- system.settings_profile_elements -s2_01294 \N \N 0 readonly 0 \N \N \N \N \N -s3_01294 \N \N 0 max_memory_usage 5000000 4000000 6000000 1 \N \N -s4_01294 \N \N 0 max_memory_usage 5000000 \N \N \N \N \N -s5_01294 \N \N 0 \N \N \N \N \N \N default -s5_01294 \N \N 1 readonly 0 \N \N \N \N \N -s5_01294 \N \N 2 max_memory_usage \N \N 6000000 0 \N \N +s2_01294 \N \N 0 readonly 0 \N \N \N \N +s3_01294 \N \N 0 max_memory_usage 5000000 4000000 6000000 CONST \N +s4_01294 \N \N 0 max_memory_usage 5000000 \N \N \N \N +s5_01294 \N \N 0 \N \N \N \N \N default +s5_01294 \N \N 1 readonly 0 \N \N \N \N +s5_01294 \N \N 2 max_memory_usage \N \N 6000000 WRITABLE \N diff --git a/tests/queries/0_stateless/01605_drop_settings_profile_while_assigned.reference b/tests/queries/0_stateless/01605_drop_settings_profile_while_assigned.reference index ac580c02596..47942812a11 100644 --- a/tests/queries/0_stateless/01605_drop_settings_profile_while_assigned.reference +++ b/tests/queries/0_stateless/01605_drop_settings_profile_while_assigned.reference @@ -1,2 +1,2 @@ -\N test_01605 \N 0 \N \N \N \N \N \N test_01605 +\N test_01605 \N 0 \N \N \N \N \N test_01605 PROFILE DROPPED diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 2cfd99ec9f4..69f2d7901e7 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -968,8 +968,7 @@ CREATE TABLE system.settings_profile_elements `value` Nullable(String), `min` Nullable(String), `max` Nullable(String), - `readonly` Nullable(UInt8), - `changeable_in_readonly` Nullable(UInt8), + `writability` Nullable(Enum8('WRITABLE' = 0, 'CONST' = 1, 'CHANGEABLE_IN_READONLY' = 2)) `inherit_profile` Nullable(String) ) ENGINE = SystemSettingsProfileElements From 0d1ef863facd8d02af1098b12f3fcf3ec645fea0 Mon Sep 17 00:00:00 2001 From: serxa Date: Thu, 15 Sep 2022 11:48:15 +0000 Subject: [PATCH 35/86] fix stateless tests --- tests/queries/0_stateless/01292_create_user.reference | 2 +- .../0_stateless/02117_show_create_table_system.reference | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01292_create_user.reference b/tests/queries/0_stateless/01292_create_user.reference index 717e8ccf5b6..f723412c636 100644 --- a/tests/queries/0_stateless/01292_create_user.reference +++ b/tests/queries/0_stateless/01292_create_user.reference @@ -111,7 +111,7 @@ u4_01292 local directory double_sha1_password {} ['::/0'] [] [] [] 1 [] ['r1_012 -- system.settings_profile_elements \N u1_01292 \N 0 readonly 1 \N \N \N \N \N u2_01292 \N 0 \N \N \N \N \N default -\N u3_01292 \N 0 max_memory_usage 5000000 4000000 6000000 0 \N +\N u3_01292 \N 0 max_memory_usage 5000000 4000000 6000000 WRITABLE \N \N u4_01292 \N 0 \N \N \N \N \N default \N u4_01292 \N 1 max_memory_usage 5000000 \N \N \N \N \N u4_01292 \N 2 readonly 1 \N \N \N \N diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 69f2d7901e7..213c754a69f 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -968,7 +968,7 @@ CREATE TABLE system.settings_profile_elements `value` Nullable(String), `min` Nullable(String), `max` Nullable(String), - `writability` Nullable(Enum8('WRITABLE' = 0, 'CONST' = 1, 'CHANGEABLE_IN_READONLY' = 2)) + `writability` Nullable(Enum8('WRITABLE' = 0, 'CONST' = 1, 'CHANGEABLE_IN_READONLY' = 2)), `inherit_profile` Nullable(String) ) ENGINE = SystemSettingsProfileElements From 77c4ea96477fc4a42d677cc13b953b2a421cf0ed Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 15 Sep 2022 14:34:31 +0200 Subject: [PATCH 36/86] fix --- src/Interpreters/InterpreterCreateQuery.cpp | 22 +++++++++------------ 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index b086adb9e73..feb100ebd2e 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1008,16 +1008,14 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) { auto database = DatabaseCatalog::instance().getDatabase(database_name); - if (database->getEngineName() == "Replicated") + if (auto * replicated = typeid_cast(database.get())) { - auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); - - if (auto * ptr = typeid_cast(database.get()); - ptr && !getContext()->getClientInfo().is_replicated_database_internal) + if (!getContext()->getClientInfo().is_replicated_database_internal) { + auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); create.setDatabase(database_name); guard->releaseTableLock(); - return ptr->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); + return replicated->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); } } @@ -1152,17 +1150,15 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (need_add_to_database) database = DatabaseCatalog::instance().getDatabase(database_name); - if (need_add_to_database && database->getEngineName() == "Replicated") + if (auto * replicated = typeid_cast(database.get())) { - chassert(!ddl_guard); - auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); - - if (auto * ptr = typeid_cast(database.get()); - ptr && !getContext()->getClientInfo().is_replicated_database_internal) + if (!getContext()->getClientInfo().is_replicated_database_internal) { + chassert(!ddl_guard); + auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); assertOrSetUUID(create, database); guard->releaseTableLock(); - return ptr->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); + return replicated->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); } } From d41bb7c2a4eaf19cc724ef78db4577bc36e17280 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 15 Sep 2022 12:40:32 +0000 Subject: [PATCH 37/86] Fix comment --- src/AggregateFunctions/IAggregateFunction.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 69d4433f5b3..18505dd17bd 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -182,10 +182,6 @@ public: */ virtual bool isState() const { return false; } - /** Same as isState but also returns true when the final value of this aggregate function contains - * State of other aggregate function inside (when some other combinators are used after -State) - */ - /** The inner loop that uses the function pointer is better than using the virtual function. * The reason is that in the case of virtual functions GCC 5.1.2 generates code, * which, at each iteration of the loop, reloads the function address (the offset value in the virtual function table) from memory to the register. From 6bb6c9e91657f260afb8aa2e1ba11655f3feae8f Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 15 Sep 2022 12:41:39 +0000 Subject: [PATCH 38/86] Fix comment --- src/AggregateFunctions/IAggregateFunction.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 18505dd17bd..7a4feebbe0f 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -179,6 +179,7 @@ public: /** Returns true for aggregate functions of type -State * They are executed as other aggregate functions, but not finalized (return an aggregation state that can be combined with another). + * Also returns true when the final value of this aggregate function contains State of other aggregate function inside. */ virtual bool isState() const { return false; } From b3b8123f7385fb1026e96080d2740446765772cd Mon Sep 17 00:00:00 2001 From: serxa Date: Thu, 15 Sep 2022 12:53:56 +0000 Subject: [PATCH 39/86] fix style --- src/Parsers/Access/ASTSettingsProfileElement.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Parsers/Access/ASTSettingsProfileElement.cpp b/src/Parsers/Access/ASTSettingsProfileElement.cpp index b224092e411..76973c428b2 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.cpp +++ b/src/Parsers/Access/ASTSettingsProfileElement.cpp @@ -52,7 +52,8 @@ void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, Form << applyVisitor(FieldVisitorToString{}, max_value); } - if (writability) { + if (writability) + { switch (*writability) { case SettingConstraintWritability::WRITABLE: From ee361c6d46cf7dbd26c546ad14f79a8491b5dfc0 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 15 Sep 2022 17:31:28 +0000 Subject: [PATCH 40/86] Enable DNS cache updater earlier. --- programs/server/Server.cpp | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 168a9a0489a..f8cd2dadb3f 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1476,6 +1476,23 @@ int Server::main(const std::vector & /*args*/) /// try set up encryption. There are some errors in config, error will be printed and server wouldn't start. CompressionCodecEncrypted::Configuration::instance().load(config(), "encryption_codecs"); + std::unique_ptr dns_cache_updater; + if (config().has("disable_internal_dns_cache") && config().getInt("disable_internal_dns_cache")) + { + /// Disable DNS caching at all + DNSResolver::instance().setDisableCacheFlag(); + LOG_DEBUG(log, "DNS caching disabled"); + } + else + { + /// Initialize a watcher periodically updating DNS cache + dns_cache_updater = std::make_unique( + global_context, config().getInt("dns_cache_update_period", 15), config().getUInt("dns_max_consecutive_failures", 5)); + } + + if (dns_cache_updater) + dns_cache_updater->start(); + SCOPE_EXIT({ /// Stop reloading of the main config. This must be done before `global_context->shutdown()` because /// otherwise the reloading may pass a changed config to some destroyed parts of ContextSharedPart. @@ -1629,20 +1646,6 @@ int Server::main(const std::vector & /*args*/) LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe)."); - std::unique_ptr dns_cache_updater; - if (config().has("disable_internal_dns_cache") && config().getInt("disable_internal_dns_cache")) - { - /// Disable DNS caching at all - DNSResolver::instance().setDisableCacheFlag(); - LOG_DEBUG(log, "DNS caching disabled"); - } - else - { - /// Initialize a watcher periodically updating DNS cache - dns_cache_updater = std::make_unique( - global_context, config().getInt("dns_cache_update_period", 15), config().getUInt("dns_max_consecutive_failures", 5)); - } - #if defined(OS_LINUX) auto tasks_stats_provider = TasksStatsCounters::findBestAvailableProvider(); if (tasks_stats_provider == TasksStatsCounters::MetricsProvider::None) @@ -1707,8 +1710,6 @@ int Server::main(const std::vector & /*args*/) main_config_reloader->start(); access_control.startPeriodicReloading(); - if (dns_cache_updater) - dns_cache_updater->start(); { LOG_INFO(log, "Available RAM: {}; physical cores: {}; logical cores: {}.", From 92979af3bf5b242b1db22693768abfac872364f6 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 15 Sep 2022 17:45:50 +0000 Subject: [PATCH 41/86] Add missing methods in combinators --- .../AggregateFunctionArray.h | 5 +++++ .../AggregateFunctionDistinct.h | 20 +++++++++++++++++++ .../AggregateFunctionForEach.h | 5 +++++ src/AggregateFunctions/AggregateFunctionIf.h | 5 +++++ src/AggregateFunctions/AggregateFunctionMap.h | 5 +++++ .../AggregateFunctionNull.h | 15 ++++++++++++++ .../AggregateFunctionResample.h | 15 ++++++++++++++ .../AggregateFunctionSimpleState.h | 5 +++++ 8 files changed, 75 insertions(+) diff --git a/src/AggregateFunctions/AggregateFunctionArray.h b/src/AggregateFunctions/AggregateFunctionArray.h index 642f98eb029..abefe8e0de1 100644 --- a/src/AggregateFunctions/AggregateFunctionArray.h +++ b/src/AggregateFunctions/AggregateFunctionArray.h @@ -64,6 +64,11 @@ public: return nested_func->isVersioned(); } + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + size_t getDefaultVersion() const override { return nested_func->getDefaultVersion(); diff --git a/src/AggregateFunctions/AggregateFunctionDistinct.h b/src/AggregateFunctions/AggregateFunctionDistinct.h index 622015ccc57..567891131af 100644 --- a/src/AggregateFunctions/AggregateFunctionDistinct.h +++ b/src/AggregateFunctions/AggregateFunctionDistinct.h @@ -251,6 +251,26 @@ public: return nested_func->isState(); } + bool isVersioned() const override + { + return nested_func->isVersioned(); + } + + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + + size_t getDefaultVersion() const override + { + return nested_func->getDefaultVersion(); + } + + bool hasTrivialDestructor() const override + { + return nested_func->hasTrivialDestructor(); + } + AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; diff --git a/src/AggregateFunctions/AggregateFunctionForEach.h b/src/AggregateFunctions/AggregateFunctionForEach.h index d25c752d42a..becd00a782e 100644 --- a/src/AggregateFunctions/AggregateFunctionForEach.h +++ b/src/AggregateFunctions/AggregateFunctionForEach.h @@ -135,6 +135,11 @@ public: return nested_func->isVersioned(); } + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + size_t getDefaultVersion() const override { return nested_func->getDefaultVersion(); diff --git a/src/AggregateFunctions/AggregateFunctionIf.h b/src/AggregateFunctions/AggregateFunctionIf.h index 58ecadfa167..6b0905d6d5e 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.h +++ b/src/AggregateFunctions/AggregateFunctionIf.h @@ -71,6 +71,11 @@ public: return nested_func->isVersioned(); } + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + size_t getDefaultVersion() const override { return nested_func->getDefaultVersion(); diff --git a/src/AggregateFunctions/AggregateFunctionMap.h b/src/AggregateFunctions/AggregateFunctionMap.h index d19b83b4cd4..3eb2599a0d4 100644 --- a/src/AggregateFunctions/AggregateFunctionMap.h +++ b/src/AggregateFunctions/AggregateFunctionMap.h @@ -94,6 +94,11 @@ public: return nested_func->isVersioned(); } + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + size_t getDefaultVersion() const override { return nested_func->getDefaultVersion(); diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index 80fd70e59f3..3bb6f6fb6d2 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -194,6 +194,21 @@ public: return nested_function->isState(); } + bool isVersioned() const override + { + return nested_function->isVersioned(); + } + + size_t getVersionFromRevision(size_t revision) const override + { + return nested_function->getVersionFromRevision(revision); + } + + size_t getDefaultVersion() const override + { + return nested_function->getDefaultVersion(); + } + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } #if USE_EMBEDDED_COMPILER diff --git a/src/AggregateFunctions/AggregateFunctionResample.h b/src/AggregateFunctions/AggregateFunctionResample.h index b29259f9c84..471a6820939 100644 --- a/src/AggregateFunctions/AggregateFunctionResample.h +++ b/src/AggregateFunctions/AggregateFunctionResample.h @@ -91,6 +91,21 @@ public: return nested_function->isState(); } + bool isVersioned() const override + { + return nested_function->isVersioned(); + } + + size_t getVersionFromRevision(size_t revision) const override + { + return nested_function->getVersionFromRevision(revision); + } + + size_t getDefaultVersion() const override + { + return nested_function->getDefaultVersion(); + } + bool allocatesMemoryInArena() const override { return nested_function->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/AggregateFunctionSimpleState.h b/src/AggregateFunctions/AggregateFunctionSimpleState.h index c9a4dafc752..f50c86c684e 100644 --- a/src/AggregateFunctions/AggregateFunctionSimpleState.h +++ b/src/AggregateFunctions/AggregateFunctionSimpleState.h @@ -61,6 +61,11 @@ public: return nested_func->isState(); } + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + void create(AggregateDataPtr __restrict place) const override { nested_func->create(place); } void destroy(AggregateDataPtr __restrict place) const noexcept override { nested_func->destroy(place); } From 03c193ccca93ca153e8de194e1f199f30afb7d2f Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 15 Sep 2022 21:15:57 +0200 Subject: [PATCH 42/86] fix ON CLUSTER with Replicated database cluster --- src/Databases/DatabaseReplicated.cpp | 20 +++++++++++ src/Databases/DatabaseReplicated.h | 2 ++ src/Databases/IDatabase.h | 2 ++ src/Interpreters/InterpreterAlterQuery.cpp | 6 +--- .../InterpreterCreateIndexQuery.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 29 ++++++--------- src/Interpreters/InterpreterDeleteQuery.cpp | 3 +- .../InterpreterDropIndexQuery.cpp | 2 +- src/Interpreters/InterpreterDropQuery.cpp | 5 +-- src/Interpreters/InterpreterRenameQuery.cpp | 2 +- src/Interpreters/executeDDLQueryOnCluster.cpp | 14 +++++--- .../test_replicated_database/test.py | 36 +++++++++++++++++++ 12 files changed, 86 insertions(+), 37 deletions(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index c6ef3867b63..f1bf56e2beb 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -1259,4 +1259,24 @@ void DatabaseReplicated::createTableRestoredFromBackup( } } +bool DatabaseReplicated::shouldReplicateQuery(const ContextPtr & query_context, const ASTPtr & query_ptr) const +{ + if (query_context->getClientInfo().is_replicated_database_internal) + return false; + + /// Some ALTERs are not replicated on database level + if (const auto * alter = query_ptr->as()) + { + return !alter->isAttachAlter() && !alter->isFetchAlter() && !alter->isDropPartitionAlter(); + } + + /// DROP DATABASE is not replicated + if (const auto * drop = query_ptr->as()) + { + return drop->table.get(); + } + + return true; +} + } diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 3aeec42b271..eba0623547c 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -75,6 +75,8 @@ public: std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override; + bool shouldReplicateQuery(const ContextPtr & query_context, const ASTPtr & query_ptr) const override; + friend struct DatabaseReplicatedTask; friend class DatabaseReplicatedDDLWorker; private: diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 65dfb80846d..5874b24bbf1 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -333,6 +333,8 @@ public: virtual bool hasReplicationThread() const { return false; } + virtual bool shouldReplicateQuery(const ContextPtr & /*query_context*/, const ASTPtr & /*query_ptr*/) const { return false; } + virtual void stopReplication() { throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine {} does not run a replication thread!", getEngineName()); diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 762afd28923..e9fcdc0059a 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -78,11 +78,7 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) query_ptr->as().setDatabase(table_id.database_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); - if (typeid_cast(database.get()) - && !getContext()->getClientInfo().is_replicated_database_internal - && !alter.isAttachAlter() - && !alter.isFetchAlter() - && !alter.isDropPartitionAlter()) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index 5117b92efdf..cb5db84b906 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -38,7 +38,7 @@ BlockIO InterpreterCreateIndexQuery::execute() query_ptr->as().setDatabase(table_id.database_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); - if (typeid_cast(database.get()) && !current_context->getClientInfo().is_replicated_database_internal) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index feb100ebd2e..3d79fc36a66 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1007,16 +1007,12 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.attach && !create.storage && !create.columns_list) { auto database = DatabaseCatalog::instance().getDatabase(database_name); - - if (auto * replicated = typeid_cast(database.get())) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { - if (!getContext()->getClientInfo().is_replicated_database_internal) - { - auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); - create.setDatabase(database_name); - guard->releaseTableLock(); - return replicated->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); - } + auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); + create.setDatabase(database_name); + guard->releaseTableLock(); + return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); } if (!create.cluster.empty()) @@ -1150,16 +1146,13 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (need_add_to_database) database = DatabaseCatalog::instance().getDatabase(database_name); - if (auto * replicated = typeid_cast(database.get())) + if (need_add_to_database && database->shouldReplicateQuery(getContext(), query_ptr)) { - if (!getContext()->getClientInfo().is_replicated_database_internal) - { - chassert(!ddl_guard); - auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); - assertOrSetUUID(create, database); - guard->releaseTableLock(); - return replicated->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); - } + chassert(!ddl_guard); + auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); + assertOrSetUUID(create, database); + guard->releaseTableLock(); + return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); } if (!create.cluster.empty()) diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index 51fb6cfb948..bd6e4bfc243 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -48,8 +48,7 @@ BlockIO InterpreterDeleteQuery::execute() throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); - if (typeid_cast(database.get()) - && !getContext()->getClientInfo().is_replicated_database_internal) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); diff --git a/src/Interpreters/InterpreterDropIndexQuery.cpp b/src/Interpreters/InterpreterDropIndexQuery.cpp index 70f35a92688..8072968e49f 100644 --- a/src/Interpreters/InterpreterDropIndexQuery.cpp +++ b/src/Interpreters/InterpreterDropIndexQuery.cpp @@ -36,7 +36,7 @@ BlockIO InterpreterDropIndexQuery::execute() query_ptr->as().setDatabase(table_id.database_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); - if (typeid_cast(database.get()) && !current_context->getClientInfo().is_replicated_database_internal) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 1f83992dbbe..6aab38fd364 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -139,9 +139,6 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue /// Prevents recursive drop from drop database query. The original query must specify a table. bool is_drop_or_detach_database = !query_ptr->as()->table; - bool is_replicated_ddl_query = typeid_cast(database.get()) && - !context_->getClientInfo().is_replicated_database_internal && - !is_drop_or_detach_database; AccessFlags drop_storage; @@ -152,7 +149,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue else drop_storage = AccessType::DROP_TABLE; - if (is_replicated_ddl_query) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { if (query.kind == ASTDropQuery::Kind::Detach) context_->checkAccess(drop_storage, table_id); diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index c22863ef8e5..5fe5b12e943 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -107,7 +107,7 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c } DatabasePtr database = database_catalog.getDatabase(elem.from_database_name); - if (typeid_cast(database.get()) && !getContext()->getClientInfo().is_replicated_database_internal) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { if (1 < descriptions.size()) throw Exception( diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 7cc4efcb64d..25e1dce4f9f 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -562,12 +562,16 @@ bool maybeRemoveOnCluster(const ASTPtr & query_ptr, ContextPtr context) if (database_name != query_on_cluster->cluster) return false; - auto db = DatabaseCatalog::instance().tryGetDatabase(database_name); - if (!db || db->getEngineName() != "Replicated") - return false; + auto database = DatabaseCatalog::instance().tryGetDatabase(database_name); + if (database && database->shouldReplicateQuery(context, query_ptr)) + { + /// It's Replicated database and query is replicated on database level, + /// so ON CLUSTER clause is redundant. + query_on_cluster->cluster.clear(); + return true; + } - query_on_cluster->cluster.clear(); - return true; + return false; } } diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 0cf237d57f3..de5433d5beb 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -354,6 +354,42 @@ def test_alter_drop_detached_part(started_cluster, engine): dummy_node.query("DROP DATABASE testdb SYNC") +@pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +def test_alter_drop_partition(started_cluster, engine): + main_node.query( + "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/clickhouse/databases/test_alter_drop_partition', 'shard1', 'replica1');" + ) + dummy_node.query( + "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/clickhouse/databases/test_alter_drop_partition', 'shard1', 'replica2');" + ) + snapshotting_node.query( + "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/clickhouse/databases/test_alter_drop_partition', 'shard2', 'replica1');" + ) + + table = f"alter_drop_partition.alter_drop_{engine}" + main_node.query( + f"CREATE TABLE {table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" + ) + main_node.query(f"INSERT INTO {table} VALUES (123)") + if engine == "MergeTree": + dummy_node.query(f"INSERT INTO {table} VALUES (456)") + snapshotting_node.query(f"INSERT INTO {table} VALUES (789)") + main_node.query( + f"ALTER TABLE {table} ON CLUSTER alter_drop_partition DROP PARTITION ID 'all'", + settings={"replication_alter_partitions_sync": 2}, + ) + assert ( + main_node.query( + f"SELECT CounterID FROM clusterAllReplicas('alter_drop_partition', {table})" + ) + == "" + ) + assert dummy_node.query(f"SELECT CounterID FROM {table}") == "" + main_node.query("DROP DATABASE alter_drop_partition") + dummy_node.query("DROP DATABASE alter_drop_partition") + snapshotting_node.query("DROP DATABASE alter_drop_partition") + + def test_alter_fetch(started_cluster): main_node.query( "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" From 456da30c9ff040051e963ee10919854e410ac26e Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 16 Sep 2022 14:06:26 +0200 Subject: [PATCH 43/86] Revert "Merge pull request #41249 from ClickHouse/revert-40968-s3-sharding-2" This reverts commit ac60300997fe6daf9f8d308e31c8e7775d5f83e0, reversing changes made to a9c272283abace749b004eac606810722462031a. --- .../ObjectStorages/S3/S3ObjectStorage.cpp | 15 +++- src/Disks/ObjectStorages/StoredObject.h | 1 + tests/integration/runner | 3 +- .../test_join_set_family_s3/test.py | 2 +- tests/integration/test_log_family_s3/test.py | 2 +- tests/integration/test_merge_tree_s3/test.py | 68 +++++++++++-------- .../test_profile_events_s3/test.py | 24 ++++--- .../test_replicated_merge_tree_s3/test.py | 8 +-- .../test.py | 8 +-- .../test_s3_zero_copy_replication/test.py | 4 +- 10 files changed, 84 insertions(+), 51 deletions(-) diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 998b521cc56..45304ac2fac 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -31,6 +31,7 @@ #include #include + namespace DB { @@ -90,7 +91,19 @@ void logIfError(const Aws::Utils::Outcome & response, std::functi std::string S3ObjectStorage::generateBlobNameForPath(const std::string & /* path */) { - return getRandomASCIIString(32); + /// Path to store the new S3 object. + + /// Total length is 32 a-z characters for enough randomness. + /// First 3 characters are used as a prefix for + /// https://aws.amazon.com/premiumsupport/knowledge-center/s3-object-key-naming-pattern/ + + constexpr size_t key_name_total_size = 32; + constexpr size_t key_name_prefix_size = 3; + + /// Path to store new S3 object. + return fmt::format("{}/{}", + getRandomASCIIString(key_name_prefix_size), + getRandomASCIIString(key_name_total_size - key_name_prefix_size)); } Aws::S3::Model::HeadObjectOutcome S3ObjectStorage::requestObjectHeadData(const std::string & bucket_from, const std::string & key) const diff --git a/src/Disks/ObjectStorages/StoredObject.h b/src/Disks/ObjectStorages/StoredObject.h index acb8a5fd127..d9faa766540 100644 --- a/src/Disks/ObjectStorages/StoredObject.h +++ b/src/Disks/ObjectStorages/StoredObject.h @@ -3,6 +3,7 @@ #include #include + namespace DB { diff --git a/tests/integration/runner b/tests/integration/runner index f0d87b23a83..e1b9a55b43e 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -350,8 +350,7 @@ if __name__ == "__main__": # randomizer, we should remove it after Sep 2022 try: subprocess.check_call( - "docker volume rm $(docker volume ls -q | " - f"grep '{VOLUME_NAME}_.*_volume')", + f"docker volume ls -q | grep '{VOLUME_NAME}_.*_volume' | xargs --no-run-if-empty docker volume rm", shell=True, ) except Exception as ex: diff --git a/tests/integration/test_join_set_family_s3/test.py b/tests/integration/test_join_set_family_s3/test.py index b09d5735628..38b56b7b15b 100644 --- a/tests/integration/test_join_set_family_s3/test.py +++ b/tests/integration/test_join_set_family_s3/test.py @@ -27,7 +27,7 @@ def cluster(): def assert_objects_count(cluster, objects_count, path="data/"): minio = cluster.minio_client - s3_objects = list(minio.list_objects(cluster.minio_bucket, path)) + s3_objects = list(minio.list_objects(cluster.minio_bucket, path, recursive=True)) if objects_count != len(s3_objects): for s3_object in s3_objects: object_meta = minio.stat_object(cluster.minio_bucket, s3_object.object_name) diff --git a/tests/integration/test_log_family_s3/test.py b/tests/integration/test_log_family_s3/test.py index 76ff0930db3..bed379d098b 100644 --- a/tests/integration/test_log_family_s3/test.py +++ b/tests/integration/test_log_family_s3/test.py @@ -25,7 +25,7 @@ def cluster(): def assert_objects_count(cluster, objects_count, path="data/"): minio = cluster.minio_client - s3_objects = list(minio.list_objects(cluster.minio_bucket, path)) + s3_objects = list(minio.list_objects(cluster.minio_bucket, path, recursive=True)) if objects_count != len(s3_objects): for s3_object in s3_objects: object_meta = minio.stat_object(cluster.minio_bucket, s3_object.object_name) diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index 4ce5fd5a069..912457afaa1 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -121,11 +121,17 @@ def run_s3_mocks(cluster): def wait_for_delete_s3_objects(cluster, expected, timeout=30): minio = cluster.minio_client while timeout > 0: - if len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == expected: + if ( + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + == expected + ): return timeout -= 1 time.sleep(1) - assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == expected + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + == expected + ) @pytest.fixture(autouse=True) @@ -141,7 +147,9 @@ def drop_table(cluster, node_name): wait_for_delete_s3_objects(cluster, 0) finally: # Remove extra objects to prevent tests cascade failing - for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): + for obj in list( + minio.list_objects(cluster.minio_bucket, "data/", recursive=True) + ): minio.remove_object(cluster.minio_bucket, obj.object_name) @@ -163,7 +171,7 @@ def test_simple_insert_select( node.query("INSERT INTO s3_test VALUES {}".format(values1)) assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values") == values1 assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + files_per_part ) @@ -174,7 +182,7 @@ def test_simple_insert_select( == values1 + "," + values2 ) assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + files_per_part * 2 ) @@ -218,7 +226,7 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name): node.query("SELECT count(distinct(id)) FROM s3_test FORMAT Values") == "(8192)" ) assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD_PER_PART_WIDE * 6 + FILES_OVERHEAD ) @@ -307,28 +315,28 @@ def test_attach_detach_partition(cluster, node_name): ) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("ALTER TABLE s3_test DETACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(4096)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("ALTER TABLE s3_test ATTACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("ALTER TABLE s3_test DROP PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(4096)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE ) @@ -339,7 +347,8 @@ def test_attach_detach_partition(cluster, node_name): ) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + == FILES_OVERHEAD ) @@ -357,21 +366,21 @@ def test_move_partition_to_another_disk(cluster, node_name): ) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("ALTER TABLE s3_test MOVE PARTITION '2020-01-04' TO DISK 'hdd'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE ) node.query("ALTER TABLE s3_test MOVE PARTITION '2020-01-04' TO DISK 's3'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) @@ -392,7 +401,7 @@ def test_table_manipulations(cluster, node_name): node.query("RENAME TABLE s3_test TO s3_renamed") assert node.query("SELECT count(*) FROM s3_renamed FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("RENAME TABLE s3_renamed TO s3_test") @@ -403,14 +412,15 @@ def test_table_manipulations(cluster, node_name): node.query("ATTACH TABLE s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("TRUNCATE TABLE s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + == FILES_OVERHEAD ) @@ -435,7 +445,7 @@ def test_move_replace_partition_to_another_table(cluster, node_name): assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 ) @@ -449,7 +459,7 @@ def test_move_replace_partition_to_another_table(cluster, node_name): assert node.query("SELECT count(*) FROM s3_clone FORMAT Values") == "(8192)" # Number of objects in S3 should be unchanged. assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 ) @@ -463,7 +473,7 @@ def test_move_replace_partition_to_another_table(cluster, node_name): assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 6 ) @@ -484,14 +494,14 @@ def test_move_replace_partition_to_another_table(cluster, node_name): assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" # Data should remain in S3 assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 ) node.query("ALTER TABLE s3_test FREEZE") # Number S3 objects should be unchanged. assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 ) @@ -500,7 +510,7 @@ def test_move_replace_partition_to_another_table(cluster, node_name): wait_for_delete_s3_objects(cluster, FILES_OVERHEAD_PER_PART_WIDE * 4) - for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): minio.remove_object(cluster.minio_bucket, obj.object_name) @@ -521,7 +531,7 @@ def test_freeze_unfreeze(cluster, node_name): node.query("TRUNCATE TABLE s3_test") assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) @@ -536,7 +546,8 @@ def test_freeze_unfreeze(cluster, node_name): # Data should be removed from S3. assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + == FILES_OVERHEAD ) @@ -559,7 +570,7 @@ def test_freeze_system_unfreeze(cluster, node_name): node.query("TRUNCATE TABLE s3_test") node.query("DROP TABLE s3_test_removed NO DELAY") assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) @@ -570,7 +581,8 @@ def test_freeze_system_unfreeze(cluster, node_name): # Data should be removed from S3. assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + == FILES_OVERHEAD ) @@ -697,7 +709,7 @@ def test_lazy_seek_optimization_for_async_read(cluster, node_name): node.query("SELECT * FROM s3_test WHERE value LIKE '%abc%' ORDER BY value LIMIT 10") node.query("DROP TABLE IF EXISTS s3_test NO DELAY") minio = cluster.minio_client - for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): minio.remove_object(cluster.minio_bucket, obj.object_name) diff --git a/tests/integration/test_profile_events_s3/test.py b/tests/integration/test_profile_events_s3/test.py index a0f664df000..18f1c5ee9ad 100644 --- a/tests/integration/test_profile_events_s3/test.py +++ b/tests/integration/test_profile_events_s3/test.py @@ -62,7 +62,7 @@ init_list = { def get_s3_events(instance): result = init_list.copy() events = instance.query( - "SELECT event,value FROM system.events WHERE event LIKE '%S3%'" + "SELECT event, value FROM system.events WHERE event LIKE '%S3%'" ).split("\n") for event in events: ev = event.split("\t") @@ -85,20 +85,20 @@ def get_minio_stat(cluster): ) ).text.split("\n") for line in stat: - x = re.search("s3_requests_total(\{.*\})?\s(\d+)(\s.*)?", line) + x = re.search(r"s3_requests_total(\{.*\})?\s(\d+)(\s.*)?", line) if x != None: y = re.search('.*api="(get|list|head|select).*', x.group(1)) if y != None: result["get_requests"] += int(x.group(2)) else: result["set_requests"] += int(x.group(2)) - x = re.search("s3_errors_total(\{.*\})?\s(\d+)(\s.*)?", line) + x = re.search(r"s3_errors_total(\{.*\})?\s(\d+)(\s.*)?", line) if x != None: result["errors"] += int(x.group(2)) - x = re.search("s3_rx_bytes_total(\{.*\})?\s([\d\.e\+\-]+)(\s.*)?", line) + x = re.search(r"s3_rx_bytes_total(\{.*\})?\s([\d\.e\+\-]+)(\s.*)?", line) if x != None: result["tx_bytes"] += float(x.group(2)) - x = re.search("s3_tx_bytes_total(\{.*\})?\s([\d\.e\+\-]+)(\s.*)?", line) + x = re.search(r"s3_tx_bytes_total(\{.*\})?\s([\d\.e\+\-]+)(\s.*)?", line) if x != None: result["rx_bytes"] += float(x.group(2)) return result @@ -128,8 +128,10 @@ def get_query_stat(instance, hint): def get_minio_size(cluster): minio = cluster.minio_client size = 0 - for obj in minio.list_objects(cluster.minio_bucket, "data/"): - size += obj.size + for obj_level1 in minio.list_objects( + cluster.minio_bucket, prefix="data/", recursive=True + ): + size += obj_level1.size return size @@ -145,7 +147,7 @@ def test_profile_events(cluster): metrics0 = get_s3_events(instance) minio0 = get_minio_stat(cluster) - query1 = "CREATE TABLE test_s3.test_s3 (key UInt32, value UInt32) ENGINE=MergeTree PRIMARY KEY key ORDER BY key SETTINGS storage_policy='s3'" + query1 = "CREATE TABLE test_s3.test_s3 (key UInt32, value UInt32) ENGINE=MergeTree PRIMARY KEY key ORDER BY key SETTINGS storage_policy = 's3'" instance.query(query1) size1 = get_minio_size(cluster) @@ -167,7 +169,7 @@ def test_profile_events(cluster): metrics1["WriteBufferFromS3Bytes"] - metrics0["WriteBufferFromS3Bytes"] == size1 ) - query2 = "INSERT INTO test_s3.test_s3 FORMAT Values" + query2 = "INSERT INTO test_s3.test_s3 VALUES" instance.query(query2 + " (1,1)") size2 = get_minio_size(cluster) @@ -182,9 +184,12 @@ def test_profile_events(cluster): metrics2["S3WriteRequestsCount"] - metrics1["S3WriteRequestsCount"] == minio2["set_requests"] - minio1["set_requests"] ) + stat2 = get_query_stat(instance, query2) + for metric in stat2: assert stat2[metric] == metrics2[metric] - metrics1[metric] + assert ( metrics2["WriteBufferFromS3Bytes"] - metrics1["WriteBufferFromS3Bytes"] == size2 - size1 @@ -205,6 +210,7 @@ def test_profile_events(cluster): == minio3["set_requests"] - minio2["set_requests"] ) stat3 = get_query_stat(instance, query3) + # With async reads profile events are not updated fully because reads are done in a separate thread. # for metric in stat3: # print(metric) diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index 37027d07969..0d978bb6967 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -113,7 +113,7 @@ def drop_table(cluster): minio = cluster.minio_client # Remove extra objects to prevent tests cascade failing - for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): minio.remove_object(cluster.minio_bucket, obj.object_name) @@ -130,9 +130,9 @@ def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_par insert(cluster, node_idxs=[1, 2, 3], verify=True) minio = cluster.minio_client - assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == 3 * ( - FILES_OVERHEAD + files_per_part * 3 - ) + assert len( + list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)) + ) == 3 * (FILES_OVERHEAD + files_per_part * 3) def test_drop_cache_on_cluster(cluster): diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index 73b611ad169..60a1b9b9746 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -87,7 +87,7 @@ def drop_table(cluster): minio = cluster.minio_client # Remove extra objects to prevent tests cascade failing - for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): minio.remove_object(cluster.minio_bucket, obj.object_name) @@ -124,6 +124,6 @@ def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_par ) minio = cluster.minio_client - assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == ( - 3 * FILES_OVERHEAD - ) + (files_per_part * 3) + assert len( + list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)) + ) == (3 * FILES_OVERHEAD) + (files_per_part * 3) diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py index 7b7fb9d21ad..860b83d4ed1 100644 --- a/tests/integration/test_s3_zero_copy_replication/test.py +++ b/tests/integration/test_s3_zero_copy_replication/test.py @@ -39,7 +39,9 @@ def cluster(): def get_large_objects_count(cluster, size=100, folder="data"): minio = cluster.minio_client counter = 0 - for obj in minio.list_objects(cluster.minio_bucket, "{}/".format(folder)): + for obj in minio.list_objects( + cluster.minio_bucket, "{}/".format(folder), recursive=True + ): if obj.size is not None and obj.size >= size: counter = counter + 1 return counter From d9c3549cc8dbe649b1b6edbe4c8e19eb5e919985 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 16 Sep 2022 14:43:43 +0200 Subject: [PATCH 44/86] Revert s3 prefixes + add retries on write requests unexpected errors --- src/Core/Settings.h | 1 + src/Disks/ObjectStorages/S3/diskSettings.cpp | 1 + src/IO/WriteBufferFromS3.cpp | 62 ++++++++++++++------ src/Storages/StorageS3Settings.cpp | 3 + src/Storages/StorageS3Settings.h | 4 +- 5 files changed, 51 insertions(+), 20 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 86fccf45a8d..6fbce95b94e 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -86,6 +86,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(UInt64, s3_upload_part_size_multiply_parts_count_threshold, 1000, "Each time this number of parts was uploaded to S3 s3_min_upload_part_size multiplied by s3_upload_part_size_multiply_factor.", 0) \ M(UInt64, s3_max_single_part_upload_size, 32*1024*1024, "The maximum size of object to upload using singlepart upload to S3.", 0) \ M(UInt64, s3_max_single_read_retries, 4, "The maximum number of retries during single S3 read.", 0) \ + M(UInt64, s3_max_unexpected_write_error_retries, 4, "The maximum number of retries in case of unexpected errors during S3 write.", 0) \ M(UInt64, s3_max_redirects, 10, "Max number of S3 redirects hops allowed.", 0) \ M(UInt64, s3_max_connections, 1024, "The maximum number of connections per server.", 0) \ M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index a93d95d91bd..b5efc11db8b 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -39,6 +39,7 @@ std::unique_ptr getSettings(const Poco::Util::AbstractC rw_settings.upload_part_size_multiply_parts_count_threshold = config.getUInt64(config_prefix + ".s3_upload_part_size_multiply_parts_count_threshold", context->getSettingsRef().s3_upload_part_size_multiply_parts_count_threshold); rw_settings.max_single_part_upload_size = config.getUInt64(config_prefix + ".s3_max_single_part_upload_size", context->getSettingsRef().s3_max_single_part_upload_size); rw_settings.check_objects_after_upload = config.getUInt64(config_prefix + ".s3_check_objects_after_upload", context->getSettingsRef().s3_check_objects_after_upload); + rw_settings.max_unexpected_write_error_retries = config.getUInt64(config_prefix + ".s3_max_unexpected_write_error_retries", context->getSettingsRef().s3_max_unexpected_write_error_retries); return std::make_unique( rw_settings, diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index b09abda85db..608f4a5db01 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -339,17 +339,29 @@ void WriteBufferFromS3::completeMultipartUpload() } req.SetMultipartUpload(multipart_upload); - - auto outcome = client_ptr->CompleteMultipartUpload(req); - - if (outcome.IsSuccess()) - LOG_TRACE(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, tags.size()); - else + size_t max_retry = std::max(s3_settings.max_unexpected_write_error_retries, 1UL); + for (size_t i = 0; i < max_retry; ++i) { - throw S3Exception( - outcome.GetError().GetErrorType(), - "Message: {}, Key: {}, Bucket: {}, Tags: {}", - outcome.GetError().GetMessage(), key, bucket, fmt::join(tags.begin(), tags.end(), " ")); + auto outcome = client_ptr->CompleteMultipartUpload(req); + + if (outcome.IsSuccess()) + { + LOG_TRACE(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, tags.size()); + break; + } + else if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) + { + /// For unknown reason, at least MinIO can respond with NO_SUCH_KEY for put requests + /// BTW, NO_SUCH_UPLOAD is expected error and we shouldn't retry it + LOG_WARNING(log, "Multipart upload failed with NO_SUCH_KEY error for Bucket: {}, Key: {}, Upload_id: {}, Parts: {}, will retry", bucket, key, multipart_upload_id, tags.size()); + } + else + { + throw S3Exception( + outcome.GetError().GetErrorType(), + "Message: {}, Key: {}, Bucket: {}, Tags: {}", + outcome.GetError().GetMessage(), key, bucket, fmt::join(tags.begin(), tags.end(), " ")); + } } } @@ -429,15 +441,27 @@ void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req) void WriteBufferFromS3::processPutRequest(const PutObjectTask & task) { - auto outcome = client_ptr->PutObject(task.req); - bool with_pool = static_cast(schedule); - if (outcome.IsSuccess()) - LOG_TRACE(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); - else - throw S3Exception( - outcome.GetError().GetErrorType(), - "Message: {}, Key: {}, Bucket: {}, Object size: {}, WithPool: {}", - outcome.GetError().GetMessage(), key, bucket, task.req.GetContentLength(), with_pool); + size_t max_retry = std::max(s3_settings.max_unexpected_write_error_retries, 1UL); + for (size_t i = 0; i < max_retry; ++i) + { + auto outcome = client_ptr->PutObject(task.req); + bool with_pool = static_cast(schedule); + if (outcome.IsSuccess()) + { + LOG_TRACE(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); + break; + } + else if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) + { + /// For unknown reason, at least MinIO can respond with NO_SUCH_KEY for put requests + LOG_WARNING(log, "Single part upload failed with NO_SUCH_KEY error for Bucket: {}, Key: {}, Object size: {}, WithPool: {}, will retry", bucket, key, task.req.GetContentLength(), with_pool); + } + else + throw S3Exception( + outcome.GetError().GetErrorType(), + "Message: {}, Key: {}, Bucket: {}, Object size: {}, WithPool: {}", + outcome.GetError().GetMessage(), key, bucket, task.req.GetContentLength(), with_pool); + } } void WriteBufferFromS3::waitForReadyBackGroundTasks() diff --git a/src/Storages/StorageS3Settings.cpp b/src/Storages/StorageS3Settings.cpp index 353e324c853..4ab3375e188 100644 --- a/src/Storages/StorageS3Settings.cpp +++ b/src/Storages/StorageS3Settings.cpp @@ -121,6 +121,7 @@ S3Settings::ReadWriteSettings::ReadWriteSettings(const Settings & settings) max_single_part_upload_size = settings.s3_max_single_part_upload_size; max_connections = settings.s3_max_connections; check_objects_after_upload = settings.s3_check_objects_after_upload; + max_unexpected_write_error_retries = settings.s3_max_unexpected_write_error_retries; } void S3Settings::ReadWriteSettings::updateFromSettingsIfEmpty(const Settings & settings) @@ -137,6 +138,8 @@ void S3Settings::ReadWriteSettings::updateFromSettingsIfEmpty(const Settings & s max_single_part_upload_size = settings.s3_max_single_part_upload_size; if (!max_connections) max_connections = settings.s3_max_connections; + if (!max_unexpected_write_error_retries) + max_unexpected_write_error_retries = settings.s3_max_unexpected_write_error_retries; check_objects_after_upload = settings.s3_check_objects_after_upload; } diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index 9ef51c77191..41136117b24 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -61,6 +61,7 @@ struct S3Settings size_t max_single_part_upload_size = 0; size_t max_connections = 0; bool check_objects_after_upload = false; + size_t max_unexpected_write_error_retries = 0; ReadWriteSettings() = default; explicit ReadWriteSettings(const Settings & settings); @@ -73,7 +74,8 @@ struct S3Settings && upload_part_size_multiply_parts_count_threshold == other.upload_part_size_multiply_parts_count_threshold && max_single_part_upload_size == other.max_single_part_upload_size && max_connections == other.max_connections - && check_objects_after_upload == other.check_objects_after_upload; + && check_objects_after_upload == other.check_objects_after_upload + && max_unexpected_write_error_retries == other.max_unexpected_write_error_retries; } void updateFromSettingsIfEmpty(const Settings & settings); From 65474830ac3ed96fad07d6fe72f62f35d62f390f Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 16 Sep 2022 16:25:32 +0200 Subject: [PATCH 45/86] make tryEnqueueReplicatedDDL virtual --- src/Databases/DatabaseReplicated.h | 2 +- src/Databases/IDatabase.h | 10 ++++++++-- src/Interpreters/InterpreterAlterQuery.cpp | 2 +- src/Interpreters/InterpreterCreateIndexQuery.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 4 ++-- src/Interpreters/InterpreterDeleteQuery.cpp | 2 +- src/Interpreters/InterpreterDropIndexQuery.cpp | 2 +- src/Interpreters/InterpreterDropQuery.cpp | 2 +- src/Interpreters/InterpreterRenameQuery.cpp | 2 +- 9 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index eba0623547c..0c9a3b77844 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -46,7 +46,7 @@ public: /// Try to execute DLL query on current host as initial query. If query is succeed, /// then it will be executed on all replicas. - BlockIO tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, bool internal = false); + BlockIO tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, bool internal) override; bool hasReplicationThread() const override { return true; } diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 5874b24bbf1..a0bb01a9550 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -333,13 +334,18 @@ public: virtual bool hasReplicationThread() const { return false; } - virtual bool shouldReplicateQuery(const ContextPtr & /*query_context*/, const ASTPtr & /*query_ptr*/) const { return false; } - virtual void stopReplication() { throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine {} does not run a replication thread!", getEngineName()); } + virtual bool shouldReplicateQuery(const ContextPtr & /*query_context*/, const ASTPtr & /*query_ptr*/) const { return false; } + + virtual BlockIO tryEnqueueReplicatedDDL(const ASTPtr & /*query*/, ContextPtr /*query_context*/, [[maybe_unused]] bool internal = false) /// NOLINT + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine {} does not have replicated DDL queue", getEngineName()); + } + /// Returns CREATE TABLE queries and corresponding tables prepared for writing to a backup. virtual std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & context) const; diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index e9fcdc0059a..82f635017c9 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -82,7 +82,7 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext()); } StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index cb5db84b906..714bcd6d356 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -42,7 +42,7 @@ BlockIO InterpreterCreateIndexQuery::execute() { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); - return assert_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, current_context); + return database->tryEnqueueReplicatedDDL(query_ptr, current_context); } StoragePtr table = DatabaseCatalog::instance().getTable(table_id, current_context); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 3d79fc36a66..21a02ba351f 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1012,7 +1012,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); create.setDatabase(database_name); guard->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); } if (!create.cluster.empty()) @@ -1152,7 +1152,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); assertOrSetUUID(create, database); guard->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); } if (!create.cluster.empty()) diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index bd6e4bfc243..b5b8ae81366 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -52,7 +52,7 @@ BlockIO InterpreterDeleteQuery::execute() { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext()); } auto table_lock = table->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); diff --git a/src/Interpreters/InterpreterDropIndexQuery.cpp b/src/Interpreters/InterpreterDropIndexQuery.cpp index 8072968e49f..98d48942487 100644 --- a/src/Interpreters/InterpreterDropIndexQuery.cpp +++ b/src/Interpreters/InterpreterDropIndexQuery.cpp @@ -40,7 +40,7 @@ BlockIO InterpreterDropIndexQuery::execute() { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); - return assert_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, current_context); + return database->tryEnqueueReplicatedDDL(query_ptr, current_context); } StoragePtr table = DatabaseCatalog::instance().getTable(table_id, current_context); diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 6aab38fd364..71d65ee7fed 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -160,7 +160,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue ddl_guard->releaseTableLock(); table.reset(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query.clone(), context_); + return database->tryEnqueueReplicatedDDL(query.clone(), context_); } if (query.kind == ASTDropQuery::Kind::Detach) diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index 5fe5b12e943..666a674b2c8 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -120,7 +120,7 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c UniqueTableName to(elem.to_database_name, elem.to_table_name); ddl_guards[from]->releaseTableLock(); ddl_guards[to]->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext()); } else { From c9a9ef5b7effed115d6a6bd635565d0f620cfc22 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 16 Sep 2022 16:12:30 +0000 Subject: [PATCH 46/86] Fix comments --- src/AggregateFunctions/AggregateFunctionForEach.h | 4 ++-- src/AggregateFunctions/AggregateFunctionMap.h | 3 +-- src/Interpreters/Aggregator.cpp | 3 +++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionForEach.h b/src/AggregateFunctions/AggregateFunctionForEach.h index becd00a782e..07713dcb304 100644 --- a/src/AggregateFunctions/AggregateFunctionForEach.h +++ b/src/AggregateFunctions/AggregateFunctionForEach.h @@ -145,7 +145,7 @@ public: return nested_func->getDefaultVersion(); } - template + template void destroyImpl(AggregateDataPtr __restrict place) const noexcept { AggregateFunctionForEachData & state = data(place); @@ -153,7 +153,7 @@ public: char * nested_state = state.array_of_aggregate_datas; for (size_t i = 0; i < state.dynamic_array_size; ++i) { - if constexpr (up_tp_state) + if constexpr (up_to_state) nested_func->destroyUpToState(nested_state); else nested_func->destroy(nested_state); diff --git a/src/AggregateFunctions/AggregateFunctionMap.h b/src/AggregateFunctions/AggregateFunctionMap.h index 3eb2599a0d4..527f35fd948 100644 --- a/src/AggregateFunctions/AggregateFunctionMap.h +++ b/src/AggregateFunctions/AggregateFunctionMap.h @@ -225,8 +225,7 @@ public: nested_func->destroy(nested_place); } - auto to_destroy = std::move(state.merged_maps); - this->data(place).~Data(); + state.~Data(); } void destroy(AggregateDataPtr __restrict place) const noexcept override diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 0eb6ce7afad..2080f36a531 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1756,6 +1756,9 @@ inline void Aggregator::insertAggregatesIntoColumns(Mapped & mapped, MutableColu * and ColumnAggregateFunction will take ownership of this state. * So, for aggregate functions with "-State" modifier, only states of all combinators that are used * after -State will be destroyed after result has been transferred to ColumnAggregateFunction. + * For example, if we have function `uniqStateForEachMap` after aggregation we should destroy all states that + * were created by combinators `-ForEach` and `-Map`, because resulting ColumnAggregateFunction will be + * responsible only for destruction of the states created by `uniq` function. * But we should mark that the data no longer owns these states. */ From 8f9f5c69daf497e3c9c5b9c6baacfbc189eee779 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 8 Aug 2022 11:32:52 +0200 Subject: [PATCH 47/86] Use MemoryAccessStorage to simplify ReplicatedAccessStorage. --- src/Access/MemoryAccessStorage.cpp | 62 ++++++++++++- src/Access/MemoryAccessStorage.h | 13 ++- src/Access/ReplicatedAccessStorage.cpp | 123 +++---------------------- src/Access/ReplicatedAccessStorage.h | 20 +--- 4 files changed, 84 insertions(+), 134 deletions(-) diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index 60669532e25..0f6be7ee96c 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -90,6 +90,9 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; auto it_by_name = entries_by_name.find(name); bool name_collision = (it_by_name != entries_by_name.end()); + UUID id_by_name; + if (name_collision) + id_by_name = it_by_name->second->id; if (name_collision && !replace_if_exists) { @@ -100,16 +103,43 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & } auto it_by_id = entries_by_id.find(id); - if (it_by_id != entries_by_id.end()) + bool id_collision = (it_by_id != entries_by_id.end()); + if (id_collision && !replace_if_exists) { const auto & existing_entry = it_by_id->second; - throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); + if (throw_if_exists) + throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); + else + return false; } - if (name_collision && replace_if_exists) + /// Remove collisions if necessary. + if (name_collision && (id_by_name != id)) { - const auto & existing_entry = *(it_by_name->second); - removeNoLock(existing_entry.id, /* throw_if_not_exists = */ false); + assert(replace_if_exists); + removeNoLock(id_by_name, /* throw_if_not_exists = */ false); + } + + if (id_collision) + { + assert(replace_if_exists); + auto & existing_entry = it_by_id->second; + if (existing_entry.entity->getType() == new_entity->getType()) + { + if (existing_entry.entity->getName() != new_entity->getName()) + { + entries_by_name.erase(existing_entry.entity->getName()); + bool inserted = entries_by_name.emplace(new_entity->getName(), &existing_entry).second; + assert(inserted); + } + if (*existing_entry.entity != *new_entity) + { + existing_entry.entity = new_entity; + changes_notifier.onEntityUpdated(id, new_entity); + } + return true; + } + removeNoLock(id, /* throw_if_not_exists = */ false); } /// Do insertion. @@ -201,6 +231,28 @@ bool MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & updat } +void MemoryAccessStorage::removeAllExcept(const std::vector & ids_to_keep) +{ + std::lock_guard lock{mutex}; + removeAllExceptNoLock(ids_to_keep); +} + +void MemoryAccessStorage::removeAllExceptNoLock(const std::vector & ids_to_keep) +{ + boost::container::flat_set ids_to_keep_set{ids_to_keep.begin(), ids_to_keep.end()}; + for (auto it = entries_by_id.begin(); it != entries_by_id.end();) + { + const auto & id = it->first; + ++it; + if (!ids_to_keep_set.contains(id)) + { + UUID id_to_remove = id; + removeNoLock(id_to_remove, /* throw_if_not_exists */ false); + } + } +} + + void MemoryAccessStorage::setAll(const std::vector & all_entities) { std::vector> entities_with_ids; diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h index 5c8d33ed443..7c439a5ea21 100644 --- a/src/Access/MemoryAccessStorage.h +++ b/src/Access/MemoryAccessStorage.h @@ -22,6 +22,14 @@ public: const char * getStorageType() const override { return STORAGE_TYPE; } + /// Inserts an entity with a specified ID. + /// If `replace_if_exists == true` it can replace an existing entry with such ID and also remove an existing entry + /// with such name & type. + bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists); + + /// Removes all entities except a specified list. + void removeAllExcept(const std::vector & ids_to_keep); + /// Sets all entities at once. void setAll(const std::vector & all_entities); void setAll(const std::vector> & all_entities); @@ -39,11 +47,12 @@ private: bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; - bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists); bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) TSA_REQUIRES(mutex); bool removeNoLock(const UUID & id, bool throw_if_not_exists) TSA_REQUIRES(mutex); bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) TSA_REQUIRES(mutex); + void removeAllExceptNoLock(const std::vector & ids_to_keep) TSA_REQUIRES(mutex); + struct Entry { UUID id; @@ -54,6 +63,6 @@ private: std::unordered_map entries_by_id TSA_GUARDED_BY(mutex); /// We want to search entries both by ID and by the pair of name and type. std::unordered_map entries_by_name_and_type[static_cast(AccessEntityType::MAX)] TSA_GUARDED_BY(mutex); AccessChangesNotifier & changes_notifier; - bool backup_allowed = false; + const bool backup_allowed = false; }; } diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index c7aec75265c..8a3b69883d4 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -46,6 +46,7 @@ ReplicatedAccessStorage::ReplicatedAccessStorage( , zookeeper_path(zookeeper_path_) , get_zookeeper(get_zookeeper_) , watched_queue(std::make_shared>(std::numeric_limits::max())) + , memory_storage(storage_name_, changes_notifier_, false) , changes_notifier(changes_notifier_) , backup_allowed(allow_backup_) { @@ -373,11 +374,6 @@ void ReplicatedAccessStorage::resetAfterError() UUID id; while (watched_queue->tryPop(id)) {} - - std::lock_guard lock{mutex}; - for (const auto type : collections::range(AccessEntityType::MAX)) - entries_by_name_and_type[static_cast(type)].clear(); - entries_by_id.clear(); } void ReplicatedAccessStorage::initializeZookeeper() @@ -437,30 +433,17 @@ void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zooke Coordination::Stat stat; const auto entity_uuid_strs = zookeeper->getChildrenWatch(zookeeper_uuids_path, &stat, watch_entities_list); - std::unordered_set entity_uuids; + std::vector entity_uuids; entity_uuids.reserve(entity_uuid_strs.size()); for (const String & entity_uuid_str : entity_uuid_strs) - entity_uuids.insert(parseUUID(entity_uuid_str)); + entity_uuids.emplace_back(parseUUID(entity_uuid_str)); std::lock_guard lock{mutex}; - - std::vector entities_to_remove; - /// Locally remove entities that were removed from ZooKeeper - for (const auto & pair : entries_by_id) + memory_storage.removeAllExcept(entity_uuids); + for (const auto & uuid : entity_uuids) { - const UUID & entity_uuid = pair.first; - if (!entity_uuids.contains(entity_uuid)) - entities_to_remove.push_back(entity_uuid); - } - for (const auto & entity_uuid : entities_to_remove) - removeEntityNoLock(entity_uuid); - - /// Locally add entities that were added to ZooKeeper - for (const auto & entity_uuid : entity_uuids) - { - const auto it = entries_by_id.find(entity_uuid); - if (it == entries_by_id.end()) - refreshEntityNoLock(zookeeper, entity_uuid); + if (!initialized || !memory_storage.exists(uuid)) + refreshEntityNoLock(zookeeper, uuid); } LOG_DEBUG(getLogger(), "Refreshing entities list finished"); @@ -500,122 +483,42 @@ void ReplicatedAccessStorage::refreshEntityNoLock(const zkutil::ZooKeeperPtr & z void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntityPtr & entity) { LOG_DEBUG(getLogger(), "Setting id {} to entity named {}", toString(id), entity->getName()); - const AccessEntityType type = entity->getType(); - const String & name = entity->getName(); - - /// If the type+name already exists and is a different entity, remove old entity - auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; - if (auto it = entries_by_name.find(name); it != entries_by_name.end() && it->second->id != id) - { - removeEntityNoLock(it->second->id); - } - - /// If the entity already exists under a different type+name, remove old type+name - bool existed_before = false; - if (auto it = entries_by_id.find(id); it != entries_by_id.end()) - { - existed_before = true; - const AccessEntityPtr & existing_entity = it->second.entity; - const AccessEntityType existing_type = existing_entity->getType(); - const String & existing_name = existing_entity->getName(); - if (existing_type != type || existing_name != name) - { - auto & existing_entries_by_name = entries_by_name_and_type[static_cast(existing_type)]; - existing_entries_by_name.erase(existing_name); - } - } - - auto & entry = entries_by_id[id]; - entry.id = id; - entry.entity = entity; - entries_by_name[name] = &entry; - - if (initialized) - { - if (existed_before) - changes_notifier.onEntityUpdated(id, entity); - else - changes_notifier.onEntityAdded(id, entity); - } + memory_storage.insertWithID(id, entity, /* replace_if_exists= */ true, /* throw_if_exists= */ false); } void ReplicatedAccessStorage::removeEntityNoLock(const UUID & id) { LOG_DEBUG(getLogger(), "Removing entity with id {}", toString(id)); - const auto it = entries_by_id.find(id); - if (it == entries_by_id.end()) - { - LOG_DEBUG(getLogger(), "Id {} not found, ignoring removal", toString(id)); - return; - } - - const Entry & entry = it->second; - const AccessEntityType type = entry.entity->getType(); - const String & name = entry.entity->getName(); - - auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; - const auto name_it = entries_by_name.find(name); - if (name_it == entries_by_name.end()) - LOG_WARNING(getLogger(), "Entity {} not found in names, ignoring removal of name", toString(id)); - else if (name_it->second != &(it->second)) - LOG_WARNING(getLogger(), "Name {} not pointing to entity {}, ignoring removal of name", name, toString(id)); - else - entries_by_name.erase(name); - - UUID removed_id = id; - entries_by_id.erase(id); - LOG_DEBUG(getLogger(), "Removed entity with id {}", toString(id)); - - changes_notifier.onEntityRemoved(removed_id, type); + memory_storage.remove(id, /* throw_if_exists= */ false); } std::optional ReplicatedAccessStorage::findImpl(AccessEntityType type, const String & name) const { std::lock_guard lock{mutex}; - const auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; - const auto it = entries_by_name.find(name); - if (it == entries_by_name.end()) - return {}; - - const Entry * entry = it->second; - return entry->id; + return memory_storage.find(type, name); } std::vector ReplicatedAccessStorage::findAllImpl(AccessEntityType type) const { std::lock_guard lock{mutex}; - std::vector result; - result.reserve(entries_by_id.size()); - for (const auto & [id, entry] : entries_by_id) - if (entry.entity->isTypeOf(type)) - result.emplace_back(id); - return result; + return memory_storage.findAll(type); } bool ReplicatedAccessStorage::exists(const UUID & id) const { std::lock_guard lock{mutex}; - return entries_by_id.contains(id); + return memory_storage.exists(id); } AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const { std::lock_guard lock{mutex}; - const auto it = entries_by_id.find(id); - if (it == entries_by_id.end()) - { - if (throw_if_not_exists) - throwNotFound(id); - else - return nullptr; - } - const Entry & entry = it->second; - return entry.entity; + return memory_storage.read(id, throw_if_not_exists); } diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h index 6311e2ac7c0..c56e55b3ccd 100644 --- a/src/Access/ReplicatedAccessStorage.h +++ b/src/Access/ReplicatedAccessStorage.h @@ -1,20 +1,13 @@ #pragma once #include -#include -#include -#include -#include - -#include -#include #include #include #include #include -#include +#include namespace DB @@ -77,20 +70,13 @@ private: void setEntityNoLock(const UUID & id, const AccessEntityPtr & entity) TSA_REQUIRES(mutex); void removeEntityNoLock(const UUID & id) TSA_REQUIRES(mutex); - struct Entry - { - UUID id; - AccessEntityPtr entity; - }; - std::optional findImpl(AccessEntityType type, const String & name) const override; std::vector findAllImpl(AccessEntityType type) const override; AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; mutable std::mutex mutex; - std::unordered_map entries_by_id TSA_GUARDED_BY(mutex); - std::unordered_map entries_by_name_and_type[static_cast(AccessEntityType::MAX)] TSA_GUARDED_BY(mutex); + MemoryAccessStorage memory_storage TSA_GUARDED_BY(mutex); AccessChangesNotifier & changes_notifier; - bool backup_allowed = false; + const bool backup_allowed = false; }; } From 646cd5569055989d927032c7ed1e859f0b527560 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 9 Aug 2022 11:43:23 +0200 Subject: [PATCH 48/86] Improve recovery of ReplicatedAccessStorage after errors. --- src/Access/IAccessStorage.h | 4 +- src/Access/MemoryAccessStorage.cpp | 2 +- src/Access/ReplicatedAccessStorage.cpp | 162 ++++++++++++++++++------- src/Access/ReplicatedAccessStorage.h | 13 +- 4 files changed, 132 insertions(+), 49 deletions(-) diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 394d3ed6358..514374a828a 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -45,10 +45,10 @@ public: /// Reloads and updates entities in this storage. This function is used to implement SYSTEM RELOAD CONFIG. virtual void reload() {} - /// Starts periodic reloading and update of entities in this storage. + /// Starts periodic reloading and updating of entities in this storage. virtual void startPeriodicReloading() {} - /// Stops periodic reloading and update of entities in this storage. + /// Stops periodic reloading and updating of entities in this storage. virtual void stopPeriodicReloading() {} /// Returns the identifiers of all the entities of a specified type contained in the storage. diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index 0f6be7ee96c..fa8bbc3a559 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -119,7 +119,7 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & assert(replace_if_exists); removeNoLock(id_by_name, /* throw_if_not_exists = */ false); } - + if (id_collision) { assert(replace_if_exists); diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index 8a3b69883d4..e5373f4df00 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -24,8 +24,8 @@ namespace DB { namespace ErrorCodes { -extern const int BAD_ARGUMENTS; -extern const int NO_ZOOKEEPER; + extern const int BAD_ARGUMENTS; + extern const int NO_ZOOKEEPER; } static UUID parseUUID(const String & text) @@ -60,7 +60,7 @@ ReplicatedAccessStorage::ReplicatedAccessStorage( if (zookeeper_path.front() != '/') zookeeper_path = "/" + zookeeper_path; - initializeZookeeper(); + initZooKeeperIfNeeded(); } ReplicatedAccessStorage::~ReplicatedAccessStorage() @@ -122,15 +122,14 @@ bool ReplicatedAccessStorage::insertWithID(const UUID & id, const AccessEntityPt const String & name = new_entity->getName(); LOG_DEBUG(getLogger(), "Inserting entity of type {} named {} with id {}", type_info.name, name, toString(id)); - auto zookeeper = get_zookeeper(); + auto zookeeper = getZooKeeper(); bool ok = false; retryOnZooKeeperUserError(10, [&]{ ok = insertZooKeeper(zookeeper, id, new_entity, replace_if_exists, throw_if_exists); }); if (!ok) return false; - std::lock_guard lock{mutex}; - refreshEntityNoLock(zookeeper, id); + refreshEntity(zookeeper, id); return true; } @@ -222,7 +221,7 @@ bool ReplicatedAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exis { LOG_DEBUG(getLogger(), "Removing entity {}", toString(id)); - auto zookeeper = get_zookeeper(); + auto zookeeper = getZooKeeper(); bool ok = false; retryOnZooKeeperUserError(10, [&] { ok = removeZooKeeper(zookeeper, id, throw_if_not_exists); }); @@ -274,15 +273,14 @@ bool ReplicatedAccessStorage::updateImpl(const UUID & id, const UpdateFunc & upd { LOG_DEBUG(getLogger(), "Updating entity {}", toString(id)); - auto zookeeper = get_zookeeper(); + auto zookeeper = getZooKeeper(); bool ok = false; retryOnZooKeeperUserError(10, [&] { ok = updateZooKeeper(zookeeper, id, update_func, throw_if_not_exists); }); if (!ok) return false; - std::lock_guard lock{mutex}; - refreshEntityNoLock(zookeeper, id); + refreshEntity(zookeeper, id); return true; } @@ -350,45 +348,82 @@ void ReplicatedAccessStorage::runWatchingThread() { LOG_DEBUG(getLogger(), "Started watching thread"); setThreadName("ReplACLWatch"); + while (watching) { + bool refreshed = false; try { - if (!initialized) - initializeZookeeper(); - if (refresh()) - changes_notifier.sendNotifications(); + initZooKeeperIfNeeded(); + refreshed = refresh(); } catch (...) { - tryLogCurrentException(getLogger(), "Unexpected error, will try to restart worker thread:"); + tryLogCurrentException(getLogger(), "Will try to restart watching thread after error"); resetAfterError(); sleepForSeconds(5); + continue; + } + + if (refreshed) + { + try + { + changes_notifier.sendNotifications(); + } + catch (...) + { + tryLogCurrentException(getLogger(), "Error while sending notifications"); + } } } } void ReplicatedAccessStorage::resetAfterError() { - initialized = false; - - UUID id; - while (watched_queue->tryPop(id)) {} + /// Make watching thread reinitialize ZooKeeper and reread everything. + std::lock_guard lock{cached_zookeeper_mutex}; + cached_zookeeper = nullptr; } -void ReplicatedAccessStorage::initializeZookeeper() +void ReplicatedAccessStorage::initZooKeeperIfNeeded() { - assert(!initialized); - auto zookeeper = get_zookeeper(); + getZooKeeper(); +} - if (!zookeeper) - throw Exception("Can't have Replicated access without ZooKeeper", ErrorCodes::NO_ZOOKEEPER); +zkutil::ZooKeeperPtr ReplicatedAccessStorage::getZooKeeper() +{ + std::lock_guard lock{cached_zookeeper_mutex}; + return getZooKeeperNoLock(); +} - createRootNodes(zookeeper); +zkutil::ZooKeeperPtr ReplicatedAccessStorage::getZooKeeperNoLock() +{ + if (!cached_zookeeper || cached_zookeeper->expired()) + { + auto zookeeper = get_zookeeper(); + if (!zookeeper) + throw Exception("Can't have Replicated access without ZooKeeper", ErrorCodes::NO_ZOOKEEPER); - refreshEntities(zookeeper); + /// It's possible that we connected to different [Zoo]Keeper instance + /// so we may read a bit stale state. + zookeeper->sync(zookeeper_path); - initialized = true; + createRootNodes(zookeeper); + refreshEntities(zookeeper, /* all= */ true); + cached_zookeeper = zookeeper; + } + return cached_zookeeper; +} + +void ReplicatedAccessStorage::reload() +{ +#if 0 + /// Reinitialize ZooKeeper and reread everything. + std::lock_guard lock{cached_zookeeper_mutex}; + cached_zookeeper = nullptr; + getZooKeeperNoLock(); +#endif } void ReplicatedAccessStorage::createRootNodes(const zkutil::ZooKeeperPtr & zookeeper) @@ -410,10 +445,10 @@ bool ReplicatedAccessStorage::refresh() if (!watched_queue->tryPop(id, /* timeout_ms: */ 10000)) return false; - auto zookeeper = get_zookeeper(); + auto zookeeper = getZooKeeper(); if (id == UUIDHelpers::Nil) - refreshEntities(zookeeper); + refreshEntities(zookeeper, /* all= */ false); else refreshEntity(zookeeper, id); @@ -421,10 +456,16 @@ bool ReplicatedAccessStorage::refresh() } -void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zookeeper) +void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zookeeper, bool all) { LOG_DEBUG(getLogger(), "Refreshing entities list"); + if (all) + { + /// It doesn't make sense to keep the queue because we will reread everything in this function. + watched_queue->clear(); + } + const String zookeeper_uuids_path = zookeeper_path + "/uuid"; auto watch_entities_list = [watched_queue = watched_queue](const Coordination::WatchResponse &) { @@ -439,47 +480,84 @@ void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zooke entity_uuids.emplace_back(parseUUID(entity_uuid_str)); std::lock_guard lock{mutex}; - memory_storage.removeAllExcept(entity_uuids); - for (const auto & uuid : entity_uuids) + + if (all) { - if (!initialized || !memory_storage.exists(uuid)) - refreshEntityNoLock(zookeeper, uuid); + /// all=true means we read & parse all access entities from ZooKeeper. + std::vector> entities; + for (const auto & uuid : entity_uuids) + { + if (auto entity = tryReadEntityFromZooKeeper(zookeeper, uuid)) + entities.emplace_back(uuid, entity); + } + memory_storage.setAll(entities); + } + else + { + /// all=false means we read & parse only new access entities from ZooKeeper. + memory_storage.removeAllExcept(entity_uuids); + for (const auto & uuid : entity_uuids) + { + if (!memory_storage.exists(uuid)) + refreshEntityNoLock(zookeeper, uuid); + } } LOG_DEBUG(getLogger(), "Refreshing entities list finished"); } + void ReplicatedAccessStorage::refreshEntity(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) { + LOG_DEBUG(getLogger(), "Refreshing entity {}", toString(id)); + + auto entity = tryReadEntityFromZooKeeper(zookeeper, id); + std::lock_guard lock{mutex}; - refreshEntityNoLock(zookeeper, id); + + if (entity) + setEntityNoLock(id, entity); + else + removeEntityNoLock(id); } void ReplicatedAccessStorage::refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) { LOG_DEBUG(getLogger(), "Refreshing entity {}", toString(id)); + auto entity = tryReadEntityFromZooKeeper(zookeeper, id); + if (entity) + setEntityNoLock(id, entity); + else + removeEntityNoLock(id); +} + +AccessEntityPtr ReplicatedAccessStorage::tryReadEntityFromZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) const +{ const auto watch_entity = [watched_queue = watched_queue, id](const Coordination::WatchResponse & response) { if (response.type == Coordination::Event::CHANGED) [[maybe_unused]] bool push_result = watched_queue->push(id); }; + Coordination::Stat entity_stat; const String entity_path = zookeeper_path + "/uuid/" + toString(id); String entity_definition; - const bool exists = zookeeper->tryGetWatch(entity_path, entity_definition, &entity_stat, watch_entity); - if (exists) + bool exists = zookeeper->tryGetWatch(entity_path, entity_definition, &entity_stat, watch_entity); + if (!exists) + return nullptr; + + try { - const AccessEntityPtr entity = deserializeAccessEntity(entity_definition, entity_path); - setEntityNoLock(id, entity); + return deserializeAccessEntity(entity_definition, entity_path); } - else + catch (...) { - removeEntityNoLock(id); + tryLogCurrentException(getLogger(), "Error while reading the definition of " + toString(id)); + return nullptr; } } - void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntityPtr & entity) { LOG_DEBUG(getLogger(), "Setting id {} to entity named {}", toString(id), entity->getName()); diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h index c56e55b3ccd..6b1bd01238d 100644 --- a/src/Access/ReplicatedAccessStorage.h +++ b/src/Access/ReplicatedAccessStorage.h @@ -25,6 +25,7 @@ public: const char * getStorageType() const override { return STORAGE_TYPE; } + void reload() override; void startPeriodicReloading() override { startWatchingThread(); } void stopPeriodicReloading() override { stopWatchingThread(); } @@ -36,9 +37,10 @@ public: private: String zookeeper_path; - zkutil::GetZooKeeper get_zookeeper; + const zkutil::GetZooKeeper get_zookeeper; - std::atomic initialized = false; + zkutil::ZooKeeperPtr cached_zookeeper TSA_GUARDED_BY(cached_zookeeper_mutex); + std::mutex cached_zookeeper_mutex; std::atomic watching = false; ThreadFromGlobalPool watching_thread; @@ -53,7 +55,9 @@ private: bool removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists); bool updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); - void initializeZookeeper(); + void initZooKeeperIfNeeded(); + zkutil::ZooKeeperPtr getZooKeeper(); + zkutil::ZooKeeperPtr getZooKeeperNoLock() TSA_REQUIRES(cached_zookeeper_mutex); void createRootNodes(const zkutil::ZooKeeperPtr & zookeeper); void startWatchingThread(); @@ -63,10 +67,11 @@ private: void resetAfterError(); bool refresh(); - void refreshEntities(const zkutil::ZooKeeperPtr & zookeeper); + void refreshEntities(const zkutil::ZooKeeperPtr & zookeeper, bool all); void refreshEntity(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id); void refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) TSA_REQUIRES(mutex); + AccessEntityPtr tryReadEntityFromZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) const; void setEntityNoLock(const UUID & id, const AccessEntityPtr & entity) TSA_REQUIRES(mutex); void removeEntityNoLock(const UUID & id) TSA_REQUIRES(mutex); From 1206e77883934563cd4154eb3eb0bc834013806d Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 9 Aug 2022 12:39:32 +0200 Subject: [PATCH 49/86] Add test checking that ReplicatedAccessStorage can continue after reloading ZooKeeper. --- .../configs/zookeeper.xml | 19 +++ .../integration/test_replicated_users/test.py | 127 +++++++++++++++++- 2 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 tests/integration/test_replicated_users/configs/zookeeper.xml diff --git a/tests/integration/test_replicated_users/configs/zookeeper.xml b/tests/integration/test_replicated_users/configs/zookeeper.xml new file mode 100644 index 00000000000..87b991f3ab3 --- /dev/null +++ b/tests/integration/test_replicated_users/configs/zookeeper.xml @@ -0,0 +1,19 @@ + + + + + zoo1 + 2181 + + + zoo2 + 2181 + + + zoo3 + 2181 + + 20000 + + + \ No newline at end of file diff --git a/tests/integration/test_replicated_users/test.py b/tests/integration/test_replicated_users/test.py index 56383f0d2df..920e6660dc3 100644 --- a/tests/integration/test_replicated_users/test.py +++ b/tests/integration/test_replicated_users/test.py @@ -1,15 +1,24 @@ import pytest +import time from dataclasses import dataclass from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry, TSV -cluster = ClickHouseCluster(__file__) +cluster = ClickHouseCluster(__file__, zookeeper_config_path="configs/zookeeper.xml") node1 = cluster.add_instance( - "node1", main_configs=["configs/config.xml"], with_zookeeper=True, stay_alive=True + "node1", + main_configs=["configs/config.xml"], + with_zookeeper=True, + stay_alive=True, ) + node2 = cluster.add_instance( - "node2", main_configs=["configs/config.xml"], with_zookeeper=True, stay_alive=True + "node2", + main_configs=["configs/config.xml"], + with_zookeeper=True, + stay_alive=True, ) all_nodes = [node1, node2] @@ -88,3 +97,115 @@ def test_rename_replicated(started_cluster, entity): f"ALTER {entity.keyword} {entity.name} {entity.options} RENAME TO {entity.name}2" ) node1.query(f"DROP {entity.keyword} {entity.name}2 {entity.options}") + + +# ReplicatedAccessStorage must be able to continue working after reloading ZooKeeper. +def test_reload_zookeeper(started_cluster): + def wait_zookeeper_node_to_start(zk_nodes, timeout=60): + start = time.time() + while time.time() - start < timeout: + try: + for instance in zk_nodes: + conn = cluster.get_kazoo_client(instance) + conn.get_children("/") + print("All instances of ZooKeeper started") + return + except Exception as ex: + print(("Can't connect to ZooKeeper " + str(ex))) + time.sleep(0.5) + + def replace_zookeeper_config(new_config): + node1.replace_config("/etc/clickhouse-server/conf.d/zookeeper.xml", new_config) + node2.replace_config("/etc/clickhouse-server/conf.d/zookeeper.xml", new_config) + node1.query("SYSTEM RELOAD CONFIG") + node2.query("SYSTEM RELOAD CONFIG") + + def get_active_zk_connections(): + return str( + node1.exec_in_container( + [ + "bash", + "-c", + "lsof -a -i4 -i6 -itcp -w | grep 2181 | grep ESTABLISHED | wc -l", + ], + privileged=True, + user="root", + ) + ).strip() + + node1.query("CREATE USER u1") + assert_eq_with_retry( + node2, "SELECT name FROM system.users WHERE name ='u1'", "u1\n" + ) + + ## remove zoo2, zoo3 from configs + replace_zookeeper_config( + """ + + + + zoo1 + 2181 + + 2000 + + +""" + ) + + ## config reloads, but can still work + node1.query("CREATE USER u2") + assert_eq_with_retry( + node2, + "SELECT name FROM system.users WHERE name IN ['u1', 'u2'] ORDER BY name", + TSV(["u1", "u2"]), + ) + + ## stop all zookeepers, users will be readonly + cluster.stop_zookeeper_nodes(["zoo1", "zoo2", "zoo3"]) + assert node2.query( + "SELECT name FROM system.users WHERE name IN ['u1', 'u2'] ORDER BY name" + ) == TSV(["u1", "u2"]) + expected_error = "Cannot resolve any of provided ZooKeeper hosts" + assert expected_error in node1.query_and_get_error("CREATE USER u3") + + ## start zoo2, zoo3, users will be readonly too, because it only connect to zoo1 + cluster.start_zookeeper_nodes(["zoo2", "zoo3"]) + wait_zookeeper_node_to_start(["zoo2", "zoo3"]) + assert node2.query( + "SELECT name FROM system.users WHERE name IN ['u1', 'u2'] ORDER BY name" + ) == TSV(["u1", "u2"]) + expected_error = "Cannot resolve any of provided ZooKeeper hosts" + assert expected_error in node1.query_and_get_error("CREATE USER u3") + + ## set config to zoo2, server will be normal + replace_zookeeper_config( + """ + + + + zoo2 + 2181 + + 2000 + + +""" + ) + + active_zk_connections = get_active_zk_connections() + assert ( + active_zk_connections == "1" + ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections) + + node1.query("CREATE USER u3") + assert_eq_with_retry( + node2, + "SELECT name FROM system.users WHERE name IN ['u1', 'u2', 'u3'] ORDER BY name", + TSV(["u1", "u2", "u3"]), + ) + + active_zk_connections = get_active_zk_connections() + assert ( + active_zk_connections == "1" + ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections) From 7e716d14cdc63b1d5df0208ba863fe6766d04138 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 9 Aug 2022 21:40:03 +0200 Subject: [PATCH 50/86] Make ReplicatedAccessStorage::insertWithID() more consistent with MemoryAccessStorage::insertWithID(). --- src/Access/MemoryAccessStorage.cpp | 16 ++-- src/Access/ReplicatedAccessStorage.cpp | 104 ++++++++++++++++--------- 2 files changed, 75 insertions(+), 45 deletions(-) diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index fa8bbc3a559..738a0d305d8 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -117,7 +117,7 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & if (name_collision && (id_by_name != id)) { assert(replace_if_exists); - removeNoLock(id_by_name, /* throw_if_not_exists = */ false); + removeNoLock(id_by_name, /* throw_if_not_exists= */ false); } if (id_collision) @@ -126,20 +126,20 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & auto & existing_entry = it_by_id->second; if (existing_entry.entity->getType() == new_entity->getType()) { - if (existing_entry.entity->getName() != new_entity->getName()) - { - entries_by_name.erase(existing_entry.entity->getName()); - bool inserted = entries_by_name.emplace(new_entity->getName(), &existing_entry).second; - assert(inserted); - } if (*existing_entry.entity != *new_entity) { + if (existing_entry.entity->getName() != new_entity->getName()) + { + entries_by_name.erase(existing_entry.entity->getName()); + [[maybe_unused]] bool inserted = entries_by_name.emplace(new_entity->getName(), &existing_entry).second; + assert(inserted); + } existing_entry.entity = new_entity; changes_notifier.onEntityUpdated(id, new_entity); } return true; } - removeNoLock(id, /* throw_if_not_exists = */ false); + removeNoLock(id, /* throw_if_not_exists= */ false); } /// Do insertion. diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index e5373f4df00..4d4d6366f49 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -163,51 +163,79 @@ bool ReplicatedAccessStorage::insertZooKeeper( if (res == Coordination::Error::ZNODEEXISTS) { + if (!throw_if_exists && !replace_if_exists) + return false; /// Couldn't insert a new entity. + + if (throw_if_exists) + { + if (responses[0]->error == Coordination::Error::ZNODEEXISTS) + { + /// To fail with a nice error message, we need info about what already exists. + /// This itself could fail if the conflicting uuid disappears in the meantime. + /// If that happens, then we'll just retry from the start. + String existing_entity_definition = zookeeper->get(entity_path); + + AccessEntityPtr existing_entity = deserializeAccessEntity(existing_entity_definition, entity_path); + AccessEntityType existing_type = existing_entity->getType(); + String existing_name = existing_entity->getName(); + throwIDCollisionCannotInsert(id, type, name, existing_type, existing_name); + } + else + { + /// Couldn't insert the new entity because there is an existing entity with such name. + throwNameCollisionCannotInsert(type, name); + } + } + + assert(replace_if_exists); + Coordination::Requests replace_ops; if (responses[0]->error == Coordination::Error::ZNODEEXISTS) { - /// The UUID already exists, simply fail. - - /// To fail with a nice error message, we need info about what already exists. - /// This itself could fail if the conflicting uuid disappears in the meantime. + /// The UUID is already associated with some existing entity, we will get rid of the conflicting entity first. + /// This itself could fail if the conflicting entity disappears in the meantime. /// If that happens, then we'll just retry from the start. - String existing_entity_definition = zookeeper->get(entity_path); + Coordination::Stat stat; + String existing_entity_definition = zookeeper->get(entity_path, &stat); + auto existing_entity = deserializeAccessEntity(existing_entity_definition, entity_path); + const String & existing_entity_name = existing_entity->getName(); + const AccessEntityType existing_entity_type = existing_entity->getType(); + const AccessEntityTypeInfo existing_entity_type_info = AccessEntityTypeInfo::get(existing_entity_type); + const String existing_name_path = zookeeper_path + "/" + existing_entity_type_info.unique_char + "/" + escapeForFileName(existing_entity_name); - AccessEntityPtr existing_entity = deserializeAccessEntity(existing_entity_definition, entity_path); - AccessEntityType existing_type = existing_entity->getType(); - String existing_name = existing_entity->getName(); - throwIDCollisionCannotInsert(id, type, name, existing_type, existing_name); - } - else if (replace_if_exists) - { - /// The name already exists for this type. - /// If asked to, we need to replace the existing entity. + if (existing_name_path != name_path) + replace_ops.emplace_back(zkutil::makeRemoveRequest(existing_name_path, -1)); - /// First get the uuid of the existing entity - /// This itself could fail if the conflicting name disappears in the meantime. - /// If that happens, then we'll just retry from the start. - Coordination::Stat name_stat; - String existing_entity_uuid = zookeeper->get(name_path, &name_stat); - - const String existing_entity_path = zookeeper_path + "/uuid/" + existing_entity_uuid; - Coordination::Requests replace_ops; - replace_ops.emplace_back(zkutil::makeRemoveRequest(existing_entity_path, -1)); - replace_ops.emplace_back(zkutil::makeCreateRequest(entity_path, new_entity_definition, zkutil::CreateMode::Persistent)); - replace_ops.emplace_back(zkutil::makeSetRequest(name_path, entity_uuid, name_stat.version)); - - /// If this fails, then we'll just retry from the start. - zookeeper->multi(replace_ops); - - /// Everything's fine, the new entity has been inserted instead of an existing entity. - return true; + replace_ops.emplace_back(zkutil::makeSetRequest(entity_path, new_entity_definition, stat.version)); } else { - /// Couldn't insert the new entity because there is an existing entity with such name. - if (throw_if_exists) - throwNameCollisionCannotInsert(type, name); - else - return false; + replace_ops.emplace_back(zkutil::makeCreateRequest(entity_path, new_entity_definition, zkutil::CreateMode::Persistent)); } + + if (responses[1]->error == Coordination::Error::ZNODEEXISTS) + { + /// The name is already associated with some existing entity, we will get rid of the conflicting entity first. + /// This itself could fail if the conflicting entity disappears in the meantime. + /// If that happens, then we'll just retry from the start. + Coordination::Stat stat; + String existing_entity_uuid = zookeeper->get(name_path, &stat); + const String existing_entity_path = zookeeper_path + "/uuid/" + existing_entity_uuid; + + if (existing_entity_path != entity_path) + replace_ops.emplace_back(zkutil::makeRemoveRequest(existing_entity_path, -1)); + + replace_ops.emplace_back(zkutil::makeSetRequest(name_path, entity_uuid, stat.version)); + } + else + { + replace_ops.emplace_back(zkutil::makeCreateRequest(name_path, entity_uuid, zkutil::CreateMode::Persistent)); + } + + /// If this fails, then we'll just retry from the start. + zookeeper->multi(replace_ops); + + /// Everything's fine, the new entity has been inserted instead of an existing entity. + return true; } /// If this fails, then we'll just retry from the start. @@ -418,6 +446,8 @@ zkutil::ZooKeeperPtr ReplicatedAccessStorage::getZooKeeperNoLock() void ReplicatedAccessStorage::reload() { + /// TODO: Disabled because reload() is called by SYSTEM RELOAD CONFIG and replicated access storage is not a config-based. + /// We need a separate SYSTEM RELOAD USES command. #if 0 /// Reinitialize ZooKeeper and reread everything. std::lock_guard lock{cached_zookeeper_mutex}; @@ -568,7 +598,7 @@ void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntit void ReplicatedAccessStorage::removeEntityNoLock(const UUID & id) { LOG_DEBUG(getLogger(), "Removing entity with id {}", toString(id)); - memory_storage.remove(id, /* throw_if_exists= */ false); + memory_storage.remove(id, /* throw_if_not_exists= */ false); } From 2ec6ef497db754e8bd7cf055310d30f6319a1584 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 10 Aug 2022 12:16:12 +0200 Subject: [PATCH 51/86] Make DiskAccessStorage::insertWithID() more consistent with MemoryAccessStorage::insertWithID(). --- src/Access/DiskAccessStorage.cpp | 199 +++++++++++++++++++++-------- src/Access/DiskAccessStorage.h | 41 +++--- src/Access/MemoryAccessStorage.cpp | 20 ++- 3 files changed, 182 insertions(+), 78 deletions(-) diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 0cbe420f345..32799f81467 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -182,8 +183,8 @@ DiskAccessStorage::DiskAccessStorage(const String & storage_name_, const String if (should_rebuild_lists) { - rebuildLists(); - writeLists(); + LOG_WARNING(getLogger(), "Recovering lists in directory {}", directory_path); + reloadAllAndRebuildLists(); } } @@ -224,63 +225,57 @@ bool DiskAccessStorage::isPathEqual(const String & directory_path_) const } -void DiskAccessStorage::clear() -{ - entries_by_id.clear(); - for (auto type : collections::range(AccessEntityType::MAX)) - entries_by_name_and_type[static_cast(type)].clear(); -} - - bool DiskAccessStorage::readLists() { - clear(); + std::vector> ids_names_types; - bool ok = true; for (auto type : collections::range(AccessEntityType::MAX)) { - auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; auto file_path = getListFilePath(directory_path, type); if (!std::filesystem::exists(file_path)) { LOG_WARNING(getLogger(), "File {} doesn't exist", file_path); - ok = false; - break; + return false; } try { for (const auto & [id, name] : readListFile(file_path)) - { - auto & entry = entries_by_id[id]; - entry.id = id; - entry.type = type; - entry.name = name; - entries_by_name[entry.name] = &entry; - } + ids_names_types.emplace_back(id, name, type); } catch (...) { tryLogCurrentException(getLogger(), "Could not read " + file_path); - ok = false; - break; + return false; } } - if (!ok) - clear(); - return ok; + entries_by_id.clear(); + for (auto type : collections::range(AccessEntityType::MAX)) + entries_by_name_and_type[static_cast(type)].clear(); + + for (const auto & [id, name, type] : ids_names_types) + { + auto & entry = entries_by_id[id]; + entry.id = id; + entry.type = type; + entry.name = name; + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + entries_by_name[entry.name] = &entry; + } + + return true; } -bool DiskAccessStorage::writeLists() +void DiskAccessStorage::writeLists() { if (failed_to_write_lists) - return false; /// We don't try to write list files after the first fail. - /// The next restart of the server will invoke rebuilding of the list files. + return; /// We don't try to write list files after the first fail. + /// The next restart of the server will invoke rebuilding of the list files. if (types_of_lists_to_write.empty()) - return true; + return; for (const auto & type : types_of_lists_to_write) { @@ -299,14 +294,13 @@ bool DiskAccessStorage::writeLists() tryLogCurrentException(getLogger(), "Could not write " + file_path); failed_to_write_lists = true; types_of_lists_to_write.clear(); - return false; + return; } } /// The list files was successfully written, we don't need the 'need_rebuild_lists.mark' file any longer. std::filesystem::remove(getNeedRebuildListsMarkFilePath(directory_path)); types_of_lists_to_write.clear(); - return true; } @@ -364,10 +358,9 @@ void DiskAccessStorage::stopListsWritingThread() /// Reads and parses all the ".sql" files from a specified directory /// and then saves the files "users.list", "roles.list", etc. to the same directory. -bool DiskAccessStorage::rebuildLists() +void DiskAccessStorage::reloadAllAndRebuildLists() { - LOG_WARNING(getLogger(), "Recovering lists in directory {}", directory_path); - clear(); + std::vector> all_entities; for (const auto & directory_entry : std::filesystem::directory_iterator(directory_path)) { @@ -386,21 +379,85 @@ bool DiskAccessStorage::rebuildLists() if (!entity) continue; - const String & name = entity->getName(); - auto type = entity->getType(); - auto & entry = entries_by_id[id]; - entry.id = id; - entry.type = type; - entry.name = name; - entry.entity = entity; - auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; - entries_by_name[entry.name] = &entry; + all_entities.emplace_back(id, entity); } + setAll(all_entities); + for (auto type : collections::range(AccessEntityType::MAX)) types_of_lists_to_write.insert(type); - return true; + failed_to_write_lists = false; /// Try again writing lists. + writeLists(); +} + +void DiskAccessStorage::setAll(const std::vector> & all_entities) +{ + /// This function is similar to MemoryAccessStorage::setAll(). + boost::container::flat_set not_used_ids; + std::vector conflicting_ids; + + /// Get the list of currently used IDs. Later we will remove those of them which are not used anymore. + for (const auto & id : entries_by_id | boost::adaptors::map_keys) + not_used_ids.emplace(id); + + /// Get the list of conflicting IDs and update the list of currently used ones. + for (const auto & [id, entity] : all_entities) + { + auto it = entries_by_id.find(id); + if (it != entries_by_id.end()) + { + not_used_ids.erase(id); /// ID is used. + + Entry & entry = it->second; + if (entry.entity->getType() != entity->getType()) + conflicting_ids.emplace_back(id); /// Conflict: same ID, different type. + } + + const auto & entries_by_name = entries_by_name_and_type[static_cast(entity->getType())]; + auto it2 = entries_by_name.find(entity->getName()); + if (it2 != entries_by_name.end()) + { + Entry & entry = *(it2->second); + if (entry.id != id) + conflicting_ids.emplace_back(entry.id); /// Conflict: same name and type, different ID. + } + } + + /// Remove entities which are not used anymore and which are in conflict with new entities. + boost::container::flat_set ids_to_remove = std::move(not_used_ids); + boost::range::copy(conflicting_ids, std::inserter(ids_to_remove, ids_to_remove.end())); + for (const auto & id : ids_to_remove) + removeNoLock(id, /* throw_if_not_exists= */ false); + + /// Insert or update entities. + for (const auto & [id, entity] : all_entities) + { + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) + { + insertNoLock(id, entity, /* replace_if_exists= */ false, /* throw_if_exists= */ true); + } + else if (!it->second.entity || (*(it->second.entity) != *entity)) + { + const AccessEntityPtr & changed_entity = entity; + updateNoLock( + id, + [&changed_entity](const AccessEntityPtr &) { return changed_entity; }, + /* throw_if_not_exists= */ true); + } + } +} + + +void DiskAccessStorage::reload() +{ + /// TODO: Disabled because reload() is called by SYSTEM RELOAD CONFIG and replicated access storage is not a config-based. + /// We need a separate SYSTEM RELOAD USES command. +#if 0 + std::lock_guard lock{mutex}; + reloadAllAndRebuildLists(); +#endif } @@ -497,6 +554,9 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; auto it_by_name = entries_by_name.find(name); bool name_collision = (it_by_name != entries_by_name.end()); + UUID id_by_name; + if (name_collision) + id_by_name = it_by_name->second->id; if (name_collision && !replace_if_exists) { @@ -507,19 +567,54 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne } auto it_by_id = entries_by_id.find(id); - if (it_by_id != entries_by_id.end()) + bool id_collision = (it_by_id != entries_by_id.end()); + if (id_collision && !replace_if_exists) { - const auto & existing_entry = it_by_id->second; - throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); + if (throw_if_exists) + { + const auto & existing_entry = it_by_id->second; + throwIDCollisionCannotInsert(id, type, name, existing_entry.type, existing_entry.name); + } + else + return false; } scheduleWriteLists(type); - writeAccessEntityToDisk(id, *new_entity); - if (name_collision && replace_if_exists) - removeNoLock(it_by_name->second->id, /* throw_if_not_exists = */ false); + /// Remove collisions if necessary. + if (name_collision && (id_by_name != id)) + { + assert(replace_if_exists); + removeNoLock(id_by_name, /* throw_if_not_exists= */ false); + } + + if (id_collision) + { + assert(replace_if_exists); + auto & existing_entry = it_by_id->second; + if (existing_entry.type == new_entity->getType()) + { + if (!existing_entry.entity || (*existing_entry.entity != *new_entity)) + { + if (existing_entry.name != new_entity->getName()) + { + entries_by_name.erase(existing_entry.name); + [[maybe_unused]] bool inserted = entries_by_name.emplace(new_entity->getName(), &existing_entry).second; + assert(inserted); + } + existing_entry.entity = new_entity; + changes_notifier.onEntityUpdated(id, new_entity); + } + return true; + } + + scheduleWriteLists(existing_entry.type); + removeNoLock(id, /* throw_if_not_exists= */ false); + } /// Do insertion. + writeAccessEntityToDisk(id, *new_entity); + auto & entry = entries_by_id[id]; entry.id = id; entry.type = type; diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h index d3bd61ff353..4e6c627443e 100644 --- a/src/Access/DiskAccessStorage.h +++ b/src/Access/DiskAccessStorage.h @@ -27,6 +27,8 @@ public: void setReadOnly(bool readonly_) { readonly = readonly_; } bool isReadOnly() const override { return readonly; } + void reload() override; + bool exists(const UUID & id) const override; bool isBackupAllowed() const override { return backup_allowed; } @@ -41,19 +43,19 @@ private: bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; - void clear(); - bool readLists(); - bool writeLists(); - void scheduleWriteLists(AccessEntityType type); - bool rebuildLists(); + bool readLists() TSA_REQUIRES(mutex); + void writeLists() TSA_REQUIRES(mutex); + void scheduleWriteLists(AccessEntityType type) TSA_REQUIRES(mutex); + void reloadAllAndRebuildLists() TSA_REQUIRES(mutex); + void setAll(const std::vector> & all_entities) TSA_REQUIRES(mutex); - void listsWritingThreadFunc(); + void listsWritingThreadFunc() TSA_NO_THREAD_SAFETY_ANALYSIS; void stopListsWritingThread(); bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists); - bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists); - bool removeNoLock(const UUID & id, bool throw_if_not_exists); - bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); + bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) TSA_REQUIRES(mutex); + bool removeNoLock(const UUID & id, bool throw_if_not_exists) TSA_REQUIRES(mutex); + bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) TSA_REQUIRES(mutex); AccessEntityPtr readAccessEntityFromDisk(const UUID & id) const; void writeAccessEntityToDisk(const UUID & id, const IAccessEntity & entity) const; @@ -69,13 +71,22 @@ private: }; String directory_path; - std::unordered_map entries_by_id; - std::unordered_map entries_by_name_and_type[static_cast(AccessEntityType::MAX)]; - boost::container::flat_set types_of_lists_to_write; - bool failed_to_write_lists = false; /// Whether writing of the list files has been failed since the recent restart of the server. - ThreadFromGlobalPool lists_writing_thread; /// List files are written in a separate thread. - std::condition_variable lists_writing_thread_should_exit; /// Signals `lists_writing_thread` to exit. + + std::unordered_map entries_by_id TSA_GUARDED_BY(mutex); + std::unordered_map entries_by_name_and_type[static_cast(AccessEntityType::MAX)] TSA_GUARDED_BY(mutex); + boost::container::flat_set types_of_lists_to_write TSA_GUARDED_BY(mutex); + + /// Whether writing of the list files has been failed since the recent restart of the server. + bool failed_to_write_lists TSA_GUARDED_BY(mutex) = false; + + /// List files are written in a separate thread. + ThreadFromGlobalPool lists_writing_thread; + + /// Signals `lists_writing_thread` to exit. + std::condition_variable lists_writing_thread_should_exit; + bool lists_writing_thread_is_waiting = false; + AccessChangesNotifier & changes_notifier; std::atomic readonly; std::atomic backup_allowed; diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index 738a0d305d8..fa6f3fd7b18 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -307,20 +307,18 @@ void MemoryAccessStorage::setAll(const std::vectorsecond.entity) != *entity) - { - const AccessEntityPtr & changed_entity = entity; - updateNoLock(id, - [&changed_entity](const AccessEntityPtr &) { return changed_entity; }, - /* throw_if_not_exists = */ true); - } - } - else + if (it == entries_by_id.end()) { insertNoLock(id, entity, /* replace_if_exists = */ false, /* throw_if_exists = */ true); } + else if (*(it->second.entity) != *entity) + { + const AccessEntityPtr & changed_entity = entity; + updateNoLock( + id, + [&changed_entity](const AccessEntityPtr &) { return changed_entity; }, + /* throw_if_not_exists = */ true); + } } } From 406c76ee4ee2bd9a349dbb3e2c22bac5a39bdfea Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 10 Aug 2022 12:22:42 +0200 Subject: [PATCH 52/86] Fix test. --- .../integration/test_replicated_users/configs/zookeeper.xml | 2 -- tests/integration/test_replicated_users/test.py | 6 ++---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_replicated_users/configs/zookeeper.xml b/tests/integration/test_replicated_users/configs/zookeeper.xml index 87b991f3ab3..3176f4959ee 100644 --- a/tests/integration/test_replicated_users/configs/zookeeper.xml +++ b/tests/integration/test_replicated_users/configs/zookeeper.xml @@ -1,4 +1,3 @@ - @@ -16,4 +15,3 @@ 20000 - \ No newline at end of file diff --git a/tests/integration/test_replicated_users/test.py b/tests/integration/test_replicated_users/test.py index 920e6660dc3..4fb2408c46f 100644 --- a/tests/integration/test_replicated_users/test.py +++ b/tests/integration/test_replicated_users/test.py @@ -166,8 +166,7 @@ def test_reload_zookeeper(started_cluster): assert node2.query( "SELECT name FROM system.users WHERE name IN ['u1', 'u2'] ORDER BY name" ) == TSV(["u1", "u2"]) - expected_error = "Cannot resolve any of provided ZooKeeper hosts" - assert expected_error in node1.query_and_get_error("CREATE USER u3") + assert "ZooKeeper" in node1.query_and_get_error("CREATE USER u3") ## start zoo2, zoo3, users will be readonly too, because it only connect to zoo1 cluster.start_zookeeper_nodes(["zoo2", "zoo3"]) @@ -175,8 +174,7 @@ def test_reload_zookeeper(started_cluster): assert node2.query( "SELECT name FROM system.users WHERE name IN ['u1', 'u2'] ORDER BY name" ) == TSV(["u1", "u2"]) - expected_error = "Cannot resolve any of provided ZooKeeper hosts" - assert expected_error in node1.query_and_get_error("CREATE USER u3") + assert "ZooKeeper" in node1.query_and_get_error("CREATE USER u3") ## set config to zoo2, server will be normal replace_zookeeper_config( From 391507b4b63d4a3500ff1ded03734e711a2d67ab Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 29 Aug 2022 15:18:05 +0200 Subject: [PATCH 53/86] Update src/Access/DiskAccessStorage.cpp Co-authored-by: Antonio Andelic --- src/Access/DiskAccessStorage.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 32799f81467..0da715b1398 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -398,7 +398,7 @@ void DiskAccessStorage::setAll(const std::vector conflicting_ids; /// Get the list of currently used IDs. Later we will remove those of them which are not used anymore. - for (const auto & id : entries_by_id | boost::adaptors::map_keys) + for (const auto & [id, _] : entries_by_id) not_used_ids.emplace(id); /// Get the list of conflicting IDs and update the list of currently used ones. @@ -409,7 +409,7 @@ void DiskAccessStorage::setAll(const std::vectorsecond; + const Entry & entry = it->second; if (entry.entity->getType() != entity->getType()) conflicting_ids.emplace_back(id); /// Conflict: same ID, different type. } @@ -418,7 +418,7 @@ void DiskAccessStorage::setAll(const std::vectorgetName()); if (it2 != entries_by_name.end()) { - Entry & entry = *(it2->second); + const Entry & entry = *(it2->second); if (entry.id != id) conflicting_ids.emplace_back(entry.id); /// Conflict: same name and type, different ID. } From fa172e21aefa177250603eee510f21358c5977fd Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 16 Sep 2022 17:47:05 +0000 Subject: [PATCH 54/86] Remove unnecessary method implementation --- src/AggregateFunctions/AggregateFunctionDistinct.h | 5 ----- src/AggregateFunctions/AggregateFunctionMap.h | 5 ----- 2 files changed, 10 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionDistinct.h b/src/AggregateFunctions/AggregateFunctionDistinct.h index 567891131af..482d21363fe 100644 --- a/src/AggregateFunctions/AggregateFunctionDistinct.h +++ b/src/AggregateFunctions/AggregateFunctionDistinct.h @@ -266,11 +266,6 @@ public: return nested_func->getDefaultVersion(); } - bool hasTrivialDestructor() const override - { - return nested_func->hasTrivialDestructor(); - } - AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; diff --git a/src/AggregateFunctions/AggregateFunctionMap.h b/src/AggregateFunctions/AggregateFunctionMap.h index 527f35fd948..4cb26fcc8d1 100644 --- a/src/AggregateFunctions/AggregateFunctionMap.h +++ b/src/AggregateFunctions/AggregateFunctionMap.h @@ -104,11 +104,6 @@ public: return nested_func->getDefaultVersion(); } - bool hasTrivialDestructor() const override - { - return false; - } - AggregateFunctionMap(AggregateFunctionPtr nested, const DataTypes & types) : Base(types, nested->getParameters()), nested_func(nested) { if (types.empty()) From bd54a6c45d7fe38741cc17a3f5164c9d9235de55 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 13 Sep 2022 22:35:10 +0200 Subject: [PATCH 55/86] tests: add perf test for lowerUTF8()/upperUTF8() Signed-off-by: Azat Khuzhin --- tests/performance/lower_upper_utf8.xml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tests/performance/lower_upper_utf8.xml diff --git a/tests/performance/lower_upper_utf8.xml b/tests/performance/lower_upper_utf8.xml new file mode 100644 index 00000000000..cc0c3af983b --- /dev/null +++ b/tests/performance/lower_upper_utf8.xml @@ -0,0 +1,4 @@ + + SELECT lowerUTF8(SearchPhrase) FROM hits_100m_single FORMAT Null + SELECT upperUTF8(SearchPhrase) FROM hits_100m_single FORMAT Null + From 01b5b9cebad80bdb8b17091814523d0ce6b909e7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 13 Sep 2022 21:08:01 +0200 Subject: [PATCH 56/86] Do not allow invalid sequences influence other rows in lowerUTF8/upperUTF8 Right now lowerUTF8() and upperUTF8() does not respect row boundaries, and so one row may break another. Signed-off-by: Azat Khuzhin --- src/Functions/LowerUpperUTF8Impl.h | 37 ++++++++++++++++--- ...71_lower_upper_utf8_row_overlaps.reference | 11 ++++++ .../02071_lower_upper_utf8_row_overlaps.sql | 10 +++++ 3 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.reference create mode 100644 tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.sql diff --git a/src/Functions/LowerUpperUTF8Impl.h b/src/Functions/LowerUpperUTF8Impl.h index a7475870dab..3a050e2bd6f 100644 --- a/src/Functions/LowerUpperUTF8Impl.h +++ b/src/Functions/LowerUpperUTF8Impl.h @@ -2,6 +2,7 @@ #include #include #include +#include #ifdef __SSE2__ #include @@ -89,9 +90,11 @@ struct LowerUpperUTF8Impl ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets) { + if (data.empty()) + return; res_data.resize(data.size()); res_offsets.assign(offsets); - array(data.data(), data.data() + data.size(), res_data.data()); + array(data.data(), data.data() + data.size(), offsets, res_data.data()); } static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &) @@ -164,8 +167,11 @@ private: static constexpr auto ascii_upper_bound = '\x7f'; static constexpr auto flip_case_mask = 'A' ^ 'a'; - static void array(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) + static void array(const UInt8 * src, const UInt8 * src_end, const ColumnString::Offsets & offsets, UInt8 * dst) { + auto offset_it = offsets.begin(); + const UInt8 * begin = src; + #ifdef __SSE2__ static constexpr auto bytes_sse = sizeof(__m128i); const auto * src_end_sse = src + (src_end - src) / bytes_sse * bytes_sse; @@ -213,10 +219,17 @@ private: else { /// UTF-8 - const auto * expected_end = src + bytes_sse; + + size_t offset_from_begin = src - begin; + while (offset_from_begin >= *offset_it) + ++offset_it; + /// Do not allow one row influence another (since row may have invalid sequence, and break the next) + const UInt8 * row_end = begin + *offset_it; + chassert(row_end >= src); + const UInt8 * expected_end = std::min(src + bytes_sse, row_end); while (src < expected_end) - toCase(src, src_end, dst); + toCase(src, expected_end, dst); /// adjust src_end_sse by pushing it forward or backward const auto diff = src - expected_end; @@ -229,10 +242,22 @@ private: } } } + + /// Find which offset src has now + while (offset_it != offsets.end() && static_cast(src - begin) >= *offset_it) + ++offset_it; #endif - /// handle remaining symbols + + /// handle remaining symbols, row by row (to avoid influence of bad UTF8 symbols from one row, to another) while (src < src_end) - toCase(src, src_end, dst); + { + const UInt8 * row_end = begin + *offset_it; + chassert(row_end >= src); + + while (src < row_end) + toCase(src, row_end, dst); + ++offset_it; + } } }; diff --git a/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.reference b/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.reference new file mode 100644 index 00000000000..2b3f8138c2b --- /dev/null +++ b/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.reference @@ -0,0 +1,11 @@ +-- { echoOn } +-- NOTE: total string size should be > 16 (sizeof(__m128i)) +insert into utf8_overlap values ('\xe2'), ('Foo⚊BarBazBam'), ('\xe2'), ('Foo⚊BarBazBam'); +-- ^ +-- MONOGRAM FOR YANG +with lowerUTF8(str) as l_, upperUTF8(str) as u_, '0x' || hex(str) as h_ +select length(str), if(l_ == '\xe2', h_, l_), if(u_ == '\xe2', h_, u_) from utf8_overlap format CSV; +1,"0xE2","0xE2" +15,"foo⚊barbazbam","FOO⚊BARBAZBAM" +1,"0xE2","0xE2" +15,"foo⚊barbazbam","FOO⚊BARBAZBAM" diff --git a/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.sql b/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.sql new file mode 100644 index 00000000000..ee0d29be699 --- /dev/null +++ b/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.sql @@ -0,0 +1,10 @@ +drop table if exists utf8_overlap; +create table utf8_overlap (str String) engine=Memory(); + +-- { echoOn } +-- NOTE: total string size should be > 16 (sizeof(__m128i)) +insert into utf8_overlap values ('\xe2'), ('Foo⚊BarBazBam'), ('\xe2'), ('Foo⚊BarBazBam'); +-- ^ +-- MONOGRAM FOR YANG +with lowerUTF8(str) as l_, upperUTF8(str) as u_, '0x' || hex(str) as h_ +select length(str), if(l_ == '\xe2', h_, l_), if(u_ == '\xe2', h_, u_) from utf8_overlap format CSV; From 84828874e0cc802a929bc2b8d1cc301662abf7d6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 17 Sep 2022 19:06:54 +0200 Subject: [PATCH 57/86] Change log level --- src/IO/WriteBufferFromS3.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 608f4a5db01..19187e7db31 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -353,7 +353,7 @@ void WriteBufferFromS3::completeMultipartUpload() { /// For unknown reason, at least MinIO can respond with NO_SUCH_KEY for put requests /// BTW, NO_SUCH_UPLOAD is expected error and we shouldn't retry it - LOG_WARNING(log, "Multipart upload failed with NO_SUCH_KEY error for Bucket: {}, Key: {}, Upload_id: {}, Parts: {}, will retry", bucket, key, multipart_upload_id, tags.size()); + LOG_INFO(log, "Multipart upload failed with NO_SUCH_KEY error for Bucket: {}, Key: {}, Upload_id: {}, Parts: {}, will retry", bucket, key, multipart_upload_id, tags.size()); } else { @@ -454,7 +454,7 @@ void WriteBufferFromS3::processPutRequest(const PutObjectTask & task) else if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { /// For unknown reason, at least MinIO can respond with NO_SUCH_KEY for put requests - LOG_WARNING(log, "Single part upload failed with NO_SUCH_KEY error for Bucket: {}, Key: {}, Object size: {}, WithPool: {}, will retry", bucket, key, task.req.GetContentLength(), with_pool); + LOG_INFO(log, "Single part upload failed with NO_SUCH_KEY error for Bucket: {}, Key: {}, Object size: {}, WithPool: {}, will retry", bucket, key, task.req.GetContentLength(), with_pool); } else throw S3Exception( From 51745dcb439484711f690f57d648c6304470d186 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 17 Sep 2022 20:20:32 +0200 Subject: [PATCH 58/86] Fix assertion in write-through cache --- .../IO/CachedOnDiskWriteBufferFromFile.cpp | 208 +++++++++++++++--- .../IO/CachedOnDiskWriteBufferFromFile.h | 53 ++++- src/Interpreters/Cache/FileSegment.cpp | 188 ---------------- src/Interpreters/Cache/FileSegment.h | 45 ---- 4 files changed, 225 insertions(+), 269 deletions(-) diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp index 53793ba2c10..987d27d2e1a 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp @@ -31,6 +31,178 @@ namespace }; } + +FileSegmentRangeWriter::FileSegmentRangeWriter( + FileCache * cache_, + const FileSegment::Key & key_, + std::shared_ptr cache_log_, + const String & query_id_, + const String & source_path_) + : cache(cache_) + , key(key_) + , cache_log(cache_log_) + , query_id(query_id_) + , source_path(source_path_) + , current_file_segment_it(file_segments_holder.file_segments.end()) +{ +} + +bool FileSegmentRangeWriter::write(const char * data, size_t size, size_t offset, bool is_persistent) +{ + if (finalized) + return false; + + auto & file_segments = file_segments_holder.file_segments; + + if (current_file_segment_it == file_segments.end()) + { + current_file_segment_it = allocateFileSegment(current_file_segment_write_offset, is_persistent); + } + else + { + auto file_segment = *current_file_segment_it; + assert(file_segment->getCurrentWriteOffset() == current_file_segment_write_offset); + + if (current_file_segment_write_offset != offset) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cannot write file segment at offset {}, because current write offset is: {}", + offset, current_file_segment_write_offset); + } + + if (file_segment->range().size() == file_segment->getDownloadedSize()) + { + completeFileSegment(*file_segment); + current_file_segment_it = allocateFileSegment(current_file_segment_write_offset, is_persistent); + } + } + + auto & file_segment = *current_file_segment_it; + + auto downloader = file_segment->getOrSetDownloader(); + if (downloader != FileSegment::getCallerId()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to set a downloader. ({})", file_segment->getInfoForLog()); + + SCOPE_EXIT({ + if (file_segment->isDownloader()) + file_segment->completePartAndResetDownloader(); + }); + + bool reserved = file_segment->reserve(size); + if (!reserved) + { + file_segment->completeWithState(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); + appendFilesystemCacheLog(*file_segment); + + LOG_DEBUG( + &Poco::Logger::get("FileSegmentRangeWriter"), + "Unsuccessful space reservation attempt (size: {}, file segment info: {}", + size, file_segment->getInfoForLog()); + + return false; + } + + try + { + file_segment->write(data, size, offset); + } + catch (...) + { + file_segment->completePartAndResetDownloader(); + throw; + } + + file_segment->completePartAndResetDownloader(); + current_file_segment_write_offset += size; + + return true; +} + +void FileSegmentRangeWriter::finalize() +{ + if (finalized) + return; + + auto & file_segments = file_segments_holder.file_segments; + if (file_segments.empty() || current_file_segment_it == file_segments.end()) + return; + + completeFileSegment(**current_file_segment_it); + finalized = true; +} + +FileSegmentRangeWriter::~FileSegmentRangeWriter() +{ + try + { + if (!finalized) + finalize(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } +} + +FileSegments::iterator FileSegmentRangeWriter::allocateFileSegment(size_t offset, bool is_persistent) +{ + /** + * Allocate a new file segment starting `offset`. + * File segment capacity will equal `max_file_segment_size`, but actual size is 0. + */ + + std::lock_guard cache_lock(cache->mutex); + + CreateFileSegmentSettings create_settings + { + .is_persistent = is_persistent, + }; + + /// We set max_file_segment_size to be downloaded, + /// if we have less size to write, file segment will be resized in complete() method. + auto file_segment = cache->createFileSegmentForDownload( + key, offset, cache->max_file_segment_size, create_settings, cache_lock); + + return file_segments_holder.add(std::move(file_segment)); +} + +void FileSegmentRangeWriter::appendFilesystemCacheLog(const FileSegment & file_segment) +{ + if (cache_log) + { + auto file_segment_range = file_segment.range(); + size_t file_segment_right_bound = file_segment_range.left + file_segment.getDownloadedSize() - 1; + + FilesystemCacheLogElement elem + { + .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()), + .query_id = query_id, + .source_file_path = source_path, + .file_segment_range = { file_segment_range.left, file_segment_right_bound }, + .requested_range = {}, + .cache_type = FilesystemCacheLogElement::CacheType::WRITE_THROUGH_CACHE, + .file_segment_size = file_segment_range.size(), + .read_from_cache_attempted = false, + .read_buffer_id = {}, + .profile_counters = nullptr, + }; + + cache_log->add(elem); + } +} + +void FileSegmentRangeWriter::completeFileSegment(FileSegment & file_segment) +{ + /// File segment can be detached if space reservation failed. + if (file_segment.isDetached()) + return; + + file_segment.completeWithoutState(); + appendFilesystemCacheLog(file_segment); +} + + CachedOnDiskWriteBufferFromFile::CachedOnDiskWriteBufferFromFile( std::unique_ptr impl_, FileCachePtr cache_, @@ -47,7 +219,6 @@ CachedOnDiskWriteBufferFromFile::CachedOnDiskWriteBufferFromFile( , is_persistent_cache_file(is_persistent_cache_file_) , query_id(query_id_) , enable_cache_log(!query_id_.empty() && settings_.enable_filesystem_cache_log) - , cache_log(Context::getGlobalContextInstance()->getFilesystemCacheLog()) { } @@ -82,8 +253,11 @@ void CachedOnDiskWriteBufferFromFile::cacheData(char * data, size_t size) if (!cache_writer) { - cache_writer = std::make_unique( - cache.get(), key, [this](const FileSegment & file_segment) { appendFilesystemCacheLog(file_segment); }); + std::shared_ptr cache_log; + if (enable_cache_log) + cache_log = Context::getGlobalContextInstance()->getFilesystemCacheLog(); + + cache_writer = std::make_unique(cache.get(), key, cache_log, query_id, source_path); } Stopwatch watch(CLOCK_MONOTONIC); @@ -119,37 +293,9 @@ void CachedOnDiskWriteBufferFromFile::cacheData(char * data, size_t size) ProfileEvents::increment(ProfileEvents::CachedWriteBufferCacheWriteBytes, size); ProfileEvents::increment(ProfileEvents::CachedWriteBufferCacheWriteMicroseconds, watch.elapsedMicroseconds()); - current_file_segment_counters.increment( - ProfileEvents::FileSegmentWriteMicroseconds, watch.elapsedMicroseconds()); - cache_in_error_state_or_disabled = false; } -void CachedOnDiskWriteBufferFromFile::appendFilesystemCacheLog(const FileSegment & file_segment) -{ - if (cache_log) - { - auto file_segment_range = file_segment.range(); - FilesystemCacheLogElement elem - { - .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()), - .query_id = query_id, - .source_file_path = source_path, - .file_segment_range = { file_segment_range.left, file_segment_range.right }, - .requested_range = {}, - .cache_type = FilesystemCacheLogElement::CacheType::WRITE_THROUGH_CACHE, - .file_segment_size = file_segment_range.size(), - .read_from_cache_attempted = false, - .read_buffer_id = {}, - .profile_counters = std::make_shared(current_file_segment_counters.getPartiallyAtomicSnapshot()), - }; - - current_file_segment_counters.reset(); - - cache_log->add(elem); - } -} - void CachedOnDiskWriteBufferFromFile::finalizeImpl() { try diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h index 7be50fe2c46..cec7305ab1b 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h @@ -13,11 +13,57 @@ class Logger; namespace DB { +/** +* We want to write eventually some size, which is not known until the very end. +* Therefore we allocate file segments lazily. Each file segment is assigned capacity +* of max_file_segment_size, but reserved_size remains 0, until call to tryReserve(). +* Once current file segment is full (reached max_file_segment_size), we allocate a +* new file segment. All allocated file segments resize in file segments holder. +* If at the end of all writes, the last file segment is not full, then it is resized. +*/ +class FileSegmentRangeWriter +{ +public: + FileSegmentRangeWriter( + FileCache * cache_, const FileSegment::Key & key_, + std::shared_ptr cache_log_, const String & query_id_, const String & source_path_); + + /** + * Write a range of file segments. Allocate file segment of `max_file_segment_size` and write to + * it until it is full and then allocate next file segment. + */ + bool write(const char * data, size_t size, size_t offset, bool is_persistent); + + void finalize(); + + ~FileSegmentRangeWriter(); + +private: + FileSegments::iterator allocateFileSegment(size_t offset, bool is_persistent); + + void appendFilesystemCacheLog(const FileSegment & file_segment); + + void completeFileSegment(FileSegment & file_segment); + + FileCache * cache; + FileSegment::Key key; + + std::shared_ptr cache_log; + String query_id; + String source_path; + + FileSegmentsHolder file_segments_holder{}; + FileSegments::iterator current_file_segment_it; + + size_t current_file_segment_write_offset = 0; + + bool finalized = false; +}; + + /** * Write buffer for filesystem caching on write operations. */ -class FileSegmentRangeWriter; - class CachedOnDiskWriteBufferFromFile final : public WriteBufferFromFileDecorator { public: @@ -36,7 +82,6 @@ public: private: void cacheData(char * data, size_t size); - void appendFilesystemCacheLog(const FileSegment & file_segment); Poco::Logger * log; @@ -49,11 +94,9 @@ private: const String query_id; bool enable_cache_log; - std::shared_ptr cache_log; bool cache_in_error_state_or_disabled = false; - ProfileEvents::Counters current_file_segment_counters; std::unique_ptr cache_writer; }; diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index e0cf581dd95..708d60f56dc 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -125,12 +125,6 @@ size_t FileSegment::getDownloadedSizeUnlocked(std::unique_lock & /* return downloaded_size; } -size_t FileSegment::getRemainingSizeToDownload() const -{ - std::unique_lock segment_lock(mutex); - return range().size() - getDownloadedSizeUnlocked(segment_lock); -} - bool FileSegment::isDownloaded() const { std::lock_guard segment_lock(mutex); @@ -836,186 +830,4 @@ String FileSegmentsHolder::toString() return ranges; } -FileSegmentRangeWriter::FileSegmentRangeWriter( - FileCache * cache_, - const FileSegment::Key & key_, - OnCompleteFileSegmentCallback && on_complete_file_segment_func_) - : cache(cache_) - , key(key_) - , current_file_segment_it(file_segments_holder.file_segments.end()) - , on_complete_file_segment_func(on_complete_file_segment_func_) -{ -} - -FileSegments::iterator FileSegmentRangeWriter::allocateFileSegment(size_t offset, bool is_persistent) -{ - /** - * Allocate a new file segment starting `offset`. - * File segment capacity will equal `max_file_segment_size`, but actual size is 0. - */ - - std::lock_guard cache_lock(cache->mutex); - - CreateFileSegmentSettings create_settings - { - .is_persistent = is_persistent, - }; - - /// We set max_file_segment_size to be downloaded, - /// if we have less size to write, file segment will be resized in complete() method. - auto file_segment = cache->createFileSegmentForDownload( - key, offset, cache->max_file_segment_size, create_settings, cache_lock); - - return file_segments_holder.add(std::move(file_segment)); -} - -void FileSegmentRangeWriter::completeFileSegment(FileSegment & file_segment) -{ - /** - * Complete file segment based on downaloaded size. - */ - - /// File segment can be detached if space reservation failed. - if (file_segment.isDetached()) - return; - - size_t current_downloaded_size = file_segment.getDownloadedSize(); - - /// file_segment->complete(DOWNLOADED) is not enough, because file segment capacity - /// was initially set with a margin as `max_file_segment_size`. => We need to always - /// resize to actual size after download finished. - if (current_downloaded_size != file_segment.range().size()) - { - /// Current file segment is downloaded as a part of write-through cache - /// and therefore cannot be concurrently accessed. Nevertheless, it can be - /// accessed by cache system tables if someone read from them, - /// therefore we need a mutex. - std::unique_lock segment_lock(file_segment.mutex); - - assert(current_downloaded_size <= file_segment.range().size()); - file_segment.segment_range = FileSegment::Range( - file_segment.segment_range.left, - file_segment.segment_range.left + current_downloaded_size - 1); - file_segment.reserved_size = current_downloaded_size; - - file_segment.setDownloadedUnlocked(segment_lock); - } - - file_segment.completeWithoutState(); - - on_complete_file_segment_func(file_segment); -} - -bool FileSegmentRangeWriter::write(const char * data, size_t size, size_t offset, bool is_persistent) -{ - /** - * Write a range of file segments. Allocate file segment of `max_file_segment_size` and write to - * it until it is full and then allocate next file segment. - */ - - if (finalized) - return false; - - auto & file_segments = file_segments_holder.file_segments; - - if (current_file_segment_it == file_segments.end()) - { - current_file_segment_it = allocateFileSegment(current_file_segment_write_offset, is_persistent); - } - else - { - if (current_file_segment_write_offset != offset) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Cannot write file segment at offset {}, because current write offset is: {}", - offset, current_file_segment_write_offset); - } - - size_t current_write_offset = (*current_file_segment_it)->getCurrentWriteOffset(); - - auto current_file_segment = *current_file_segment_it; - if (current_file_segment->getRemainingSizeToDownload() == 0) - { - completeFileSegment(*current_file_segment); - current_file_segment_it = allocateFileSegment(current_file_segment_write_offset, is_persistent); - } - else if (current_write_offset != offset) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Cannot file segment download offset {} does not match current write offset {}", - current_write_offset, offset); - } - } - - auto & file_segment = *current_file_segment_it; - - auto downloader = file_segment->getOrSetDownloader(); - if (downloader != FileSegment::getCallerId()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to set a downloader. ({})", file_segment->getInfoForLog()); - - SCOPE_EXIT({ - if (file_segment->isDownloader()) - { - file_segment->completePartAndResetDownloader(); - } - }); - - bool reserved = file_segment->reserve(size); - if (!reserved) - { - file_segment->completeWithState(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); - on_complete_file_segment_func(*file_segment); - - LOG_DEBUG( - &Poco::Logger::get("FileSegmentRangeWriter"), - "Unsuccessful space reservation attempt (size: {}, file segment info: {}", - size, file_segment->getInfoForLog()); - - return false; - } - - try - { - file_segment->write(data, size, offset); - } - catch (...) - { - file_segment->completePartAndResetDownloader(); - throw; - } - - file_segment->completePartAndResetDownloader(); - current_file_segment_write_offset += size; - - return true; -} - -void FileSegmentRangeWriter::finalize() -{ - if (finalized) - return; - - auto & file_segments = file_segments_holder.file_segments; - if (file_segments.empty() || current_file_segment_it == file_segments.end()) - return; - - completeFileSegment(**current_file_segment_it); - finalized = true; -} - -FileSegmentRangeWriter::~FileSegmentRangeWriter() -{ - try - { - if (!finalized) - finalize(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } -} - } diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 9f6c3697960..617e7173c2f 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -217,8 +217,6 @@ public: void resetRemoteFileReader(); - size_t getRemainingSizeToDownload() const; - private: size_t getFirstNonDownloadedOffsetUnlocked(std::unique_lock & segment_lock) const; size_t getCurrentWriteOffsetUnlocked(std::unique_lock & segment_lock) const; @@ -327,47 +325,4 @@ struct FileSegmentsHolder : private boost::noncopyable FileSegments file_segments{}; }; -/** - * We want to write eventually some size, which is not known until the very end. - * Therefore we allocate file segments lazily. Each file segment is assigned capacity - * of max_file_segment_size, but reserved_size remains 0, until call to tryReserve(). - * Once current file segment is full (reached max_file_segment_size), we allocate a - * new file segment. All allocated file segments resize in file segments holder. - * If at the end of all writes, the last file segment is not full, then it is resized. - */ -class FileSegmentRangeWriter -{ -public: - using OnCompleteFileSegmentCallback = std::function; - - FileSegmentRangeWriter( - FileCache * cache_, - const FileSegment::Key & key_, - /// A callback which is called right after each file segment is completed. - /// It is used to write into filesystem cache log. - OnCompleteFileSegmentCallback && on_complete_file_segment_func_); - - ~FileSegmentRangeWriter(); - - bool write(const char * data, size_t size, size_t offset, bool is_persistent); - - void finalize(); - -private: - FileSegments::iterator allocateFileSegment(size_t offset, bool is_persistent); - void completeFileSegment(FileSegment & file_segment); - - FileCache * cache; - FileSegment::Key key; - - FileSegmentsHolder file_segments_holder; - FileSegments::iterator current_file_segment_it; - - size_t current_file_segment_write_offset = 0; - - bool finalized = false; - - OnCompleteFileSegmentCallback on_complete_file_segment_func; -}; - } From 1b40f94c6bf60209574c015300a8945f818df924 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 14 Sep 2022 19:18:08 +0200 Subject: [PATCH 59/86] Move common code from MemoryAccessStorage and DiskAccessStorage to IAccessStorage. --- src/Access/DiskAccessStorage.cpp | 121 +++++++++++-------------- src/Access/DiskAccessStorage.h | 11 ++- src/Access/IAccessStorage.cpp | 57 ++++++++++++ src/Access/IAccessStorage.h | 1 + src/Access/MemoryAccessStorage.cpp | 79 ++++------------ src/Access/MemoryAccessStorage.h | 4 +- src/Access/ReplicatedAccessStorage.cpp | 2 +- 7 files changed, 139 insertions(+), 136 deletions(-) diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 0da715b1398..118b38f7ddc 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -382,7 +382,7 @@ void DiskAccessStorage::reloadAllAndRebuildLists() all_entities.emplace_back(id, entity); } - setAll(all_entities); + setAllInMemory(all_entities); for (auto type : collections::range(AccessEntityType::MAX)) types_of_lists_to_write.insert(type); @@ -391,69 +391,41 @@ void DiskAccessStorage::reloadAllAndRebuildLists() writeLists(); } -void DiskAccessStorage::setAll(const std::vector> & all_entities) +void DiskAccessStorage::setAllInMemory(const std::vector> & all_entities) { - /// This function is similar to MemoryAccessStorage::setAll(). - boost::container::flat_set not_used_ids; - std::vector conflicting_ids; + /// Remove conflicting entities from the specified list. + auto entities_without_conflicts = all_entities; + clearConflictsInEntitiesList(entities_without_conflicts, getLogger()); - /// Get the list of currently used IDs. Later we will remove those of them which are not used anymore. - for (const auto & [id, _] : entries_by_id) - not_used_ids.emplace(id); - - /// Get the list of conflicting IDs and update the list of currently used ones. - for (const auto & [id, entity] : all_entities) - { - auto it = entries_by_id.find(id); - if (it != entries_by_id.end()) - { - not_used_ids.erase(id); /// ID is used. - - const Entry & entry = it->second; - if (entry.entity->getType() != entity->getType()) - conflicting_ids.emplace_back(id); /// Conflict: same ID, different type. - } - - const auto & entries_by_name = entries_by_name_and_type[static_cast(entity->getType())]; - auto it2 = entries_by_name.find(entity->getName()); - if (it2 != entries_by_name.end()) - { - const Entry & entry = *(it2->second); - if (entry.id != id) - conflicting_ids.emplace_back(entry.id); /// Conflict: same name and type, different ID. - } - } - - /// Remove entities which are not used anymore and which are in conflict with new entities. - boost::container::flat_set ids_to_remove = std::move(not_used_ids); - boost::range::copy(conflicting_ids, std::inserter(ids_to_remove, ids_to_remove.end())); - for (const auto & id : ids_to_remove) - removeNoLock(id, /* throw_if_not_exists= */ false); + /// Remove entities which are not used anymore. + boost::container::flat_set ids_to_keep; + ids_to_keep.reserve(entities_without_conflicts.size()); + for (const auto & [id, _] : entities_without_conflicts) + ids_to_keep.insert(id); + removeAllExceptInMemory(ids_to_keep); /// Insert or update entities. - for (const auto & [id, entity] : all_entities) + for (const auto & [id, entity] : entities_without_conflicts) + insertNoLock(id, entity, /* replace_if_exists = */ true, /* throw_if_exists = */ false, /* write_on_disk= */ false); +} + +void DiskAccessStorage::removeAllExceptInMemory(const boost::container::flat_set & ids_to_keep) +{ + for (auto it = entries_by_id.begin(); it != entries_by_id.end();) { - auto it = entries_by_id.find(id); - if (it == entries_by_id.end()) - { - insertNoLock(id, entity, /* replace_if_exists= */ false, /* throw_if_exists= */ true); - } - else if (!it->second.entity || (*(it->second.entity) != *entity)) - { - const AccessEntityPtr & changed_entity = entity; - updateNoLock( - id, - [&changed_entity](const AccessEntityPtr &) { return changed_entity; }, - /* throw_if_not_exists= */ true); - } + const auto & id = it->first; + ++it; + if (!ids_to_keep.contains(id)) + removeNoLock(id, /* throw_if_not_exists */ true, /* write_on_disk= */ false); } } + void DiskAccessStorage::reload() { /// TODO: Disabled because reload() is called by SYSTEM RELOAD CONFIG and replicated access storage is not a config-based. - /// We need a separate SYSTEM RELOAD USES command. + /// We need a separate SYSTEM RELOAD USERS command. #if 0 std::lock_guard lock{mutex}; reloadAllAndRebuildLists(); @@ -528,21 +500,21 @@ std::optional> DiskAccessStorage::readNameWi std::optional DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) { UUID id = generateRandomID(); - if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists)) + if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists, /* write_on_disk= */ true)) return id; return std::nullopt; } -bool DiskAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) +bool DiskAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk) { std::lock_guard lock{mutex}; - return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists); + return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, write_on_disk); } -bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) +bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk) { const String & name = new_entity->getName(); AccessEntityType type = new_entity->getType(); @@ -579,13 +551,14 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne return false; } - scheduleWriteLists(type); + if (write_on_disk) + scheduleWriteLists(type); /// Remove collisions if necessary. if (name_collision && (id_by_name != id)) { assert(replace_if_exists); - removeNoLock(id_by_name, /* throw_if_not_exists= */ false); + removeNoLock(id_by_name, /* throw_if_not_exists= */ false, write_on_disk); } if (id_collision) @@ -596,6 +569,8 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne { if (!existing_entry.entity || (*existing_entry.entity != *new_entity)) { + if (write_on_disk) + writeAccessEntityToDisk(id, *new_entity); if (existing_entry.name != new_entity->getName()) { entries_by_name.erase(existing_entry.name); @@ -608,12 +583,12 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne return true; } - scheduleWriteLists(existing_entry.type); - removeNoLock(id, /* throw_if_not_exists= */ false); + removeNoLock(id, /* throw_if_not_exists= */ false, write_on_disk); } /// Do insertion. - writeAccessEntityToDisk(id, *new_entity); + if (write_on_disk) + writeAccessEntityToDisk(id, *new_entity); auto & entry = entries_by_id[id]; entry.id = id; @@ -630,11 +605,11 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne bool DiskAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists) { std::lock_guard lock{mutex}; - return removeNoLock(id, throw_if_not_exists); + return removeNoLock(id, throw_if_not_exists, /* write_on_disk= */ true); } -bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists) +bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists, bool write_on_disk) { auto it = entries_by_id.find(id); if (it == entries_by_id.end()) @@ -651,8 +626,11 @@ bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists) if (readonly) throwReadonlyCannotRemove(type, entry.name); - scheduleWriteLists(type); - deleteAccessEntityOnDisk(id); + if (write_on_disk) + { + scheduleWriteLists(type); + deleteAccessEntityOnDisk(id); + } /// Do removing. UUID removed_id = id; @@ -668,11 +646,11 @@ bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists) bool DiskAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) { std::lock_guard lock{mutex}; - return updateNoLock(id, update_func, throw_if_not_exists); + return updateNoLock(id, update_func, throw_if_not_exists, /* write_on_disk= */ true); } -bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) +bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, bool write_on_disk) { auto it = entries_by_id.find(id); if (it == entries_by_id.end()) @@ -708,10 +686,13 @@ bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ { if (entries_by_name.contains(new_name)) throwNameCollisionCannotRename(type, old_name, new_name); - scheduleWriteLists(type); + if (write_on_disk) + scheduleWriteLists(type); } - writeAccessEntityToDisk(id, *new_entity); + if (write_on_disk) + writeAccessEntityToDisk(id, *new_entity); + entry.entity = new_entity; if (name_changed) @@ -763,7 +744,7 @@ void DiskAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) restorer.addDataRestoreTask([this, entities = std::move(entities), replace_if_exists, throw_if_exists] { for (const auto & [id, entity] : entities) - insertWithID(id, entity, replace_if_exists, throw_if_exists); + insertWithID(id, entity, replace_if_exists, throw_if_exists, /* write_on_disk= */ true); }); } diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h index 4e6c627443e..327627470e1 100644 --- a/src/Access/DiskAccessStorage.h +++ b/src/Access/DiskAccessStorage.h @@ -47,15 +47,16 @@ private: void writeLists() TSA_REQUIRES(mutex); void scheduleWriteLists(AccessEntityType type) TSA_REQUIRES(mutex); void reloadAllAndRebuildLists() TSA_REQUIRES(mutex); - void setAll(const std::vector> & all_entities) TSA_REQUIRES(mutex); + void setAllInMemory(const std::vector> & all_entities) TSA_REQUIRES(mutex); + void removeAllExceptInMemory(const boost::container::flat_set & ids_to_keep) TSA_REQUIRES(mutex); void listsWritingThreadFunc() TSA_NO_THREAD_SAFETY_ANALYSIS; void stopListsWritingThread(); - bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists); - bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) TSA_REQUIRES(mutex); - bool removeNoLock(const UUID & id, bool throw_if_not_exists) TSA_REQUIRES(mutex); - bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) TSA_REQUIRES(mutex); + bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk); + bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk) TSA_REQUIRES(mutex); + bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex); + bool removeNoLock(const UUID & id, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex); AccessEntityPtr readAccessEntityFromDisk(const UUID & id) const; void writeAccessEntityToDisk(const UUID & id, const IAccessEntity & entity) const; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 2be99dfb38b..f4092368857 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -562,6 +563,62 @@ UUID IAccessStorage::generateRandomID() } +void IAccessStorage::clearConflictsInEntitiesList(std::vector> & entities, const Poco::Logger * log_) +{ + std::unordered_map positions_by_id; + std::unordered_map positions_by_type_and_name[static_cast(AccessEntityType::MAX)]; + std::vector positions_to_remove; + + for (size_t pos = 0; pos != entities.size(); ++pos) + { + const auto & [id, entity] = entities[pos]; + + if (auto it = positions_by_id.find(id); it == positions_by_id.end()) + { + positions_by_id[id] = pos; + } + else if (it->second != pos) + { + /// Conflict: same ID is used for multiple entities. We will ignore them. + positions_to_remove.emplace_back(pos); + positions_to_remove.emplace_back(it->second); + } + + std::string_view entity_name = entity->getName(); + auto & positions_by_name = positions_by_type_and_name[static_cast(entity->getType())]; + if (auto it = positions_by_name.find(entity_name); it == positions_by_name.end()) + { + positions_by_name[entity_name] = pos; + } + else if (it->second != pos) + { + /// Conflict: same name and type are used for multiple entities. We will ignore them. + positions_to_remove.emplace_back(pos); + positions_to_remove.emplace_back(it->second); + } + } + + if (positions_to_remove.empty()) + return; + + std::sort(positions_to_remove.begin(), positions_to_remove.end()); + positions_to_remove.erase(std::unique(positions_to_remove.begin(), positions_to_remove.end()), positions_to_remove.end()); + + for (size_t pos : positions_to_remove) + { + LOG_WARNING( + log_, + "Skipping {} (id={}) due to conflicts with other access entities", + entities[pos].second->formatTypeWithName(), + toString(entities[pos].first)); + } + + /// Remove conflicting entities. + for (size_t pos : positions_to_remove | boost::adaptors::reversed) /// Must remove in reversive order. + entities.erase(entities.begin() + pos); +} + + Poco::Logger * IAccessStorage::getLogger() const { Poco::Logger * ptr = log.load(); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 514374a828a..205eb281cae 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -177,6 +177,7 @@ protected: static UUID generateRandomID(); Poco::Logger * getLogger() const; static String formatEntityTypeWithName(AccessEntityType type, const String & name) { return AccessEntityTypeInfo::get(type).formatEntityNameWithType(name); } + static void clearConflictsInEntitiesList(std::vector> & entities, const Poco::Logger * log_); [[noreturn]] void throwNotFound(const UUID & id) const; [[noreturn]] void throwNotFound(AccessEntityType type, const String & name) const; [[noreturn]] static void throwBadCast(const UUID & id, AccessEntityType type, const String & name, AccessEntityType required_type); diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index fa6f3fd7b18..a9bd6cacd10 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -5,7 +5,6 @@ #include #include #include -#include namespace DB @@ -117,7 +116,7 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & if (name_collision && (id_by_name != id)) { assert(replace_if_exists); - removeNoLock(id_by_name, /* throw_if_not_exists= */ false); + removeNoLock(id_by_name, /* throw_if_not_exists= */ true); } if (id_collision) @@ -139,7 +138,7 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & } return true; } - removeNoLock(id, /* throw_if_not_exists= */ false); + removeNoLock(id, /* throw_if_not_exists= */ true); } /// Do insertion. @@ -239,16 +238,17 @@ void MemoryAccessStorage::removeAllExcept(const std::vector & ids_to_keep) void MemoryAccessStorage::removeAllExceptNoLock(const std::vector & ids_to_keep) { - boost::container::flat_set ids_to_keep_set{ids_to_keep.begin(), ids_to_keep.end()}; + removeAllExceptNoLock(boost::container::flat_set{ids_to_keep.begin(), ids_to_keep.end()}); +} + +void MemoryAccessStorage::removeAllExceptNoLock(const boost::container::flat_set & ids_to_keep) +{ for (auto it = entries_by_id.begin(); it != entries_by_id.end();) { const auto & id = it->first; ++it; - if (!ids_to_keep_set.contains(id)) - { - UUID id_to_remove = id; - removeNoLock(id_to_remove, /* throw_if_not_exists */ false); - } + if (!ids_to_keep.contains(id)) + removeNoLock(id, /* throw_if_not_exists */ true); } } @@ -267,59 +267,20 @@ void MemoryAccessStorage::setAll(const std::vector not_used_ids; - std::vector conflicting_ids; + /// Remove conflicting entities from the specified list. + auto entities_without_conflicts = all_entities; + clearConflictsInEntitiesList(entities_without_conflicts, getLogger()); - /// Get the list of currently used IDs. Later we will remove those of them which are not used anymore. - for (const auto & id : entries_by_id | boost::adaptors::map_keys) - not_used_ids.emplace(id); - - /// Get the list of conflicting IDs and update the list of currently used ones. - for (const auto & [id, entity] : all_entities) - { - auto it = entries_by_id.find(id); - if (it != entries_by_id.end()) - { - not_used_ids.erase(id); /// ID is used. - - Entry & entry = it->second; - if (entry.entity->getType() != entity->getType()) - conflicting_ids.emplace_back(id); /// Conflict: same ID, different type. - } - - const auto & entries_by_name = entries_by_name_and_type[static_cast(entity->getType())]; - auto it2 = entries_by_name.find(entity->getName()); - if (it2 != entries_by_name.end()) - { - Entry & entry = *(it2->second); - if (entry.id != id) - conflicting_ids.emplace_back(entry.id); /// Conflict: same name and type, different ID. - } - } - - /// Remove entities which are not used anymore and which are in conflict with new entities. - boost::container::flat_set ids_to_remove = std::move(not_used_ids); - boost::range::copy(conflicting_ids, std::inserter(ids_to_remove, ids_to_remove.end())); - for (const auto & id : ids_to_remove) - removeNoLock(id, /* throw_if_not_exists = */ false); + /// Remove entities which are not used anymore. + boost::container::flat_set ids_to_keep; + ids_to_keep.reserve(entities_without_conflicts.size()); + for (const auto & [id, _] : entities_without_conflicts) + ids_to_keep.insert(id); + removeAllExceptNoLock(ids_to_keep); /// Insert or update entities. - for (const auto & [id, entity] : all_entities) - { - auto it = entries_by_id.find(id); - if (it == entries_by_id.end()) - { - insertNoLock(id, entity, /* replace_if_exists = */ false, /* throw_if_exists = */ true); - } - else if (*(it->second.entity) != *entity) - { - const AccessEntityPtr & changed_entity = entity; - updateNoLock( - id, - [&changed_entity](const AccessEntityPtr &) { return changed_entity; }, - /* throw_if_not_exists = */ true); - } - } + for (const auto & [id, entity] : entities_without_conflicts) + insertNoLock(id, entity, /* replace_if_exists = */ true, /* throw_if_exists = */ false); } diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h index 7c439a5ea21..b63132147da 100644 --- a/src/Access/MemoryAccessStorage.h +++ b/src/Access/MemoryAccessStorage.h @@ -27,7 +27,8 @@ public: /// with such name & type. bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists); - /// Removes all entities except a specified list. + /// Removes all entities except the specified list `ids_to_keep`. + /// The function skips IDs not contained in the storage. void removeAllExcept(const std::vector & ids_to_keep); /// Sets all entities at once. @@ -52,6 +53,7 @@ private: bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) TSA_REQUIRES(mutex); void removeAllExceptNoLock(const std::vector & ids_to_keep) TSA_REQUIRES(mutex); + void removeAllExceptNoLock(const boost::container::flat_set & ids_to_keep) TSA_REQUIRES(mutex); struct Entry { diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index 4d4d6366f49..90fe6d61dfd 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -447,7 +447,7 @@ zkutil::ZooKeeperPtr ReplicatedAccessStorage::getZooKeeperNoLock() void ReplicatedAccessStorage::reload() { /// TODO: Disabled because reload() is called by SYSTEM RELOAD CONFIG and replicated access storage is not a config-based. - /// We need a separate SYSTEM RELOAD USES command. + /// We need a separate SYSTEM RELOAD USERS command. #if 0 /// Reinitialize ZooKeeper and reread everything. std::lock_guard lock{cached_zookeeper_mutex}; From 5365b105ccc835485215502783856158288c2b91 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 16 Sep 2022 13:19:39 +0200 Subject: [PATCH 60/86] Add SYSTEM RELOAD USERS command. --- programs/server/Server.cpp | 2 +- src/Access/AccessControl.cpp | 4 ++-- src/Access/AccessControl.h | 4 ++-- src/Access/Common/AccessType.h | 1 + src/Access/DiskAccessStorage.cpp | 10 ++++------ src/Access/DiskAccessStorage.h | 2 +- src/Access/IAccessStorage.h | 17 ++++++++++++++--- src/Access/MultipleAccessStorage.cpp | 14 +++++++------- src/Access/MultipleAccessStorage.h | 2 +- src/Access/QuotaCache.cpp | 1 + src/Access/ReplicatedAccessStorage.cpp | 9 ++++----- src/Access/ReplicatedAccessStorage.h | 2 +- src/Access/UsersConfigAccessStorage.cpp | 14 +++++++------- src/Access/UsersConfigAccessStorage.h | 2 +- src/Interpreters/InterpreterSystemQuery.cpp | 10 ++++++++++ src/Parsers/ASTSystemQuery.h | 1 + .../0_stateless/01271_show_privileges.reference | 1 + .../02117_show_create_table_system.reference | 6 +++--- 18 files changed, 62 insertions(+), 40 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 5c09ba5b52e..915e3e2893f 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1381,7 +1381,7 @@ int Server::main(const std::vector & /*args*/) global_context->setConfigReloadCallback([&]() { main_config_reloader->reload(); - access_control.reload(); + access_control.reload(AccessControl::ReloadMode::USERS_CONFIG_ONLY); }); /// Limit on total number of concurrently executed queries. diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 89292fe9272..3c38ba64fc1 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -390,9 +390,9 @@ void AccessControl::addStoragesFromMainConfig( } -void AccessControl::reload() +void AccessControl::reload(ReloadMode reload_mode) { - MultipleAccessStorage::reload(); + MultipleAccessStorage::reload(reload_mode); changes_notifier->sendNotifications(); } diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index ab9cdba9ad1..1505aca03aa 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -99,8 +99,8 @@ public: const String & config_path, const zkutil::GetZooKeeper & get_zookeeper_function); - /// Reloads and updates entities in this storage. This function is used to implement SYSTEM RELOAD CONFIG. - void reload() override; + /// Reloads and updates all access entities. + void reload(ReloadMode reload_mode) override; using OnChangedHandler = std::function; diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index e8d3b453188..de44874238e 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -143,6 +143,7 @@ enum class AccessType M(SYSTEM_DROP_SCHEMA_CACHE, "SYSTEM DROP SCHEMA CACHE, DROP SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_CACHE, "DROP CACHE", GROUP, SYSTEM) \ M(SYSTEM_RELOAD_CONFIG, "RELOAD CONFIG", GLOBAL, SYSTEM_RELOAD) \ + M(SYSTEM_RELOAD_USERS, "RELOAD USERS", GLOBAL, SYSTEM_RELOAD) \ M(SYSTEM_RELOAD_SYMBOLS, "RELOAD SYMBOLS", GLOBAL, SYSTEM_RELOAD) \ M(SYSTEM_RELOAD_DICTIONARY, "SYSTEM RELOAD DICTIONARIES, RELOAD DICTIONARY, RELOAD DICTIONARIES", GLOBAL, SYSTEM_RELOAD) \ M(SYSTEM_RELOAD_MODEL, "SYSTEM RELOAD MODELS, RELOAD MODEL, RELOAD MODELS", GLOBAL, SYSTEM_RELOAD) \ diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 118b38f7ddc..0544e4a0c0a 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -421,15 +421,13 @@ void DiskAccessStorage::removeAllExceptInMemory(const boost::container::flat_set } - -void DiskAccessStorage::reload() +void DiskAccessStorage::reload(ReloadMode reload_mode) { - /// TODO: Disabled because reload() is called by SYSTEM RELOAD CONFIG and replicated access storage is not a config-based. - /// We need a separate SYSTEM RELOAD USERS command. -#if 0 + if (reload_mode != ReloadMode::ALL) + return; + std::lock_guard lock{mutex}; reloadAllAndRebuildLists(); -#endif } diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h index 327627470e1..b1ef1d10ba7 100644 --- a/src/Access/DiskAccessStorage.h +++ b/src/Access/DiskAccessStorage.h @@ -27,7 +27,7 @@ public: void setReadOnly(bool readonly_) { readonly = readonly_; } bool isReadOnly() const override { return readonly; } - void reload() override; + void reload(ReloadMode reload_mode) override; bool exists(const UUID & id) const override; diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 205eb281cae..aa3947201e7 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -42,15 +42,26 @@ public: /// Returns true if this entity is readonly. virtual bool isReadOnly(const UUID &) const { return isReadOnly(); } - /// Reloads and updates entities in this storage. This function is used to implement SYSTEM RELOAD CONFIG. - virtual void reload() {} - /// Starts periodic reloading and updating of entities in this storage. virtual void startPeriodicReloading() {} /// Stops periodic reloading and updating of entities in this storage. virtual void stopPeriodicReloading() {} + enum class ReloadMode + { + /// Try to reload all access storages (including users.xml, local(disk) access storage, replicated(in zk) access storage. + /// This mode is invoked by the SYSTEM RELOAD USERS command. + ALL, + + /// Only reloads users.xml + /// This mode is invoked by the SYSTEM RELOAD CONFIG command. + USERS_CONFIG_ONLY, + }; + + /// Makes this storage to reload and update access entities right now. + virtual void reload(ReloadMode /* reload_mode */) {} + /// Returns the identifiers of all the entities of a specified type contained in the storage. std::vector findAll(AccessEntityType type) const; diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index e7151cc7b4b..f910c99d6b3 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -223,13 +223,6 @@ bool MultipleAccessStorage::isReadOnly(const UUID & id) const } -void MultipleAccessStorage::reload() -{ - auto storages = getStoragesInternal(); - for (const auto & storage : *storages) - storage->reload(); -} - void MultipleAccessStorage::startPeriodicReloading() { auto storages = getStoragesInternal(); @@ -244,6 +237,13 @@ void MultipleAccessStorage::stopPeriodicReloading() storage->stopPeriodicReloading(); } +void MultipleAccessStorage::reload(ReloadMode reload_mode) +{ + auto storages = getStoragesInternal(); + for (const auto & storage : *storages) + storage->reload(reload_mode); +} + std::optional MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) { diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index bc9a4705785..6a0c1bdfc02 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -25,9 +25,9 @@ public: bool isReadOnly() const override; bool isReadOnly(const UUID & id) const override; - void reload() override; void startPeriodicReloading() override; void stopPeriodicReloading() override; + void reload(ReloadMode reload_mode) override; void setStorages(const std::vector & storages); void addStorage(const StoragePtr & new_storage); diff --git a/src/Access/QuotaCache.cpp b/src/Access/QuotaCache.cpp index 43ab4268b0c..af47819b8ce 100644 --- a/src/Access/QuotaCache.cpp +++ b/src/Access/QuotaCache.cpp @@ -301,4 +301,5 @@ std::vector QuotaCache::getAllQuotasUsage() const } return all_usage; } + } diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index 90fe6d61dfd..b32954e69a5 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -444,16 +444,15 @@ zkutil::ZooKeeperPtr ReplicatedAccessStorage::getZooKeeperNoLock() return cached_zookeeper; } -void ReplicatedAccessStorage::reload() +void ReplicatedAccessStorage::reload(ReloadMode reload_mode) { - /// TODO: Disabled because reload() is called by SYSTEM RELOAD CONFIG and replicated access storage is not a config-based. - /// We need a separate SYSTEM RELOAD USERS command. -#if 0 + if (reload_mode != ReloadMode::ALL) + return; + /// Reinitialize ZooKeeper and reread everything. std::lock_guard lock{cached_zookeeper_mutex}; cached_zookeeper = nullptr; getZooKeeperNoLock(); -#endif } void ReplicatedAccessStorage::createRootNodes(const zkutil::ZooKeeperPtr & zookeeper) diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h index 6b1bd01238d..7a8aca52942 100644 --- a/src/Access/ReplicatedAccessStorage.h +++ b/src/Access/ReplicatedAccessStorage.h @@ -25,9 +25,9 @@ public: const char * getStorageType() const override { return STORAGE_TYPE; } - void reload() override; void startPeriodicReloading() override { startWatchingThread(); } void stopPeriodicReloading() override { stopWatchingThread(); } + void reload(ReloadMode reload_mode) override; bool exists(const UUID & id) const override; diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 1d755fdf1da..e3182612b5b 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -635,13 +635,6 @@ void UsersConfigAccessStorage::load( /* already_loaded = */ false); } -void UsersConfigAccessStorage::reload() -{ - std::lock_guard lock{load_mutex}; - if (config_reloader) - config_reloader->reload(); -} - void UsersConfigAccessStorage::startPeriodicReloading() { std::lock_guard lock{load_mutex}; @@ -656,6 +649,13 @@ void UsersConfigAccessStorage::stopPeriodicReloading() config_reloader->stop(); } +void UsersConfigAccessStorage::reload(ReloadMode /* reload_mode */) +{ + std::lock_guard lock{load_mutex}; + if (config_reloader) + config_reloader->reload(); +} + std::optional UsersConfigAccessStorage::findImpl(AccessEntityType type, const String & name) const { return memory_storage.find(type, name); diff --git a/src/Access/UsersConfigAccessStorage.h b/src/Access/UsersConfigAccessStorage.h index 3fa8b4185a8..b533ccbf200 100644 --- a/src/Access/UsersConfigAccessStorage.h +++ b/src/Access/UsersConfigAccessStorage.h @@ -38,9 +38,9 @@ public: const String & preprocessed_dir = {}, const zkutil::GetZooKeeper & get_zookeeper_function = {}); - void reload() override; void startPeriodicReloading() override; void stopPeriodicReloading() override; + void reload(ReloadMode reload_mode) override; bool exists(const UUID & id) const override; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 34504933436..56e87d6a4fb 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -422,6 +423,10 @@ BlockIO InterpreterSystemQuery::execute() getContext()->checkAccess(AccessType::SYSTEM_RELOAD_CONFIG); system_context->reloadConfig(); break; + case Type::RELOAD_USERS: + getContext()->checkAccess(AccessType::SYSTEM_RELOAD_USERS); + system_context->getAccessControl().reload(AccessControl::ReloadMode::ALL); + break; case Type::RELOAD_SYMBOLS: { #if defined(__ELF__) && !defined(OS_FREEBSD) @@ -900,6 +905,11 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() required_access.emplace_back(AccessType::SYSTEM_RELOAD_CONFIG); break; } + case Type::RELOAD_USERS: + { + required_access.emplace_back(AccessType::SYSTEM_RELOAD_USERS); + break; + } case Type::RELOAD_SYMBOLS: { required_access.emplace_back(AccessType::SYSTEM_RELOAD_SYMBOLS); diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index f2c3fa8ece5..33f2fcb708c 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -47,6 +47,7 @@ public: RELOAD_FUNCTIONS, RELOAD_EMBEDDED_DICTIONARIES, RELOAD_CONFIG, + RELOAD_USERS, RELOAD_SYMBOLS, RESTART_DISK, STOP_MERGES, diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 49f4d8245a2..5c9e1a839b2 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -97,6 +97,7 @@ SYSTEM DROP FILESYSTEM CACHE ['SYSTEM DROP FILESYSTEM CACHE','DROP FILESYSTEM CA SYSTEM DROP SCHEMA CACHE ['SYSTEM DROP SCHEMA CACHE','DROP SCHEMA CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP CACHE ['DROP CACHE'] \N SYSTEM SYSTEM RELOAD CONFIG ['RELOAD CONFIG'] GLOBAL SYSTEM RELOAD +SYSTEM RELOAD USERS ['RELOAD USERS'] GLOBAL SYSTEM RELOAD SYSTEM RELOAD SYMBOLS ['RELOAD SYMBOLS'] GLOBAL SYSTEM RELOAD SYSTEM RELOAD DICTIONARY ['SYSTEM RELOAD DICTIONARIES','RELOAD DICTIONARY','RELOAD DICTIONARIES'] GLOBAL SYSTEM RELOAD SYSTEM RELOAD MODEL ['SYSTEM RELOAD MODELS','RELOAD MODEL','RELOAD MODELS'] GLOBAL SYSTEM RELOAD diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index d087bb55622..5a339ed46da 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -279,7 +279,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD SYMBOLS' = 99, 'SYSTEM RELOAD DICTIONARY' = 100, 'SYSTEM RELOAD MODEL' = 101, 'SYSTEM RELOAD FUNCTION' = 102, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 103, 'SYSTEM RELOAD' = 104, 'SYSTEM RESTART DISK' = 105, 'SYSTEM MERGES' = 106, 'SYSTEM TTL MERGES' = 107, 'SYSTEM FETCHES' = 108, 'SYSTEM MOVES' = 109, 'SYSTEM DISTRIBUTED SENDS' = 110, 'SYSTEM REPLICATED SENDS' = 111, 'SYSTEM SENDS' = 112, 'SYSTEM REPLICATION QUEUES' = 113, 'SYSTEM DROP REPLICA' = 114, 'SYSTEM SYNC REPLICA' = 115, 'SYSTEM RESTART REPLICA' = 116, 'SYSTEM RESTORE REPLICA' = 117, 'SYSTEM SYNC DATABASE REPLICA' = 118, 'SYSTEM SYNC TRANSACTION LOG' = 119, 'SYSTEM FLUSH DISTRIBUTED' = 120, 'SYSTEM FLUSH LOGS' = 121, 'SYSTEM FLUSH' = 122, 'SYSTEM THREAD FUZZER' = 123, 'SYSTEM UNFREEZE' = 124, 'SYSTEM' = 125, 'dictGet' = 126, 'addressToLine' = 127, 'addressToLineWithInlines' = 128, 'addressToSymbol' = 129, 'demangle' = 130, 'INTROSPECTION' = 131, 'FILE' = 132, 'URL' = 133, 'REMOTE' = 134, 'MONGO' = 135, 'MEILISEARCH' = 136, 'MYSQL' = 137, 'POSTGRES' = 138, 'SQLITE' = 139, 'ODBC' = 140, 'JDBC' = 141, 'HDFS' = 142, 'S3' = 143, 'HIVE' = 144, 'SOURCES' = 145, 'CLUSTER' = 146, 'ALL' = 147, 'NONE' = 148), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD USERS' = 99, 'SYSTEM RELOAD SYMBOLS' = 100, 'SYSTEM RELOAD DICTIONARY' = 101, 'SYSTEM RELOAD MODEL' = 102, 'SYSTEM RELOAD FUNCTION' = 103, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 104, 'SYSTEM RELOAD' = 105, 'SYSTEM RESTART DISK' = 106, 'SYSTEM MERGES' = 107, 'SYSTEM TTL MERGES' = 108, 'SYSTEM FETCHES' = 109, 'SYSTEM MOVES' = 110, 'SYSTEM DISTRIBUTED SENDS' = 111, 'SYSTEM REPLICATED SENDS' = 112, 'SYSTEM SENDS' = 113, 'SYSTEM REPLICATION QUEUES' = 114, 'SYSTEM DROP REPLICA' = 115, 'SYSTEM SYNC REPLICA' = 116, 'SYSTEM RESTART REPLICA' = 117, 'SYSTEM RESTORE REPLICA' = 118, 'SYSTEM SYNC DATABASE REPLICA' = 119, 'SYSTEM SYNC TRANSACTION LOG' = 120, 'SYSTEM FLUSH DISTRIBUTED' = 121, 'SYSTEM FLUSH LOGS' = 122, 'SYSTEM FLUSH' = 123, 'SYSTEM THREAD FUZZER' = 124, 'SYSTEM UNFREEZE' = 125, 'SYSTEM' = 126, 'dictGet' = 127, 'addressToLine' = 128, 'addressToLineWithInlines' = 129, 'addressToSymbol' = 130, 'demangle' = 131, 'INTROSPECTION' = 132, 'FILE' = 133, 'URL' = 134, 'REMOTE' = 135, 'MONGO' = 136, 'MEILISEARCH' = 137, 'MYSQL' = 138, 'POSTGRES' = 139, 'SQLITE' = 140, 'ODBC' = 141, 'JDBC' = 142, 'HDFS' = 143, 'S3' = 144, 'HIVE' = 145, 'SOURCES' = 146, 'CLUSTER' = 147, 'ALL' = 148, 'NONE' = 149), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -541,10 +541,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD SYMBOLS' = 99, 'SYSTEM RELOAD DICTIONARY' = 100, 'SYSTEM RELOAD MODEL' = 101, 'SYSTEM RELOAD FUNCTION' = 102, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 103, 'SYSTEM RELOAD' = 104, 'SYSTEM RESTART DISK' = 105, 'SYSTEM MERGES' = 106, 'SYSTEM TTL MERGES' = 107, 'SYSTEM FETCHES' = 108, 'SYSTEM MOVES' = 109, 'SYSTEM DISTRIBUTED SENDS' = 110, 'SYSTEM REPLICATED SENDS' = 111, 'SYSTEM SENDS' = 112, 'SYSTEM REPLICATION QUEUES' = 113, 'SYSTEM DROP REPLICA' = 114, 'SYSTEM SYNC REPLICA' = 115, 'SYSTEM RESTART REPLICA' = 116, 'SYSTEM RESTORE REPLICA' = 117, 'SYSTEM SYNC DATABASE REPLICA' = 118, 'SYSTEM SYNC TRANSACTION LOG' = 119, 'SYSTEM FLUSH DISTRIBUTED' = 120, 'SYSTEM FLUSH LOGS' = 121, 'SYSTEM FLUSH' = 122, 'SYSTEM THREAD FUZZER' = 123, 'SYSTEM UNFREEZE' = 124, 'SYSTEM' = 125, 'dictGet' = 126, 'addressToLine' = 127, 'addressToLineWithInlines' = 128, 'addressToSymbol' = 129, 'demangle' = 130, 'INTROSPECTION' = 131, 'FILE' = 132, 'URL' = 133, 'REMOTE' = 134, 'MONGO' = 135, 'MEILISEARCH' = 136, 'MYSQL' = 137, 'POSTGRES' = 138, 'SQLITE' = 139, 'ODBC' = 140, 'JDBC' = 141, 'HDFS' = 142, 'S3' = 143, 'HIVE' = 144, 'SOURCES' = 145, 'CLUSTER' = 146, 'ALL' = 147, 'NONE' = 148), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD USERS' = 99, 'SYSTEM RELOAD SYMBOLS' = 100, 'SYSTEM RELOAD DICTIONARY' = 101, 'SYSTEM RELOAD MODEL' = 102, 'SYSTEM RELOAD FUNCTION' = 103, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 104, 'SYSTEM RELOAD' = 105, 'SYSTEM RESTART DISK' = 106, 'SYSTEM MERGES' = 107, 'SYSTEM TTL MERGES' = 108, 'SYSTEM FETCHES' = 109, 'SYSTEM MOVES' = 110, 'SYSTEM DISTRIBUTED SENDS' = 111, 'SYSTEM REPLICATED SENDS' = 112, 'SYSTEM SENDS' = 113, 'SYSTEM REPLICATION QUEUES' = 114, 'SYSTEM DROP REPLICA' = 115, 'SYSTEM SYNC REPLICA' = 116, 'SYSTEM RESTART REPLICA' = 117, 'SYSTEM RESTORE REPLICA' = 118, 'SYSTEM SYNC DATABASE REPLICA' = 119, 'SYSTEM SYNC TRANSACTION LOG' = 120, 'SYSTEM FLUSH DISTRIBUTED' = 121, 'SYSTEM FLUSH LOGS' = 122, 'SYSTEM FLUSH' = 123, 'SYSTEM THREAD FUZZER' = 124, 'SYSTEM UNFREEZE' = 125, 'SYSTEM' = 126, 'dictGet' = 127, 'addressToLine' = 128, 'addressToLineWithInlines' = 129, 'addressToSymbol' = 130, 'demangle' = 131, 'INTROSPECTION' = 132, 'FILE' = 133, 'URL' = 134, 'REMOTE' = 135, 'MONGO' = 136, 'MEILISEARCH' = 137, 'MYSQL' = 138, 'POSTGRES' = 139, 'SQLITE' = 140, 'ODBC' = 141, 'JDBC' = 142, 'HDFS' = 143, 'S3' = 144, 'HIVE' = 145, 'SOURCES' = 146, 'CLUSTER' = 147, 'ALL' = 148, 'NONE' = 149), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD SYMBOLS' = 99, 'SYSTEM RELOAD DICTIONARY' = 100, 'SYSTEM RELOAD MODEL' = 101, 'SYSTEM RELOAD FUNCTION' = 102, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 103, 'SYSTEM RELOAD' = 104, 'SYSTEM RESTART DISK' = 105, 'SYSTEM MERGES' = 106, 'SYSTEM TTL MERGES' = 107, 'SYSTEM FETCHES' = 108, 'SYSTEM MOVES' = 109, 'SYSTEM DISTRIBUTED SENDS' = 110, 'SYSTEM REPLICATED SENDS' = 111, 'SYSTEM SENDS' = 112, 'SYSTEM REPLICATION QUEUES' = 113, 'SYSTEM DROP REPLICA' = 114, 'SYSTEM SYNC REPLICA' = 115, 'SYSTEM RESTART REPLICA' = 116, 'SYSTEM RESTORE REPLICA' = 117, 'SYSTEM SYNC DATABASE REPLICA' = 118, 'SYSTEM SYNC TRANSACTION LOG' = 119, 'SYSTEM FLUSH DISTRIBUTED' = 120, 'SYSTEM FLUSH LOGS' = 121, 'SYSTEM FLUSH' = 122, 'SYSTEM THREAD FUZZER' = 123, 'SYSTEM UNFREEZE' = 124, 'SYSTEM' = 125, 'dictGet' = 126, 'addressToLine' = 127, 'addressToLineWithInlines' = 128, 'addressToSymbol' = 129, 'demangle' = 130, 'INTROSPECTION' = 131, 'FILE' = 132, 'URL' = 133, 'REMOTE' = 134, 'MONGO' = 135, 'MEILISEARCH' = 136, 'MYSQL' = 137, 'POSTGRES' = 138, 'SQLITE' = 139, 'ODBC' = 140, 'JDBC' = 141, 'HDFS' = 142, 'S3' = 143, 'HIVE' = 144, 'SOURCES' = 145, 'CLUSTER' = 146, 'ALL' = 147, 'NONE' = 148)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD USERS' = 99, 'SYSTEM RELOAD SYMBOLS' = 100, 'SYSTEM RELOAD DICTIONARY' = 101, 'SYSTEM RELOAD MODEL' = 102, 'SYSTEM RELOAD FUNCTION' = 103, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 104, 'SYSTEM RELOAD' = 105, 'SYSTEM RESTART DISK' = 106, 'SYSTEM MERGES' = 107, 'SYSTEM TTL MERGES' = 108, 'SYSTEM FETCHES' = 109, 'SYSTEM MOVES' = 110, 'SYSTEM DISTRIBUTED SENDS' = 111, 'SYSTEM REPLICATED SENDS' = 112, 'SYSTEM SENDS' = 113, 'SYSTEM REPLICATION QUEUES' = 114, 'SYSTEM DROP REPLICA' = 115, 'SYSTEM SYNC REPLICA' = 116, 'SYSTEM RESTART REPLICA' = 117, 'SYSTEM RESTORE REPLICA' = 118, 'SYSTEM SYNC DATABASE REPLICA' = 119, 'SYSTEM SYNC TRANSACTION LOG' = 120, 'SYSTEM FLUSH DISTRIBUTED' = 121, 'SYSTEM FLUSH LOGS' = 122, 'SYSTEM FLUSH' = 123, 'SYSTEM THREAD FUZZER' = 124, 'SYSTEM UNFREEZE' = 125, 'SYSTEM' = 126, 'dictGet' = 127, 'addressToLine' = 128, 'addressToLineWithInlines' = 129, 'addressToSymbol' = 130, 'demangle' = 131, 'INTROSPECTION' = 132, 'FILE' = 133, 'URL' = 134, 'REMOTE' = 135, 'MONGO' = 136, 'MEILISEARCH' = 137, 'MYSQL' = 138, 'POSTGRES' = 139, 'SQLITE' = 140, 'ODBC' = 141, 'JDBC' = 142, 'HDFS' = 143, 'S3' = 144, 'HIVE' = 145, 'SOURCES' = 146, 'CLUSTER' = 147, 'ALL' = 148, 'NONE' = 149)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' From 69996c960c8d94e463a168dabc5ea6c9f58cfecf Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 16 Sep 2022 16:56:21 +0200 Subject: [PATCH 61/86] Add retries for the initialization of ReplicatedAccessStorage. --- src/Access/ReplicatedAccessStorage.cpp | 29 +++++++++++++++++++++++++- src/Access/ReplicatedAccessStorage.h | 1 + 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index b32954e69a5..747c354b28f 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -60,7 +60,7 @@ ReplicatedAccessStorage::ReplicatedAccessStorage( if (zookeeper_path.front() != '/') zookeeper_path = "/" + zookeeper_path; - initZooKeeperIfNeeded(); + initZooKeeperWithRetries(/* max_retries= */ 2); } ReplicatedAccessStorage::~ReplicatedAccessStorage() @@ -414,6 +414,33 @@ void ReplicatedAccessStorage::resetAfterError() cached_zookeeper = nullptr; } +void ReplicatedAccessStorage::initZooKeeperWithRetries(size_t max_retries) +{ + for (size_t attempt = 0; attempt < max_retries; ++attempt) + { + try + { + initZooKeeperIfNeeded(); + break; /// If we're here the initialization has been successful. + } + catch (const Exception & e) + { + bool need_another_attempt = false; + + if (const auto * coordination_exception = dynamic_cast(&e); + coordination_exception && Coordination::isHardwareError(coordination_exception->code)) + { + /// In case of a network error we'll try to initialize again. + LOG_ERROR(getLogger(), "Initialization failed. Error: {}", e.message()); + need_another_attempt = (attempt + 1 < max_retries); + } + + if (!need_another_attempt) + throw; + } + } +} + void ReplicatedAccessStorage::initZooKeeperIfNeeded() { getZooKeeper(); diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h index 7a8aca52942..d9d4b628f8d 100644 --- a/src/Access/ReplicatedAccessStorage.h +++ b/src/Access/ReplicatedAccessStorage.h @@ -55,6 +55,7 @@ private: bool removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists); bool updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); + void initZooKeeperWithRetries(size_t max_retries); void initZooKeeperIfNeeded(); zkutil::ZooKeeperPtr getZooKeeper(); zkutil::ZooKeeperPtr getZooKeeperNoLock() TSA_REQUIRES(cached_zookeeper_mutex); From e7e51ab2d9fadbe380179e6fe43021c1de6c7b00 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 16 Sep 2022 14:58:46 +0200 Subject: [PATCH 62/86] Add comments. --- src/Access/DiskAccessStorage.cpp | 6 +++--- src/Access/IAccessStorage.cpp | 2 +- src/Access/MemoryAccessStorage.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 0544e4a0c0a..ff51ae1cfd4 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -254,12 +254,12 @@ bool DiskAccessStorage::readLists() for (auto type : collections::range(AccessEntityType::MAX)) entries_by_name_and_type[static_cast(type)].clear(); - for (const auto & [id, name, type] : ids_names_types) + for (auto & [id, name, type] : ids_names_types) { auto & entry = entries_by_id[id]; entry.id = id; entry.type = type; - entry.name = name; + entry.name = std::move(name); auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; entries_by_name[entry.name] = &entry; } @@ -414,7 +414,7 @@ void DiskAccessStorage::removeAllExceptInMemory(const boost::container::flat_set for (auto it = entries_by_id.begin(); it != entries_by_id.end();) { const auto & id = it->first; - ++it; + ++it; /// We must go to the next element in the map `entries_by_id` here because otherwise removeNoLock() can invalidate our iterator. if (!ids_to_keep.contains(id)) removeNoLock(id, /* throw_if_not_exists */ true, /* write_on_disk= */ false); } diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index f4092368857..f562a6ebeec 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -572,7 +572,7 @@ void IAccessStorage::clearConflictsInEntitiesList(std::vectorfirst; - ++it; + ++it; /// We must go to the next element in the map `entries_by_id` here because otherwise removeNoLock() can invalidate our iterator. if (!ids_to_keep.contains(id)) removeNoLock(id, /* throw_if_not_exists */ true); } From d6748b182f122b2a25d80b21e3112a7d1489691f Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Sun, 18 Sep 2022 14:48:24 +0200 Subject: [PATCH 63/86] Update CachedOnDiskWriteBufferFromFile.cpp --- src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp index 987d27d2e1a..994bb743c5f 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp @@ -11,12 +11,16 @@ namespace ProfileEvents { extern const Event CachedWriteBufferCacheWriteBytes; extern const Event CachedWriteBufferCacheWriteMicroseconds; - extern const Event FileSegmentWriteMicroseconds; } namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace { class SwapHelper From 09b75901f06dec3ec67180fcc4a8cd71f29c8881 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 19 Sep 2022 14:07:31 +0800 Subject: [PATCH 64/86] Fix nullptr deref --- src/Disks/VolumeJBOD.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Disks/VolumeJBOD.h b/src/Disks/VolumeJBOD.h index 21d61e6dd8d..81da64c488d 100644 --- a/src/Disks/VolumeJBOD.h +++ b/src/Disks/VolumeJBOD.h @@ -82,6 +82,9 @@ private: ReservationPtr reserve(uint64_t bytes) { ReservationPtr reservation = disk->reserve(bytes); + if (!reservation) + return {}; + /// Not just subtract bytes, but update the value, /// since some reservations may be done directly via IDisk, or not by ClickHouse. free_size = reservation->getUnreservedSpace(); From d2601b81928a5cf6bee5dc2f4370d902f1a2698c Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 19 Sep 2022 14:41:34 +0800 Subject: [PATCH 65/86] Add a comment --- src/Disks/IDisk.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index bfbdba0e050..998e00fba45 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -71,6 +71,7 @@ public: virtual const String & getName() const = 0; /// Reserve the specified number of bytes. + /// Returns valid reservation or nullptr when failure. virtual ReservationPtr reserve(UInt64 bytes) = 0; virtual ~Space() = default; From c33e231ce93d11731491e12a8da4b4dc1040e6b9 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 9 Sep 2022 11:28:50 +0000 Subject: [PATCH 66/86] Throw exception on precision lost during decimal to decimal convertion --- src/DataTypes/DataTypesDecimal.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/DataTypes/DataTypesDecimal.h b/src/DataTypes/DataTypesDecimal.h index 7bcc6593435..1d8b1fbc8c4 100644 --- a/src/DataTypes/DataTypesDecimal.h +++ b/src/DataTypes/DataTypesDecimal.h @@ -123,7 +123,18 @@ inline ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & v } } else - converted_value = value.value / DecimalUtils::scaleMultiplier(scale_from - scale_to); + { + auto multiplier = DecimalUtils::scaleMultiplier(scale_from - scale_to); + if (value.value % multiplier != 0) + { + if constexpr (throw_exception) + throw Exception(std::string(ToDataType::family_name) + " convert overflow", + ErrorCodes::DECIMAL_OVERFLOW); + else + return ReturnType(false); + } + converted_value = value.value / multiplier; + } if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType)) { From c2634c9fae4297ebfeb213654f8e058455fd9ac3 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 9 Sep 2022 13:40:37 +0000 Subject: [PATCH 67/86] Revert "Throw exception on precision lost during decimal to decimal convertion" This reverts commit d4233cf3c658adf2541070b1bbba678ad7c769db. --- src/DataTypes/DataTypesDecimal.h | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/DataTypes/DataTypesDecimal.h b/src/DataTypes/DataTypesDecimal.h index 1d8b1fbc8c4..7bcc6593435 100644 --- a/src/DataTypes/DataTypesDecimal.h +++ b/src/DataTypes/DataTypesDecimal.h @@ -123,18 +123,7 @@ inline ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & v } } else - { - auto multiplier = DecimalUtils::scaleMultiplier(scale_from - scale_to); - if (value.value % multiplier != 0) - { - if constexpr (throw_exception) - throw Exception(std::string(ToDataType::family_name) + " convert overflow", - ErrorCodes::DECIMAL_OVERFLOW); - else - return ReturnType(false); - } - converted_value = value.value / multiplier; - } + converted_value = value.value / DecimalUtils::scaleMultiplier(scale_from - scale_to); if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType)) { From 5c9bfbc116f5e4d98dc1e6a2ea726fff0688814e Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 9 Sep 2022 13:52:02 +0000 Subject: [PATCH 68/86] Fix incorrect result in case of decimal precision loss in IN operator --- src/Interpreters/ActionsVisitor.cpp | 36 +++++++++++++++---- .../queries/0_stateless/00862_decimal_in.sql | 12 ++++--- ...decimal_in_precision_issue_41125.reference | 10 ++++++ ...02421_decimal_in_precision_issue_41125.sql | 23 ++++++++++++ 4 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.reference create mode 100644 tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.sql diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 8e135d325e6..0c251d49335 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -1,8 +1,9 @@ #include + #include #include -#include -#include +#include + #include #include @@ -22,9 +23,12 @@ #include #include #include +#include -#include +#include #include +#include +#include #include @@ -87,6 +91,22 @@ static size_t getTypeDepth(const DataTypePtr & type) return 0; } +/// Applies stricter rules than convertFieldToType: +/// Doesn't allow : +/// - loss of precision with `Decimals` +static bool convertFieldToTypeStrict(const Field & from_value, const IDataType & to_type, Field & result_value) +{ + result_value = convertFieldToType(from_value, to_type); + if (Field::isDecimal(from_value.getType()) && Field::isDecimal(result_value.getType())) + return applyVisitor(FieldVisitorAccurateEquals{}, from_value, result_value); + + return true; +} + +/// The `convertFieldToTypeStrict` is used to prevent unexpected results in case of conversion with loss of precision. +/// Example: `SELECT 33.3 :: Decimal(9, 1) AS a WHERE a IN (33.33 :: Decimal(9, 2))` +/// 33.33 in the set is converted to 33.3, but it is not equal to 33.3 in the column, so the result should still be empty. +/// We can not include values that don't represent any possible value from the type of filtered column to the set. template static Block createBlockFromCollection(const Collection & collection, const DataTypes & types, bool transform_null_in) { @@ -103,9 +123,10 @@ static Block createBlockFromCollection(const Collection & collection, const Data { if (columns_num == 1) { - auto field = convertFieldToType(value, *types[0]); + Field field; + bool is_conversion_ok = convertFieldToTypeStrict(value, *types[0], field); bool need_insert_null = transform_null_in && types[0]->isNullable(); - if (!field.isNull() || need_insert_null) + if (is_conversion_ok && (!field.isNull() || need_insert_null)) columns[0]->insert(std::move(field)); } else @@ -127,7 +148,10 @@ static Block createBlockFromCollection(const Collection & collection, const Data size_t i = 0; for (; i < tuple_size; ++i) { - tuple_values[i] = convertFieldToType(tuple[i], *types[i]); + bool is_conversion_ok = convertFieldToTypeStrict(tuple[i], *types[i], tuple_values[i]); + if (!is_conversion_ok) + break; + bool need_insert_null = transform_null_in && types[i]->isNullable(); if (tuple_values[i].isNull() && !need_insert_null) break; diff --git a/tests/queries/0_stateless/00862_decimal_in.sql b/tests/queries/0_stateless/00862_decimal_in.sql index 2e24ea6d6ac..b5c058119a2 100644 --- a/tests/queries/0_stateless/00862_decimal_in.sql +++ b/tests/queries/0_stateless/00862_decimal_in.sql @@ -1,18 +1,17 @@ DROP TABLE IF EXISTS temp; CREATE TABLE temp ( - x Decimal(38,2), - y Nullable(Decimal(38,2)) + x Decimal(38, 2), + y Nullable(Decimal(38, 2)) ) ENGINE = Memory; INSERT INTO temp VALUES (32, 32), (64, 64), (128, 128); -SELECT * FROM temp WHERE x IN (toDecimal128(128, 2)); -SELECT * FROM temp WHERE y IN (toDecimal128(128, 2)); - SELECT * FROM temp WHERE x IN (toDecimal128(128, 1)); +SELECT * FROM temp WHERE x IN (toDecimal128(128, 2)); SELECT * FROM temp WHERE x IN (toDecimal128(128, 3)); SELECT * FROM temp WHERE y IN (toDecimal128(128, 1)); +SELECT * FROM temp WHERE y IN (toDecimal128(128, 2)); SELECT * FROM temp WHERE y IN (toDecimal128(128, 3)); SELECT * FROM temp WHERE x IN (toDecimal32(32, 1)); @@ -29,4 +28,7 @@ SELECT * FROM temp WHERE y IN (toDecimal64(64, 1)); SELECT * FROM temp WHERE y IN (toDecimal64(64, 2)); SELECT * FROM temp WHERE y IN (toDecimal64(64, 3)); +SELECT * FROM temp WHERE x IN (toDecimal256(256, 1)); -- { serverError 53 } +SELECT * FROM temp WHERE y IN (toDecimal256(256, 1)); -- { serverError 53 } + DROP TABLE IF EXISTS temp; diff --git a/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.reference b/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.reference new file mode 100644 index 00000000000..d3d171221e8 --- /dev/null +++ b/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.reference @@ -0,0 +1,10 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.sql b/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.sql new file mode 100644 index 00000000000..f5978a34061 --- /dev/null +++ b/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS dtest; + +SELECT count() == 0 FROM (SELECT '33.3' :: Decimal(9, 1) AS a WHERE a IN ('33.33' :: Decimal(9, 2))); + +CREATE TABLE dtest ( `a` Decimal(18, 0), `b` Decimal(18, 1), `c` Decimal(36, 0) ) ENGINE = Memory; +INSERT INTO dtest VALUES ('33', '44.4', '35'); + +SELECT count() == 0 FROM dtest WHERE a IN toDecimal32('33.3000', 4); +SELECT count() == 0 FROM dtest WHERE a IN toDecimal64('33.3000', 4); +SELECT count() == 0 FROM dtest WHERE a IN toDecimal128('33.3000', 4); +SELECT count() == 0 FROM dtest WHERE a IN toDecimal256('33.3000', 4); -- { serverError 53 } + +SELECT count() == 0 FROM dtest WHERE b IN toDecimal32('44.4000', 0); +SELECT count() == 0 FROM dtest WHERE b IN toDecimal64('44.4000', 0); +SELECT count() == 0 FROM dtest WHERE b IN toDecimal128('44.4000', 0); +SELECT count() == 0 FROM dtest WHERE b IN toDecimal256('44.4000', 0); -- { serverError 53 } + +SELECT count() == 1 FROM dtest WHERE b IN toDecimal32('44.4000', 4); +SELECT count() == 1 FROM dtest WHERE b IN toDecimal64('44.4000', 4); +SELECT count() == 1 FROM dtest WHERE b IN toDecimal128('44.4000', 4); +SELECT count() == 1 FROM dtest WHERE b IN toDecimal256('44.4000', 4); -- { serverError 53 } + +DROP TABLE IF EXISTS dtest; From a6dc99983df1fc2d81a0b4b89a62807b3136ab19 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 14 Sep 2022 09:15:01 +0000 Subject: [PATCH 69/86] Fix tidy build --- src/Interpreters/ActionsVisitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 0c251d49335..54faf37f236 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -127,7 +127,7 @@ static Block createBlockFromCollection(const Collection & collection, const Data bool is_conversion_ok = convertFieldToTypeStrict(value, *types[0], field); bool need_insert_null = transform_null_in && types[0]->isNullable(); if (is_conversion_ok && (!field.isNull() || need_insert_null)) - columns[0]->insert(std::move(field)); + columns[0]->insert(field); } else { From fc4a0e0bdaff474255f14ff596f44e7d68ba35a3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Sep 2022 09:34:29 +0000 Subject: [PATCH 70/86] Update version_date.tsv and changelogs after v22.6.8.35-stable --- docs/changelogs/v22.6.8.35-stable.md | 34 ++++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 1 + 2 files changed, 35 insertions(+) create mode 100644 docs/changelogs/v22.6.8.35-stable.md diff --git a/docs/changelogs/v22.6.8.35-stable.md b/docs/changelogs/v22.6.8.35-stable.md new file mode 100644 index 00000000000..ffc5e03e489 --- /dev/null +++ b/docs/changelogs/v22.6.8.35-stable.md @@ -0,0 +1,34 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + +### ClickHouse release v22.6.8.35-stable (b91dc59a565) FIXME as compared to v22.6.7.7-stable (8eae2af3b9a) + +#### New Feature +* Backported in [#40868](https://github.com/ClickHouse/ClickHouse/issues/40868): Add setting to disable limit on kafka_num_consumers. Closes [#40331](https://github.com/ClickHouse/ClickHouse/issues/40331). [#40670](https://github.com/ClickHouse/ClickHouse/pull/40670) ([Kruglov Pavel](https://github.com/Avogar)). + +#### Bug Fix +* Backported in [#41274](https://github.com/ClickHouse/ClickHouse/issues/41274): Fix memory safety issues with functions `encrypt` and `contingency` if Array of Nullable is used as an argument. This fixes [#41004](https://github.com/ClickHouse/ClickHouse/issues/41004). [#40195](https://github.com/ClickHouse/ClickHouse/pull/40195) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#41282](https://github.com/ClickHouse/ClickHouse/issues/41282): Fix unused unknown columns introduced by WITH statement. This fixes [#37812](https://github.com/ClickHouse/ClickHouse/issues/37812) . [#39131](https://github.com/ClickHouse/ClickHouse/pull/39131) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#40905](https://github.com/ClickHouse/ClickHouse/issues/40905): Fix potential deadlock in WriteBufferFromS3 during task scheduling failure. [#40070](https://github.com/ClickHouse/ClickHouse/pull/40070) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#40864](https://github.com/ClickHouse/ClickHouse/issues/40864): - Fix crash while parsing values of type `Object` that contains arrays of variadic dimension. [#40483](https://github.com/ClickHouse/ClickHouse/pull/40483) ([Duc Canh Le](https://github.com/canhld94)). +* Backported in [#40803](https://github.com/ClickHouse/ClickHouse/issues/40803): During insertion of a new query to the `ProcessList` allocations happen. If we reach the memory limit during these allocations we can not use `OvercommitTracker`, because `ProcessList::mutex` is already acquired. Fixes [#40611](https://github.com/ClickHouse/ClickHouse/issues/40611). [#40677](https://github.com/ClickHouse/ClickHouse/pull/40677) ([Dmitry Novik](https://github.com/novikd)). +* Backported in [#40891](https://github.com/ClickHouse/ClickHouse/issues/40891): Fix memory leak while pushing to MVs w/o query context (from Kafka/...). [#40732](https://github.com/ClickHouse/ClickHouse/pull/40732) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#41133](https://github.com/ClickHouse/ClickHouse/issues/41133): Fix access rights for `DESCRIBE TABLE url()` and some other `DESCRIBE TABLE ()`. [#40975](https://github.com/ClickHouse/ClickHouse/pull/40975) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#41360](https://github.com/ClickHouse/ClickHouse/issues/41360): Fix incorrect logical error `Expected relative path` in disk object storage. Related to [#41246](https://github.com/ClickHouse/ClickHouse/issues/41246). [#41297](https://github.com/ClickHouse/ClickHouse/pull/41297) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#41357](https://github.com/ClickHouse/ClickHouse/issues/41357): Add column type check before UUID insertion in MsgPack format. [#41309](https://github.com/ClickHouse/ClickHouse/pull/41309) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* use ROBOT_CLICKHOUSE_COMMIT_TOKEN for create-pull-request [#40067](https://github.com/ClickHouse/ClickHouse/pull/40067) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* use input token instead of env var [#40421](https://github.com/ClickHouse/ClickHouse/pull/40421) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Migrate artifactory [#40831](https://github.com/ClickHouse/ClickHouse/pull/40831) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Docker server version [#41256](https://github.com/ClickHouse/ClickHouse/pull/41256) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Increase open files limit [#41345](https://github.com/ClickHouse/ClickHouse/pull/41345) ([Eugene Konkov](https://github.com/ekonkov)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index f7df0345842..399f6066f15 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -8,6 +8,7 @@ v22.7.4.16-stable 2022-08-23 v22.7.3.5-stable 2022-08-10 v22.7.2.15-stable 2022-08-03 v22.7.1.2484-stable 2022-07-21 +v22.6.8.35-stable 2022-09-19 v22.6.7.7-stable 2022-08-29 v22.6.6.16-stable 2022-08-23 v22.6.5.22-stable 2022-08-09 From d0ff4da7812ed067403e157d928a49d1a92dd53d Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 19 Sep 2022 10:11:56 +0000 Subject: [PATCH 71/86] fix create_role stateless test reference --- .../queries/0_stateless/01293_create_role.reference | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/01293_create_role.reference b/tests/queries/0_stateless/01293_create_role.reference index d33693e61fa..9b3c4eabd47 100644 --- a/tests/queries/0_stateless/01293_create_role.reference +++ b/tests/queries/0_stateless/01293_create_role.reference @@ -30,9 +30,9 @@ CREATE ROLE r2_01293 SETTINGS readonly = 1 -- system.roles r1_01293 local directory -- system.settings_profile_elements -\N \N r1_01293 0 readonly 1 \N \N \N \N \N -\N \N r2_01293 0 \N \N \N \N \N \N default -\N \N r3_01293 0 max_memory_usage 5000000 4000000 6000000 0 \N \N -\N \N r4_01293 0 \N \N \N \N \N \N default -\N \N r4_01293 1 max_memory_usage 5000000 \N \N \N \N \N -\N \N r4_01293 2 readonly 1 \N \N \N \N \N +\N \N r1_01293 0 readonly 1 \N \N \N \N +\N \N r2_01293 0 \N \N \N \N \N default +\N \N r3_01293 0 max_memory_usage 5000000 4000000 6000000 WRITABLE \N +\N \N r4_01293 0 \N \N \N \N \N default +\N \N r4_01293 1 max_memory_usage 5000000 \N \N \N \N +\N \N r4_01293 2 readonly 1 \N \N \N \N From a0bfa801e5089a10683059726ec5ec13e300e806 Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 19 Sep 2022 10:52:08 +0000 Subject: [PATCH 72/86] explicit ctor --- src/Access/SettingsConstraints.cpp | 2 +- src/Access/SettingsConstraints.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index e8a532ff48b..d97a78c78ab 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -277,7 +277,7 @@ SettingsConstraints::Checker SettingsConstraints::getChecker(const Settings & cu if (it == constraints.end()) return Checker(); // Allowed } - return it->second; + return Checker(it->second); } bool SettingsConstraints::Constraint::operator==(const Constraint & other) const diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 3c4038d92c4..822bf42861b 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -115,7 +115,7 @@ private: {} // Allow or forbid depending on range defined by constraint, also used to return stored constraint - Checker(const Constraint & constraint_) + explicit Checker(const Constraint & constraint_) : constraint(constraint_) {} From f8665c83da5f5200c3bdd7e6837af04fdd2c5bc3 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 19 Sep 2022 14:26:38 +0200 Subject: [PATCH 73/86] Add retries for nats --- src/Storages/NATS/NATSSettings.h | 3 ++- src/Storages/NATS/StorageNATS.cpp | 22 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Storages/NATS/NATSSettings.h b/src/Storages/NATS/NATSSettings.h index 6029aaea9f6..9bf9b969387 100644 --- a/src/Storages/NATS/NATSSettings.h +++ b/src/Storages/NATS/NATSSettings.h @@ -24,7 +24,8 @@ class ASTStorage; M(Milliseconds, nats_flush_interval_ms, 0, "Timeout for flushing data from NATS.", 0) \ M(String, nats_username, "", "NATS username", 0) \ M(String, nats_password, "", "NATS password", 0) \ - M(String, nats_token, "", "NATS token", 0) + M(String, nats_token, "", "NATS token", 0) \ + M(UInt64, nats_startup_connect_tries, 5, "Number of connect tries at startup", 0) \ #define LIST_OF_NATS_SETTINGS(M) \ NATS_RELATED_SETTINGS(M) \ diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp index fc3079a7aa7..4a3ba973e67 100644 --- a/src/Storages/NATS/StorageNATS.cpp +++ b/src/Storages/NATS/StorageNATS.cpp @@ -92,10 +92,24 @@ StorageNATS::StorageNATS( try { - connection = std::make_shared(configuration, log); - if (!connection->connect()) - throw Exception(ErrorCodes::CANNOT_CONNECT_NATS, "Cannot connect to {}. Nats last error: {}", - connection->connectionInfoForLog(), nats_GetLastError(nullptr)); + size_t num_tries = nats_settings->nats_startup_connect_tries; + for (size_t i = 0; i < num_tries; ++i) + { + connection = std::make_shared(configuration, log); + + if (connection->connect()) + break; + + if (i == num_tries - 1) + { + throw Exception( + ErrorCodes::CANNOT_CONNECT_NATS, + "Cannot connect to {}. Nats last error: {}", + connection->connectionInfoForLog(), nats_GetLastError(nullptr)); + } + + LOG_DEBUG(log, "Connect attempt #{} failed, error: {}. Reconnecting...", i + 1, nats_GetLastError(nullptr)); + } } catch (...) { From e89dffbb95647570b2372e2eecbeb5e2990cdfe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 19 Sep 2022 14:34:08 +0200 Subject: [PATCH 74/86] Fix AggregateFunctionNullVariadic batch with null_is_skipped=false --- .../AggregateFunctionNull.h | 10 ++++++-- .../02428_batch_nullable_assert.reference | 1 + .../02428_batch_nullable_assert.sql | 24 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/02428_batch_nullable_assert.reference create mode 100644 tests/queries/0_stateless/02428_batch_nullable_assert.sql diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index 1e2c9326142..431faf09052 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -468,9 +468,15 @@ public: } } - bool found_one = false; + /// We can have 0 nullable filters if we don't skip nulls + if (nullable_filters.size() == 0) + { + this->setFlag(place); + this->nested_function->addBatchSinglePlace(row_begin, row_end, this->nestedPlace(place), nested_columns, arena, -1); + return; + } - chassert(nullable_filters.size() > 0); /// We work under the assumption that we reach this because one argument was NULL + bool found_one = false; if (nullable_filters.size() == 1) { /// We can avoid making copies of the only filter but we still need to check that there is data to be added diff --git a/tests/queries/0_stateless/02428_batch_nullable_assert.reference b/tests/queries/0_stateless/02428_batch_nullable_assert.reference new file mode 100644 index 00000000000..823ebfc421c --- /dev/null +++ b/tests/queries/0_stateless/02428_batch_nullable_assert.reference @@ -0,0 +1 @@ +100,-9223372036854775808,nan diff --git a/tests/queries/0_stateless/02428_batch_nullable_assert.sql b/tests/queries/0_stateless/02428_batch_nullable_assert.sql new file mode 100644 index 00000000000..eed9b9d34c7 --- /dev/null +++ b/tests/queries/0_stateless/02428_batch_nullable_assert.sql @@ -0,0 +1,24 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/41470 +SELECT + roundBankers(100), + -9223372036854775808, + roundBankers(result.2, 256) +FROM + ( + SELECT studentTTest(sample, variant) AS result + FROM + ( + SELECT + toFloat64(number) % NULL AS sample, + 0 AS variant + FROM system.numbers + LIMIT 1025 + UNION ALL + SELECT + (toFloat64(number) % 9223372036854775807) + nan AS sample, + -9223372036854775808 AS variant + FROM system.numbers + LIMIT 1024 + ) + ) +FORMAT CSV From 26da32f53f263b4a7bf9cb60be74a51be42b3d91 Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Mon, 19 Sep 2022 14:35:23 +0200 Subject: [PATCH 75/86] Update tests/integration/test_profile_events_s3/test.py --- tests/integration/test_profile_events_s3/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_profile_events_s3/test.py b/tests/integration/test_profile_events_s3/test.py index 34dc25e5232..28016264c9c 100644 --- a/tests/integration/test_profile_events_s3/test.py +++ b/tests/integration/test_profile_events_s3/test.py @@ -66,7 +66,7 @@ init_list = { "AbortS3MultipartUpload": 0, "CompleteS3MultipartUpload": 0, "PutS3ObjectRequest": 0, - "GetS3ObjectRequest" 0, + "GetS3ObjectRequest": 0, } From f75dd93965307cca57d6153cd8ce73262f2dc498 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 19 Sep 2022 02:02:09 +0200 Subject: [PATCH 76/86] Mask some information in logs. --- src/Interpreters/executeQuery.cpp | 8 +- src/Parsers/wipePasswordFromQuery.cpp | 32 ++++++ src/Parsers/wipePasswordFromQuery.h | 14 +++ .../test_mask_queries_in_logs/__init__.py | 0 .../test_mask_queries_in_logs/test.py | 99 +++++++++++++++++++ .../01702_system_query_log.reference | 2 +- 6 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 src/Parsers/wipePasswordFromQuery.cpp create mode 100644 src/Parsers/wipePasswordFromQuery.h create mode 100644 tests/integration/test_mask_queries_in_logs/__init__.py create mode 100644 tests/integration/test_mask_queries_in_logs/test.py diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index c501c1722ba..05f5568a472 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -111,8 +112,11 @@ static String prepareQueryForLogging(const String & query, ContextPtr context) { String res = query; - // wiping sensitive data before cropping query by log_queries_cut_to_length, - // otherwise something like credit card without last digit can go to log + // Wiping a password or its hash from CREATE/ALTER USER query because we don't want it to go to logs. + res = wipePasswordFromQuery(res); + + // Wiping sensitive data before cropping query by log_queries_cut_to_length, + // otherwise something like credit card without last digit can go to log. if (auto * masker = SensitiveDataMasker::getInstance()) { auto matches = masker->wipeSensitiveData(res); diff --git a/src/Parsers/wipePasswordFromQuery.cpp b/src/Parsers/wipePasswordFromQuery.cpp new file mode 100644 index 00000000000..87b297d6fcc --- /dev/null +++ b/src/Parsers/wipePasswordFromQuery.cpp @@ -0,0 +1,32 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +String wipePasswordFromQuery(const String & query) +{ + String error_message; + const char * begin = query.data(); + const char * end = begin + query.size(); + + { + ParserCreateUserQuery parser; + const char * pos = begin; + if (auto ast = tryParseQuery(parser, pos, end, error_message, false, "", false, 0, 0)) + { + auto create_query = typeid_cast>(ast); + create_query->show_password = false; + return serializeAST(*create_query); + } + } + + return query; +} + +} diff --git a/src/Parsers/wipePasswordFromQuery.h b/src/Parsers/wipePasswordFromQuery.h new file mode 100644 index 00000000000..cdd8518de53 --- /dev/null +++ b/src/Parsers/wipePasswordFromQuery.h @@ -0,0 +1,14 @@ +#pragma once +#include + + +namespace DB +{ + +/// Removes a password or its hash from a query if it's specified there or replaces it with some placeholder. +/// This function is used to prepare a query for storing in logs (we don't want logs to contain sensitive information). +/// The function changes only following types of queries: +/// CREATE/ALTER USER. +String wipePasswordFromQuery(const String & query); + +} diff --git a/tests/integration/test_mask_queries_in_logs/__init__.py b/tests/integration/test_mask_queries_in_logs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_mask_queries_in_logs/test.py b/tests/integration/test_mask_queries_in_logs/test.py new file mode 100644 index 00000000000..9ab13f41c9e --- /dev/null +++ b/tests/integration/test_mask_queries_in_logs/test.py @@ -0,0 +1,99 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node") + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +# Passwords in CREATE/ALTER queries must be hidden in logs. +def test_create_alter_user(): + node.query("CREATE USER u1 IDENTIFIED BY 'qwe123' SETTINGS custom_a = 'a'") + node.query("ALTER USER u1 IDENTIFIED BY '123qwe' SETTINGS custom_b = 'b'") + node.query( + "CREATE USER u2 IDENTIFIED WITH plaintext_password BY 'plainpasswd' SETTINGS custom_c = 'c'" + ) + + assert ( + node.query("SHOW CREATE USER u1") + == "CREATE USER u1 IDENTIFIED WITH sha256_password SETTINGS custom_b = \\'b\\'\n" + ) + assert ( + node.query("SHOW CREATE USER u2") + == "CREATE USER u2 IDENTIFIED WITH plaintext_password SETTINGS custom_c = \\'c\\'\n" + ) + + node.query("SYSTEM FLUSH LOGS") + + assert node.contains_in_log("CREATE USER u1") + assert node.contains_in_log("ALTER USER u1") + assert node.contains_in_log("CREATE USER u2") + assert not node.contains_in_log("qwe123") + assert not node.contains_in_log("123qwe") + assert not node.contains_in_log("plainpasswd") + assert not node.contains_in_log("IDENTIFIED WITH sha256_password BY") + assert not node.contains_in_log("IDENTIFIED WITH sha256_hash BY") + assert not node.contains_in_log("IDENTIFIED WITH plaintext_password BY") + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'CREATE USER u1%IDENTIFIED WITH sha256_password%'" + ).strip() + ) + >= 1 + ) + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'CREATE USER u1%IDENTIFIED WITH sha256_password BY%'" + ).strip() + ) + == 0 + ) + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'ALTER USER u1%IDENTIFIED WITH sha256_password%'" + ).strip() + ) + >= 1 + ) + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'ALTER USER u1%IDENTIFIED WITH sha256_password BY%'" + ).strip() + ) + == 0 + ) + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'CREATE USER u2%IDENTIFIED WITH plaintext_password%'" + ).strip() + ) + >= 1 + ) + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'CREATE USER u2%IDENTIFIED WITH plaintext_password BY%'" + ).strip() + ) + == 0 + ) diff --git a/tests/queries/0_stateless/01702_system_query_log.reference b/tests/queries/0_stateless/01702_system_query_log.reference index 1f329feac22..4b9eeb139f4 100644 --- a/tests/queries/0_stateless/01702_system_query_log.reference +++ b/tests/queries/0_stateless/01702_system_query_log.reference @@ -21,7 +21,7 @@ Create CREATE DATABASE sqllt; Create CREATE TABLE sqllt.table\n(\n i UInt8, s String\n)\nENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple(); Create CREATE VIEW sqllt.view AS SELECT i, s FROM sqllt.table; Create CREATE DICTIONARY sqllt.dictionary (key UInt64, value UInt64) PRIMARY KEY key SOURCE(CLICKHOUSE(DB \'sqllt\' TABLE \'table\' HOST \'localhost\' PORT 9001)) LIFETIME(0) LAYOUT(FLAT()); - CREATE USER sqllt_user IDENTIFIED WITH PLAINTEXT_PASSWORD BY \'password\'; + CREATE USER sqllt_user IDENTIFIED WITH plaintext_password CREATE ROLE sqllt_role; CREATE POLICY sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary AS PERMISSIVE TO ALL; CREATE POLICY sqllt_row_policy ON sqllt.table, sqllt.view, sqllt.dictionary AS PERMISSIVE TO ALL; From 0f92e5e576cc4f07d56ef56f8c185c9de3bc0103 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 19 Sep 2022 14:10:00 +0000 Subject: [PATCH 77/86] Add option for loading only system tables from path using clickhouse-local --- docker/test/stateless/run.sh | 8 ++++---- programs/local/LocalServer.cpp | 11 +++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 72b0eb5bda1..8151ec9d5e4 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -179,17 +179,17 @@ pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhous # for files >64MB, we want this files to be compressed explicitly for table in query_log zookeeper_log trace_log transactions_info_log do - clickhouse-local --path /var/lib/clickhouse/ -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz ||: + clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz ||: if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then - clickhouse-local --path /var/lib/clickhouse1/ -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.1.tsv.gz ||: - clickhouse-local --path /var/lib/clickhouse2/ -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.2.tsv.gz ||: + clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.1.tsv.gz ||: + clickhouse-local --path /var/lib/clickhouse2/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.2.tsv.gz ||: fi done # Also export trace log in flamegraph-friendly format. for trace_type in CPU Memory Real do - clickhouse-local --path /var/lib/clickhouse/ -q " + clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q " select arrayStringConcat((arrayMap(x -> concat(splitByChar('/', addressToLine(x))[-1], '#', demangle(addressToSymbol(x)) ), trace)), ';') AS stack, count(*) AS samples diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 7a15e46eefa..2b9d819f5eb 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -622,9 +622,13 @@ void LocalServer::processConfig() attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)); - loadMetadata(global_context); startupSystemTables(); - DatabaseCatalog::instance().loadDatabases(); + + if (!config().has("only-system-tables")) + { + loadMetadata(global_context); + DatabaseCatalog::instance().loadDatabases(); + } LOG_DEBUG(log, "Loaded metadata."); } @@ -715,6 +719,7 @@ void LocalServer::addOptions(OptionsDescription & options_description) ("no-system-tables", "do not attach system tables (better startup time)") ("path", po::value(), "Storage path") + ("only-system-tables", "attach only system tables from specified path") ("top_level_domains_path", po::value(), "Path to lists with custom TLDs") ; } @@ -743,6 +748,8 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp config().setString("table-structure", options["structure"].as()); if (options.count("no-system-tables")) config().setBool("no-system-tables", true); + if (options.count("only-system-tables")) + config().setBool("only-system-tables", true); if (options.count("input-format")) config().setString("table-data-format", options["input-format"].as()); From e26976fff2023d1cd9f5825a92343172cb4fa391 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 18 Sep 2022 11:51:33 +0200 Subject: [PATCH 78/86] Add a test for hungs in case of parallel KILL QUERY (with distributed queries) Signed-off-by: Azat Khuzhin --- ..._kill_distributed_query_deadlock.reference | 0 .../02450_kill_distributed_query_deadlock.sh | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/queries/0_stateless/02450_kill_distributed_query_deadlock.reference create mode 100755 tests/queries/0_stateless/02450_kill_distributed_query_deadlock.sh diff --git a/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.reference b/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.sh b/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.sh new file mode 100755 index 00000000000..11ca3f43d8f --- /dev/null +++ b/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Tags: long, no-backward-compatibility-check + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Test that running distributed query and cancel it ASAP, +# this can trigger a hung/deadlock in ProcessorList. +for i in {1..100}; do + query_id="$CLICKHOUSE_TEST_UNIQUE_NAME-$i" + $CLICKHOUSE_CLIENT --format Null --query_id "$query_id" -q "select * from remote('127.{1|2|3|4|5|6}', numbers(1e12))" 2>/dev/null & + while :; do + killed_queries="$($CLICKHOUSE_CLIENT -q "kill query where query_id = '$query_id' sync" | wc -l)" + if [[ "$killed_queries" -ge 1 ]]; then + break + fi + done + wait -n + query_return_status=$? + if [[ $query_return_status -eq 0 ]]; then + echo "Query $query_id should be cancelled, however it returns successfully" + fi +done From cb613b86756e9b40f316ff144f7b1b1990abf981 Mon Sep 17 00:00:00 2001 From: Derek Chia Date: Mon, 19 Sep 2022 22:37:03 +0800 Subject: [PATCH 79/86] Change input_format_skip_unknown_fields default to 1 As per PR https://github.com/ClickHouse/ClickHouse/pull/37192, `input_format_skip_unknown_fields` is enabled by default. Likewise, it is also set as `true` in the factory settings https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L696 Updating the docs to reflect the change. --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index dde40acb91a..99dc1885354 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3433,7 +3433,7 @@ Possible values: - 0 — Disabled. - 1 — Enabled. -Default value: 0. +Default value: 1. ## input_format_with_names_use_header {#input_format_with_names_use_header} From f76f14a99c040f2a4672eff62d79e409d29fd741 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 18 Sep 2022 15:46:14 +0200 Subject: [PATCH 80/86] Fix possible hung/deadlock on query cancellation Right due to graceful cancellation of the query, it is possible to hung accepting new queries or even have a deadlock, this is because cancellation is done while acquiring ProcessListBase::mutex. So this patch makes query cancellation lock-free, and now the lock will be acquired only for preparing the query and after cancel is done. Signed-off-by: Azat Khuzhin --- src/Interpreters/ProcessList.cpp | 58 ++++++++++++++++++++++++++++---- src/Interpreters/ProcessList.h | 8 +++++ 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index e21d3814086..b48e3610d6b 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -297,6 +297,9 @@ ProcessListEntry::~ProcessListEntry() } } + /// Wait for the query if it is in the cancellation right now. + parent.cancelled_cv.wait(lock.lock, [&]() { return it->is_cancelling == false; }); + /// This removes the memory_tracker of one request. parent.processes.erase(it); @@ -430,12 +433,35 @@ QueryStatus * ProcessList::tryGetProcessListElement(const String & current_query CancellationCode ProcessList::sendCancelToQuery(const String & current_query_id, const String & current_user, bool kill) { - auto lock = safeLock(); + QueryStatus * elem; - QueryStatus * elem = tryGetProcessListElement(current_query_id, current_user); + /// Cancelling the query should be done without the lock. + /// + /// Since it may be not that trivial, for example in case of distributed + /// queries it tries to cancel the query gracefully on shards and this can + /// take a while, so acquiring a lock during this time will lead to wait + /// all new queries for this cancellation. + /// + /// Another problem is that it can lead to a deadlock, because of + /// OvercommitTracker. + /// + /// So here we first set is_cancelling, and later reset it. + /// The ProcessListEntry cannot be destroy if is_cancelling is true. + { + auto lock = safeLock(); + elem = tryGetProcessListElement(current_query_id, current_user); + if (!elem) + return CancellationCode::NotFound; + elem->is_cancelling = true; + } - if (!elem) - return CancellationCode::NotFound; + SCOPE_EXIT({ + DENY_ALLOCATIONS_IN_SCOPE; + + auto lock = unsafeLock(); + elem->is_cancelling = false; + cancelled_cv.notify_all(); + }); return elem->cancelQuery(kill); } @@ -443,10 +469,28 @@ CancellationCode ProcessList::sendCancelToQuery(const String & current_query_id, void ProcessList::killAllQueries() { - auto lock = safeLock(); + std::vector cancelled_processes; + + SCOPE_EXIT({ + auto lock = safeLock(); + for (auto & cancelled_process : cancelled_processes) + cancelled_process->is_cancelling = false; + cancelled_cv.notify_all(); + }); + + { + auto lock = safeLock(); + cancelled_processes.reserve(processes.size()); + for (auto & process : processes) + { + cancelled_processes.push_back(&process); + process.is_cancelling = true; + } + } + + for (auto & cancelled_process : cancelled_processes) + cancelled_process->cancelQuery(true); - for (auto & process : processes) - process.cancelQuery(true); } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 203a1f114df..e7ad4e70712 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -100,6 +100,11 @@ protected: QueryPriorities::Handle priority_handle = nullptr; + /// True if query cancellation is in progress right now + /// ProcessListEntry should not be destroyed if is_cancelling is true + /// Flag changes is synced with ProcessListBase::mutex and notified with ProcessList::cancelled_cv + bool is_cancelling { false }; + /// KILL was send to the query std::atomic is_killed { false }; /// All data to the client already had been sent. @@ -331,6 +336,9 @@ protected: /// List of queries Container processes; + /// Notify about cancelled queries (done with ProcessListBase::mutex acquired). + mutable std::condition_variable cancelled_cv; + size_t max_size = 0; /// 0 means no limit. Otherwise, when limit exceeded, an exception is thrown. /// Stores per-user info: queries, statistics and limits From ba8cb7c086f5f29af196ff3f2a91322cdefb8799 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 19 Sep 2022 12:33:23 +0200 Subject: [PATCH 81/86] Fix --- tests/queries/0_stateless/02417_load_marks_async.reference | 1 - tests/queries/0_stateless/02417_load_marks_async.sh | 6 ------ 2 files changed, 7 deletions(-) diff --git a/tests/queries/0_stateless/02417_load_marks_async.reference b/tests/queries/0_stateless/02417_load_marks_async.reference index 541dab48def..7326d960397 100644 --- a/tests/queries/0_stateless/02417_load_marks_async.reference +++ b/tests/queries/0_stateless/02417_load_marks_async.reference @@ -1,2 +1 @@ Ok -Ok diff --git a/tests/queries/0_stateless/02417_load_marks_async.sh b/tests/queries/0_stateless/02417_load_marks_async.sh index 310258e6b3a..a5cbcd08f75 100755 --- a/tests/queries/0_stateless/02417_load_marks_async.sh +++ b/tests/queries/0_stateless/02417_load_marks_async.sh @@ -34,12 +34,6 @@ function test ${CLICKHOUSE_CLIENT} --query_id "${QUERY_ID}" -q "SELECT * FROM test SETTINGS load_marks_asynchronously=$1 FORMAT Null" ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS" - result=$(${CLICKHOUSE_CLIENT} -q "SELECT ProfileEvents['WaitMarksLoadMicroseconds'] FROM system.query_log WHERE query_id = '${QUERY_ID}' AND type = 'QueryFinish' AND current_database = currentDatabase()") - if [[ $result -ne 0 ]]; then - echo 'Ok' - else - echo 'F' - fi result=$(${CLICKHOUSE_CLIENT} -q "SELECT ProfileEvents['BackgroundLoadingMarksTasks'] FROM system.query_log WHERE query_id = '${QUERY_ID}' AND type = 'QueryFinish' AND current_database = currentDatabase()") if [[ $result -ne 0 ]]; then echo 'Ok' From 6ed53585abdf9c76c012bfe6cdd51bdf88d572b6 Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 19 Sep 2022 16:21:09 +0000 Subject: [PATCH 82/86] rearrange and fix profile event descriptions --- src/Common/ProfileEvents.cpp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 8f37fcde50d..2590d636028 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -301,6 +301,18 @@ The server successfully detected this situation and will download merged part fr M(DiskS3WriteRequestsThrottling, "Number of 429 and 503 errors in POST, DELETE, PUT and PATCH requests to DiskS3 storage.") \ M(DiskS3WriteRequestsRedirects, "Number of redirects in POST, DELETE, PUT and PATCH requests to DiskS3 storage.") \ \ + M(DeleteS3Objects, "Number of S3 API DeleteObjects calls.") \ + M(CopyS3Object, "Number of S3 API CopyObject calls.") \ + M(ListS3Objects, "Number of S3 API ListObjects calls.") \ + M(HeadS3Object, "Number of S3 API HeadObject calls.") \ + M(CreateS3MultipartUpload, "Number of S3 API CreateMultipartUpload calls.") \ + M(UploadS3PartCopy, "Number of S3 API UploadPartCopy calls.") \ + M(UploadS3Part, "Number of S3 API UploadPart calls.") \ + M(AbortS3MultipartUpload, "Number of S3 API AbortMultipartUpload calls.") \ + M(CompleteS3MultipartUpload, "Number of S3 API CompleteMultipartUpload calls.") \ + M(PutS3ObjectRequest, "Number of S3 API PutObject calls.") \ + M(GetS3ObjectRequest, "Number of S3 API GetObject calls.") \ + \ M(ReadBufferFromS3Microseconds, "Time spend in reading from S3.") \ M(ReadBufferFromS3Bytes, "Bytes read from S3.") \ M(ReadBufferFromS3RequestsErrors, "Number of exceptions while reading from S3.") \ @@ -410,17 +422,7 @@ The server successfully detected this situation and will download merged part fr M(OverflowBreak, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'break' and the result is incomplete.") \ M(OverflowThrow, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'throw' and exception was thrown.") \ M(OverflowAny, "Number of times approximate GROUP BY was in effect: when aggregation was performed only on top of first 'max_rows_to_group_by' unique keys and other keys were ignored due to 'group_by_overflow_mode' = 'any'.") \ - M(DeleteS3Objects, "Number of s3 API DeleteObjects be called") \ - M(CopyS3Object, "Number of s3 API CopyObject be called") \ - M(ListS3Objects, "Number of s3 API ListObjects be called") \ - M(HeadS3Object, "Number of s3 API HeadObject be called") \ - M(CreateS3MultipartUpload, "Number of s3 API CreateMultipartUpload be called") \ - M(UploadS3PartCopy, "Number of s3 API UploadPartCopy be called") \ - M(UploadS3Part, "Number of s3 API UploadS3Part be called") \ - M(AbortS3MultipartUpload, "Number of s3 API AbortMultipartUpload be called") \ - M(CompleteS3MultipartUpload, "Number of s3 API CompleteS3MultipartUpload be called") \ - M(PutS3ObjectRequest, "Number of s3 API PutS3ObjectRequest be called") \ - M(GetS3ObjectRequest, "Number of s3 API GetS3ObjectRequest be called") + namespace ProfileEvents { From f8aa738511b313fe7cbd06b12c2413b22f526aa4 Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 19 Sep 2022 17:23:22 +0000 Subject: [PATCH 83/86] more conventional profile events names --- src/Common/ProfileEvents.cpp | 22 ++++++------ .../ObjectStorages/S3/S3ObjectStorage.cpp | 34 +++++++++---------- src/IO/ReadBufferFromS3.cpp | 4 +-- src/IO/WriteBufferFromS3.cpp | 12 +++---- src/Storages/StorageS3.cpp | 10 +++--- .../test_profile_events_s3/test.py | 22 ++++++------ 6 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 2590d636028..a5eda3ca2b1 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -301,17 +301,17 @@ The server successfully detected this situation and will download merged part fr M(DiskS3WriteRequestsThrottling, "Number of 429 and 503 errors in POST, DELETE, PUT and PATCH requests to DiskS3 storage.") \ M(DiskS3WriteRequestsRedirects, "Number of redirects in POST, DELETE, PUT and PATCH requests to DiskS3 storage.") \ \ - M(DeleteS3Objects, "Number of S3 API DeleteObjects calls.") \ - M(CopyS3Object, "Number of S3 API CopyObject calls.") \ - M(ListS3Objects, "Number of S3 API ListObjects calls.") \ - M(HeadS3Object, "Number of S3 API HeadObject calls.") \ - M(CreateS3MultipartUpload, "Number of S3 API CreateMultipartUpload calls.") \ - M(UploadS3PartCopy, "Number of S3 API UploadPartCopy calls.") \ - M(UploadS3Part, "Number of S3 API UploadPart calls.") \ - M(AbortS3MultipartUpload, "Number of S3 API AbortMultipartUpload calls.") \ - M(CompleteS3MultipartUpload, "Number of S3 API CompleteMultipartUpload calls.") \ - M(PutS3ObjectRequest, "Number of S3 API PutObject calls.") \ - M(GetS3ObjectRequest, "Number of S3 API GetObject calls.") \ + M(S3DeleteObjects, "Number of S3 API DeleteObjects calls.") \ + M(S3CopyObject, "Number of S3 API CopyObject calls.") \ + M(S3ListObjects, "Number of S3 API ListObjects calls.") \ + M(S3HeadObject, "Number of S3 API HeadObject calls.") \ + M(S3CreateMultipartUpload, "Number of S3 API CreateMultipartUpload calls.") \ + M(S3UploadPartCopy, "Number of S3 API UploadPartCopy calls.") \ + M(S3UploadPart, "Number of S3 API UploadPart calls.") \ + M(S3AbortMultipartUpload, "Number of S3 API AbortMultipartUpload calls.") \ + M(S3CompleteMultipartUpload, "Number of S3 API CompleteMultipartUpload calls.") \ + M(S3PutObject, "Number of S3 API PutObject calls.") \ + M(S3GetObject, "Number of S3 API GetObject calls.") \ \ M(ReadBufferFromS3Microseconds, "Time spend in reading from S3.") \ M(ReadBufferFromS3Bytes, "Bytes read from S3.") \ diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index faa97c4848f..ae493ad30ac 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -34,14 +34,14 @@ namespace ProfileEvents { - extern const Event DeleteS3Objects; - extern const Event HeadS3Object; - extern const Event ListS3Objects; - extern const Event CopyS3Object; - extern const Event CreateS3MultipartUpload; - extern const Event UploadS3PartCopy; - extern const Event AbortS3MultipartUpload; - extern const Event CompleteS3MultipartUpload; + extern const Event S3DeleteObjects; + extern const Event S3HeadObject; + extern const Event S3ListObjects; + extern const Event S3CopyObject; + extern const Event S3CreateMultipartUpload; + extern const Event S3UploadPartCopy; + extern const Event S3AbortMultipartUpload; + extern const Event S3CompleteMultipartUpload; } namespace DB @@ -110,7 +110,7 @@ Aws::S3::Model::HeadObjectOutcome S3ObjectStorage::requestObjectHeadData(const s { auto client_ptr = client.get(); - ProfileEvents::increment(ProfileEvents::HeadS3Object); + ProfileEvents::increment(ProfileEvents::S3HeadObject); Aws::S3::Model::HeadObjectRequest request; request.SetBucket(bucket_from); request.SetKey(key); @@ -226,7 +226,7 @@ void S3ObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize auto settings_ptr = s3_settings.get(); auto client_ptr = client.get(); - ProfileEvents::increment(ProfileEvents::ListS3Objects); + ProfileEvents::increment(ProfileEvents::S3ListObjects); Aws::S3::Model::ListObjectsV2Request request; request.SetBucket(bucket); request.SetPrefix(path); @@ -255,7 +255,7 @@ void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exis { auto client_ptr = client.get(); - ProfileEvents::increment(ProfileEvents::DeleteS3Objects); + ProfileEvents::increment(ProfileEvents::S3DeleteObjects); Aws::S3::Model::DeleteObjectRequest request; request.SetBucket(bucket); request.SetKey(object.absolute_path); @@ -302,7 +302,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e Aws::S3::Model::Delete delkeys; delkeys.SetObjects(current_chunk); - ProfileEvents::increment(ProfileEvents::DeleteS3Objects); + ProfileEvents::increment(ProfileEvents::S3DeleteObjects); Aws::S3::Model::DeleteObjectsRequest request; request.SetBucket(bucket); request.SetDelete(delkeys); @@ -377,7 +377,7 @@ void S3ObjectStorage::copyObjectImpl( { auto client_ptr = client.get(); - ProfileEvents::increment(ProfileEvents::CopyS3Object); + ProfileEvents::increment(ProfileEvents::S3CopyObject); Aws::S3::Model::CopyObjectRequest request; request.SetCopySource(src_bucket + "/" + src_key); request.SetBucket(dst_bucket); @@ -426,7 +426,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( String multipart_upload_id; { - ProfileEvents::increment(ProfileEvents::CreateS3MultipartUpload); + ProfileEvents::increment(ProfileEvents::S3CreateMultipartUpload); Aws::S3::Model::CreateMultipartUploadRequest request; request.SetBucket(dst_bucket); request.SetKey(dst_key); @@ -445,7 +445,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( size_t upload_part_size = settings_ptr->s3_settings.min_upload_part_size; for (size_t position = 0, part_number = 1; position < size; ++part_number, position += upload_part_size) { - ProfileEvents::increment(ProfileEvents::UploadS3PartCopy); + ProfileEvents::increment(ProfileEvents::S3UploadPartCopy); Aws::S3::Model::UploadPartCopyRequest part_request; part_request.SetCopySource(src_bucket + "/" + src_key); part_request.SetBucket(dst_bucket); @@ -457,7 +457,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( auto outcome = client_ptr->UploadPartCopy(part_request); if (!outcome.IsSuccess()) { - ProfileEvents::increment(ProfileEvents::AbortS3MultipartUpload); + ProfileEvents::increment(ProfileEvents::S3AbortMultipartUpload); Aws::S3::Model::AbortMultipartUploadRequest abort_request; abort_request.SetBucket(dst_bucket); abort_request.SetKey(dst_key); @@ -472,7 +472,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( } { - ProfileEvents::increment(ProfileEvents::CompleteS3MultipartUpload); + ProfileEvents::increment(ProfileEvents::S3CompleteMultipartUpload); Aws::S3::Model::CompleteMultipartUploadRequest req; req.SetBucket(dst_bucket); req.SetKey(dst_key); diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index ded1333e887..eaad6089dab 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -24,7 +24,7 @@ namespace ProfileEvents extern const Event ReadBufferFromS3Bytes; extern const Event ReadBufferFromS3RequestsErrors; extern const Event ReadBufferSeekCancelConnection; - extern const Event GetS3ObjectRequest; + extern const Event S3GetObject; } namespace DB @@ -276,7 +276,7 @@ SeekableReadBuffer::Range ReadBufferFromS3::getRemainingReadRange() const std::unique_ptr ReadBufferFromS3::initialize() { - ProfileEvents::increment(ProfileEvents::GetS3ObjectRequest); + ProfileEvents::increment(ProfileEvents::S3GetObject); Aws::S3::Model::GetObjectRequest req; req.SetBucket(bucket); req.SetKey(key); diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 0917b5e03b1..6deccebc7f6 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -26,9 +26,9 @@ namespace ProfileEvents { extern const Event WriteBufferFromS3Bytes; extern const Event S3WriteBytes; - extern const Event CompleteS3MultipartUpload; - extern const Event UploadS3Part; - extern const Event PutS3ObjectRequest; + extern const Event S3CompleteMultipartUpload; + extern const Event S3UploadPart; + extern const Event S3PutObject; } namespace DB @@ -308,7 +308,7 @@ void WriteBufferFromS3::fillUploadRequest(Aws::S3::Model::UploadPartRequest & re void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) { - ProfileEvents::increment(ProfileEvents::UploadS3Part); + ProfileEvents::increment(ProfileEvents::S3UploadPart); auto outcome = client_ptr->UploadPart(task.req); if (outcome.IsSuccess()) @@ -332,7 +332,7 @@ void WriteBufferFromS3::completeMultipartUpload() if (tags.empty()) throw Exception("Failed to complete multipart upload. No parts have uploaded", ErrorCodes::S3_ERROR); - ProfileEvents::increment(ProfileEvents::CompleteS3MultipartUpload); + ProfileEvents::increment(ProfileEvents::S3CompleteMultipartUpload); Aws::S3::Model::CompleteMultipartUploadRequest req; req.SetBucket(bucket); req.SetKey(key); @@ -436,7 +436,7 @@ void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req) void WriteBufferFromS3::processPutRequest(const PutObjectTask & task) { - ProfileEvents::increment(ProfileEvents::PutS3ObjectRequest); + ProfileEvents::increment(ProfileEvents::S3PutObject); auto outcome = client_ptr->PutObject(task.req); bool with_pool = static_cast(schedule); if (outcome.IsSuccess()) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 7a24d63f186..62184a9419c 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -66,8 +66,8 @@ static const String PARTITION_ID_WILDCARD = "{_partition_id}"; namespace ProfileEvents { - extern const Event DeleteS3Objects; - extern const Event ListS3Objects; + extern const Event S3DeleteObjects; + extern const Event S3ListObjects; } namespace DB @@ -171,7 +171,7 @@ private: { buffer.clear(); - ProfileEvents::increment(ProfileEvents::ListS3Objects); + ProfileEvents::increment(ProfileEvents::S3ListObjects); outcome = client.ListObjectsV2(request); if (!outcome.IsSuccess()) throw Exception(ErrorCodes::S3_ERROR, "Could not list objects in bucket {} with prefix {}, S3 exception: {}, message: {}", @@ -567,7 +567,7 @@ static bool checkIfObjectExists(const std::shared_ptr & request.SetPrefix(key); while (!is_finished) { - ProfileEvents::increment(ProfileEvents::ListS3Objects); + ProfileEvents::increment(ProfileEvents::S3ListObjects); outcome = client->ListObjectsV2(request); if (!outcome.IsSuccess()) throw Exception( @@ -1045,7 +1045,7 @@ void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, delkeys.AddObjects(std::move(obj)); } - ProfileEvents::increment(ProfileEvents::DeleteS3Objects); + ProfileEvents::increment(ProfileEvents::S3DeleteObjects); Aws::S3::Model::DeleteObjectsRequest request; request.SetBucket(s3_configuration.uri.bucket); request.SetDelete(delkeys); diff --git a/tests/integration/test_profile_events_s3/test.py b/tests/integration/test_profile_events_s3/test.py index 28016264c9c..768498eb9ee 100644 --- a/tests/integration/test_profile_events_s3/test.py +++ b/tests/integration/test_profile_events_s3/test.py @@ -56,17 +56,17 @@ init_list = { "DiskS3WriteRequestsErrorsTotal": 0, "DiskS3WriteRequestsErrors503": 0, "DiskS3WriteRequestsRedirects": 0, - "DeleteS3Objects": 0, - "CopyS3Object": 0, - "ListS3Objects": 0, - "HeadS3Object": 0, - "CreateS3MultipartUpload": 0, - "UploadS3PartCopy": 0, - "UploadS3Part": 0, - "AbortS3MultipartUpload": 0, - "CompleteS3MultipartUpload": 0, - "PutS3ObjectRequest": 0, - "GetS3ObjectRequest": 0, + "S3DeleteObjects": 0, + "S3CopyObject": 0, + "S3ListObjects": 0, + "S3HeadObject": 0, + "S3CreateMultipartUpload": 0, + "S3UploadPartCopy": 0, + "S3UploadPart": 0, + "S3AbortMultipartUpload": 0, + "S3CompleteMultipartUpload": 0, + "S3PutObject": 0, + "S3GetObject": 0, } From 7a1d4a404af4c6165fa18257bf00c93d98f50453 Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 19 Sep 2022 18:10:47 +0000 Subject: [PATCH 84/86] add DiskS3* profile events per S3 API calls --- src/Common/ProfileEvents.cpp | 14 +++++++++++++- src/Disks/ObjectStorages/IObjectStorage.cpp | 2 ++ src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp | 17 +++++++++++++++++ src/IO/ReadBufferFromS3.cpp | 4 ++++ src/IO/ReadSettings.h | 3 +++ src/IO/WriteBufferFromS3.cpp | 12 ++++++++++++ src/IO/WriteSettings.h | 3 +++ .../integration/test_profile_events_s3/test.py | 11 +++++++++++ 8 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index a5eda3ca2b1..46bec669626 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -301,7 +301,7 @@ The server successfully detected this situation and will download merged part fr M(DiskS3WriteRequestsThrottling, "Number of 429 and 503 errors in POST, DELETE, PUT and PATCH requests to DiskS3 storage.") \ M(DiskS3WriteRequestsRedirects, "Number of redirects in POST, DELETE, PUT and PATCH requests to DiskS3 storage.") \ \ - M(S3DeleteObjects, "Number of S3 API DeleteObjects calls.") \ + M(S3DeleteObjects, "Number of S3 API DeleteObject(s) calls.") \ M(S3CopyObject, "Number of S3 API CopyObject calls.") \ M(S3ListObjects, "Number of S3 API ListObjects calls.") \ M(S3HeadObject, "Number of S3 API HeadObject calls.") \ @@ -313,6 +313,18 @@ The server successfully detected this situation and will download merged part fr M(S3PutObject, "Number of S3 API PutObject calls.") \ M(S3GetObject, "Number of S3 API GetObject calls.") \ \ + M(DiskS3DeleteObjects, "Number of DiskS3 API DeleteObject(s) calls.") \ + M(DiskS3CopyObject, "Number of DiskS3 API CopyObject calls.") \ + M(DiskS3ListObjects, "Number of DiskS3 API ListObjects calls.") \ + M(DiskS3HeadObject, "Number of DiskS3 API HeadObject calls.") \ + M(DiskS3CreateMultipartUpload, "Number of DiskS3 API CreateMultipartUpload calls.") \ + M(DiskS3UploadPartCopy, "Number of DiskS3 API UploadPartCopy calls.") \ + M(DiskS3UploadPart, "Number of DiskS3 API UploadPart calls.") \ + M(DiskS3AbortMultipartUpload, "Number of DiskS3 API AbortMultipartUpload calls.") \ + M(DiskS3CompleteMultipartUpload, "Number of DiskS3 API CompleteMultipartUpload calls.") \ + M(DiskS3PutObject, "Number of DiskS3 API PutObject calls.") \ + M(DiskS3GetObject, "Number of DiskS3 API GetObject calls.") \ + \ M(ReadBufferFromS3Microseconds, "Time spend in reading from S3.") \ M(ReadBufferFromS3Bytes, "Bytes read from S3.") \ M(ReadBufferFromS3RequestsErrors, "Number of exceptions while reading from S3.") \ diff --git a/src/Disks/ObjectStorages/IObjectStorage.cpp b/src/Disks/ObjectStorages/IObjectStorage.cpp index 1a128770015..65720ec3937 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.cpp +++ b/src/Disks/ObjectStorages/IObjectStorage.cpp @@ -61,6 +61,7 @@ ReadSettings IObjectStorage::patchSettings(const ReadSettings & read_settings) c std::unique_lock lock{throttlers_mutex}; ReadSettings settings{read_settings}; settings.remote_throttler = remote_read_throttler; + settings.for_object_storage = true; return settings; } @@ -69,6 +70,7 @@ WriteSettings IObjectStorage::patchSettings(const WriteSettings & write_settings std::unique_lock lock{throttlers_mutex}; WriteSettings settings{write_settings}; settings.remote_throttler = remote_write_throttler; + settings.for_object_storage = true; return settings; } diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index ae493ad30ac..9b6976e7423 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -42,6 +42,15 @@ namespace ProfileEvents extern const Event S3UploadPartCopy; extern const Event S3AbortMultipartUpload; extern const Event S3CompleteMultipartUpload; + + extern const Event DiskS3DeleteObjects; + extern const Event DiskS3HeadObject; + extern const Event DiskS3ListObjects; + extern const Event DiskS3CopyObject; + extern const Event DiskS3CreateMultipartUpload; + extern const Event DiskS3UploadPartCopy; + extern const Event DiskS3AbortMultipartUpload; + extern const Event DiskS3CompleteMultipartUpload; } namespace DB @@ -111,6 +120,7 @@ Aws::S3::Model::HeadObjectOutcome S3ObjectStorage::requestObjectHeadData(const s auto client_ptr = client.get(); ProfileEvents::increment(ProfileEvents::S3HeadObject); + ProfileEvents::increment(ProfileEvents::DiskS3HeadObject); Aws::S3::Model::HeadObjectRequest request; request.SetBucket(bucket_from); request.SetKey(key); @@ -227,6 +237,7 @@ void S3ObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize auto client_ptr = client.get(); ProfileEvents::increment(ProfileEvents::S3ListObjects); + ProfileEvents::increment(ProfileEvents::DiskS3ListObjects); Aws::S3::Model::ListObjectsV2Request request; request.SetBucket(bucket); request.SetPrefix(path); @@ -256,6 +267,7 @@ void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exis auto client_ptr = client.get(); ProfileEvents::increment(ProfileEvents::S3DeleteObjects); + ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); Aws::S3::Model::DeleteObjectRequest request; request.SetBucket(bucket); request.SetKey(object.absolute_path); @@ -303,6 +315,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e delkeys.SetObjects(current_chunk); ProfileEvents::increment(ProfileEvents::S3DeleteObjects); + ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); Aws::S3::Model::DeleteObjectsRequest request; request.SetBucket(bucket); request.SetDelete(delkeys); @@ -378,6 +391,7 @@ void S3ObjectStorage::copyObjectImpl( auto client_ptr = client.get(); ProfileEvents::increment(ProfileEvents::S3CopyObject); + ProfileEvents::increment(ProfileEvents::DiskS3CopyObject); Aws::S3::Model::CopyObjectRequest request; request.SetCopySource(src_bucket + "/" + src_key); request.SetBucket(dst_bucket); @@ -427,6 +441,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( { ProfileEvents::increment(ProfileEvents::S3CreateMultipartUpload); + ProfileEvents::increment(ProfileEvents::DiskS3CreateMultipartUpload); Aws::S3::Model::CreateMultipartUploadRequest request; request.SetBucket(dst_bucket); request.SetKey(dst_key); @@ -446,6 +461,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( for (size_t position = 0, part_number = 1; position < size; ++part_number, position += upload_part_size) { ProfileEvents::increment(ProfileEvents::S3UploadPartCopy); + ProfileEvents::increment(ProfileEvents::DiskS3UploadPartCopy); Aws::S3::Model::UploadPartCopyRequest part_request; part_request.SetCopySource(src_bucket + "/" + src_key); part_request.SetBucket(dst_bucket); @@ -458,6 +474,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( if (!outcome.IsSuccess()) { ProfileEvents::increment(ProfileEvents::S3AbortMultipartUpload); + ProfileEvents::increment(ProfileEvents::DiskS3AbortMultipartUpload); Aws::S3::Model::AbortMultipartUploadRequest abort_request; abort_request.SetBucket(dst_bucket); abort_request.SetKey(dst_key); diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index eaad6089dab..556ba960352 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -25,6 +25,7 @@ namespace ProfileEvents extern const Event ReadBufferFromS3RequestsErrors; extern const Event ReadBufferSeekCancelConnection; extern const Event S3GetObject; + extern const Event DiskS3GetObject; } namespace DB @@ -277,6 +278,9 @@ SeekableReadBuffer::Range ReadBufferFromS3::getRemainingReadRange() const std::unique_ptr ReadBufferFromS3::initialize() { ProfileEvents::increment(ProfileEvents::S3GetObject); + if (read_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3GetObject); + Aws::S3::Model::GetObjectRequest req; req.SetBucket(bucket); req.SetKey(key); diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 1307d0faa7f..1fa4aa637f5 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -105,6 +105,9 @@ struct ReadSettings size_t http_retry_max_backoff_ms = 1600; bool http_skip_not_found_url_for_globs = true; + /// Monitoring + bool for_object_storage = false; // to choose which profile events should be incremented + ReadSettings adjustBufferSize(size_t file_size) const { ReadSettings res = *this; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 6deccebc7f6..88e8462d52f 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -26,9 +26,14 @@ namespace ProfileEvents { extern const Event WriteBufferFromS3Bytes; extern const Event S3WriteBytes; + extern const Event S3CompleteMultipartUpload; extern const Event S3UploadPart; extern const Event S3PutObject; + + extern const Event DiskS3CompleteMultipartUpload; + extern const Event DiskS3UploadPart; + extern const Event DiskS3PutObject; } namespace DB @@ -309,6 +314,9 @@ void WriteBufferFromS3::fillUploadRequest(Aws::S3::Model::UploadPartRequest & re void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) { ProfileEvents::increment(ProfileEvents::S3UploadPart); + if (write_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3UploadPart); + auto outcome = client_ptr->UploadPart(task.req); if (outcome.IsSuccess()) @@ -333,6 +341,8 @@ void WriteBufferFromS3::completeMultipartUpload() throw Exception("Failed to complete multipart upload. No parts have uploaded", ErrorCodes::S3_ERROR); ProfileEvents::increment(ProfileEvents::S3CompleteMultipartUpload); + if (write_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Aws::S3::Model::CompleteMultipartUploadRequest req; req.SetBucket(bucket); req.SetKey(key); @@ -437,6 +447,8 @@ void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req) void WriteBufferFromS3::processPutRequest(const PutObjectTask & task) { ProfileEvents::increment(ProfileEvents::S3PutObject); + if (write_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3PutObject); auto outcome = client_ptr->PutObject(task.req); bool with_pool = static_cast(schedule); if (outcome.IsSuccess()) diff --git a/src/IO/WriteSettings.h b/src/IO/WriteSettings.h index ec2b5d17ec2..38a706997cf 100644 --- a/src/IO/WriteSettings.h +++ b/src/IO/WriteSettings.h @@ -15,6 +15,9 @@ struct WriteSettings bool enable_filesystem_cache_on_write_operations = false; bool enable_filesystem_cache_log = false; bool is_file_cache_persistent = false; + + /// Monitoring + bool for_object_storage = false; // to choose which profile events should be incremented }; } diff --git a/tests/integration/test_profile_events_s3/test.py b/tests/integration/test_profile_events_s3/test.py index 768498eb9ee..7b857c20f3a 100644 --- a/tests/integration/test_profile_events_s3/test.py +++ b/tests/integration/test_profile_events_s3/test.py @@ -67,6 +67,17 @@ init_list = { "S3CompleteMultipartUpload": 0, "S3PutObject": 0, "S3GetObject": 0, + "DiskS3DeleteObjects": 0, + "DiskS3CopyObject": 0, + "DiskS3ListObjects": 0, + "DiskS3HeadObject": 0, + "DiskS3CreateMultipartUpload": 0, + "DiskS3UploadPartCopy": 0, + "DiskS3UploadPart": 0, + "DiskS3AbortMultipartUpload": 0, + "DiskS3CompleteMultipartUpload": 0, + "DiskS3PutObject": 0, + "DiskS3GetObject": 0, } From 2ef696ffe1d0a6153f0d47281732fb30ce6b542d Mon Sep 17 00:00:00 2001 From: serxa Date: Mon, 19 Sep 2022 18:40:32 +0000 Subject: [PATCH 85/86] fix issues --- .../ObjectStorages/S3/S3ObjectStorage.cpp | 5 +++-- src/IO/ReadBufferFromS3.cpp | 10 +++++----- src/IO/S3Common.cpp | 16 ++++++++++++--- src/IO/S3Common.h | 4 ++-- src/IO/WriteBufferFromS3.cpp | 20 ++++++++++++++++--- src/Storages/StorageS3.cpp | 4 ++-- 6 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 9b6976e7423..ca9996c8e48 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -236,8 +236,6 @@ void S3ObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize auto settings_ptr = s3_settings.get(); auto client_ptr = client.get(); - ProfileEvents::increment(ProfileEvents::S3ListObjects); - ProfileEvents::increment(ProfileEvents::DiskS3ListObjects); Aws::S3::Model::ListObjectsV2Request request; request.SetBucket(bucket); request.SetPrefix(path); @@ -246,6 +244,8 @@ void S3ObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize Aws::S3::Model::ListObjectsV2Outcome outcome; do { + ProfileEvents::increment(ProfileEvents::S3ListObjects); + ProfileEvents::increment(ProfileEvents::DiskS3ListObjects); outcome = client_ptr->ListObjectsV2(request); throwIfError(outcome); @@ -490,6 +490,7 @@ void S3ObjectStorage::copyObjectMultipartImpl( { ProfileEvents::increment(ProfileEvents::S3CompleteMultipartUpload); + ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Aws::S3::Model::CompleteMultipartUploadRequest req; req.SetBucket(dst_bucket); req.SetKey(dst_key); diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 556ba960352..c17bf731c62 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -250,7 +250,7 @@ size_t ReadBufferFromS3::getFileSize() if (file_size) return *file_size; - auto object_size = S3::getObjectSize(client_ptr, bucket, key, version_id); + auto object_size = S3::getObjectSize(client_ptr, bucket, key, version_id, true, read_settings.for_object_storage); file_size = object_size; return *file_size; @@ -277,10 +277,6 @@ SeekableReadBuffer::Range ReadBufferFromS3::getRemainingReadRange() const std::unique_ptr ReadBufferFromS3::initialize() { - ProfileEvents::increment(ProfileEvents::S3GetObject); - if (read_settings.for_object_storage) - ProfileEvents::increment(ProfileEvents::DiskS3GetObject); - Aws::S3::Model::GetObjectRequest req; req.SetBucket(bucket); req.SetKey(key); @@ -321,6 +317,10 @@ std::unique_ptr ReadBufferFromS3::initialize() offset); } + ProfileEvents::increment(ProfileEvents::S3GetObject); + if (read_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3GetObject); + Aws::S3::Model::GetObjectOutcome outcome = client_ptr->GetObject(req); if (outcome.IsSuccess()) diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index e97fa707c13..f807c8bc9b6 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -35,6 +35,12 @@ # include +namespace ProfileEvents +{ + extern const Event S3HeadObject; + extern const Event DiskS3HeadObject; +} + namespace DB { @@ -812,8 +818,12 @@ namespace S3 } - S3::ObjectInfo getObjectInfo(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error) + S3::ObjectInfo getObjectInfo(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3) { + ProfileEvents::increment(ProfileEvents::S3HeadObject); + if (for_disk_s3) + ProfileEvents::increment(ProfileEvents::DiskS3HeadObject); + Aws::S3::Model::HeadObjectRequest req; req.SetBucket(bucket); req.SetKey(key); @@ -835,9 +845,9 @@ namespace S3 return {}; } - size_t getObjectSize(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error) + size_t getObjectSize(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3) { - return getObjectInfo(client_ptr, bucket, key, version_id, throw_on_error).size; + return getObjectInfo(client_ptr, bucket, key, version_id, throw_on_error, for_disk_s3).size; } } diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index ddc23b7aa8a..05391b58403 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -124,9 +124,9 @@ struct ObjectInfo time_t last_modification_time = 0; }; -S3::ObjectInfo getObjectInfo(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id = {}, bool throw_on_error = true); +S3::ObjectInfo getObjectInfo(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3); -size_t getObjectSize(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id = {}, bool throw_on_error = true); +size_t getObjectSize(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3); } diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 88e8462d52f..d78ce533273 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -27,10 +27,14 @@ namespace ProfileEvents extern const Event WriteBufferFromS3Bytes; extern const Event S3WriteBytes; + extern const Event S3HeadObject; + extern const Event S3CreateMultipartUpload; extern const Event S3CompleteMultipartUpload; extern const Event S3UploadPart; extern const Event S3PutObject; + extern const Event DiskS3HeadObject; + extern const Event DiskS3CreateMultipartUpload; extern const Event DiskS3CompleteMultipartUpload; extern const Event DiskS3UploadPart; extern const Event DiskS3PutObject; @@ -177,10 +181,15 @@ void WriteBufferFromS3::finalizeImpl() { LOG_TRACE(log, "Checking object {} exists after upload", key); + Aws::S3::Model::HeadObjectRequest request; request.SetBucket(bucket); request.SetKey(key); + ProfileEvents::increment(ProfileEvents::S3HeadObject); + if (write_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3HeadObject); + auto response = client_ptr->HeadObject(request); if (!response.IsSuccess()) @@ -202,6 +211,10 @@ void WriteBufferFromS3::createMultipartUpload() if (object_metadata.has_value()) req.SetMetadata(object_metadata.value()); + ProfileEvents::increment(ProfileEvents::S3CreateMultipartUpload); + if (write_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3CreateMultipartUpload); + auto outcome = client_ptr->CreateMultipartUpload(req); if (outcome.IsSuccess()) @@ -340,9 +353,6 @@ void WriteBufferFromS3::completeMultipartUpload() if (tags.empty()) throw Exception("Failed to complete multipart upload. No parts have uploaded", ErrorCodes::S3_ERROR); - ProfileEvents::increment(ProfileEvents::S3CompleteMultipartUpload); - if (write_settings.for_object_storage) - ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Aws::S3::Model::CompleteMultipartUploadRequest req; req.SetBucket(bucket); req.SetKey(key); @@ -357,6 +367,10 @@ void WriteBufferFromS3::completeMultipartUpload() req.SetMultipartUpload(multipart_upload); + ProfileEvents::increment(ProfileEvents::S3CompleteMultipartUpload); + if (write_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); + auto outcome = client_ptr->CompleteMultipartUpload(req); if (outcome.IsSuccess()) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 62184a9419c..ad91a30aa82 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -483,7 +483,7 @@ std::unique_ptr StorageS3Source::createS3ReadBuffer(const String & k if (it != object_infos.end()) object_size = it->second.size; else - object_size = DB::S3::getObjectSize(client, bucket, key, version_id, false); + object_size = DB::S3::getObjectSize(client, bucket, key, version_id, false, false); auto download_buffer_size = getContext()->getSettings().max_download_buffer_size; const bool use_parallel_download = download_buffer_size > 0 && download_thread_num > 1; @@ -1389,7 +1389,7 @@ std::optional StorageS3::tryGetColumnsFromCache( /// Note that in case of exception in getObjectInfo returned info will be empty, /// but schema cache will handle this case and won't return columns from cache /// because we can't say that it's valid without last modification time. - info = S3::getObjectInfo(s3_configuration.client, s3_configuration.uri.bucket, *it, s3_configuration.uri.version_id, false); + info = S3::getObjectInfo(s3_configuration.client, s3_configuration.uri.bucket, *it, s3_configuration.uri.version_id, false, false); if (object_infos) (*object_infos)[path] = info; } From 5d8b8a7db6abbfbaa06895cb2efc47052b0755db Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 19 Sep 2022 20:42:02 +0200 Subject: [PATCH 86/86] Add deb Source for packages --- docker/packager/binary/Dockerfile | 2 +- packages/clickhouse-client.yaml | 4 ++++ packages/clickhouse-common-static-dbg.yaml | 4 ++++ packages/clickhouse-common-static.yaml | 4 ++++ packages/clickhouse-keeper-dbg.yaml | 4 ++++ packages/clickhouse-keeper.yaml | 3 +++ packages/clickhouse-server.yaml | 4 ++++ 7 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index c4244504923..db55c950241 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -46,7 +46,7 @@ RUN apt-get install binutils-riscv64-linux-gnu # Architecture of the image when BuildKit/buildx is used ARG TARGETARCH -ARG NFPM_VERSION=2.16.0 +ARG NFPM_VERSION=2.18.1 RUN arch=${TARGETARCH:-amd64} \ && curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \ diff --git a/packages/clickhouse-client.yaml b/packages/clickhouse-client.yaml index 642d66f5475..459a09ee0b8 100644 --- a/packages/clickhouse-client.yaml +++ b/packages/clickhouse-client.yaml @@ -30,6 +30,10 @@ overrides: depends: - clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING} +deb: + fields: + Source: clickhouse + contents: - src: root/etc/clickhouse-client/config.xml dst: /etc/clickhouse-client/config.xml diff --git a/packages/clickhouse-common-static-dbg.yaml b/packages/clickhouse-common-static-dbg.yaml index 12a1594bd30..b2d2b3aaf26 100644 --- a/packages/clickhouse-common-static-dbg.yaml +++ b/packages/clickhouse-common-static-dbg.yaml @@ -20,6 +20,10 @@ description: | debugging symbols for clickhouse-common-static This package contains the debugging symbols for clickhouse-common. +deb: + fields: + Source: clickhouse + contents: - src: root/usr/lib/debug/usr/bin/clickhouse.debug dst: /usr/lib/debug/usr/bin/clickhouse.debug diff --git a/packages/clickhouse-common-static.yaml b/packages/clickhouse-common-static.yaml index 527b6a24703..c77914d0d69 100644 --- a/packages/clickhouse-common-static.yaml +++ b/packages/clickhouse-common-static.yaml @@ -26,6 +26,10 @@ description: | that allows generating analytical data reports in real time. This package provides common files for both clickhouse server and client +deb: + fields: + Source: clickhouse + contents: - src: root/usr/bin/clickhouse dst: /usr/bin/clickhouse diff --git a/packages/clickhouse-keeper-dbg.yaml b/packages/clickhouse-keeper-dbg.yaml index 2c70b7ad4aa..a6be9ec9e97 100644 --- a/packages/clickhouse-keeper-dbg.yaml +++ b/packages/clickhouse-keeper-dbg.yaml @@ -14,6 +14,10 @@ description: | debugging symbols for clickhouse-keeper This package contains the debugging symbols for clickhouse-keeper. +deb: + fields: + Source: clickhouse + contents: - src: root/usr/lib/debug/usr/bin/clickhouse-keeper.debug dst: /usr/lib/debug/usr/bin/clickhouse-keeper.debug diff --git a/packages/clickhouse-keeper.yaml b/packages/clickhouse-keeper.yaml index e99ac30f944..7803729c469 100644 --- a/packages/clickhouse-keeper.yaml +++ b/packages/clickhouse-keeper.yaml @@ -22,6 +22,9 @@ description: | Static clickhouse-keeper binary A stand-alone clickhouse-keeper package +deb: + fields: + Source: clickhouse contents: - src: root/etc/clickhouse-keeper diff --git a/packages/clickhouse-server.yaml b/packages/clickhouse-server.yaml index 28995689754..a94ad1e9169 100644 --- a/packages/clickhouse-server.yaml +++ b/packages/clickhouse-server.yaml @@ -37,6 +37,10 @@ overrides: depends: - clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING} +deb: + fields: + Source: clickhouse + contents: - src: root/etc/clickhouse-server dst: /etc/clickhouse-server