From cd94eda70457bac2e13534d8bf58d42c855af649 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 14 Nov 2023 13:25:47 +0000 Subject: [PATCH 001/289] Simplify --- src/Functions/toStartOfInterval.cpp | 130 +++++++++++++++++----------- 1 file changed, 78 insertions(+), 52 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 48bf88cb14c..0a3a409ea9f 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -324,38 +324,61 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - bool first_argument_is_date = false; + bool value_is_date = false; auto check_first_argument = [&] { - if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) + const DataTypePtr & type_arg1 = arguments[0].type; + if (!isDate(type_arg1) && !isDateTime(type_arg1) && !isDateTime64(type_arg1)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " - "Should be a date or a date with time", arguments[0].type->getName(), getName()); - first_argument_is_date = isDate(arguments[0].type); + "Should be a date or a date with time", type_arg1->getName(), getName()); + value_is_date = isDate(type_arg1); }; const DataTypeInterval * interval_type = nullptr; - bool result_type_is_date = false; - bool result_type_is_datetime = false; - auto check_interval_argument = [&] + enum class ResultType { - interval_type = checkAndGetDataType(arguments[1].type.get()); + Date, + DateTime, + DateTime64 + }; + ResultType result_type; + auto check_second_argument = [&] + { + const DataTypePtr & type_arg2 = arguments[1].type; + interval_type = checkAndGetDataType(type_arg2.get()); if (!interval_type) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " - "Should be an interval of time", arguments[1].type->getName(), getName()); - result_type_is_date = (interval_type->getKind() == IntervalKind::Year) - || (interval_type->getKind() == IntervalKind::Quarter) || (interval_type->getKind() == IntervalKind::Month) - || (interval_type->getKind() == IntervalKind::Week); - result_type_is_datetime = (interval_type->getKind() == IntervalKind::Day) || (interval_type->getKind() == IntervalKind::Hour) - || (interval_type->getKind() == IntervalKind::Minute) || (interval_type->getKind() == IntervalKind::Second); + "Should be an interval of time", type_arg2->getName(), getName()); + switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) + { + case IntervalKind::Nanosecond: + case IntervalKind::Microsecond: + case IntervalKind::Millisecond: + result_type = ResultType::DateTime64; + break; + case IntervalKind::Second: + case IntervalKind::Minute: + case IntervalKind::Hour: + case IntervalKind::Day: + result_type = ResultType::DateTime; + break; + case IntervalKind::Week: + case IntervalKind::Month: + case IntervalKind::Quarter: + case IntervalKind::Year: + result_type = ResultType::Date; + break; + } }; - auto check_timezone_argument = [&] + auto check_third_argument = [&] { - if (!WhichDataType(arguments[2].type).isString()) + const DataTypePtr & type_arg3 = arguments[2].type; + if (!isString(type_arg3)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " "This argument is optional and must be a constant string with timezone name", - arguments[2].type->getName(), getName()); - if (first_argument_is_date && result_type_is_date) + type_arg3->getName(), getName()); + if (value_is_date && result_type == ResultType::Date) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The timezone argument of function {} with interval type {} is allowed only when the 1st argument " "has the type DateTime or DateTime64", @@ -365,13 +388,13 @@ public: if (arguments.size() == 2) { check_first_argument(); - check_interval_argument(); + check_second_argument(); } else if (arguments.size() == 3) { check_first_argument(); - check_interval_argument(); - check_timezone_argument(); + check_second_argument(); + check_third_argument(); } else { @@ -380,24 +403,27 @@ public: getName(), arguments.size()); } - if (result_type_is_date) - return std::make_shared(); - else if (result_type_is_datetime) - return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, false)); - else + switch (result_type) { - auto scale = 0; + case ResultType::Date: + return std::make_shared(); + case ResultType::DateTime: + return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, false)); + case ResultType::DateTime64: + { + UInt32 scale = 0; + if (interval_type->getKind() == IntervalKind::Nanosecond) + scale = 9; + else if (interval_type->getKind() == IntervalKind::Microsecond) + scale = 6; + else if (interval_type->getKind() == IntervalKind::Millisecond) + scale = 3; - if (interval_type->getKind() == IntervalKind::Nanosecond) - scale = 9; - else if (interval_type->getKind() == IntervalKind::Microsecond) - scale = 6; - else if (interval_type->getKind() == IntervalKind::Millisecond) - scale = 3; - - return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, false)); + return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, false)); + } } + std::unreachable(); } bool useDefaultImplementationForConstants() const override { return true; } @@ -426,34 +452,34 @@ private: ColumnPtr dispatchForColumns( const ColumnWithTypeAndName & time_column, const ColumnWithTypeAndName & interval_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone) const { - const auto & from_datatype = *time_column.type.get(); - const auto which_type = WhichDataType(from_datatype); + const auto & time_column_type = *time_column.type.get(); + const auto & time_column_col = *time_column.column.get(); - if (which_type.isDateTime64()) + if (isDateTime64(time_column_type)) { - const auto * time_column_vec = checkAndGetColumn(time_column.column.get()); - auto scale = assert_cast(from_datatype).getScale(); + const auto * time_column_vec = checkAndGetColumn(time_column_col); + auto scale = assert_cast(time_column_type).getScale(); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, result_type, time_zone, scale); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, result_type, time_zone, scale); } - if (which_type.isDateTime()) + else if (isDateTime(time_column_type)) { - const auto * time_column_vec = checkAndGetColumn(time_column.column.get()); + const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, result_type, time_zone); } - if (which_type.isDate()) + else if (isDate(time_column_type)) { - const auto * time_column_vec = checkAndGetColumn(time_column.column.get()); + const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, result_type, time_zone); } - if (which_type.isDate32()) + else if (isDate32(time_column_type)) { - const auto * time_column_vec = checkAndGetColumn(time_column.column.get()); + const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, result_type, time_zone); } throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. " "Must contain dates or dates with time", getName()); @@ -502,7 +528,7 @@ private: return execute(from, time_column, num_units, result_type, time_zone, scale); } - UNREACHABLE(); + std::unreachable(); } template @@ -515,7 +541,7 @@ private: size_t size = time_data.size(); auto result_col = result_type->createColumn(); - auto *col_to = assert_cast(result_col.get()); + auto * col_to = assert_cast(result_col.get()); auto & result_data = col_to->getData(); result_data.resize(size); From aef1ea3df71d7d19c6267f6bb03ce994b7e14909 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 14 Nov 2023 17:49:08 +0000 Subject: [PATCH 002/289] Extend with origin argument, pt. I --- src/Functions/toStartOfInterval.cpp | 128 +++++++++++++++++++++------- 1 file changed, 95 insertions(+), 33 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 0a3a409ea9f..4d716637932 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -321,6 +321,11 @@ public: size_t getNumberOfArguments() const override { return 0; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2, 3}; } + + bool hasInformationAboutMonotonicity() const override { return true; } + Monotonicity getMonotonicityForRange(const IDataType &, const Field &, const Field &) const override { return { .is_monotonic = true, .is_always_monotonic = true }; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -329,7 +334,7 @@ public: { const DataTypePtr & type_arg1 = arguments[0].type; if (!isDate(type_arg1) && !isDateTime(type_arg1) && !isDateTime64(type_arg1)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 1st argument of function {}. " "Should be a date or a date with time", type_arg1->getName(), getName()); value_is_date = isDate(type_arg1); }; @@ -347,7 +352,7 @@ public: const DataTypePtr & type_arg2 = arguments[1].type; interval_type = checkAndGetDataType(type_arg2.get()); if (!interval_type) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 2nd argument of function {}. " "Should be an interval of time", type_arg2->getName(), getName()); switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) { @@ -371,18 +376,48 @@ public: } }; + enum class ThirdArgument + { + IsTimezone, + IsOrigin + }; + ThirdArgument third_argument; /// valid only if 3rd argument is given auto check_third_argument = [&] { const DataTypePtr & type_arg3 = arguments[2].type; - if (!isString(type_arg3)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " - "This argument is optional and must be a constant string with timezone name", + if (isString(type_arg3)) + { + third_argument = ThirdArgument::IsTimezone; + if (value_is_date && result_type == ResultType::Date) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The timezone argument of function {} with interval type {} is allowed only when the 1st argument has the type DateTime or DateTime64", + getName(), interval_type->getKind().toString()); + } + else if (isDateOrDate32OrDateTimeOrDateTime64(type_arg3)) + third_argument = ThirdArgument::IsOrigin; + else + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 3rd argument of function {}. " + "This argument is optional and must be a constant String with timezone name or a Date/Date32/DateTime/DateTime64 with a constant origin", type_arg3->getName(), getName()); + + }; + + auto check_fourth_argument = [&] + { + if (third_argument != ThirdArgument::IsOrigin) /// sanity check + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 3rd argument of function {}. " + "The third argument must a Date/Date32/DateTime/DateTime64 with a constant origin", + arguments[2].type->getName(), getName()); + + const DataTypePtr & type_arg4 = arguments[3].type; + if (!isString(type_arg4)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 4th argument of function {}. " + "This argument is optional and must be a constant String with timezone name", + type_arg4->getName(), getName()); if (value_is_date && result_type == ResultType::Date) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "The timezone argument of function {} with interval type {} is allowed only when the 1st argument " - "has the type DateTime or DateTime64", - getName(), interval_type->getKind().toString()); + "The timezone argument of function {} with interval type {} is allowed only when the 1st argument has the type DateTime or DateTime64", + getName(), interval_type->getKind().toString()); }; if (arguments.size() == 2) @@ -396,10 +431,17 @@ public: check_second_argument(); check_third_argument(); } + else if (arguments.size() == 4) + { + check_first_argument(); + check_second_argument(); + check_third_argument(); + check_fourth_argument(); + } else { throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3", + "Number of arguments for function {} doesn't match: passed {}, should be 2, 3 or 4", getName(), arguments.size()); } @@ -408,7 +450,10 @@ public: case ResultType::Date: return std::make_shared(); case ResultType::DateTime: - return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, false)); + { + const size_t time_zone_arg_num = (arguments.size() == 2 || (arguments.size() == 3 && third_argument == ThirdArgument::IsTimezone)) ? 2 : 3; + return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, time_zone_arg_num, 0, false)); + } case ResultType::DateTime64: { UInt32 scale = 0; @@ -419,7 +464,8 @@ public: else if (interval_type->getKind() == IntervalKind::Millisecond) scale = 3; - return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, false)); + const size_t time_zone_arg_num = (arguments.size() == 2 || (arguments.size() == 3 && third_argument == ThirdArgument::IsTimezone)) ? 2 : 3; + return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, time_zone_arg_num, 0, false)); } } @@ -433,8 +479,16 @@ public: { const auto & time_column = arguments[0]; const auto & interval_column = arguments[1]; - const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, 2, 0); - auto result_column = dispatchForColumns(time_column, interval_column, result_type, time_zone); + + ColumnWithTypeAndName origin_column; + const bool has_origin_arg = (arguments.size() == 3 && isDateOrDate32OrDateTimeOrDateTime64(arguments[2].type)) || arguments.size() == 4; + if (has_origin_arg) + origin_column = arguments[2]; + + const size_t time_zone_arg_num = (arguments.size() == 2 || (arguments.size() == 3 && isString(arguments[2].type))) ? 2 : 3; + const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, time_zone_arg_num, 0); + + auto result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); return result_column; } @@ -449,8 +503,8 @@ public: } private: - ColumnPtr dispatchForColumns( - const ColumnWithTypeAndName & time_column, const ColumnWithTypeAndName & interval_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone) const + ColumnPtr dispatchForTimeColumn( + const ColumnWithTypeAndName & time_column, const ColumnWithTypeAndName & interval_column, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone) const { const auto & time_column_type = *time_column.type.get(); const auto & time_column_col = *time_column.column.get(); @@ -461,25 +515,25 @@ private: auto scale = assert_cast(time_column_type).getScale(); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, result_type, time_zone, scale); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone, scale); } else if (isDateTime(time_column_type)) { const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDate(time_column_type)) { const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDate32(time_column_type)) { const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. " "Must contain dates or dates with time", getName()); @@ -487,7 +541,7 @@ private: template ColumnPtr dispatchForIntervalColumn( - const FromDataType & from, const ColumnType & time_column, const ColumnWithTypeAndName & interval_column, + const TimeDataType & time_data_type, const TimeColumnType & time_column, const ColumnWithTypeAndName & interval_column, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale = 1) const { const auto * interval_type = checkAndGetDataType(interval_column.type.get()); @@ -505,34 +559,34 @@ private: switch (interval_type->getKind()) { case IntervalKind::Nanosecond: - return execute(from, time_column, num_units, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Microsecond: - return execute(from, time_column, num_units, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Millisecond: - return execute(from, time_column, num_units, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Second: - return execute(from, time_column, num_units, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Minute: - return execute(from, time_column, num_units, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Hour: - return execute(from, time_column, num_units, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Day: - return execute(from, time_column, num_units, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Week: - return execute(from, time_column, num_units, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Month: - return execute(from, time_column, num_units, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Quarter: - return execute(from, time_column, num_units, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Year: - return execute(from, time_column, num_units, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); } std::unreachable(); } - template - ColumnPtr execute(const FromDataType &, const ColumnType & time_column_type, Int64 num_units, const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale) const + template + ColumnPtr execute(const TimeDataType &, const ColumnType & time_column_type, Int64 num_units, [[maybe_unused]] const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale) const { using ToColumnType = typename ToDataType::ColumnType; using ToFieldType = typename ToDataType::FieldType; @@ -547,6 +601,14 @@ private: Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale); + /// TODO: This part is missing. origin_column is either {} (<-- to check, you could do `origin_column.column == nullptr`) or not {} + /// In the former case, we can execute below existing code. + /// In the latter case, we need to read the actual origin value. As per `getArgumentsThatAreAlwaysConstant()` (see above), we + /// can be sure that origin_column is a `ColumnConst`. The second assumption we can reasonable make is that it has the same + /// type (Date/Date32/DateTime/DateTime64) as the time column (1st argument). Since the method we are in is already + /// templatized on the data type of the time column, we can use `checkAndGetColumnConst(...)` to cast + /// `origin_column.column` to a const column and then read the (const) value from it, and proceed with the calculations. + for (size_t i = 0; i != size; ++i) result_data[i] = static_cast( Transform::execute(time_data[i], num_units, time_zone, scale_multiplier)); From 188c469318e8a7a7e339bcf4048952181e4e324c Mon Sep 17 00:00:00 2001 From: yariks5s Date: Fri, 17 Nov 2023 14:22:39 +0000 Subject: [PATCH 003/289] done suggestings about logic --- src/Functions/toStartOfInterval.cpp | 341 +++------------------------- 1 file changed, 27 insertions(+), 314 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 4d716637932..29f8faa810c 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include #include #include @@ -21,300 +23,18 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ARGUMENT_OUT_OF_BOUND; - extern const int DECIMAL_OVERFLOW; } namespace { - constexpr auto function_name = "toStartOfInterval"; - - template - struct Transform; - - template <> - struct Transform - { - static UInt16 execute(UInt16 d, Int64 years, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfYearInterval(DayNum(d), years); - } - - static UInt16 execute(Int32 d, Int64 years, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfYearInterval(ExtendedDayNum(d), years); - } - - static UInt16 execute(UInt32 t, Int64 years, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); - } - - static UInt16 execute(Int64 t, Int64 years, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfYearInterval(time_zone.toDayNum(t / scale_multiplier), years); - } - }; - - template <> - struct Transform - { - static UInt16 execute(UInt16 d, Int64 quarters, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfQuarterInterval(DayNum(d), quarters); - } - - static UInt16 execute(Int32 d, Int64 quarters, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfQuarterInterval(ExtendedDayNum(d), quarters); - } - - static UInt16 execute(UInt32 t, Int64 quarters, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); - } - - static UInt16 execute(Int64 t, Int64 quarters, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t / scale_multiplier), quarters); - } - }; - - template <> - struct Transform - { - static UInt16 execute(UInt16 d, Int64 months, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfMonthInterval(DayNum(d), months); - } - - static UInt16 execute(Int32 d, Int64 months, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfMonthInterval(ExtendedDayNum(d), months); - } - - static UInt16 execute(UInt32 t, Int64 months, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); - } - - static UInt16 execute(Int64 t, Int64 months, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t / scale_multiplier), months); - } - }; - - template <> - struct Transform - { - static UInt16 execute(UInt16 d, Int64 weeks, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfWeekInterval(DayNum(d), weeks); - } - - static UInt16 execute(Int32 d, Int64 weeks, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfWeekInterval(ExtendedDayNum(d), weeks); - } - - static UInt16 execute(UInt32 t, Int64 weeks, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); - } - - static UInt16 execute(Int64 t, Int64 weeks, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t / scale_multiplier), weeks); - } - }; - - template <> - struct Transform - { - static UInt32 execute(UInt16 d, Int64 days, const DateLUTImpl & time_zone, Int64) - { - return static_cast(time_zone.toStartOfDayInterval(ExtendedDayNum(d), days)); - } - - static UInt32 execute(Int32 d, Int64 days, const DateLUTImpl & time_zone, Int64) - { - return static_cast(time_zone.toStartOfDayInterval(ExtendedDayNum(d), days)); - } - - static UInt32 execute(UInt32 t, Int64 days, const DateLUTImpl & time_zone, Int64) - { - return static_cast(time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days)); - } - - static Int64 execute(Int64 t, Int64 days, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfDayInterval(time_zone.toDayNum(t / scale_multiplier), days); - } - }; - - template <> - struct Transform - { - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32 t, Int64 hours, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfHourInterval(t, hours); - } - - static Int64 execute(Int64 t, Int64 hours, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfHourInterval(t / scale_multiplier, hours); - } - }; - - template <> - struct Transform - { - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32 t, Int64 minutes, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfMinuteInterval(t, minutes); - } - - static Int64 execute(Int64 t, Int64 minutes, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfMinuteInterval(t / scale_multiplier, minutes); - } - }; - - template <> - struct Transform - { - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32 t, Int64 seconds, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfSecondInterval(t, seconds); - } - - static Int64 execute(Int64 t, Int64 seconds, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfSecondInterval(t / scale_multiplier, seconds); - } - }; - - template <> - struct Transform - { - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { throwDateTimeIsNotSupported(function_name); } - - static Int64 execute(Int64 t, Int64 milliseconds, const DateLUTImpl &, Int64 scale_multiplier) - { - if (scale_multiplier < 1000) - { - Int64 t_milliseconds = 0; - if (common::mulOverflow(t, static_cast(1000) / scale_multiplier, t_milliseconds)) - throw DB::Exception(ErrorCodes::DECIMAL_OVERFLOW, "Numeric overflow"); - if (likely(t >= 0)) - return t_milliseconds / milliseconds * milliseconds; - else - return ((t_milliseconds + 1) / milliseconds - 1) * milliseconds; - } - else if (scale_multiplier > 1000) - { - Int64 scale_diff = scale_multiplier / static_cast(1000); - if (likely(t >= 0)) - return t / milliseconds / scale_diff * milliseconds; - else - return ((t + 1) / milliseconds / scale_diff - 1) * milliseconds; - } - else - if (likely(t >= 0)) - return t / milliseconds * milliseconds; - else - return ((t + 1) / milliseconds - 1) * milliseconds; - } - }; - - template <> - struct Transform - { - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { throwDateTimeIsNotSupported(function_name); } - - static Int64 execute(Int64 t, Int64 microseconds, const DateLUTImpl &, Int64 scale_multiplier) - { - if (scale_multiplier < 1000000) - { - Int64 t_microseconds = 0; - if (common::mulOverflow(t, static_cast(1000000) / scale_multiplier, t_microseconds)) - throw DB::Exception(ErrorCodes::DECIMAL_OVERFLOW, "Numeric overflow"); - if (likely(t >= 0)) - return t_microseconds / microseconds * microseconds; - else - return ((t_microseconds + 1) / microseconds - 1) * microseconds; - } - else if (scale_multiplier > 1000000) - { - Int64 scale_diff = scale_multiplier / static_cast(1000000); - if (likely(t >= 0)) - return t / microseconds / scale_diff * microseconds; - else - return ((t + 1) / microseconds / scale_diff - 1) * microseconds; - } - else - if (likely(t >= 0)) - return t / microseconds * microseconds; - else - return ((t + 1) / microseconds - 1) * microseconds; - } - }; - - template <> - struct Transform - { - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { throwDateTimeIsNotSupported(function_name); } - - static Int64 execute(Int64 t, Int64 nanoseconds, const DateLUTImpl &, Int64 scale_multiplier) - { - if (scale_multiplier < 1000000000) - { - Int64 t_nanoseconds = 0; - if (common::mulOverflow(t, (static_cast(1000000000) / scale_multiplier), t_nanoseconds)) - throw DB::Exception(ErrorCodes::DECIMAL_OVERFLOW, "Numeric overflow"); - if (likely(t >= 0)) - return t_nanoseconds / nanoseconds * nanoseconds; - else - return ((t_nanoseconds + 1) / nanoseconds - 1) * nanoseconds; - } - else - if (likely(t >= 0)) - return t / nanoseconds * nanoseconds; - else - return ((t + 1) / nanoseconds - 1) * nanoseconds; - } - }; class FunctionToStartOfInterval : public IFunction { public: static FunctionPtr create(ContextPtr) { return std::make_shared(); } - static constexpr auto name = function_name; + static constexpr auto name = "toStartOfInterval"; String getName() const override { return name; } bool isVariadic() const override { return true; } @@ -472,9 +192,6 @@ public: std::unreachable(); } - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /* input_rows_count */) const override { const auto & time_column = arguments[0]; @@ -485,23 +202,13 @@ public: if (has_origin_arg) origin_column = arguments[2]; - const size_t time_zone_arg_num = (arguments.size() == 2 || (arguments.size() == 3 && isString(arguments[2].type))) ? 2 : 3; + const size_t time_zone_arg_num = (arguments.size() == 3 && isString(arguments[2].type)) ? 2 : 3; const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, time_zone_arg_num, 0); auto result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); return result_column; } - bool hasInformationAboutMonotonicity() const override - { - return true; - } - - Monotonicity getMonotonicityForRange(const IDataType &, const Field &, const Field &) const override - { - return { .is_monotonic = true, .is_always_monotonic = true }; - } - private: ColumnPtr dispatchForTimeColumn( const ColumnWithTypeAndName & time_column, const ColumnWithTypeAndName & interval_column, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone) const @@ -535,11 +242,10 @@ private: if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. " - "Must contain dates or dates with time", getName()); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. Must contain dates or dates with time", getName()); } - template + template ColumnPtr dispatchForIntervalColumn( const TimeDataType & time_data_type, const TimeColumnType & time_column, const ColumnWithTypeAndName & interval_column, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale = 1) const @@ -547,16 +253,16 @@ private: const auto * interval_type = checkAndGetDataType(interval_column.type.get()); if (!interval_type) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for second argument of function {}, must be an interval of time.", getName()); + const auto * interval_column_const_int64 = checkAndGetColumnConst(interval_column.column.get()); if (!interval_column_const_int64) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, - "Illegal column for second argument of function {}, must be a const interval of time.", - getName()); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for second argument of function {}, must be a const interval of time.", getName()); + Int64 num_units = interval_column_const_int64->getValue(); if (num_units <= 0) throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Value for second argument of function {} must be positive.", getName()); - switch (interval_type->getKind()) + switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) { case IntervalKind::Nanosecond: return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); @@ -601,17 +307,24 @@ private: Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale); - /// TODO: This part is missing. origin_column is either {} (<-- to check, you could do `origin_column.column == nullptr`) or not {} - /// In the former case, we can execute below existing code. - /// In the latter case, we need to read the actual origin value. As per `getArgumentsThatAreAlwaysConstant()` (see above), we - /// can be sure that origin_column is a `ColumnConst`. The second assumption we can reasonable make is that it has the same - /// type (Date/Date32/DateTime/DateTime64) as the time column (1st argument). Since the method we are in is already - /// templatized on the data type of the time column, we can use `checkAndGetColumnConst(...)` to cast - /// `origin_column.column` to a const column and then read the (const) value from it, and proceed with the calculations. + if (origin_column.column == nullptr) + for (size_t i = 0; i != size; ++i) + result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_multiplier)); + else + { + UInt64 od = origin_column.column->get64(0); + + for (size_t i = 0; i != size; ++i) + { + auto td = time_data[i]; + if (od > size_t(td)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The origin must be before the end date/datetime"); + td -= od; + result_data[i] = static_cast(ToStartOfInterval::execute(td, num_units, time_zone, scale_multiplier)); - for (size_t i = 0; i != size; ++i) - result_data[i] = static_cast( - Transform::execute(time_data[i], num_units, time_zone, scale_multiplier)); + result_data[i] += scale_multiplier == 10 ? od : od / scale_multiplier; + } + } return result_col; } From 6605a375b642d0bc8b6ffccbd0400eb4ccee223e Mon Sep 17 00:00:00 2001 From: yariks5s Date: Fri, 17 Nov 2023 17:35:04 +0000 Subject: [PATCH 004/289] style check, docs, tests and modified logic --- .../functions/date-time-functions.md | 7 +- src/Functions/toStartOfInterval.cpp | 306 +++++++++++++++++- ...tart_of_interval_origin_overload.reference | 15 + ...6_to_start_of_interval_origin_overload.sql | 12 + 4 files changed, 335 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.reference create mode 100644 tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.sql diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 0364a610404..3f522eeb164 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -590,7 +590,7 @@ Rounds down a date with time to the start of the ten-minute interval. Rounds down the date with time to the start of the fifteen-minute interval. -## toStartOfInterval(time_or_data, INTERVAL x unit \[, time_zone\]) +## toStartOfInterval(time_or_data, INTERVAL x unit \[, origin_time_or_data\] \[, time_zone\]) This is a generalization of other functions named `toStartOf*`. For example, `toStartOfInterval(t, INTERVAL 1 year)` returns the same as `toStartOfYear(t)`, @@ -598,6 +598,11 @@ This is a generalization of other functions named `toStartOf*`. For example, `toStartOfInterval(t, INTERVAL 1 day)` returns the same as `toStartOfDay(t)`, `toStartOfInterval(t, INTERVAL 15 minute)` returns the same as `toStartOfFifteenMinutes(t)` etc. +Also it has an overload including `origin_time_or_data` column which emulates TimescaleDB's `time_bucket()` function, respectively PostgreSQL's `date_bin()` function: +``` SQL +SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), toIntervalMinute(1), toDateTime('2023-01-01 14:35:30')); +``` + ## toTime Converts a date with time to a certain fixed date, while preserving the time. diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 29f8faa810c..8529643ee70 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -23,11 +24,292 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ARGUMENT_OUT_OF_BOUND; + extern const int BAD_ARGUMENTS; } namespace { +constexpr auto function_name = "toStartOfInterval"; + +template +struct Transform; + +template <> +struct Transform +{ + static UInt16 execute(UInt16 d, Int64 years, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfYearInterval(DayNum(d), years); + } + + static UInt16 execute(Int32 d, Int64 years, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfYearInterval(ExtendedDayNum(d), years); + } + + static UInt16 execute(UInt32 t, Int64 years, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); + } + + static UInt16 execute(Int64 t, Int64 years, const DateLUTImpl & time_zone, Int64 scale_multiplier) + { + return time_zone.toStartOfYearInterval(time_zone.toDayNum(t / scale_multiplier), years); + } +}; + +template <> +struct Transform +{ + static UInt16 execute(UInt16 d, Int64 quarters, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfQuarterInterval(DayNum(d), quarters); + } + + static UInt16 execute(Int32 d, Int64 quarters, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfQuarterInterval(ExtendedDayNum(d), quarters); + } + + static UInt16 execute(UInt32 t, Int64 quarters, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); + } + + static UInt16 execute(Int64 t, Int64 quarters, const DateLUTImpl & time_zone, Int64 scale_multiplier) + { + return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t / scale_multiplier), quarters); + } +}; + +template <> +struct Transform +{ + static UInt16 execute(UInt16 d, Int64 months, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfMonthInterval(DayNum(d), months); + } + + static UInt16 execute(Int32 d, Int64 months, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfMonthInterval(ExtendedDayNum(d), months); + } + + static UInt16 execute(UInt32 t, Int64 months, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); + } + + static UInt16 execute(Int64 t, Int64 months, const DateLUTImpl & time_zone, Int64 scale_multiplier) + { + return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t / scale_multiplier), months); + } +}; + +template <> +struct Transform +{ + static UInt16 execute(UInt16 d, Int64 weeks, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfWeekInterval(DayNum(d), weeks); + } +static UInt16 execute(Int32 d, Int64 weeks, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfWeekInterval(ExtendedDayNum(d), weeks); + } + + static UInt16 execute(UInt32 t, Int64 weeks, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); + } + + static UInt16 execute(Int64 t, Int64 weeks, const DateLUTImpl & time_zone, Int64 scale_multiplier) + { + return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t / scale_multiplier), weeks); + } +}; + +template <> +struct Transform +{ + static UInt32 execute(UInt16 d, Int64 days, const DateLUTImpl & time_zone, Int64) + { + return static_cast(time_zone.toStartOfDayInterval(ExtendedDayNum(d), days)); + } + + static UInt32 execute(Int32 d, Int64 days, const DateLUTImpl & time_zone, Int64) + { + return static_cast(time_zone.toStartOfDayInterval(ExtendedDayNum(d), days)); + } + + static UInt32 execute(UInt32 t, Int64 days, const DateLUTImpl & time_zone, Int64) + { + return static_cast(time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days)); + } + + static Int64 execute(Int64 t, Int64 days, const DateLUTImpl & time_zone, Int64 scale_multiplier) + { + return time_zone.toStartOfDayInterval(time_zone.toDayNum(t / scale_multiplier), days); + } +}; + +template <> +struct Transform +{ + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(UInt32 t, Int64 hours, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfHourInterval(t, hours); + } + + static Int64 execute(Int64 t, Int64 hours, const DateLUTImpl & time_zone, Int64 scale_multiplier) + { + return time_zone.toStartOfHourInterval(t / scale_multiplier, hours); + } +}; + +template <> +struct Transform +{ + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(UInt32 t, Int64 minutes, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfMinuteInterval(t, minutes); + } + + static Int64 execute(Int64 t, Int64 minutes, const DateLUTImpl & time_zone, Int64 scale_multiplier) + { + return time_zone.toStartOfMinuteInterval(t / scale_multiplier, minutes); + } +}; + +template <> +struct Transform +{ + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(UInt32 t, Int64 seconds, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfSecondInterval(t, seconds); + } + + static Int64 execute(Int64 t, Int64 seconds, const DateLUTImpl & time_zone, Int64 scale_multiplier) + { + return time_zone.toStartOfSecondInterval(t / scale_multiplier, seconds); + } +}; + +template <> +struct Transform +{ + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { throwDateTimeIsNotSupported(function_name); } + + static Int64 execute(Int64 t, Int64 milliseconds, const DateLUTImpl &, Int64 scale_multiplier) + { + if (scale_multiplier < 1000) + { + Int64 t_milliseconds = 0; + if (common::mulOverflow(t, static_cast(1000) / scale_multiplier, t_milliseconds)) + throw DB::Exception(ErrorCodes::DECIMAL_OVERFLOW, "Numeric overflow"); + if (likely(t >= 0)) + return t_milliseconds / milliseconds * milliseconds; + else + return ((t_milliseconds + 1) / milliseconds - 1) * milliseconds; + } + else if (scale_multiplier > 1000) + { + Int64 scale_diff = scale_multiplier / static_cast(1000); + if (likely(t >= 0)) + return t / milliseconds / scale_diff * milliseconds; + else + return ((t + 1) / milliseconds / scale_diff - 1) * milliseconds; + } + else + if (likely(t >= 0)) + return t / milliseconds * milliseconds; + else + return ((t + 1) / milliseconds - 1) * milliseconds; + } +}; + +template <> +struct Transform +{ + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { throwDateTimeIsNotSupported(function_name); } + + static Int64 execute(Int64 t, Int64 microseconds, const DateLUTImpl &, Int64 scale_multiplier) + { + if (scale_multiplier < 1000000) + { + Int64 t_microseconds = 0; + if (common::mulOverflow(t, static_cast(1000000) / scale_multiplier, t_microseconds)) + throw DB::Exception(ErrorCodes::DECIMAL_OVERFLOW, "Numeric overflow"); + if (likely(t >= 0)) + return t_microseconds / microseconds * microseconds; + else + return ((t_microseconds + 1) / microseconds - 1) * microseconds; + } + else if (scale_multiplier > 1000000) + { + Int64 scale_diff = scale_multiplier / static_cast(1000000); + if (likely(t >= 0)) + return t / microseconds / scale_diff * microseconds; + else + return ((t + 1) / microseconds / scale_diff - 1) * microseconds; + } + else + if (likely(t >= 0)) + return t / microseconds * microseconds; + else + return ((t + 1) / microseconds - 1) * microseconds; + } +}; + +template <> +struct Transform +{ + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } + + static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { throwDateTimeIsNotSupported(function_name); } + + static Int64 execute(Int64 t, Int64 nanoseconds, const DateLUTImpl &, Int64 scale_multiplier) + { + if (scale_multiplier < 1000000000) + { + Int64 t_nanoseconds = 0; + if (common::mulOverflow(t, (static_cast(1000000000) / scale_multiplier), t_nanoseconds)) + throw DB::Exception(ErrorCodes::DECIMAL_OVERFLOW, "Numeric overflow"); + if (likely(t >= 0)) + return t_nanoseconds / nanoseconds * nanoseconds; + else + return ((t_nanoseconds + 1) / nanoseconds - 1) * nanoseconds; + } + else + if (likely(t >= 0)) + return t / nanoseconds * nanoseconds; + else + return ((t + 1) / nanoseconds - 1) * nanoseconds; + } +}; class FunctionToStartOfInterval : public IFunction { @@ -218,6 +500,10 @@ private: if (isDateTime64(time_column_type)) { + if (origin_column.column != nullptr) + if (!isDateTime64(origin_column.type.get())) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); + const auto * time_column_vec = checkAndGetColumn(time_column_col); auto scale = assert_cast(time_column_type).getScale(); @@ -226,18 +512,30 @@ private: } else if (isDateTime(time_column_type)) { + if (origin_column.column != nullptr) + if (!isDateTime(origin_column.type.get())) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); + const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDate(time_column_type)) { + if (origin_column.column != nullptr) + if (!isDate(origin_column.type.get())) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); + const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDate32(time_column_type)) { + if (origin_column.column != nullptr) + if (!isDate32(origin_column.type.get())) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); + const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); @@ -292,7 +590,7 @@ private: } template - ColumnPtr execute(const TimeDataType &, const ColumnType & time_column_type, Int64 num_units, [[maybe_unused]] const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale) const + ColumnPtr execute(const TimeDataType &, const ColumnType & time_column_type, Int64 num_units, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale) const { using ToColumnType = typename ToDataType::ColumnType; using ToFieldType = typename ToDataType::FieldType; @@ -309,18 +607,18 @@ private: if (origin_column.column == nullptr) for (size_t i = 0; i != size; ++i) - result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_multiplier)); + result_data[i] = static_cast(Transform::execute(time_data[i], num_units, time_zone, scale_multiplier)); else { UInt64 od = origin_column.column->get64(0); - + for (size_t i = 0; i != size; ++i) { auto td = time_data[i]; if (od > size_t(td)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The origin must be before the end date/datetime"); td -= od; - result_data[i] = static_cast(ToStartOfInterval::execute(td, num_units, time_zone, scale_multiplier)); + result_data[i] = static_cast(Transform::execute(td, num_units, time_zone, scale_multiplier)); result_data[i] += scale_multiplier == 10 ? od : od / scale_multiplier; } diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.reference b/tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.reference new file mode 100644 index 00000000000..7213925fb64 --- /dev/null +++ b/tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.reference @@ -0,0 +1,15 @@ +2023-03-01 16:55:00 +2023-02-01 16:55:00 +2023-03-01 16:55:00 +2023-02-01 16:55:00 +2023-03-01 16:55:00 +2023-03-01 16:55:00 +2023-02-01 16:55:00 +2023-03-01 16:55:00 +2023-02-01 16:55:00 +2023-03-01 16:55:00 +2023-01-02 15:44:30 +2023-01-02 15:44:30 +2023-01-02 14:45:30 +2023-01-02 14:45:30 +2023-01-02 diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.sql b/tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.sql new file mode 100644 index 00000000000..ce4c8f87811 --- /dev/null +++ b/tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.sql @@ -0,0 +1,12 @@ +set session_timezone = 'UTC'; +SELECT toStartOfInterval(number % 2 == 0 ? toDateTime64('2023-03-01 15:55:00', 2) : toDateTime64('2023-02-01 15:55:00', 2), toIntervalMinute(1), toDateTime64('2023-01-01 13:55:00', 2), 'Europe/Amsterdam') from numbers(5); +SELECT toStartOfInterval(number % 2 == 0 ? toDateTime('2023-03-01 15:55:00') : toDateTime('2023-02-01 15:55:00'), toIntervalMinute(1), toDateTime('2023-01-01 13:55:00'), 'Europe/Amsterdam') from numbers(5); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalHour(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalMinute(1), toDateTime64('2023-01-02 14:44:30', 2)); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30')); +SELECT toStartOfInterval(toDate('2023-01-02 14:45:50'), toIntervalWeek(1), toDate('2023-01-02 14:44:30')); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime64('2023-01-02 14:44:30', 2)); -- { serverError 43 } +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalMinute(1), toDate('2023-01-02 14:44:30')); -- { serverError 43 } +SELECT toStartOfInterval(toDateTime('2023-01-02 14:42:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30')); -- { serverError 36 } +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), number % 2 == 0 ? toDateTime('2023-02-01 15:55:00') : toDateTime('2023-01-01 15:55:00'), 'Europe/Amsterdam') from numbers(1); -- { serverError 44 } From f596ae7a2c02e7915917daca82dad717a7df2640 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 18 Nov 2023 16:24:56 +0000 Subject: [PATCH 005/289] Fix style --- src/Functions/toStartOfInterval.cpp | 7 ++++--- utils/check-style/aspell-ignore/en/aspell-dict.txt | 9 +++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 8529643ee70..10792922afb 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -20,11 +20,12 @@ namespace DB { namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ARGUMENT_OUT_OF_BOUND; extern const int BAD_ARGUMENTS; + extern const int DECIMAL_OVERFLOW; + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 5d11185ff76..7eaafe8a777 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -261,6 +261,7 @@ FOSDEM FQDN Failover FarmHash +FileLog FilesystemCacheBytes FilesystemCacheElements FilesystemCacheFiles @@ -278,7 +279,6 @@ FilesystemMainPathTotalBytes FilesystemMainPathTotalINodes FilesystemMainPathUsedBytes FilesystemMainPathUsedINodes -FileLog FixedString Flink ForEach @@ -571,13 +571,13 @@ NetworkSendPackets NodeJs NuRaft NumHexagons +NumPy NumToString NumToStringClassC NumberOfDatabases NumberOfDetachedByUserParts NumberOfDetachedParts NumberOfTables -NumPy OFNS OLAP OLTP @@ -588,10 +588,10 @@ OSGuestNiceTimeNormalized OSGuestTime OSGuestTimeCPU OSGuestTimeNormalized +OSIOWaitMicroseconds OSIOWaitTime OSIOWaitTimeCPU OSIOWaitTimeNormalized -OSIOWaitMicroseconds OSIdleTime OSIdleTimeCPU OSIdleTimeNormalized @@ -900,6 +900,7 @@ ThreadPoolRemoteFSReaderThreads ThreadPoolRemoteFSReaderThreadsActive ThreadsActive ThreadsInOvercommitTracker +TimescaleDB's Timeunit TinyLog Tkachenko @@ -1470,12 +1471,12 @@ fastops fcoverage fibonacci fifo +filelog filesystem filesystemAvailable filesystemCapacity filesystemFree filesystems -filelog finalizeAggregation fips firstLine From ea2ba82c1001febabdd8753246b64143e4bf1f6f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 18 Nov 2023 16:37:50 +0000 Subject: [PATCH 006/289] Update docs --- .../functions/date-time-functions.md | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 1291b570da4..989b39e46c1 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1345,7 +1345,7 @@ toStartOfTenMinutes(toDateTime('2023-04-21 10:23:00')): 2023-04-21 10:20:00 Rounds down the date with time to the start of the fifteen-minute interval. -## toStartOfInterval(date_or_date_with_time, INTERVAL x unit \[, origin_time_or_data\] \[, time_zone\]) +## toStartOfInterval This function generalizes other `toStartOf*()` functions. For example, - `toStartOfInterval(t, INTERVAL 1 year)` returns the same as `toStartOfYear(t)`, @@ -1372,21 +1372,21 @@ The calculation is performed relative to specific points in time: (*) hour intervals are special: the calculation is always performed relative to 00:00:00 (midnight) of the current day. As a result, only hour values between 1 and 23 are useful. -Also it has an overload including `origin_time_or_data` column which emulates TimescaleDB's `time_bucket()` function, respectively PostgreSQL's `date_bin()` function: -``` SQL -SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), toIntervalMinute(1), toDateTime('2023-01-01 14:35:30')); +**Syntax** + +```sql +toStartOfInterval(value, INTERVAL x unit[, time_zone]) +toStartOfInterval(value, INTERVAL x unit[, origin[, time_zone]]) ``` +The second overload emulates TimescaleDB's `time_bucket()` function, respectively PostgreSQL's `date_bin()` function, e.g. + +``` SQL +SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30')); +``` **See Also** - - - [date_trunc](#date_trunc) -Also it has an overload including `origin_time_or_data` column which emulates TimescaleDB's `time_bucket()` function, respectively PostgreSQL's `date_bin()` function: -``` SQL -SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), toIntervalMinute(1), toDateTime('2023-01-01 14:35:30')); -``` - ## toTime Converts a date with time to a certain fixed date, while preserving the time. @@ -2462,19 +2462,29 @@ SELECT └──────────────────────────┴───────────────────────────────┴──────────────────────────────────────┘ ``` -## timeSlots(StartTime, Duration,\[, Size\]) +## timeSlots For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. This is necessary, for example, when searching for pageviews in the corresponding session. Accepts DateTime and DateTime64 as ’StartTime’ argument. For DateTime, ’Duration’ and ’Size’ arguments must be `UInt32`. For ’DateTime64’ they must be `Decimal64`. Returns an array of DateTime/DateTime64 (return type matches the type of ’StartTime’). For DateTime64, the return value's scale can differ from the scale of ’StartTime’ --- the highest scale among all given arguments is taken. -Example: +**Syntax** + +```sql +timeSlots(StartTime, Duration,\[, Size\]) +``` + +**Example** + ```sql SELECT timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600)); SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299); SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0)); ``` + +Result: + ``` text ┌─timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600))─┐ │ ['2012-01-01 12:00:00','2012-01-01 12:30:00'] │ From 74449872180542ebdcc26e7410100b6d897efc65 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 20 Nov 2023 18:23:18 +0000 Subject: [PATCH 007/289] need to deal with week+ time intervals --- src/Functions/toStartOfInterval.cpp | 311 +----------------- ...to_start_of_interva_with_origin.reference} | 0 ...6_to_start_of_interval_origin_overload.sql | 12 - ...02916_to_start_of_interval_with_origin.sql | 42 +++ 4 files changed, 58 insertions(+), 307 deletions(-) rename tests/queries/0_stateless/{02916_to_start_of_interval_origin_overload.reference => 02916_to_start_of_interva_with_origin.reference} (100%) delete mode 100644 tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.sql create mode 100644 tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 10792922afb..fd472d453b5 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -20,297 +20,16 @@ namespace DB { namespace ErrorCodes { - extern const int ARGUMENT_OUT_OF_BOUND; - extern const int BAD_ARGUMENTS; - extern const int DECIMAL_OVERFLOW; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ARGUMENT_OUT_OF_BOUND; + extern const int BAD_ARGUMENTS; } namespace { -constexpr auto function_name = "toStartOfInterval"; - -template -struct Transform; - -template <> -struct Transform -{ - static UInt16 execute(UInt16 d, Int64 years, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfYearInterval(DayNum(d), years); - } - - static UInt16 execute(Int32 d, Int64 years, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfYearInterval(ExtendedDayNum(d), years); - } - - static UInt16 execute(UInt32 t, Int64 years, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); - } - - static UInt16 execute(Int64 t, Int64 years, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfYearInterval(time_zone.toDayNum(t / scale_multiplier), years); - } -}; - -template <> -struct Transform -{ - static UInt16 execute(UInt16 d, Int64 quarters, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfQuarterInterval(DayNum(d), quarters); - } - - static UInt16 execute(Int32 d, Int64 quarters, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfQuarterInterval(ExtendedDayNum(d), quarters); - } - - static UInt16 execute(UInt32 t, Int64 quarters, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); - } - - static UInt16 execute(Int64 t, Int64 quarters, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t / scale_multiplier), quarters); - } -}; - -template <> -struct Transform -{ - static UInt16 execute(UInt16 d, Int64 months, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfMonthInterval(DayNum(d), months); - } - - static UInt16 execute(Int32 d, Int64 months, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfMonthInterval(ExtendedDayNum(d), months); - } - - static UInt16 execute(UInt32 t, Int64 months, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); - } - - static UInt16 execute(Int64 t, Int64 months, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t / scale_multiplier), months); - } -}; - -template <> -struct Transform -{ - static UInt16 execute(UInt16 d, Int64 weeks, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfWeekInterval(DayNum(d), weeks); - } -static UInt16 execute(Int32 d, Int64 weeks, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfWeekInterval(ExtendedDayNum(d), weeks); - } - - static UInt16 execute(UInt32 t, Int64 weeks, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); - } - - static UInt16 execute(Int64 t, Int64 weeks, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t / scale_multiplier), weeks); - } -}; - -template <> -struct Transform -{ - static UInt32 execute(UInt16 d, Int64 days, const DateLUTImpl & time_zone, Int64) - { - return static_cast(time_zone.toStartOfDayInterval(ExtendedDayNum(d), days)); - } - - static UInt32 execute(Int32 d, Int64 days, const DateLUTImpl & time_zone, Int64) - { - return static_cast(time_zone.toStartOfDayInterval(ExtendedDayNum(d), days)); - } - - static UInt32 execute(UInt32 t, Int64 days, const DateLUTImpl & time_zone, Int64) - { - return static_cast(time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days)); - } - - static Int64 execute(Int64 t, Int64 days, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfDayInterval(time_zone.toDayNum(t / scale_multiplier), days); - } -}; - -template <> -struct Transform -{ - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32 t, Int64 hours, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfHourInterval(t, hours); - } - - static Int64 execute(Int64 t, Int64 hours, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfHourInterval(t / scale_multiplier, hours); - } -}; - -template <> -struct Transform -{ - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32 t, Int64 minutes, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfMinuteInterval(t, minutes); - } - - static Int64 execute(Int64 t, Int64 minutes, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfMinuteInterval(t / scale_multiplier, minutes); - } -}; - -template <> -struct Transform -{ - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32 t, Int64 seconds, const DateLUTImpl & time_zone, Int64) - { - return time_zone.toStartOfSecondInterval(t, seconds); - } - - static Int64 execute(Int64 t, Int64 seconds, const DateLUTImpl & time_zone, Int64 scale_multiplier) - { - return time_zone.toStartOfSecondInterval(t / scale_multiplier, seconds); - } -}; - -template <> -struct Transform -{ - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { throwDateTimeIsNotSupported(function_name); } - - static Int64 execute(Int64 t, Int64 milliseconds, const DateLUTImpl &, Int64 scale_multiplier) - { - if (scale_multiplier < 1000) - { - Int64 t_milliseconds = 0; - if (common::mulOverflow(t, static_cast(1000) / scale_multiplier, t_milliseconds)) - throw DB::Exception(ErrorCodes::DECIMAL_OVERFLOW, "Numeric overflow"); - if (likely(t >= 0)) - return t_milliseconds / milliseconds * milliseconds; - else - return ((t_milliseconds + 1) / milliseconds - 1) * milliseconds; - } - else if (scale_multiplier > 1000) - { - Int64 scale_diff = scale_multiplier / static_cast(1000); - if (likely(t >= 0)) - return t / milliseconds / scale_diff * milliseconds; - else - return ((t + 1) / milliseconds / scale_diff - 1) * milliseconds; - } - else - if (likely(t >= 0)) - return t / milliseconds * milliseconds; - else - return ((t + 1) / milliseconds - 1) * milliseconds; - } -}; - -template <> -struct Transform -{ - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { throwDateTimeIsNotSupported(function_name); } - - static Int64 execute(Int64 t, Int64 microseconds, const DateLUTImpl &, Int64 scale_multiplier) - { - if (scale_multiplier < 1000000) - { - Int64 t_microseconds = 0; - if (common::mulOverflow(t, static_cast(1000000) / scale_multiplier, t_microseconds)) - throw DB::Exception(ErrorCodes::DECIMAL_OVERFLOW, "Numeric overflow"); - if (likely(t >= 0)) - return t_microseconds / microseconds * microseconds; - else - return ((t_microseconds + 1) / microseconds - 1) * microseconds; - } - else if (scale_multiplier > 1000000) - { - Int64 scale_diff = scale_multiplier / static_cast(1000000); - if (likely(t >= 0)) - return t / microseconds / scale_diff * microseconds; - else - return ((t + 1) / microseconds / scale_diff - 1) * microseconds; - } - else - if (likely(t >= 0)) - return t / microseconds * microseconds; - else - return ((t + 1) / microseconds - 1) * microseconds; - } -}; - -template <> -struct Transform -{ - static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { throwDateIsNotSupported(function_name); } - - static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { throwDateTimeIsNotSupported(function_name); } - - static Int64 execute(Int64 t, Int64 nanoseconds, const DateLUTImpl &, Int64 scale_multiplier) - { - if (scale_multiplier < 1000000000) - { - Int64 t_nanoseconds = 0; - if (common::mulOverflow(t, (static_cast(1000000000) / scale_multiplier), t_nanoseconds)) - throw DB::Exception(ErrorCodes::DECIMAL_OVERFLOW, "Numeric overflow"); - if (likely(t >= 0)) - return t_nanoseconds / nanoseconds * nanoseconds; - else - return ((t_nanoseconds + 1) / nanoseconds - 1) * nanoseconds; - } - else - if (likely(t >= 0)) - return t / nanoseconds * nanoseconds; - else - return ((t + 1) / nanoseconds - 1) * nanoseconds; - } -}; class FunctionToStartOfInterval : public IFunction { @@ -485,7 +204,7 @@ public: if (has_origin_arg) origin_column = arguments[2]; - const size_t time_zone_arg_num = (arguments.size() == 3 && isString(arguments[2].type)) ? 2 : 3; + const size_t time_zone_arg_num = (arguments.size() == 2 || (arguments.size() == 3 && isString(arguments[2].type))) ? 2 : 3; const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, time_zone_arg_num, 0); auto result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); @@ -501,9 +220,8 @@ private: if (isDateTime64(time_column_type)) { - if (origin_column.column != nullptr) - if (!isDateTime64(origin_column.type.get())) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); + if (!isDateTime64(origin_column.type.get()) && origin_column.column != nullptr) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); const auto * time_column_vec = checkAndGetColumn(time_column_col); auto scale = assert_cast(time_column_type).getScale(); @@ -608,20 +326,23 @@ private: if (origin_column.column == nullptr) for (size_t i = 0; i != size; ++i) - result_data[i] = static_cast(Transform::execute(time_data[i], num_units, time_zone, scale_multiplier)); + result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_multiplier)); else { - UInt64 od = origin_column.column->get64(0); + UInt64 origin = origin_column.column->get64(0); + std::cerr << "origin: " << origin << std::endl; + std::cerr << "scale_multiplier: " << scale_multiplier << std::endl; for (size_t i = 0; i != size; ++i) { auto td = time_data[i]; - if (od > size_t(td)) + if (origin > size_t(td)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The origin must be before the end date/datetime"); - td -= od; - result_data[i] = static_cast(Transform::execute(td, num_units, time_zone, scale_multiplier)); - - result_data[i] += scale_multiplier == 10 ? od : od / scale_multiplier; + td -= origin; + result_data[i] = static_cast(ToStartOfInterval::execute(td, num_units, time_zone, scale_multiplier)); + if (!(unit == IntervalKind::Millisecond || unit == IntervalKind::Microsecond || unit == IntervalKind::Nanosecond) && scale_multiplier != 10) + origin = origin / scale_multiplier; + result_data[i] += origin; } } diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.reference b/tests/queries/0_stateless/02916_to_start_of_interva_with_origin.reference similarity index 100% rename from tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.reference rename to tests/queries/0_stateless/02916_to_start_of_interva_with_origin.reference diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.sql b/tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.sql deleted file mode 100644 index ce4c8f87811..00000000000 --- a/tests/queries/0_stateless/02916_to_start_of_interval_origin_overload.sql +++ /dev/null @@ -1,12 +0,0 @@ -set session_timezone = 'UTC'; -SELECT toStartOfInterval(number % 2 == 0 ? toDateTime64('2023-03-01 15:55:00', 2) : toDateTime64('2023-02-01 15:55:00', 2), toIntervalMinute(1), toDateTime64('2023-01-01 13:55:00', 2), 'Europe/Amsterdam') from numbers(5); -SELECT toStartOfInterval(number % 2 == 0 ? toDateTime('2023-03-01 15:55:00') : toDateTime('2023-02-01 15:55:00'), toIntervalMinute(1), toDateTime('2023-01-01 13:55:00'), 'Europe/Amsterdam') from numbers(5); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalHour(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalMinute(1), toDateTime64('2023-01-02 14:44:30', 2)); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30')); -SELECT toStartOfInterval(toDate('2023-01-02 14:45:50'), toIntervalWeek(1), toDate('2023-01-02 14:44:30')); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime64('2023-01-02 14:44:30', 2)); -- { serverError 43 } -SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalMinute(1), toDate('2023-01-02 14:44:30')); -- { serverError 43 } -SELECT toStartOfInterval(toDateTime('2023-01-02 14:42:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30')); -- { serverError 36 } -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), number % 2 == 0 ? toDateTime('2023-02-01 15:55:00') : toDateTime('2023-01-01 15:55:00'), 'Europe/Amsterdam') from numbers(1); -- { serverError 44 } diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql new file mode 100644 index 00000000000..5f5f941047a --- /dev/null +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql @@ -0,0 +1,42 @@ +set session_timezone = 'UTC'; + +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.917122341', 9), toIntervalNanosecond(10000), toDateTime64('2023-01-02 14:44:30.500600700', 9)); +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.917122', 6), toIntervalMicrosecond(10000), toDateTime64('2023-01-02 14:44:30.500600', 6)); +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.91', 3), toIntervalMillisecond(100), toDateTime64('2023-01-02 14:44:30.501', 3)); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalSecond(2), toDateTime('2023-01-02 14:44:30')); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30')); +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2)); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalDay(1), toDateTime('2023-01-02 14:44:30')); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalWeek(1), toDateTime('2023-01-02 14:44:30')); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMonth(1), toDateTime('2022-01-02 14:44:30')); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalQuarter(1), toDateTime('2022-01-02 14:44:30')); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-02 14:44:30')); + +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.917122341', 9), toIntervalNanosecond(10000), toDateTime64('2023-01-02 14:44:30.500600700', 9), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.917122', 6), toIntervalMicrosecond(10000), toDateTime64('2023-01-02 14:44:30.500600', 6), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.91', 3), toIntervalMillisecond(100), toDateTime64('2023-01-02 14:44:30.501', 3), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalSecond(2), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalDay(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalWeek(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMonth(1), toDateTime('2022-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalQuarter(1), toDateTime('2022-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-02 14:44:30'), 'Europe/Amsterdam'); + +SELECT toStartOfInterval(number % 2 == 0 ? toDateTime64('2023-03-01 15:55:00', 2) : toDateTime64('2023-02-01 15:55:00', 2), toIntervalMinute(1), toDateTime64('2023-01-01 13:55:00', 2), 'Europe/Amsterdam') from numbers(5); +SELECT toStartOfInterval(number % 2 == 0 ? toDateTime('2023-03-01 15:55:00') : toDateTime('2023-02-01 15:55:00'), toIntervalHour(1), toDateTime('2023-01-01 13:55:00'), 'Europe/Amsterdam') from numbers(5); +SELECT toStartOfInterval(materialize(toDateTime('2023-01-02 14:45:50')), toIntervalHour(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(materialize(toDateTime64('2023-02-01 15:45:50', 2)), toIntervalHour(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); + +SELECT toStartOfInterval(toDate('2023-01-02 14:45:50'), toIntervalSecond(5), toDate32('2023-01-02 14:44:30')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate('2023-01-02 14:45:50'), toIntervalMillisecond(12), toDateTime('2023-01-02 14:44:30')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate32('2023-01-02 14:45:50'), toIntervalHour(5), toDate('2023-01-02 14:44:30')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime64('2023-01-02 14:44:30', 2)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalMinute(1), toDate('2023-01-02 14:44:30')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDateTime('2023-01-02 14:42:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30')); -- { serverError BAD_ARGUMENTS } +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), number % 2 == 0 ? toDateTime('2023-02-01 15:55:00') : toDateTime('2023-01-01 15:55:00'), 'Europe/Amsterdam') from numbers(1); -- { serverError ILLEGAL_COLUMN } +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), 'Europe/Amsterdam', 'a'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-02 14:44:30'), 'Europe/Amsterdam', 5); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), 5, 'Europe/Amsterdam'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalHour(1), materialize(toDateTime('2023-01-02 14:44:30')), 'Europe/Amsterdam'); -- { serverError ILLEGAL_COLUMN } From 8cd4d35ead1dc7ea5fff3a794522ffb5dbc75726 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 29 Nov 2023 17:35:33 +0000 Subject: [PATCH 008/289] fixed review --- src/Functions/toStartOfInterval.cpp | 26 ++++++++------ ..._to_start_of_interva_with_origin.reference | 35 ++++++++++++++----- ...02916_to_start_of_interval_with_origin.sql | 18 +++++----- 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index fd472d453b5..c4f60d6d6db 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -1,6 +1,8 @@ #include +#include "Common/IntervalKind.h" #include #include +#include "base/types.h" #include #include #include @@ -220,7 +222,7 @@ private: if (isDateTime64(time_column_type)) { - if (!isDateTime64(origin_column.type.get()) && origin_column.column != nullptr) + if (origin_column.column != nullptr && !isDateTime64(origin_column.type.get())) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); const auto * time_column_vec = checkAndGetColumn(time_column_col); @@ -231,9 +233,8 @@ private: } else if (isDateTime(time_column_type)) { - if (origin_column.column != nullptr) - if (!isDateTime(origin_column.type.get())) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); + if (origin_column.column != nullptr && !isDateTime(origin_column.type.get())) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) @@ -241,9 +242,8 @@ private: } else if (isDate(time_column_type)) { - if (origin_column.column != nullptr) - if (!isDate(origin_column.type.get())) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); + if (origin_column.column != nullptr && !isDate(origin_column.type.get())) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) @@ -330,19 +330,23 @@ private: else { UInt64 origin = origin_column.column->get64(0); - std::cerr << "origin: " << origin << std::endl; - std::cerr << "scale_multiplier: " << scale_multiplier << std::endl; for (size_t i = 0; i != size; ++i) { auto td = time_data[i]; + result_data[i] = 0; if (origin > size_t(td)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The origin must be before the end date/datetime"); + td -= origin; - result_data[i] = static_cast(ToStartOfInterval::execute(td, num_units, time_zone, scale_multiplier)); + auto res = static_cast(ToStartOfInterval::execute(td, num_units, time_zone, scale_multiplier)); + if (!(unit == IntervalKind::Millisecond || unit == IntervalKind::Microsecond || unit == IntervalKind::Nanosecond) && scale_multiplier != 10) origin = origin / scale_multiplier; - result_data[i] += origin; + if (unit == IntervalKind::Week || unit == IntervalKind::Month || unit == IntervalKind::Quarter || unit == IntervalKind::Year) + result_data[i] = UInt16(origin/86400 + res); + else + result_data[i] += origin + res; } } diff --git a/tests/queries/0_stateless/02916_to_start_of_interva_with_origin.reference b/tests/queries/0_stateless/02916_to_start_of_interva_with_origin.reference index 7213925fb64..78e3b117cb8 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interva_with_origin.reference +++ b/tests/queries/0_stateless/02916_to_start_of_interva_with_origin.reference @@ -1,15 +1,34 @@ +2023-01-02 14:45:50.917120700 +2023-01-02 14:45:50.910600 +2023-01-02 14:45:50.901 +2023-01-02 14:45:50 +2023-01-02 14:45:30 +2023-01-02 14:44:30 +2023-01-02 14:44:30 +2023-01-06 +2023-03-02 +2023-07-02 +2022-01-02 +2023-01-02 15:45:50.917120700 +2023-01-02 15:45:50.910600 +2023-01-02 15:45:50.901 +2023-01-02 15:45:50 +2023-01-02 15:45:30 +2023-01-02 15:44:30 +2023-01-03 14:44:30 +2023-01-06 +2023-03-02 +2023-07-02 +2022-01-02 2023-03-01 16:55:00 -2023-02-01 16:55:00 -2023-03-01 16:55:00 -2023-02-01 16:55:00 -2023-03-01 16:55:00 +2023-02-01 16:54:33 +2023-03-01 16:54:38 +2023-02-01 16:54:52 +2023-03-01 16:54:16 2023-03-01 16:55:00 2023-02-01 16:55:00 2023-03-01 16:55:00 2023-02-01 16:55:00 2023-03-01 16:55:00 2023-01-02 15:44:30 -2023-01-02 15:44:30 -2023-01-02 14:45:30 -2023-01-02 14:45:30 -2023-01-02 +2023-02-01 16:44:30 diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql index 5f5f941047a..390b36a4cf1 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql @@ -7,10 +7,10 @@ SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalSecond(2), SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30')); SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2)); SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalDay(1), toDateTime('2023-01-02 14:44:30')); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalWeek(1), toDateTime('2023-01-02 14:44:30')); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMonth(1), toDateTime('2022-01-02 14:44:30')); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalQuarter(1), toDateTime('2022-01-02 14:44:30')); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-02 14:44:30')); +SELECT toStartOfInterval(toDateTime('2023-01-08 14:45:50'), toIntervalWeek(1), toDateTime('2023-01-02 14:44:30')); +SELECT toStartOfInterval(toDateTime('2023-03-03 14:45:50'), toIntervalMonth(1), toDateTime('2022-01-02 14:44:30')); +SELECT toStartOfInterval(toDateTime('2023-08-02 14:45:50'), toIntervalQuarter(1), toDateTime('2022-01-02 14:44:30')); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-03 14:44:30')); SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.917122341', 9), toIntervalNanosecond(10000), toDateTime64('2023-01-02 14:44:30.500600700', 9), 'Europe/Amsterdam'); SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.917122', 6), toIntervalMicrosecond(10000), toDateTime64('2023-01-02 14:44:30.500600', 6), 'Europe/Amsterdam'); @@ -19,15 +19,15 @@ SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalSecond(2), SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2), 'Europe/Amsterdam'); SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalDay(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalWeek(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMonth(1), toDateTime('2022-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalQuarter(1), toDateTime('2022-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime('2023-01-08 14:45:50'), toIntervalWeek(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime('2023-03-03 14:45:50'), toIntervalMonth(1), toDateTime('2022-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime('2023-08-02 14:45:50'), toIntervalQuarter(1), toDateTime('2022-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-03 14:44:30'), 'Europe/Amsterdam'); SELECT toStartOfInterval(number % 2 == 0 ? toDateTime64('2023-03-01 15:55:00', 2) : toDateTime64('2023-02-01 15:55:00', 2), toIntervalMinute(1), toDateTime64('2023-01-01 13:55:00', 2), 'Europe/Amsterdam') from numbers(5); SELECT toStartOfInterval(number % 2 == 0 ? toDateTime('2023-03-01 15:55:00') : toDateTime('2023-02-01 15:55:00'), toIntervalHour(1), toDateTime('2023-01-01 13:55:00'), 'Europe/Amsterdam') from numbers(5); SELECT toStartOfInterval(materialize(toDateTime('2023-01-02 14:45:50')), toIntervalHour(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(materialize(toDateTime64('2023-02-01 15:45:50', 2)), toIntervalHour(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(materialize(toDateTime64('2023-02-01 15:45:50', 2)), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2), 'Europe/Amsterdam'); SELECT toStartOfInterval(toDate('2023-01-02 14:45:50'), toIntervalSecond(5), toDate32('2023-01-02 14:44:30')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT toStartOfInterval(toDate('2023-01-02 14:45:50'), toIntervalMillisecond(12), toDateTime('2023-01-02 14:44:30')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } From b298bed1cfdb67e21da89e2b8a4bb4affc44f14b Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 29 Nov 2023 23:08:19 +0100 Subject: [PATCH 009/289] Rename 02916_to_start_of_interva_with_origin.reference to 02916_to_start_of_interval_with_origin.reference --- ...reference => 02916_to_start_of_interval_with_origin.reference} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{02916_to_start_of_interva_with_origin.reference => 02916_to_start_of_interval_with_origin.reference} (100%) diff --git a/tests/queries/0_stateless/02916_to_start_of_interva_with_origin.reference b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference similarity index 100% rename from tests/queries/0_stateless/02916_to_start_of_interva_with_origin.reference rename to tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference From 450f609227fe0cedefbf2473a1a2c4b4e7b65d5f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 2 Dec 2023 22:58:52 +0000 Subject: [PATCH 010/289] Various updates --- src/Functions/toStartOfInterval.cpp | 29 +++-- ...to_start_of_interval_with_origin.reference | 74 +++++++------ ...02916_to_start_of_interval_with_origin.sql | 103 ++++++++++++------ 3 files changed, 128 insertions(+), 78 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index c4f60d6d6db..ec0deee8abd 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -1,21 +1,21 @@ -#include #include "Common/IntervalKind.h" -#include -#include -#include "base/types.h" -#include -#include #include #include +#include +#include #include #include #include #include #include +#include #include #include +#include #include #include +#include +#include namespace DB @@ -325,26 +325,31 @@ private: Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale); if (origin_column.column == nullptr) + { for (size_t i = 0; i != size; ++i) result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_multiplier)); + } else { UInt64 origin = origin_column.column->get64(0); for (size_t i = 0; i != size; ++i) { - auto td = time_data[i]; - result_data[i] = 0; - if (origin > size_t(td)) + auto t = time_data[i]; + if (origin > static_cast(t)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The origin must be before the end date/datetime"); - td -= origin; - auto res = static_cast(ToStartOfInterval::execute(td, num_units, time_zone, scale_multiplier)); + t -= origin; + auto res = static_cast(ToStartOfInterval::execute(t, num_units, time_zone, scale_multiplier)); if (!(unit == IntervalKind::Millisecond || unit == IntervalKind::Microsecond || unit == IntervalKind::Nanosecond) && scale_multiplier != 10) origin = origin / scale_multiplier; + + static constexpr size_t SECONDS_PER_DAY = 86400; + + result_data[i] = 0; if (unit == IntervalKind::Week || unit == IntervalKind::Month || unit == IntervalKind::Quarter || unit == IntervalKind::Year) - result_data[i] = UInt16(origin/86400 + res); + result_data[i] = static_cast(origin/SECONDS_PER_DAY + res); else result_data[i] += origin + res; } diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference index 78e3b117cb8..3574da8f685 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference @@ -1,34 +1,40 @@ -2023-01-02 14:45:50.917120700 -2023-01-02 14:45:50.910600 -2023-01-02 14:45:50.901 -2023-01-02 14:45:50 -2023-01-02 14:45:30 -2023-01-02 14:44:30 -2023-01-02 14:44:30 -2023-01-06 -2023-03-02 -2023-07-02 -2022-01-02 -2023-01-02 15:45:50.917120700 -2023-01-02 15:45:50.910600 -2023-01-02 15:45:50.901 -2023-01-02 15:45:50 -2023-01-02 15:45:30 -2023-01-02 15:44:30 -2023-01-03 14:44:30 -2023-01-06 -2023-03-02 -2023-07-02 -2022-01-02 -2023-03-01 16:55:00 -2023-02-01 16:54:33 -2023-03-01 16:54:38 -2023-02-01 16:54:52 -2023-03-01 16:54:16 -2023-03-01 16:55:00 -2023-02-01 16:55:00 -2023-03-01 16:55:00 -2023-02-01 16:55:00 -2023-03-01 16:55:00 -2023-01-02 15:44:30 -2023-02-01 16:44:30 +-- Negative tests +Time and origin as Time +1971-01-01 +1971-07-01 +1970-02-01 +1970-01-05 +1970-01-02 05:27:18 +Time and origin as DateTime +2023-02-01 +2023-08-01 +2023-10-09 +2023-10-05 +2023-10-09 09:08:07 +2023-10-09 10:10:07 +2023-10-09 10:11:11 +2023-10-09 10:11:12 +Time and origin as DateTime64(9) +2023-02-01 +2023-08-01 +2023-10-09 +2023-10-05 +2023-10-09 09:08:07 +2023-10-09 10:10:07 +2023-10-09 10:11:11 +2023-10-09 10:11:12 +2299-12-31 23:57:37.653 +2299-12-31 23:42:04.320986 +2023-10-09 10:11:12.987654321 +Time and origin as DateTime64(3) +2023-02-01 +2023-08-01 +2023-10-09 +2023-10-05 +2023-10-09 09:08:07 +2023-10-09 10:10:07 +2023-10-09 10:11:11 +2023-10-09 10:11:12 +2023-10-09 10:11:12.987 +1970-01-20 15:20:47.136123 +1970-01-01 00:28:17.710272123 diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql index 390b36a4cf1..864ef56e7ea 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql @@ -1,42 +1,81 @@ set session_timezone = 'UTC'; -SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.917122341', 9), toIntervalNanosecond(10000), toDateTime64('2023-01-02 14:44:30.500600700', 9)); -SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.917122', 6), toIntervalMicrosecond(10000), toDateTime64('2023-01-02 14:44:30.500600', 6)); -SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.91', 3), toIntervalMillisecond(100), toDateTime64('2023-01-02 14:44:30.501', 3)); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalSecond(2), toDateTime('2023-01-02 14:44:30')); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30')); -SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2)); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalDay(1), toDateTime('2023-01-02 14:44:30')); -SELECT toStartOfInterval(toDateTime('2023-01-08 14:45:50'), toIntervalWeek(1), toDateTime('2023-01-02 14:44:30')); -SELECT toStartOfInterval(toDateTime('2023-03-03 14:45:50'), toIntervalMonth(1), toDateTime('2022-01-02 14:44:30')); -SELECT toStartOfInterval(toDateTime('2023-08-02 14:45:50'), toIntervalQuarter(1), toDateTime('2022-01-02 14:44:30')); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-03 14:44:30')); - -SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.917122341', 9), toIntervalNanosecond(10000), toDateTime64('2023-01-02 14:44:30.500600700', 9), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.917122', 6), toIntervalMicrosecond(10000), toDateTime64('2023-01-02 14:44:30.500600', 6), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50.91', 3), toIntervalMillisecond(100), toDateTime64('2023-01-02 14:44:30.501', 3), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalSecond(2), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalDay(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime('2023-01-08 14:45:50'), toIntervalWeek(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime('2023-03-03 14:45:50'), toIntervalMonth(1), toDateTime('2022-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime('2023-08-02 14:45:50'), toIntervalQuarter(1), toDateTime('2022-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-03 14:44:30'), 'Europe/Amsterdam'); - -SELECT toStartOfInterval(number % 2 == 0 ? toDateTime64('2023-03-01 15:55:00', 2) : toDateTime64('2023-02-01 15:55:00', 2), toIntervalMinute(1), toDateTime64('2023-01-01 13:55:00', 2), 'Europe/Amsterdam') from numbers(5); -SELECT toStartOfInterval(number % 2 == 0 ? toDateTime('2023-03-01 15:55:00') : toDateTime('2023-02-01 15:55:00'), toIntervalHour(1), toDateTime('2023-01-01 13:55:00'), 'Europe/Amsterdam') from numbers(5); -SELECT toStartOfInterval(materialize(toDateTime('2023-01-02 14:45:50')), toIntervalHour(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); -SELECT toStartOfInterval(materialize(toDateTime64('2023-02-01 15:45:50', 2)), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2), 'Europe/Amsterdam'); +SELECT '-- Negative tests'; +-- time and origin arguments must have the same type SELECT toStartOfInterval(toDate('2023-01-02 14:45:50'), toIntervalSecond(5), toDate32('2023-01-02 14:44:30')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT toStartOfInterval(toDate('2023-01-02 14:45:50'), toIntervalMillisecond(12), toDateTime('2023-01-02 14:44:30')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT toStartOfInterval(toDate32('2023-01-02 14:45:50'), toIntervalHour(5), toDate('2023-01-02 14:44:30')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), toDateTime64('2023-01-02 14:44:30', 2)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT toStartOfInterval(toDateTime64('2023-01-02 14:45:50', 2), toIntervalMinute(1), toDate('2023-01-02 14:44:30')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +-- the origin must be before the time SELECT toStartOfInterval(toDateTime('2023-01-02 14:42:50'), toIntervalMinute(1), toDateTime('2023-01-02 14:44:30')); -- { serverError BAD_ARGUMENTS } -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), number % 2 == 0 ? toDateTime('2023-02-01 15:55:00') : toDateTime('2023-01-01 15:55:00'), 'Europe/Amsterdam') from numbers(1); -- { serverError ILLEGAL_COLUMN } -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), 'Europe/Amsterdam', 'a'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-02 14:44:30'), 'Europe/Amsterdam', 5); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } -SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), 5, 'Europe/Amsterdam'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +-- the origin must be constant +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalMinute(1), number % 2 == 0 ? toDateTime('2023-02-01 15:55:00') : toDateTime('2023-01-01 15:55:00')) from numbers(1); -- { serverError ILLEGAL_COLUMN } SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalHour(1), materialize(toDateTime('2023-01-02 14:44:30')), 'Europe/Amsterdam'); -- { serverError ILLEGAL_COLUMN } + +-- with 4 arguments, the 3rd one must not be a string or an integer +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), 'Europe/Amsterdam', 'Europe/Amsterdam'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), 5, 'Europe/Amsterdam'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +-- too many arguments +SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-02 14:44:30'), 'Europe/Amsterdam', 5); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +SELECT 'Time and origin as Time'; +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalYear(1), toDate('2022-02-01')); -- broken +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalQuarter(1), toDate('2022-02-01')); -- broken +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalMonth(1), toDate('2023-09-08')); -- broken +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalWeek(1), toDate('2023-10-01')); -- broken +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalDay(1), toDate('2023-10-08')); -- broken +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalHour(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalMinute(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalSecond(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalMillisecond(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalMicrosecond(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalNanosecond(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +SELECT 'Time and origin as DateTime'; +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalYear(1), toDateTime('2022-02-01 09:08:07')); -- broken, should that not return 2023-02-01 09:08:07? +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalQuarter(1), toDateTime('2022-02-01 09:08:07')); -- broken, should that not return 2023-08-01 09:08:07? +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMonth(1), toDateTime('2023-09-08 09:08:07')); -- broken, should that not return 2023-10-09 09:08:07? +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalWeek(1), toDateTime('2023-10-01 09:08:07')); -- broken, should that not return 2023-10-05 09:08:07? +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalDay(1), toDateTime('2023-10-08 09:08:07')); +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalHour(1), toDateTime('2023-10-09 09:10:07')); +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMinute(1), toDateTime('2023-10-09 10:10:11')); +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalSecond(1), toDateTime('2023-10-09 10:11:10')); +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMillisecond(1), toDateTime('2023-10-09 10:11:12')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMicrosecond(1), toDateTime('2023-10-09 10:11:12')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalNanosecond(1), toDateTime('2023-10-09 10:11:12')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +SELECT 'Time and origin as DateTime64(9)'; +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalYear(1), toDateTime64('2022-02-01 09:08:07.123456789', 9)); -- broken, see above +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalQuarter(1), toDateTime64('2022-02-01 09:08:07.123456789', 9)); -- broken, see above +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMonth(1), toDateTime64('2023-09-08 09:08:07.123456789', 9)); -- broken, see above +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalWeek(1), toDateTime64('2023-10-01 09:08:07.123456789', 9)); -- broken, see above +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalDay(1), toDateTime64('2023-10-08 09:08:07.123456789', 9)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalHour(1), toDateTime64('2023-10-09 09:10:07.123456789', 9)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMinute(1), toDateTime64('2023-10-09 10:10:11.123456789', 9)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalSecond(1), toDateTime64('2023-10-09 10:11:10.123456789', 9)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMillisecond(1), toDateTime64('2023-10-09 10:11:12.123456789', 9)); -- broken (2299) +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMicrosecond(1), toDateTime64('2023-10-09 10:11:12.123456789', 9)); -- broken (2299) +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalNanosecond(1), toDateTime64('2023-10-09 10:11:12.123456789', 9)); -- broken (2299) + +SELECT 'Time and origin as DateTime64(3)'; +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalYear(1), toDateTime64('2022-02-01 09:08:07.123', 3)); -- broken, see above +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalQuarter(1), toDateTime64('2022-02-01 09:08:07.123', 3)); -- broken, see above +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalMonth(1), toDateTime64('2023-09-08 09:08:07.123', 3)); -- broken, see above +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalWeek(1), toDateTime64('2023-10-01 09:08:07.123', 3)); -- broken, see above +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalDay(1), toDateTime64('2023-10-08 09:08:07.123', 3)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalHour(1), toDateTime64('2023-10-09 09:10:07.123', 3)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalMinute(1), toDateTime64('2023-10-09 10:10:11.123', 3)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalSecond(1), toDateTime64('2023-10-09 10:11:10.123', 3)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalMillisecond(1), toDateTime64('2023-10-09 10:11:12.123', 3)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalMicrosecond(1), toDateTime64('2023-10-09 10:11:12.123', 3)); -- broken (1970) +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalNanosecond(1), toDateTime64('2023-10-09 10:11:12.123', 3)); -- broken (1970) + +-- SELECT toStartOfInterval(number % 2 == 0 ? toDateTime64('2023-03-01 15:55:00', 2) : toDateTime64('2023-02-01 15:55:00', 2), toIntervalMinute(1), toDateTime64('2023-01-01 13:55:00', 2), 'Europe/Amsterdam') from numbers(5); +-- SELECT toStartOfInterval(number % 2 == 0 ? toDateTime('2023-03-01 15:55:00') : toDateTime('2023-02-01 15:55:00'), toIntervalHour(1), toDateTime('2023-01-01 13:55:00'), 'Europe/Amsterdam') from numbers(5); +-- SELECT toStartOfInterval(materialize(toDateTime('2023-01-02 14:45:50')), toIntervalHour(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); +-- SELECT toStartOfInterval(materialize(toDateTime64('2023-02-01 15:45:50', 2)), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2), 'Europe/Amsterdam'); From 01036a21f59925b78f45415cacb1953481292e31 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:44:40 +0100 Subject: [PATCH 011/289] Fixed bugs regarding precision --- src/Functions/toStartOfInterval.cpp | 124 +++++++++++++++++++++------- 1 file changed, 95 insertions(+), 29 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index ec0deee8abd..da4eba9a594 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -1,8 +1,10 @@ -#include "Common/IntervalKind.h" +#include +#include #include #include #include #include +#include "DataTypes/IDataType.h" #include #include #include @@ -14,7 +16,6 @@ #include #include #include -#include #include @@ -116,9 +117,22 @@ public: throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The timezone argument of function {} with interval type {} is allowed only when the 1st argument has the type DateTime or DateTime64", getName(), interval_type->getKind().toString()); + + if (arguments[0].type.get() != arguments[2].type.get()) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); } else if (isDateOrDate32OrDateTimeOrDateTime64(type_arg3)) + { third_argument = ThirdArgument::IsOrigin; + if (isDateTime64(arguments[0].type) && isDateTime64(arguments[2].type)) + result_type = ResultType::DateTime64; + else if (isDateTime(arguments[0].type) && isDateTime(arguments[2].type)) + result_type = ResultType::DateTime; + else if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) && (isDate(arguments[2].type) || isDate32(arguments[2].type))) + result_type = ResultType::Date; + else + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); + } else throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 3rd argument of function {}. " "This argument is optional and must be a constant String with timezone name or a Date/Date32/DateTime/DateTime64 with a constant origin", @@ -180,13 +194,14 @@ public: } case ResultType::DateTime64: { - UInt32 scale = 0; + auto scale_date_time = assert_cast(*arguments[0].type.get()).getScale(); + UInt32 scale = scale_date_time; if (interval_type->getKind() == IntervalKind::Nanosecond) - scale = 9; + scale = 9 > scale_date_time ? 9 : scale_date_time; else if (interval_type->getKind() == IntervalKind::Microsecond) - scale = 6; + scale = 6 > scale_date_time ? 6 : scale_date_time; else if (interval_type->getKind() == IntervalKind::Millisecond) - scale = 3; + scale = 3 > scale_date_time ? 3 : scale_date_time; const size_t time_zone_arg_num = (arguments.size() == 2 || (arguments.size() == 3 && third_argument == ThirdArgument::IsTimezone)) ? 2 : 3; return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, time_zone_arg_num, 0, false)); @@ -209,11 +224,19 @@ public: const size_t time_zone_arg_num = (arguments.size() == 2 || (arguments.size() == 3 && isString(arguments[2].type))) ? 2 : 3; const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, time_zone_arg_num, 0); - auto result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); + ColumnPtr result_column = nullptr; + if (isDateTime64(result_type)) + result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); + else if (isDateTime(result_type)) + result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); + else + result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); + return result_column; } private: + template ColumnPtr dispatchForTimeColumn( const ColumnWithTypeAndName & time_column, const ColumnWithTypeAndName & interval_column, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone) const { @@ -229,7 +252,7 @@ private: auto scale = assert_cast(time_column_type).getScale(); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone, scale); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone, scale); } else if (isDateTime(time_column_type)) { @@ -238,7 +261,7 @@ private: const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDate(time_column_type)) { @@ -247,7 +270,7 @@ private: const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDate32(time_column_type)) { @@ -257,12 +280,12 @@ private: const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. Must contain dates or dates with time", getName()); } - template + template ColumnPtr dispatchForIntervalColumn( const TimeDataType & time_data_type, const TimeColumnType & time_column, const ColumnWithTypeAndName & interval_column, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale = 1) const @@ -282,32 +305,52 @@ private: switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) { case IntervalKind::Nanosecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Microsecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Millisecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Second: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Minute: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Hour: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Day: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Week: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Month: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Quarter: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Year: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); } std::unreachable(); } + template + Int64 decideScaleOnPrecision(const UInt16 scale) const + { + static constexpr Int64 MILLISECOND_SCALE = 1000; + static constexpr Int64 MICROSECOND_SCALE = 1000000; + static constexpr Int64 NANOSECOND_SCALE = 1000000000; + Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale); + switch (unit) + { + case IntervalKind::Millisecond: + return MILLISECOND_SCALE; + case IntervalKind::Microsecond: + return MICROSECOND_SCALE; + case IntervalKind::Nanosecond: + return NANOSECOND_SCALE; + default: + return scale_multiplier; + } + } + template ColumnPtr execute(const TimeDataType &, const ColumnType & time_column_type, Int64 num_units, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale) const { @@ -323,6 +366,8 @@ private: result_data.resize(size); Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale); + Int64 scale_on_precision = decideScaleOnPrecision(scale); + Int64 scale_diff = scale_on_precision > scale_multiplier ? scale_on_precision / scale_multiplier : scale_multiplier / scale_on_precision; if (origin_column.column == nullptr) { @@ -342,19 +387,40 @@ private: t -= origin; auto res = static_cast(ToStartOfInterval::execute(t, num_units, time_zone, scale_multiplier)); - if (!(unit == IntervalKind::Millisecond || unit == IntervalKind::Microsecond || unit == IntervalKind::Nanosecond) && scale_multiplier != 10) - origin = origin / scale_multiplier; - static constexpr size_t SECONDS_PER_DAY = 86400; result_data[i] = 0; if (unit == IntervalKind::Week || unit == IntervalKind::Month || unit == IntervalKind::Quarter || unit == IntervalKind::Year) - result_data[i] = static_cast(origin/SECONDS_PER_DAY + res); + { + if (isDate(result_type) || isDate32(result_type)) + { + result_data[i] += origin + res; + } + else if (isDateTime64(result_type)) + { + result_data[i] += origin + (res * SECONDS_PER_DAY * scale_multiplier); + } + else + { + result_data[i] += origin + res * SECONDS_PER_DAY; + } + } else - result_data[i] += origin + res; + { + if (isDate(result_type) || isDate32(result_type)) + res = res / SECONDS_PER_DAY; + + if (scale_on_precision > scale_multiplier) + { + result_data[i] += (origin + res / scale_diff) * scale_diff; + } + else + { + result_data[i] += origin + res * scale_diff; + } + } } } - return result_col; } }; From 65f34394cf84fa12f6852e9988c8a3d62c155701 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:45:29 +0100 Subject: [PATCH 012/289] Update 02916_to_start_of_interval_with_origin.reference --- ...to_start_of_interval_with_origin.reference | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference index 3574da8f685..806330743d7 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference @@ -1,40 +1,40 @@ -- Negative tests Time and origin as Time -1971-01-01 -1971-07-01 -1970-02-01 -1970-01-05 -1970-01-02 05:27:18 -Time and origin as DateTime 2023-02-01 2023-08-01 2023-10-09 2023-10-05 +2023-10-08 +Time and origin as DateTime +2023-02-01 09:08:07 +2023-08-01 09:08:07 +2023-10-09 09:08:07 +2023-10-05 09:08:07 2023-10-09 09:08:07 2023-10-09 10:10:07 2023-10-09 10:11:11 2023-10-09 10:11:12 Time and origin as DateTime64(9) -2023-02-01 -2023-08-01 -2023-10-09 -2023-10-05 -2023-10-09 09:08:07 -2023-10-09 10:10:07 -2023-10-09 10:11:11 -2023-10-09 10:11:12 -2299-12-31 23:57:37.653 -2299-12-31 23:42:04.320986 +2023-02-01 09:08:07.123456789 +2023-08-01 09:08:07.123456789 +2023-09-10 09:08:07.123456789 +2023-10-05 09:08:07.123456789 +2023-10-08 09:08:07.123543189 +2023-10-09 09:10:07.123460389 +2023-10-09 10:10:11.123456849 +2023-10-09 10:11:10.123456791 +2023-10-09 10:11:12.987456789 +2023-10-09 10:11:12.987653789 2023-10-09 10:11:12.987654321 Time and origin as DateTime64(3) -2023-02-01 -2023-08-01 -2023-10-09 -2023-10-05 -2023-10-09 09:08:07 -2023-10-09 10:10:07 -2023-10-09 10:11:11 -2023-10-09 10:11:12 +2023-02-01 09:08:07.123 +2023-08-01 09:08:07.123 +2023-10-09 09:08:07.123 +2023-10-05 09:08:07.123 +2023-10-08 09:09:33.523 +2023-10-09 09:10:10.723 +2023-10-09 10:10:11.183 +2023-10-09 10:11:10.125 2023-10-09 10:11:12.987 -1970-01-20 15:20:47.136123 -1970-01-01 00:28:17.710272123 +2023-10-09 10:11:12.987000 +2023-10-09 10:11:12.987000000 From bd105d51ce469d9ed0c741a7a45012d506e34ddd Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:46:00 +0100 Subject: [PATCH 013/289] fixed tests --- ...02916_to_start_of_interval_with_origin.sql | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql index 864ef56e7ea..853103ecc77 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql @@ -24,11 +24,11 @@ SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), 5 SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-02 14:44:30'), 'Europe/Amsterdam', 5); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT 'Time and origin as Time'; -SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalYear(1), toDate('2022-02-01')); -- broken -SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalQuarter(1), toDate('2022-02-01')); -- broken -SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalMonth(1), toDate('2023-09-08')); -- broken -SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalWeek(1), toDate('2023-10-01')); -- broken -SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalDay(1), toDate('2023-10-08')); -- broken +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalYear(1), toDate('2022-02-01')); +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalQuarter(1), toDate('2022-02-01')); +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalMonth(1), toDate('2023-09-08')); +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalWeek(1), toDate('2023-10-01')); +SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalDay(1), toDate('2023-10-08')); SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalHour(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalMinute(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalSecond(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } @@ -37,10 +37,10 @@ SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalMicrosecond(1), toDate( SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalNanosecond(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT 'Time and origin as DateTime'; -SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalYear(1), toDateTime('2022-02-01 09:08:07')); -- broken, should that not return 2023-02-01 09:08:07? -SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalQuarter(1), toDateTime('2022-02-01 09:08:07')); -- broken, should that not return 2023-08-01 09:08:07? -SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMonth(1), toDateTime('2023-09-08 09:08:07')); -- broken, should that not return 2023-10-09 09:08:07? -SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalWeek(1), toDateTime('2023-10-01 09:08:07')); -- broken, should that not return 2023-10-05 09:08:07? +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalYear(1), toDateTime('2022-02-01 09:08:07')); +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalQuarter(1), toDateTime('2022-02-01 09:08:07')); +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMonth(1), toDateTime('2023-09-08 09:08:07')); +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalWeek(1), toDateTime('2023-10-01 09:08:07')); SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalDay(1), toDateTime('2023-10-08 09:08:07')); SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalHour(1), toDateTime('2023-10-09 09:10:07')); SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMinute(1), toDateTime('2023-10-09 10:10:11')); @@ -50,30 +50,30 @@ SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMicrosecon SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalNanosecond(1), toDateTime('2023-10-09 10:11:12')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT 'Time and origin as DateTime64(9)'; -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalYear(1), toDateTime64('2022-02-01 09:08:07.123456789', 9)); -- broken, see above -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalQuarter(1), toDateTime64('2022-02-01 09:08:07.123456789', 9)); -- broken, see above -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMonth(1), toDateTime64('2023-09-08 09:08:07.123456789', 9)); -- broken, see above -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalWeek(1), toDateTime64('2023-10-01 09:08:07.123456789', 9)); -- broken, see above +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalYear(1), toDateTime64('2022-02-01 09:08:07.123456789', 9)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalQuarter(1), toDateTime64('2022-02-01 09:08:07.123456789', 9)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMonth(1), toDateTime64('2023-09-10 09:08:07.123456789', 9)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalWeek(1), toDateTime64('2023-10-01 09:08:07.123456789', 9)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalDay(1), toDateTime64('2023-10-08 09:08:07.123456789', 9)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalHour(1), toDateTime64('2023-10-09 09:10:07.123456789', 9)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMinute(1), toDateTime64('2023-10-09 10:10:11.123456789', 9)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalSecond(1), toDateTime64('2023-10-09 10:11:10.123456789', 9)); -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMillisecond(1), toDateTime64('2023-10-09 10:11:12.123456789', 9)); -- broken (2299) -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMicrosecond(1), toDateTime64('2023-10-09 10:11:12.123456789', 9)); -- broken (2299) -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalNanosecond(1), toDateTime64('2023-10-09 10:11:12.123456789', 9)); -- broken (2299) +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMillisecond(1), toDateTime64('2023-10-09 10:11:12.123456789', 9)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMicrosecond(1), toDateTime64('2023-10-09 10:11:12.123456789', 9)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalNanosecond(1), toDateTime64('2023-10-09 10:11:12.123456789', 9)); SELECT 'Time and origin as DateTime64(3)'; -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalYear(1), toDateTime64('2022-02-01 09:08:07.123', 3)); -- broken, see above -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalQuarter(1), toDateTime64('2022-02-01 09:08:07.123', 3)); -- broken, see above -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalMonth(1), toDateTime64('2023-09-08 09:08:07.123', 3)); -- broken, see above -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalWeek(1), toDateTime64('2023-10-01 09:08:07.123', 3)); -- broken, see above +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalYear(1), toDateTime64('2022-02-01 09:08:07.123', 3)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalQuarter(1), toDateTime64('2022-02-01 09:08:07.123', 3)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalMonth(1), toDateTime64('2023-09-08 09:08:07.123', 3)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalWeek(1), toDateTime64('2023-10-01 09:08:07.123', 3)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalDay(1), toDateTime64('2023-10-08 09:08:07.123', 3)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalHour(1), toDateTime64('2023-10-09 09:10:07.123', 3)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalMinute(1), toDateTime64('2023-10-09 10:10:11.123', 3)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalSecond(1), toDateTime64('2023-10-09 10:11:10.123', 3)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalMillisecond(1), toDateTime64('2023-10-09 10:11:12.123', 3)); -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalMicrosecond(1), toDateTime64('2023-10-09 10:11:12.123', 3)); -- broken (1970) -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalNanosecond(1), toDateTime64('2023-10-09 10:11:12.123', 3)); -- broken (1970) +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalMicrosecond(1), toDateTime64('2023-10-09 10:11:12.123', 3)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalNanosecond(1), toDateTime64('2023-10-09 10:11:12.123', 3)); -- SELECT toStartOfInterval(number % 2 == 0 ? toDateTime64('2023-03-01 15:55:00', 2) : toDateTime64('2023-02-01 15:55:00', 2), toIntervalMinute(1), toDateTime64('2023-01-01 13:55:00', 2), 'Europe/Amsterdam') from numbers(5); -- SELECT toStartOfInterval(number % 2 == 0 ? toDateTime('2023-03-01 15:55:00') : toDateTime('2023-02-01 15:55:00'), toIntervalHour(1), toDateTime('2023-01-01 13:55:00'), 'Europe/Amsterdam') from numbers(5); From 885e44c50e1968d660866e53f25110cb662e27a2 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 6 Dec 2023 18:17:41 +0100 Subject: [PATCH 014/289] style fix --- src/Functions/toStartOfInterval.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index da4eba9a594..4fae8cb6bb6 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -231,7 +231,6 @@ public: result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); else result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); - return result_column; } From e5fdad21609704baa3b90eb07161461120364fb2 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 6 Dec 2023 19:09:55 +0100 Subject: [PATCH 015/289] fix non-const arguments --- src/Functions/toStartOfInterval.cpp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 4fae8cb6bb6..94514d1e1a0 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -392,17 +392,11 @@ private: if (unit == IntervalKind::Week || unit == IntervalKind::Month || unit == IntervalKind::Quarter || unit == IntervalKind::Year) { if (isDate(result_type) || isDate32(result_type)) - { result_data[i] += origin + res; - } else if (isDateTime64(result_type)) - { result_data[i] += origin + (res * SECONDS_PER_DAY * scale_multiplier); - } else - { result_data[i] += origin + res * SECONDS_PER_DAY; - } } else { @@ -410,13 +404,11 @@ private: res = res / SECONDS_PER_DAY; if (scale_on_precision > scale_multiplier) - { result_data[i] += (origin + res / scale_diff) * scale_diff; - } + else if (scale_on_precision == scale_multiplier && scale_on_precision != 10) /// scale == 10 is default case + result_data[i] += origin + (res * scale_on_precision); else - { result_data[i] += origin + res * scale_diff; - } } } } From ac8d80da515144d2d4fe3ed0c47e2bab1d171927 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 6 Dec 2023 19:28:06 +0100 Subject: [PATCH 016/289] fix --- src/Functions/toStartOfInterval.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 94514d1e1a0..e67d3378f6e 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -405,7 +405,8 @@ private: if (scale_on_precision > scale_multiplier) result_data[i] += (origin + res / scale_diff) * scale_diff; - else if (scale_on_precision == scale_multiplier && scale_on_precision != 10) /// scale == 10 is default case + else if (scale_on_precision == scale_multiplier && scale_on_precision % 1000 != 0 && scale_multiplier != 10) /// when it's not a default case with DateTime + /// and when precision is not sub-scale result_data[i] += origin + (res * scale_on_precision); else result_data[i] += origin + res * scale_diff; From 7be47eca5e3ee31ef958f5b9e9804ceb15f35f48 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 6 Dec 2023 19:28:27 +0100 Subject: [PATCH 017/289] Update 02916_to_start_of_interval_with_origin.reference --- ...02916_to_start_of_interval_with_origin.reference | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference index 806330743d7..870853bc371 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference @@ -38,3 +38,16 @@ Time and origin as DateTime64(3) 2023-10-09 10:11:12.987 2023-10-09 10:11:12.987000 2023-10-09 10:11:12.987000000 +Non-const arguments +2023-03-01 16:55:00.00 +2023-02-01 16:55:00.00 +2023-03-01 16:55:00.00 +2023-02-01 16:55:00.00 +2023-03-01 16:55:00.00 +2023-03-01 16:55:00 +2023-02-01 16:55:00 +2023-03-01 16:55:00 +2023-02-01 16:55:00 +2023-03-01 16:55:00 +2023-01-02 15:44:30 +2023-02-01 16:44:30.00 From 969c7f36a5db55ef53cbce2def91820b619c23ea Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 6 Dec 2023 19:28:45 +0100 Subject: [PATCH 018/289] fix tests --- .../02916_to_start_of_interval_with_origin.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql index 853103ecc77..71f5fb7fb36 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql @@ -75,7 +75,8 @@ SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalM SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalMicrosecond(1), toDateTime64('2023-10-09 10:11:12.123', 3)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987', 3), toIntervalNanosecond(1), toDateTime64('2023-10-09 10:11:12.123', 3)); --- SELECT toStartOfInterval(number % 2 == 0 ? toDateTime64('2023-03-01 15:55:00', 2) : toDateTime64('2023-02-01 15:55:00', 2), toIntervalMinute(1), toDateTime64('2023-01-01 13:55:00', 2), 'Europe/Amsterdam') from numbers(5); --- SELECT toStartOfInterval(number % 2 == 0 ? toDateTime('2023-03-01 15:55:00') : toDateTime('2023-02-01 15:55:00'), toIntervalHour(1), toDateTime('2023-01-01 13:55:00'), 'Europe/Amsterdam') from numbers(5); --- SELECT toStartOfInterval(materialize(toDateTime('2023-01-02 14:45:50')), toIntervalHour(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); --- SELECT toStartOfInterval(materialize(toDateTime64('2023-02-01 15:45:50', 2)), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2), 'Europe/Amsterdam'); +SELECT 'Non-const arguments'; +SELECT toStartOfInterval(number % 2 == 0 ? toDateTime64('2023-03-01 15:55:00', 2) : toDateTime64('2023-02-01 15:55:00', 2), toIntervalMinute(1), toDateTime64('2023-01-01 13:55:00', 2), 'Europe/Amsterdam') from numbers(5); +SELECT toStartOfInterval(number % 2 == 0 ? toDateTime('2023-03-01 15:55:00') : toDateTime('2023-02-01 15:55:00'), toIntervalHour(1), toDateTime('2023-01-01 13:55:00'), 'Europe/Amsterdam') from numbers(5); +SELECT toStartOfInterval(materialize(toDateTime('2023-01-02 14:45:50')), toIntervalHour(1), toDateTime('2023-01-02 14:44:30'), 'Europe/Amsterdam'); +SELECT toStartOfInterval(materialize(toDateTime64('2023-02-01 15:45:50', 2)), toIntervalHour(1), toDateTime64('2023-01-02 14:44:30', 2), 'Europe/Amsterdam'); From 3027f3a04f99beafdc8ead3641aa84fdf4f6657a Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 7 Dec 2023 15:29:11 +0000 Subject: [PATCH 019/289] fix tests --- src/Functions/toStartOfInterval.cpp | 23 ++++++++++--------- .../02207_subseconds_intervals.reference | 16 ++++++------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index e67d3378f6e..7c75d7aa8e9 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -117,9 +117,6 @@ public: throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The timezone argument of function {} with interval type {} is allowed only when the 1st argument has the type DateTime or DateTime64", getName(), interval_type->getKind().toString()); - - if (arguments[0].type.get() != arguments[2].type.get()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); } else if (isDateOrDate32OrDateTimeOrDateTime64(type_arg3)) { @@ -137,7 +134,6 @@ public: throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 3rd argument of function {}. " "This argument is optional and must be a constant String with timezone name or a Date/Date32/DateTime/DateTime64 with a constant origin", type_arg3->getName(), getName()); - }; auto check_fourth_argument = [&] @@ -365,13 +361,19 @@ private: result_data.resize(size); Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale); - Int64 scale_on_precision = decideScaleOnPrecision(scale); - Int64 scale_diff = scale_on_precision > scale_multiplier ? scale_on_precision / scale_multiplier : scale_multiplier / scale_on_precision; + Int64 scale_on_interval = decideScaleOnPrecision(scale); + Int64 scale_diff = scale_on_interval > scale_multiplier ? scale_on_interval / scale_multiplier : scale_multiplier / scale_on_interval; if (origin_column.column == nullptr) { for (size_t i = 0; i != size; ++i) - result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_multiplier)); + { + result_data[i] = 0; + if (scale_on_interval < scale_multiplier) + result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_multiplier)) * scale_diff; + else + result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_multiplier)); + } } else { @@ -403,11 +405,10 @@ private: if (isDate(result_type) || isDate32(result_type)) res = res / SECONDS_PER_DAY; - if (scale_on_precision > scale_multiplier) + if (scale_on_interval > scale_multiplier) result_data[i] += (origin + res / scale_diff) * scale_diff; - else if (scale_on_precision == scale_multiplier && scale_on_precision % 1000 != 0 && scale_multiplier != 10) /// when it's not a default case with DateTime - /// and when precision is not sub-scale - result_data[i] += origin + (res * scale_on_precision); + else if (scale_on_interval == scale_multiplier && scale_on_interval % 1000 != 0 && scale_multiplier != 10) + result_data[i] += origin + (res * scale_on_interval); else result_data[i] += origin + res * scale_diff; } diff --git a/tests/queries/0_stateless/02207_subseconds_intervals.reference b/tests/queries/0_stateless/02207_subseconds_intervals.reference index 91f0ecb8606..b0edbda5e76 100644 --- a/tests/queries/0_stateless/02207_subseconds_intervals.reference +++ b/tests/queries/0_stateless/02207_subseconds_intervals.reference @@ -10,25 +10,25 @@ test intervals - test microseconds 1980-12-12 12:12:12.123456 1980-12-12 12:12:12.123400 -1980-12-12 12:12:12.123456 -1980-12-12 12:12:12.123456 +1980-12-12 12:12:12.12345600 +1980-12-12 12:12:12.12345600 1930-12-12 12:12:12.123456 1930-12-12 12:12:12.123400 -1930-12-12 12:12:12.123456 +1930-12-12 12:12:12.12345600 2220-12-12 12:12:12.123456 2220-12-12 12:12:12.123400 -2220-12-12 12:12:12.123456 +2220-12-12 12:12:12.12345600 - test milliseconds 1980-12-12 12:12:12.123 1980-12-12 12:12:12.120 -1980-12-12 12:12:12.123 -1980-12-12 12:12:12.123 +1980-12-12 12:12:12.123000 +1980-12-12 12:12:12.123000 1930-12-12 12:12:12.123 1930-12-12 12:12:12.120 -1930-12-12 12:12:12.123 +1930-12-12 12:12:12.123000 2220-12-12 12:12:12.123 2220-12-12 12:12:12.120 -2220-12-12 12:12:12.123 +2220-12-12 12:12:12.123000 test add[...]seconds() - test nanoseconds 1980-12-12 12:12:12.123456790 From d08fd931bec203f1b86d89227b766973ad3612c9 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 8 Dec 2023 13:42:00 +0000 Subject: [PATCH 020/289] fix error --- src/Functions/toStartOfInterval.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 7c75d7aa8e9..e0301f45ded 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -190,8 +190,10 @@ public: } case ResultType::DateTime64: { - auto scale_date_time = assert_cast(*arguments[0].type.get()).getScale(); - UInt32 scale = scale_date_time; + UInt32 scale = 0; + auto scale_date_time = 0; + if (third_argument == ThirdArgument::IsOrigin) + scale_date_time = assert_cast(*arguments[0].type.get()).getScale(); if (interval_type->getKind() == IntervalKind::Nanosecond) scale = 9 > scale_date_time ? 9 : scale_date_time; else if (interval_type->getKind() == IntervalKind::Microsecond) From e13eec9c5a9cb49004aef8d635034e9ccfd697d4 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:07:39 +0000 Subject: [PATCH 021/289] fix --- src/Functions/toStartOfInterval.cpp | 11 +++++------ .../02207_subseconds_intervals.reference | 16 ++++++++-------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index e0301f45ded..e90f965a6f9 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -191,15 +191,14 @@ public: case ResultType::DateTime64: { UInt32 scale = 0; - auto scale_date_time = 0; - if (third_argument == ThirdArgument::IsOrigin) - scale_date_time = assert_cast(*arguments[0].type.get()).getScale(); + if (isDate32(arguments[0].type) || isDateTime(arguments[0].type) || isDateTime64(arguments[0].type)) + scale = assert_cast(*arguments[0].type.get()).getScale(); if (interval_type->getKind() == IntervalKind::Nanosecond) - scale = 9 > scale_date_time ? 9 : scale_date_time; + scale = 9 > scale ? 9 : scale; else if (interval_type->getKind() == IntervalKind::Microsecond) - scale = 6 > scale_date_time ? 6 : scale_date_time; + scale = 6 > scale ? 6 : scale; else if (interval_type->getKind() == IntervalKind::Millisecond) - scale = 3 > scale_date_time ? 3 : scale_date_time; + scale = 3 > scale ? 3 : scale; const size_t time_zone_arg_num = (arguments.size() == 2 || (arguments.size() == 3 && third_argument == ThirdArgument::IsTimezone)) ? 2 : 3; return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, time_zone_arg_num, 0, false)); diff --git a/tests/queries/0_stateless/02207_subseconds_intervals.reference b/tests/queries/0_stateless/02207_subseconds_intervals.reference index b0edbda5e76..91f0ecb8606 100644 --- a/tests/queries/0_stateless/02207_subseconds_intervals.reference +++ b/tests/queries/0_stateless/02207_subseconds_intervals.reference @@ -10,25 +10,25 @@ test intervals - test microseconds 1980-12-12 12:12:12.123456 1980-12-12 12:12:12.123400 -1980-12-12 12:12:12.12345600 -1980-12-12 12:12:12.12345600 +1980-12-12 12:12:12.123456 +1980-12-12 12:12:12.123456 1930-12-12 12:12:12.123456 1930-12-12 12:12:12.123400 -1930-12-12 12:12:12.12345600 +1930-12-12 12:12:12.123456 2220-12-12 12:12:12.123456 2220-12-12 12:12:12.123400 -2220-12-12 12:12:12.12345600 +2220-12-12 12:12:12.123456 - test milliseconds 1980-12-12 12:12:12.123 1980-12-12 12:12:12.120 -1980-12-12 12:12:12.123000 -1980-12-12 12:12:12.123000 +1980-12-12 12:12:12.123 +1980-12-12 12:12:12.123 1930-12-12 12:12:12.123 1930-12-12 12:12:12.120 -1930-12-12 12:12:12.123000 +1930-12-12 12:12:12.123 2220-12-12 12:12:12.123 2220-12-12 12:12:12.120 -2220-12-12 12:12:12.123000 +2220-12-12 12:12:12.123 test add[...]seconds() - test nanoseconds 1980-12-12 12:12:12.123456790 From 4c83b7e46ffe50409aab35b889839fb6cb92a18e Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:18:43 +0000 Subject: [PATCH 022/289] style fix --- src/Functions/toStartOfInterval.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index e90f965a6f9..b55d92f809b 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -191,7 +191,7 @@ public: case ResultType::DateTime64: { UInt32 scale = 0; - if (isDate32(arguments[0].type) || isDateTime(arguments[0].type) || isDateTime64(arguments[0].type)) + if (isDate32(arguments[0].type) || isDateTime(arguments[0].type) || isDateTime64(arguments[0].type)) scale = assert_cast(*arguments[0].type.get()).getScale(); if (interval_type->getKind() == IntervalKind::Nanosecond) scale = 9 > scale ? 9 : scale; From 5793725bc7ae8cfd66bea9c9f82ef0e30c124314 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 8 Dec 2023 16:45:45 +0000 Subject: [PATCH 023/289] tests --- .../02207_subseconds_intervals.reference | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/queries/0_stateless/02207_subseconds_intervals.reference b/tests/queries/0_stateless/02207_subseconds_intervals.reference index 91f0ecb8606..bedd5d4878b 100644 --- a/tests/queries/0_stateless/02207_subseconds_intervals.reference +++ b/tests/queries/0_stateless/02207_subseconds_intervals.reference @@ -10,25 +10,25 @@ test intervals - test microseconds 1980-12-12 12:12:12.123456 1980-12-12 12:12:12.123400 -1980-12-12 12:12:12.123456 -1980-12-12 12:12:12.123456 +1980-12-12 12:12:12.12345600 +1980-12-12 12:12:12.12345600 1930-12-12 12:12:12.123456 1930-12-12 12:12:12.123400 -1930-12-12 12:12:12.123456 +1930-12-12 12:12:12.12345600 2220-12-12 12:12:12.123456 2220-12-12 12:12:12.123400 -2220-12-12 12:12:12.123456 +2220-12-12 12:12:12.12345600 - test milliseconds 1980-12-12 12:12:12.123 1980-12-12 12:12:12.120 -1980-12-12 12:12:12.123 -1980-12-12 12:12:12.123 +1980-12-12 12:12:12.123000 +1980-12-12 12:12:12.123000 1930-12-12 12:12:12.123 1930-12-12 12:12:12.120 -1930-12-12 12:12:12.123 +1930-12-12 12:12:12.123000 2220-12-12 12:12:12.123 2220-12-12 12:12:12.120 -2220-12-12 12:12:12.123 +2220-12-12 12:12:12.123000 test add[...]seconds() - test nanoseconds 1980-12-12 12:12:12.123456790 @@ -75,4 +75,4 @@ test subtract[...]seconds() 2022-12-31 23:59:59.999 2022-12-31 23:59:59.900 2023-01-01 00:00:00.001 -2023-01-01 00:00:00.100 +2023-01-01 00:00:00.100 \ No newline at end of file From 53ef9c0cb80c734caa449430ff4600f32b900134 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 8 Dec 2023 18:38:25 +0100 Subject: [PATCH 024/289] Update 02207_subseconds_intervals.reference --- tests/queries/0_stateless/02207_subseconds_intervals.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02207_subseconds_intervals.reference b/tests/queries/0_stateless/02207_subseconds_intervals.reference index bedd5d4878b..b0edbda5e76 100644 --- a/tests/queries/0_stateless/02207_subseconds_intervals.reference +++ b/tests/queries/0_stateless/02207_subseconds_intervals.reference @@ -75,4 +75,4 @@ test subtract[...]seconds() 2022-12-31 23:59:59.999 2022-12-31 23:59:59.900 2023-01-01 00:00:00.001 -2023-01-01 00:00:00.100 \ No newline at end of file +2023-01-01 00:00:00.100 From 87bda03da17cdf0e3878a1df73175c92bae834c6 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 8 Dec 2023 21:36:24 +0100 Subject: [PATCH 025/289] logical error --- src/Functions/toStartOfInterval.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index b55d92f809b..b6a3a9389d6 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -191,7 +191,7 @@ public: case ResultType::DateTime64: { UInt32 scale = 0; - if (isDate32(arguments[0].type) || isDateTime(arguments[0].type) || isDateTime64(arguments[0].type)) + if (isDate32(arguments[0].type) || isDateTime64(arguments[0].type)) scale = assert_cast(*arguments[0].type.get()).getScale(); if (interval_type->getKind() == IntervalKind::Nanosecond) scale = 9 > scale ? 9 : scale; From d1c49cc9bcb869030361821e60349e8054a51c4b Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 12 Dec 2023 17:55:53 +0100 Subject: [PATCH 026/289] Added comments, simplified and fixed review --- src/Functions/toStartOfInterval.cpp | 138 ++++++++++++++++------------ 1 file changed, 81 insertions(+), 57 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index b6a3a9389d6..6c71b357590 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -1,9 +1,7 @@ -#include -#include #include #include #include -#include +#include #include "DataTypes/IDataType.h" #include #include @@ -16,7 +14,7 @@ #include #include #include -#include +#include namespace DB @@ -31,21 +29,23 @@ namespace ErrorCodes } -namespace -{ - class FunctionToStartOfInterval : public IFunction { public: + enum class Overload + { + Default, /// toStartOfInterval(time, interval) or toStartOfInterval(time, interval, timezone) + Origin /// toStartOfInterval(time, interval, origin) or toStartOfInterval(time, interval, origin, timezone) + }; + mutable Overload overload; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } static constexpr auto name = "toStartOfInterval"; String getName() const override { return name; } - bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2, 3}; } @@ -59,8 +59,9 @@ public: { const DataTypePtr & type_arg1 = arguments[0].type; if (!isDate(type_arg1) && !isDateTime(type_arg1) && !isDateTime64(type_arg1)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 1st argument of function {}. " - "Should be a date or a date with time", type_arg1->getName(), getName()); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of 1st argument of function {}, expected a Date, DateTime or DateTime64", + type_arg1->getName(), getName()); value_is_date = isDate(type_arg1); }; @@ -75,10 +76,14 @@ public: auto check_second_argument = [&] { const DataTypePtr & type_arg2 = arguments[1].type; + interval_type = checkAndGetDataType(type_arg2.get()); if (!interval_type) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 2nd argument of function {}. " - "Should be an interval of time", type_arg2->getName(), getName()); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of 2nd argument of function {}, expected a time interval", + type_arg2->getName(), getName()); + + /// Result here is determined for default overload (without origin) switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) { case IntervalKind::Nanosecond: @@ -89,7 +94,7 @@ public: case IntervalKind::Second: case IntervalKind::Minute: case IntervalKind::Hour: - case IntervalKind::Day: + case IntervalKind::Day: /// weird why Day leads to DateTime but too afraid to change it result_type = ResultType::DateTime; break; case IntervalKind::Week: @@ -101,31 +106,26 @@ public: } }; - enum class ThirdArgument - { - IsTimezone, - IsOrigin - }; - ThirdArgument third_argument; /// valid only if 3rd argument is given auto check_third_argument = [&] { const DataTypePtr & type_arg3 = arguments[2].type; if (isString(type_arg3)) { - third_argument = ThirdArgument::IsTimezone; + overload = Overload::Default; + if (value_is_date && result_type == ResultType::Date) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The timezone argument of function {} with interval type {} is allowed only when the 1st argument has the type DateTime or DateTime64", getName(), interval_type->getKind().toString()); } - else if (isDateOrDate32OrDateTimeOrDateTime64(type_arg3)) + else if (isDateTimeOrDateTime64(type_arg3) || isDate(type_arg3)) { - third_argument = ThirdArgument::IsOrigin; + overload = Overload::Origin; if (isDateTime64(arguments[0].type) && isDateTime64(arguments[2].type)) result_type = ResultType::DateTime64; else if (isDateTime(arguments[0].type) && isDateTime(arguments[2].type)) result_type = ResultType::DateTime; - else if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) && (isDate(arguments[2].type) || isDate32(arguments[2].type))) + else if (isDate(arguments[0].type) && isDate(arguments[2].type)) result_type = ResultType::Date; else throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); @@ -138,7 +138,7 @@ public: auto check_fourth_argument = [&] { - if (third_argument != ThirdArgument::IsOrigin) /// sanity check + if (overload != Overload::Origin) /// sanity check throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 3rd argument of function {}. " "The third argument must a Date/Date32/DateTime/DateTime64 with a constant origin", arguments[2].type->getName(), getName()); @@ -185,7 +185,7 @@ public: return std::make_shared(); case ResultType::DateTime: { - const size_t time_zone_arg_num = (arguments.size() == 2 || (arguments.size() == 3 && third_argument == ThirdArgument::IsTimezone)) ? 2 : 3; + const size_t time_zone_arg_num = (overload == Overload::Default) ? 2 : 3; return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, time_zone_arg_num, 0, false)); } case ResultType::DateTime64: @@ -200,7 +200,7 @@ public: else if (interval_type->getKind() == IntervalKind::Millisecond) scale = 3 > scale ? 3 : scale; - const size_t time_zone_arg_num = (arguments.size() == 2 || (arguments.size() == 3 && third_argument == ThirdArgument::IsTimezone)) ? 2 : 3; + const size_t time_zone_arg_num = (overload == Overload::Default) ? 2 : 3; return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, time_zone_arg_num, 0, false)); } } @@ -278,25 +278,25 @@ private: if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. Must contain dates or dates with time", getName()); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for 1st argument of function {}, expected a Date, DateTime or DateTime64", getName()); } template ColumnPtr dispatchForIntervalColumn( const TimeDataType & time_data_type, const TimeColumnType & time_column, const ColumnWithTypeAndName & interval_column, const ColumnWithTypeAndName & origin_column, - const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale = 1) const + const DataTypePtr & result_type, const DateLUTImpl & time_zone, UInt16 scale = 1) const { const auto * interval_type = checkAndGetDataType(interval_column.type.get()); if (!interval_type) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for second argument of function {}, must be an interval of time.", getName()); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for 2nd argument of function {}, must be a time interval", getName()); const auto * interval_column_const_int64 = checkAndGetColumnConst(interval_column.column.get()); if (!interval_column_const_int64) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for second argument of function {}, must be a const interval of time.", getName()); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for 2nd argument of function {}, must be a const time interval", getName()); - Int64 num_units = interval_column_const_int64->getValue(); + const Int64 num_units = interval_column_const_int64->getValue(); if (num_units <= 0) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Value for second argument of function {} must be positive.", getName()); + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Value for 2nd argument of function {} must be positive", getName()); switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) { @@ -328,12 +328,11 @@ private: } template - Int64 decideScaleOnPrecision(const UInt16 scale) const + Int64 decideScaleOnPrecision() const { static constexpr Int64 MILLISECOND_SCALE = 1000; static constexpr Int64 MICROSECOND_SCALE = 1000000; static constexpr Int64 NANOSECOND_SCALE = 1000000000; - Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale); switch (unit) { case IntervalKind::Millisecond: @@ -343,37 +342,41 @@ private: case IntervalKind::Nanosecond: return NANOSECOND_SCALE; default: - return scale_multiplier; + return 1; } } - template + template ColumnPtr execute(const TimeDataType &, const ColumnType & time_column_type, Int64 num_units, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale) const { - using ToColumnType = typename ToDataType::ColumnType; - using ToFieldType = typename ToDataType::FieldType; + using ResultColumnType = typename ResultDataType::ColumnType; + using ResultFieldType = typename ResultDataType::FieldType; const auto & time_data = time_column_type.getData(); size_t size = time_data.size(); auto result_col = result_type->createColumn(); - auto * col_to = assert_cast(result_col.get()); + auto * col_to = assert_cast(result_col.get()); auto & result_data = col_to->getData(); result_data.resize(size); - Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale); - Int64 scale_on_interval = decideScaleOnPrecision(scale); - Int64 scale_diff = scale_on_interval > scale_multiplier ? scale_on_interval / scale_multiplier : scale_multiplier / scale_on_interval; + Int64 scale_on_time = DecimalUtils::scaleMultiplier(scale); // scale that depends on type of arguments + Int64 scale_on_interval = decideScaleOnPrecision(); // scale that depends on the Interval + /// In case if we have a difference between time arguments and Interval, we need to calculate the difference between them + /// to get the right precision for the result. + Int64 scale_diff = scale_on_interval > scale_on_time ? scale_on_interval / scale_on_time : scale_on_time / scale_on_interval; if (origin_column.column == nullptr) { for (size_t i = 0; i != size; ++i) { result_data[i] = 0; - if (scale_on_interval < scale_multiplier) - result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_multiplier)) * scale_diff; + if (scale_on_interval < scale_on_time) + /// if we have a time argument that has bigger scale than the interval can contain, we need + /// to return a value with bigger precision and thus we should multiply result on the scale difference. + result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_on_interval)) * scale_diff; else - result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_multiplier)); + result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_on_time)); } } else @@ -387,31 +390,54 @@ private: throw Exception(ErrorCodes::BAD_ARGUMENTS, "The origin must be before the end date/datetime"); t -= origin; - auto res = static_cast(ToStartOfInterval::execute(t, num_units, time_zone, scale_multiplier)); + auto res = static_cast(ToStartOfInterval::execute(t, num_units, time_zone, scale_on_time)); static constexpr size_t SECONDS_PER_DAY = 86400; result_data[i] = 0; if (unit == IntervalKind::Week || unit == IntervalKind::Month || unit == IntervalKind::Quarter || unit == IntervalKind::Year) { + /// By default, when we use week, month, quarter or year interval, we get date return type. So, simply add values. if (isDate(result_type) || isDate32(result_type)) result_data[i] += origin + res; - else if (isDateTime64(result_type)) - result_data[i] += origin + (res * SECONDS_PER_DAY * scale_multiplier); - else + /// When we use DateTime arguments, we should keep in mind that we also have hours, minutes and seconds there, + /// so we need to multiply result by amount of seconds per day. + else if (isDateTime(result_type)) result_data[i] += origin + res * SECONDS_PER_DAY; + /// When we use DateTime64 arguments, we also should multiply it on right scale. + else + result_data[i] += origin + (res * SECONDS_PER_DAY * scale_on_time); } else { + /// In this case result will be calculated as datetime, so we need to get the amount of days if the arguments are Date. if (isDate(result_type) || isDate32(result_type)) res = res / SECONDS_PER_DAY; - if (scale_on_interval > scale_multiplier) - result_data[i] += (origin + res / scale_diff) * scale_diff; - else if (scale_on_interval == scale_multiplier && scale_on_interval % 1000 != 0 && scale_multiplier != 10) - result_data[i] += origin + (res * scale_on_interval); + /// Case when Interval has default scale + if (scale_on_interval == 1) + { + /// Case when the arguments are DateTime64 with precision like 4,5,7,8. Here res has right precision and origin doesn't. + if (scale_on_time % 1000 != 0 && scale_on_time >= 1000) + result_data[i] += (origin + res / scale_on_time) * scale_on_time; + /// Special case when the arguments are DateTime64 with precision 2. Here origin has right precision and res doesn't + else if (scale_on_time == 100) + result_data[i] += (origin + res * scale_on_time); + /// Cases when precision of DateTime64 is 1, 3, 6, 9 e.g. has right precision in res and origin. + else + result_data[i] += (origin + res); + } + /// Case when Interval has some specific scale (3,6,9) else - result_data[i] += origin + res * scale_diff; + { + /// If we have a time argument that has bigger scale than the interval can contain, we need + /// to return a value with bigger precision and thus we should multiply result on the scale difference. + if (scale_on_interval < scale_on_time) + result_data[i] += origin + res * scale_diff; + /// The other case: interval has bigger scale than the interval or they have the same scale, so res has the right precision and origin doesn't + else + result_data[i] += (origin + res / scale_diff) * scale_diff; + } } } } @@ -419,8 +445,6 @@ private: } }; -} - REGISTER_FUNCTION(ToStartOfInterval) { factory.registerFunction(); From 36fa954f5000c8130f2556abba6052392dd16fa7 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 12 Dec 2023 18:19:11 +0100 Subject: [PATCH 027/289] Stly check --- src/Functions/toStartOfInterval.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index f4171748b92..fb071c952ab 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -371,7 +371,7 @@ private: { result_data[i] = 0; if (scale_on_interval < scale_on_time) - /// if we have a time argument that has bigger scale than the interval can contain, we need + /// if we have a time argument that has bigger scale than the interval can contain, we need /// to return a value with bigger precision and thus we should multiply result on the scale difference. result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_on_interval)) * scale_diff; else @@ -429,7 +429,7 @@ private: /// Case when Interval has some specific scale (3,6,9) else { - /// If we have a time argument that has bigger scale than the interval can contain, we need + /// If we have a time argument that has bigger scale than the interval can contain, we need /// to return a value with bigger precision and thus we should multiply result on the scale difference. if (scale_on_interval < scale_on_time) result_data[i] += origin + res * scale_diff; From 202ca21e3f1366479ec4f0d786adeab6e54cc3d4 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:16:51 +0100 Subject: [PATCH 028/289] fix tests --- src/Functions/toStartOfInterval.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index fb071c952ab..56e721b7601 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -370,12 +370,12 @@ private: for (size_t i = 0; i != size; ++i) { result_data[i] = 0; - if (scale_on_interval < scale_on_time) - /// if we have a time argument that has bigger scale than the interval can contain, we need + if (scale_on_interval < scale_on_time && scale_on_interval != 1) + /// If we have a time argument that has bigger scale than the interval can contain and interval is not default, we need /// to return a value with bigger precision and thus we should multiply result on the scale difference. - result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_on_interval)) * scale_diff; + result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_on_time)) * scale_diff; else - result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_on_time)); + result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_on_time)); } } else From 174309821a9813dba1d2a090768332e9de9470e3 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 19 Dec 2023 20:48:30 +0000 Subject: [PATCH 029/289] Small fixups --- src/Functions/toStartOfInterval.cpp | 181 +++++++++++++--------------- 1 file changed, 85 insertions(+), 96 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 56e721b7601..81a2fd0a75d 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -31,7 +31,7 @@ namespace ErrorCodes class FunctionToStartOfInterval : public IFunction { -public: +private: enum class Overload { Default, /// toStartOfInterval(time, interval) or toStartOfInterval(time, interval, timezone) @@ -39,6 +39,7 @@ public: }; mutable Overload overload; +public: static FunctionPtr create(ContextPtr) { return std::make_shared(); } static constexpr auto name = "toStartOfInterval"; @@ -82,7 +83,9 @@ public: "Illegal type {} of 2nd argument of function {}, expected a time interval", type_arg2->getName(), getName()); - /// Result here is determined for default overload (without origin) + overload = Overload::Default; + + /// Determine result type for default overload (no origin) switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) { case IntervalKind::Nanosecond: @@ -110,21 +113,20 @@ public: const DataTypePtr & type_arg3 = arguments[2].type; if (isString(type_arg3)) { - overload = Overload::Default; - if (value_is_date && result_type == ResultType::Date) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "The timezone argument of function {} with interval type {} is allowed only when the 1st argument has the type DateTime or DateTime64", + "A timezone argument of function {} with interval type {} is allowed only when the 1st argument has the type DateTime or DateTime64", getName(), interval_type->getKind().toString()); } - else if (isDateTimeOrDateTime64(type_arg3) || isDate(type_arg3)) + else if (isDate(type_arg3) || isDateTime(type_arg3) || isDateTime64(type_arg3)) { overload = Overload::Origin; - if (isDateTime64(arguments[0].type) && isDateTime64(arguments[2].type)) + const DataTypePtr & type_arg1 = arguments[0].type; + if (isDateTime64(type_arg1) && isDateTime64(type_arg3)) result_type = ResultType::DateTime64; - else if (isDateTime(arguments[0].type) && isDateTime(arguments[2].type)) + else if (isDateTime(type_arg1) && isDateTime(type_arg3)) result_type = ResultType::DateTime; - else if (isDate(arguments[0].type) && isDate(arguments[2].type)) + else if (isDate(type_arg1) && isDate(type_arg3)) result_type = ResultType::Date; else throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); @@ -149,7 +151,7 @@ public: type_arg4->getName(), getName()); if (value_is_date && result_type == ResultType::Date) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "The timezone argument of function {} with interval type {} is allowed only when the 1st argument has the type DateTime or DateTime64", + "A timezone argument of function {} with interval type {} is allowed only when the 1st argument has the type DateTime or DateTime64", getName(), interval_type->getKind().toString()); }; @@ -190,14 +192,14 @@ public: case ResultType::DateTime64: { UInt32 scale = 0; - if (isDate32(arguments[0].type) || isDateTime64(arguments[0].type)) + if (isDateTime64(arguments[0].type)) scale = assert_cast(*arguments[0].type.get()).getScale(); if (interval_type->getKind() == IntervalKind::Nanosecond) - scale = 9 > scale ? 9 : scale; + scale = (9 > scale) ? 9 : scale; else if (interval_type->getKind() == IntervalKind::Microsecond) - scale = 6 > scale ? 6 : scale; + scale = (6 > scale) ? 6 : scale; else if (interval_type->getKind() == IntervalKind::Millisecond) - scale = 3 > scale ? 3 : scale; + scale = (3 > scale) ? 3 : scale; const size_t time_zone_arg_num = (overload == Overload::Default) ? 2 : 3; return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, time_zone_arg_num, 0, false)); @@ -213,20 +215,19 @@ public: const auto & interval_column = arguments[1]; ColumnWithTypeAndName origin_column; - const bool has_origin_arg = (arguments.size() == 3 && isDateOrDate32OrDateTimeOrDateTime64(arguments[2].type)) || arguments.size() == 4; - if (has_origin_arg) + if (overload == Overload::Origin) origin_column = arguments[2]; - const size_t time_zone_arg_num = (arguments.size() == 2 || (arguments.size() == 3 && isString(arguments[2].type))) ? 2 : 3; + const size_t time_zone_arg_num = (overload == Overload::Origin) ? 3 : 2; const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, time_zone_arg_num, 0); - ColumnPtr result_column = nullptr; - if (isDateTime64(result_type)) - result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); + ColumnPtr result_column; + if (isDate(result_type)) + result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); else if (isDateTime(result_type)) result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); - else - result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); + else if (isDateTime64(result_type)) + result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); return result_column; } @@ -238,44 +239,24 @@ private: const auto & time_column_type = *time_column.type.get(); const auto & time_column_col = *time_column.column.get(); - if (isDateTime64(time_column_type)) + if (isDate(time_column_type)) { - if (origin_column.column != nullptr && !isDateTime64(origin_column.type.get())) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); - - const auto * time_column_vec = checkAndGetColumn(time_column_col); - auto scale = assert_cast(time_column_type).getScale(); - - if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone, scale); - } - else if (isDateTime(time_column_type)) - { - if (origin_column.column != nullptr && !isDateTime(origin_column.type.get())) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); - - const auto * time_column_vec = checkAndGetColumn(time_column_col); - if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); - } - else if (isDate(time_column_type)) - { - if (origin_column.column != nullptr && !isDate(origin_column.type.get())) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); - const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } - else if (isDate32(time_column_type)) + else if (isDateTime(time_column_type)) { - if (origin_column.column != nullptr) - if (!isDate32(origin_column.type.get())) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); - - const auto * time_column_vec = checkAndGetColumn(time_column_col); + const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); + } + else if (isDateTime64(time_column_type)) + { + const auto * time_column_vec = checkAndGetColumn(time_column_col); + auto scale = assert_cast(time_column_type).getScale(); + if (time_column_vec) + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone, scale); } throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for 1st argument of function {}, expected a Date, DateTime or DateTime64", getName()); } @@ -327,26 +308,23 @@ private: } template - Int64 decideScaleOnPrecision() const + static Int64 scaleFromInterval() { - static constexpr Int64 MILLISECOND_SCALE = 1000; - static constexpr Int64 MICROSECOND_SCALE = 1000000; - static constexpr Int64 NANOSECOND_SCALE = 1000000000; switch (unit) { case IntervalKind::Millisecond: - return MILLISECOND_SCALE; + return 1'000; case IntervalKind::Microsecond: - return MICROSECOND_SCALE; + return 1'000'000; case IntervalKind::Nanosecond: - return NANOSECOND_SCALE; + return 1'000'000'000; default: return 1; } } template - ColumnPtr execute(const TimeDataType &, const ColumnType & time_column_type, Int64 num_units, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale) const + ColumnPtr execute(const TimeDataType &, const ColumnType & time_column_type, Int64 num_units, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, UInt16 scale) const { using ResultColumnType = typename ResultDataType::ColumnType; using ResultFieldType = typename ResultDataType::FieldType; @@ -359,23 +337,29 @@ private: auto & result_data = col_to->getData(); result_data.resize(size); - Int64 scale_on_time = DecimalUtils::scaleMultiplier(scale); // scale that depends on type of arguments - Int64 scale_on_interval = decideScaleOnPrecision(); // scale that depends on the Interval + const Int64 scale_time = DecimalUtils::scaleMultiplier(scale); + const Int64 scale_interval = scaleFromInterval(); + /// In case if we have a difference between time arguments and Interval, we need to calculate the difference between them /// to get the right precision for the result. - Int64 scale_diff = scale_on_interval > scale_on_time ? scale_on_interval / scale_on_time : scale_on_time / scale_on_interval; + const Int64 scale_diff = (scale_interval > scale_time) ? (scale_interval / scale_time) : (scale_time / scale_interval); if (origin_column.column == nullptr) { - for (size_t i = 0; i != size; ++i) + if (scale_time > scale_interval && scale_interval != 1) { - result_data[i] = 0; - if (scale_on_interval < scale_on_time && scale_on_interval != 1) + for (size_t i = 0; i != size; ++i) + { /// If we have a time argument that has bigger scale than the interval can contain and interval is not default, we need /// to return a value with bigger precision and thus we should multiply result on the scale difference. - result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_on_time)) * scale_diff; - else - result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_on_time)); + result_data[i] = 0; + result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_time)) * scale_diff; + } + } + else + { + for (size_t i = 0; i != size; ++i) + result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_time)); } } else @@ -386,55 +370,60 @@ private: { auto t = time_data[i]; if (origin > static_cast(t)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "The origin must be before the end date/datetime"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The origin must be before the end date / date with time"); + + /// The trick to calculate the interval starting from an offset is to + /// 1. subtract the offset, + /// 2. perform the calculation, and + /// 3. add the offset to the result. t -= origin; - auto res = static_cast(ToStartOfInterval::execute(t, num_units, time_zone, scale_on_time)); + auto res = static_cast(ToStartOfInterval::execute(t, num_units, time_zone, scale_time)); - static constexpr size_t SECONDS_PER_DAY = 86400; + static constexpr size_t SECONDS_PER_DAY = 86'400; result_data[i] = 0; if (unit == IntervalKind::Week || unit == IntervalKind::Month || unit == IntervalKind::Quarter || unit == IntervalKind::Year) { - /// By default, when we use week, month, quarter or year interval, we get date return type. So, simply add values. - if (isDate(result_type) || isDate32(result_type)) + /// For such intervals, ToStartOfInterval::execute() returns days + if (isDate(result_type)) result_data[i] += origin + res; - /// When we use DateTime arguments, we should keep in mind that we also have hours, minutes and seconds there, - /// so we need to multiply result by amount of seconds per day. else if (isDateTime(result_type)) result_data[i] += origin + res * SECONDS_PER_DAY; - /// When we use DateTime64 arguments, we also should multiply it on right scale. - else - result_data[i] += origin + (res * SECONDS_PER_DAY * scale_on_time); + else if (isDateTime64(result_type)) + result_data[i] += origin + (res * SECONDS_PER_DAY * scale_time); } else { - /// In this case result will be calculated as datetime, so we need to get the amount of days if the arguments are Date. - if (isDate(result_type) || isDate32(result_type)) + /// ToStartOfInterval::execute() returns seconds + + if (isDate(result_type)) res = res / SECONDS_PER_DAY; - /// Case when Interval has default scale - if (scale_on_interval == 1) + if (scale_interval == 1) { - /// Case when the arguments are DateTime64 with precision like 4,5,7,8. Here res has right precision and origin doesn't. - if (scale_on_time % 1000 != 0 && scale_on_time >= 1000) - result_data[i] += (origin + res / scale_on_time) * scale_on_time; - /// Special case when the arguments are DateTime64 with precision 2. Here origin has right precision and res doesn't - else if (scale_on_time == 100) - result_data[i] += (origin + res * scale_on_time); - /// Cases when precision of DateTime64 is 1, 3, 6, 9 e.g. has right precision in res and origin. + /// Interval has default scale, i.e. Year - Second + + if (scale_time % 1000 != 0 && scale_time >= 1000) + /// The arguments are DateTime64 with precision like 4,5,7,8. Here res has right precision and origin doesn't. + result_data[i] += (origin + res / scale_time) * scale_time; + else if (scale_time == 100) + /// The arguments are DateTime64 with precision 2. Here origin has right precision and res doesn't + result_data[i] += (origin + res * scale_time); else + /// Precision of DateTime64 is 1, 3, 6, 9, e.g. has right precision in res and origin. result_data[i] += (origin + res); } - /// Case when Interval has some specific scale (3,6,9) else { - /// If we have a time argument that has bigger scale than the interval can contain, we need - /// to return a value with bigger precision and thus we should multiply result on the scale difference. - if (scale_on_interval < scale_on_time) + /// Interval has some specific scale (3,6,9), i.e. Millisecond - Nanosecond + + if (scale_interval < scale_time) + /// If we have a time argument that has bigger scale than the interval can contain, we need + /// to return a value with bigger precision and thus we should multiply result on the scale difference. result_data[i] += origin + res * scale_diff; - /// The other case: interval has bigger scale than the interval or they have the same scale, so res has the right precision and origin doesn't else + /// The other case: interval has bigger scale than the interval or they have the same scale, so res has the right precision and origin doesn't result_data[i] += (origin + res / scale_diff) * scale_diff; } } From 861421d27ac74fe11921c4ba901dcccb94df76e8 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 5 Jan 2024 16:03:22 +0000 Subject: [PATCH 030/289] fixes --- src/Functions/DateTimeTransforms.h | 57 ++++++--- src/Functions/toStartOfInterval.cpp | 117 ++++++++++++------ ...to_start_of_interval_with_origin.reference | 32 ++--- ...02916_to_start_of_interval_with_origin.sql | 8 +- 4 files changed, 139 insertions(+), 75 deletions(-) diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 74b37e18907..dbe2b11d7b2 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -487,7 +487,7 @@ struct ToStartOfInterval { throwDateTimeIsNotSupported(TO_START_OF_INTERVAL_NAME); } - static Int64 execute(Int64 t, Int64 nanoseconds, const DateLUTImpl &, Int64 scale_multiplier) + static Int64 execute(Int64 t, Int64 nanoseconds, const DateLUTImpl &, Int64 scale_multiplier, Int64 /*origin*/ = 0) { if (scale_multiplier < 1000000000) { @@ -522,7 +522,7 @@ struct ToStartOfInterval { throwDateTimeIsNotSupported(TO_START_OF_INTERVAL_NAME); } - static Int64 execute(Int64 t, Int64 microseconds, const DateLUTImpl &, Int64 scale_multiplier) + static Int64 execute(Int64 t, Int64 microseconds, const DateLUTImpl &, Int64 scale_multiplier, Int64 /*origin*/ = 0) { if (scale_multiplier < 1000000) { @@ -565,7 +565,7 @@ struct ToStartOfInterval { throwDateTimeIsNotSupported(TO_START_OF_INTERVAL_NAME); } - static Int64 execute(Int64 t, Int64 milliseconds, const DateLUTImpl &, Int64 scale_multiplier) + static Int64 execute(Int64 t, Int64 milliseconds, const DateLUTImpl &, Int64 scale_multiplier, Int64 /*origin*/ = 0) { if (scale_multiplier < 1000) { @@ -608,7 +608,7 @@ struct ToStartOfInterval { return time_zone.toStartOfSecondInterval(t, seconds); } - static Int64 execute(Int64 t, Int64 seconds, const DateLUTImpl & time_zone, Int64 scale_multiplier) + static Int64 execute(Int64 t, Int64 seconds, const DateLUTImpl & time_zone, Int64 scale_multiplier, Int64 /*origin*/ = 0) { return time_zone.toStartOfSecondInterval(t / scale_multiplier, seconds); } @@ -629,7 +629,7 @@ struct ToStartOfInterval { return time_zone.toStartOfMinuteInterval(t, minutes); } - static Int64 execute(Int64 t, Int64 minutes, const DateLUTImpl & time_zone, Int64 scale_multiplier) + static Int64 execute(Int64 t, Int64 minutes, const DateLUTImpl & time_zone, Int64 scale_multiplier, Int64 /*origin*/ = 0) { return time_zone.toStartOfMinuteInterval(t / scale_multiplier, minutes); } @@ -650,7 +650,7 @@ struct ToStartOfInterval { return time_zone.toStartOfHourInterval(t, hours); } - static Int64 execute(Int64 t, Int64 hours, const DateLUTImpl & time_zone, Int64 scale_multiplier) + static Int64 execute(Int64 t, Int64 hours, const DateLUTImpl & time_zone, Int64 scale_multiplier, Int64 /*origin*/ = 0) { return time_zone.toStartOfHourInterval(t / scale_multiplier, hours); } @@ -671,7 +671,7 @@ struct ToStartOfInterval { return static_cast(time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days)); } - static Int64 execute(Int64 t, Int64 days, const DateLUTImpl & time_zone, Int64 scale_multiplier) + static Int64 execute(Int64 t, Int64 days, const DateLUTImpl & time_zone, Int64 scale_multiplier, Int64 /*origin*/ = 0) { return time_zone.toStartOfDayInterval(time_zone.toDayNum(t / scale_multiplier), days); } @@ -692,9 +692,12 @@ struct ToStartOfInterval { return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); } - static UInt16 execute(Int64 t, Int64 weeks, const DateLUTImpl & time_zone, Int64 scale_multiplier) + static Int64 execute(Int64 t, Int64 weeks, const DateLUTImpl & time_zone, Int64 scale_multiplier, Int64 origin = 0) { - return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t / scale_multiplier), weeks); + if (origin == 0) + return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t / scale_multiplier), weeks); + else + return ToStartOfInterval::execute(t, weeks * 7, time_zone, scale_multiplier, origin); } }; @@ -713,9 +716,24 @@ struct ToStartOfInterval { return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); } - static UInt16 execute(Int64 t, Int64 months, const DateLUTImpl & time_zone, Int64 scale_multiplier) + static Int64 execute(Int64 t, Int64 months, const DateLUTImpl & time_zone, Int64 scale_multiplier, Int64 origin = 0) { - return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t / scale_multiplier), months); + if (origin == 0) + return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t / scale_multiplier), months); + else + { + Int64 days = time_zone.toDayOfMonth(t / scale_multiplier + origin) - time_zone.toDayOfMonth(origin); + Int64 months_to_add = time_zone.toMonth(t / scale_multiplier + origin) - time_zone.toMonth(origin); + Int64 years = time_zone.toYear(t / scale_multiplier + origin) - time_zone.toYear(origin); + months_to_add = days < 0 ? months_to_add - 1 : months_to_add; + months_to_add += years * 12; + Int64 month_multiplier = (months_to_add / months) * months; + Int64 a = 0; + + a = time_zone.addMonths(time_zone.toDate(origin), month_multiplier); + // a += time_zone.toTime(origin); + return a - time_zone.toDate(origin); + } } }; @@ -734,9 +752,12 @@ struct ToStartOfInterval { return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); } - static UInt16 execute(Int64 t, Int64 quarters, const DateLUTImpl & time_zone, Int64 scale_multiplier) + static Int64 execute(Int64 t, Int64 quarters, const DateLUTImpl & time_zone, Int64 scale_multiplier, Int64 origin = 0) { - return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t / scale_multiplier), quarters); + if (origin == 0) + return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t / scale_multiplier), quarters); + else + return ToStartOfInterval::execute(t, quarters * 3, time_zone, scale_multiplier, origin); } }; @@ -755,9 +776,15 @@ struct ToStartOfInterval { return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); } - static UInt16 execute(Int64 t, Int64 years, const DateLUTImpl & time_zone, Int64 scale_multiplier) + static Int64 execute(Int64 t, Int64 years, const DateLUTImpl & time_zone, Int64 scale_multiplier, Int64 origin = 0) { - return time_zone.toStartOfYearInterval(time_zone.toDayNum(t / scale_multiplier), years); + if (origin == 0) + return time_zone.toStartOfYearInterval(time_zone.toDayNum(t / scale_multiplier), years); + else + { + auto a = ToStartOfInterval::execute(t, years * 12, time_zone, scale_multiplier, origin); + return a; + } } }; diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 81a2fd0a75d..1ba7fed4bee 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -1,17 +1,17 @@ +#include +#include +#include #include #include #include #include -#include "DataTypes/IDataType.h" #include #include #include #include #include -#include #include #include -#include #include #include #include @@ -270,6 +270,27 @@ private: if (!interval_type) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for 2nd argument of function {}, must be a time interval", getName()); + if (isDate(time_data_type) || isDateTime(time_data_type)) + { + switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) + { + case IntervalKind::Nanosecond: + case IntervalKind::Microsecond: + case IntervalKind::Millisecond: + if (isDate(time_data_type) || isDateTime(time_data_type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal interval kind for argument data type {}", isDate(time_data_type) ? "Date" : "DateTime"); + break; + case IntervalKind::Second: + case IntervalKind::Minute: + case IntervalKind::Hour: + if (isDate(time_data_type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal interval kind for argument data type Date"); + break; + default: + break; + } + } + const auto * interval_column_const_int64 = checkAndGetColumnConst(interval_column.column.get()); if (!interval_column_const_int64) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for 2nd argument of function {}, must be a const time interval", getName()); @@ -337,94 +358,110 @@ private: auto & result_data = col_to->getData(); result_data.resize(size); - const Int64 scale_time = DecimalUtils::scaleMultiplier(scale); + const Int64 scale_endtime = DecimalUtils::scaleMultiplier(scale); const Int64 scale_interval = scaleFromInterval(); /// In case if we have a difference between time arguments and Interval, we need to calculate the difference between them /// to get the right precision for the result. - const Int64 scale_diff = (scale_interval > scale_time) ? (scale_interval / scale_time) : (scale_time / scale_interval); + const Int64 scale_diff = (scale_interval > scale_endtime) ? (scale_interval / scale_endtime) : (scale_endtime / scale_interval); if (origin_column.column == nullptr) { - if (scale_time > scale_interval && scale_interval != 1) + if (scale_endtime > scale_interval && scale_interval != 1) { for (size_t i = 0; i != size; ++i) { /// If we have a time argument that has bigger scale than the interval can contain and interval is not default, we need /// to return a value with bigger precision and thus we should multiply result on the scale difference. result_data[i] = 0; - result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_time)) * scale_diff; + result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_endtime)) * scale_diff; } } else { for (size_t i = 0; i != size; ++i) - result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_time)); + result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_endtime)); } } else { UInt64 origin = origin_column.column->get64(0); + Int64 origin_scale = 1; + if (isDateTime64(origin_column.type.get())) + origin_scale = assert_cast(*origin_column.type.get()).getScale(); for (size_t i = 0; i != size; ++i) { - auto t = time_data[i]; - if (origin > static_cast(t)) + UInt64 end_time = time_data[i]; + + if (origin > static_cast(end_time) && origin_scale == scale) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The origin must be before the end date / date with time"); + else if (origin_scale > scale) + origin /= static_cast(std::pow(10, origin_scale - scale)); /// If aguments have different scales, we make + else if (origin_scale < scale) /// origin argument to have the same scale as the first argument. + origin *= static_cast(std::pow(10, scale - origin_scale)); /// The trick to calculate the interval starting from an offset is to /// 1. subtract the offset, /// 2. perform the calculation, and /// 3. add the offset to the result. - t -= origin; - auto res = static_cast(ToStartOfInterval::execute(t, num_units, time_zone, scale_time)); - static constexpr size_t SECONDS_PER_DAY = 86'400; - result_data[i] = 0; - if (unit == IntervalKind::Week || unit == IntervalKind::Month || unit == IntervalKind::Quarter || unit == IntervalKind::Year) + + if (isDate(origin_column.type.get())) /// We need to perform calculations on dateTime (dateTime64) values only. { - /// For such intervals, ToStartOfInterval::execute() returns days - if (isDate(result_type)) - result_data[i] += origin + res; - else if (isDateTime(result_type)) - result_data[i] += origin + res * SECONDS_PER_DAY; - else if (isDateTime64(result_type)) - result_data[i] += origin + (res * SECONDS_PER_DAY * scale_time); + end_time *= SECONDS_PER_DAY; + origin *= SECONDS_PER_DAY; + } + + Int64 delta = (end_time - origin) * (isDateTime64(origin_column.type.get()) ? 1 : scale_endtime); /// No need to multiply on scale endtime if we have dateTime64 argument. + Int64 offset = 0; + + { + auto origin_data = isDateTime64(result_type) ? origin / scale_endtime : origin; + offset = static_cast(ToStartOfInterval::execute(delta, num_units, time_zone, scale_endtime, origin_data)); + } + + + if (isDate(result_type)) /// The result should be a date and the calculations were as datetime. + result_data[i] += (origin + offset) / SECONDS_PER_DAY; + else if (unit == IntervalKind::Week || unit == IntervalKind::Month || unit == IntervalKind::Quarter || unit == IntervalKind::Year) + { + if (isDateTime64(result_type)) /// We need to have the right scale for offset, origin already has the right scale. + offset *= scale_endtime; + + result_data[i] += origin + offset; } else { - /// ToStartOfInterval::execute() returns seconds - - if (isDate(result_type)) - res = res / SECONDS_PER_DAY; + /// ToStartOfInterval::execute() returns seconds. if (scale_interval == 1) { - /// Interval has default scale, i.e. Year - Second + if (isDateTime64(result_type)) /// We need to have the right scale for offset, origin already has the right scale. + offset *= scale_endtime; - if (scale_time % 1000 != 0 && scale_time >= 1000) - /// The arguments are DateTime64 with precision like 4,5,7,8. Here res has right precision and origin doesn't. - result_data[i] += (origin + res / scale_time) * scale_time; - else if (scale_time == 100) - /// The arguments are DateTime64 with precision 2. Here origin has right precision and res doesn't - result_data[i] += (origin + res * scale_time); + /// Interval has default scale, i.e. Year - Second. + + if (scale_endtime % 1000 != 0 && scale_endtime >= 1000) + /// The arguments are DateTime64 with precision like 4,5,7,8. Here offset has right precision and origin doesn't. + result_data[i] += (origin + offset / scale_endtime) * scale_endtime; else - /// Precision of DateTime64 is 1, 3, 6, 9, e.g. has right precision in res and origin. - result_data[i] += (origin + res); + /// Precision of DateTime64 is 1, 2, 3, 6, 9, e.g. has right precision in offset and origin. + result_data[i] += (origin + offset); } else { - /// Interval has some specific scale (3,6,9), i.e. Millisecond - Nanosecond + /// Interval has some specific scale (3,6,9), i.e. Millisecond - Nanosecond. - if (scale_interval < scale_time) + if (scale_interval < scale_endtime) /// If we have a time argument that has bigger scale than the interval can contain, we need /// to return a value with bigger precision and thus we should multiply result on the scale difference. - result_data[i] += origin + res * scale_diff; + result_data[i] += origin + offset * scale_diff; else - /// The other case: interval has bigger scale than the interval or they have the same scale, so res has the right precision and origin doesn't - result_data[i] += (origin + res / scale_diff) * scale_diff; + /// The other case: interval has bigger scale than the interval or they have the same scale, so offset has the right precision and origin doesn't. + result_data[i] += (origin + offset / scale_diff) * scale_diff; } } } diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference index 870853bc371..969e2726902 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference @@ -1,40 +1,40 @@ -- Negative tests -Time and origin as Time +Time and origin as Date 2023-02-01 2023-08-01 -2023-10-09 -2023-10-05 2023-10-08 +2023-10-08 +2023-10-09 Time and origin as DateTime 2023-02-01 09:08:07 2023-08-01 09:08:07 -2023-10-09 09:08:07 -2023-10-05 09:08:07 +2023-10-08 09:08:07 +2023-10-08 09:08:07 2023-10-09 09:08:07 2023-10-09 10:10:07 -2023-10-09 10:11:11 +2023-10-09 10:11:07 2023-10-09 10:11:12 Time and origin as DateTime64(9) 2023-02-01 09:08:07.123456789 2023-08-01 09:08:07.123456789 2023-09-10 09:08:07.123456789 -2023-10-05 09:08:07.123456789 -2023-10-08 09:08:07.123543189 -2023-10-09 09:10:07.123460389 -2023-10-09 10:10:11.123456849 -2023-10-09 10:11:10.123456791 +2023-10-08 09:08:07.123456789 +2023-10-09 09:08:07.123456789 +2023-10-09 10:10:07.123456789 +2023-10-09 10:11:11.123456789 +2023-10-09 10:11:12.123456789 2023-10-09 10:11:12.987456789 2023-10-09 10:11:12.987653789 2023-10-09 10:11:12.987654321 Time and origin as DateTime64(3) 2023-02-01 09:08:07.123 2023-08-01 09:08:07.123 +2023-10-08 09:08:07.123 +2023-10-08 09:08:07.123 2023-10-09 09:08:07.123 -2023-10-05 09:08:07.123 -2023-10-08 09:09:33.523 -2023-10-09 09:10:10.723 -2023-10-09 10:10:11.183 -2023-10-09 10:11:10.125 +2023-10-09 10:10:07.123 +2023-10-09 10:11:11.123 +2023-10-09 10:11:12.123 2023-10-09 10:11:12.987 2023-10-09 10:11:12.987000 2023-10-09 10:11:12.987000000 diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql index 71f5fb7fb36..4f8a96b093d 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql @@ -23,7 +23,7 @@ SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), 5 -- too many arguments SELECT toStartOfInterval(toDateTime('2023-01-02 14:45:50'), toIntervalYear(1), toDateTime('2020-01-02 14:44:30'), 'Europe/Amsterdam', 5); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } -SELECT 'Time and origin as Time'; +SELECT 'Time and origin as Date'; SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalYear(1), toDate('2022-02-01')); SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalQuarter(1), toDate('2022-02-01')); SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalMonth(1), toDate('2023-09-08')); @@ -43,8 +43,8 @@ SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMonth(1), SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalWeek(1), toDateTime('2023-10-01 09:08:07')); SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalDay(1), toDateTime('2023-10-08 09:08:07')); SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalHour(1), toDateTime('2023-10-09 09:10:07')); -SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMinute(1), toDateTime('2023-10-09 10:10:11')); -SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalSecond(1), toDateTime('2023-10-09 10:11:10')); +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMinute(1), toDateTime('2023-10-09 09:10:07')); +SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalSecond(1), toDateTime('2023-10-09 09:10:07')); SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMillisecond(1), toDateTime('2023-10-09 10:11:12')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalMicrosecond(1), toDateTime('2023-10-09 10:11:12')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalNanosecond(1), toDateTime('2023-10-09 10:11:12')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } @@ -56,7 +56,7 @@ SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toInt SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalWeek(1), toDateTime64('2023-10-01 09:08:07.123456789', 9)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalDay(1), toDateTime64('2023-10-08 09:08:07.123456789', 9)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalHour(1), toDateTime64('2023-10-09 09:10:07.123456789', 9)); -SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMinute(1), toDateTime64('2023-10-09 10:10:11.123456789', 9)); +SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMinute(1), toDateTime64('2023-10-09 09:10:11.123456789', 9)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalSecond(1), toDateTime64('2023-10-09 10:11:10.123456789', 9)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMillisecond(1), toDateTime64('2023-10-09 10:11:12.123456789', 9)); SELECT toStartOfInterval(toDateTime64('2023-10-09 10:11:12.987654321', 9), toIntervalMicrosecond(1), toDateTime64('2023-10-09 10:11:12.123456789', 9)); From cb645f82198250b119d734291456afe8dcdde27f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 5 Jan 2024 17:18:56 +0100 Subject: [PATCH 031/289] fix style --- src/Functions/toStartOfInterval.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 1ba7fed4bee..ffabf38ef20 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -397,7 +397,7 @@ private: if (origin > static_cast(end_time) && origin_scale == scale) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The origin must be before the end date / date with time"); else if (origin_scale > scale) - origin /= static_cast(std::pow(10, origin_scale - scale)); /// If aguments have different scales, we make + origin /= static_cast(std::pow(10, origin_scale - scale)); /// If arguments have different scales, we make else if (origin_scale < scale) /// origin argument to have the same scale as the first argument. origin *= static_cast(std::pow(10, scale - origin_scale)); @@ -439,16 +439,16 @@ private: if (scale_interval == 1) { - if (isDateTime64(result_type)) /// We need to have the right scale for offset, origin already has the right scale. + if (isDateTime64(result_type)) /// We need to have the right scale for offset, origin already has the correct scale. offset *= scale_endtime; /// Interval has default scale, i.e. Year - Second. if (scale_endtime % 1000 != 0 && scale_endtime >= 1000) - /// The arguments are DateTime64 with precision like 4,5,7,8. Here offset has right precision and origin doesn't. + /// The arguments are DateTime64 with precision like 4,5,7,8. Here offset has correct precision and origin doesn't. result_data[i] += (origin + offset / scale_endtime) * scale_endtime; else - /// Precision of DateTime64 is 1, 2, 3, 6, 9, e.g. has right precision in offset and origin. + /// Precision of DateTime64 is 1, 2, 3, 6, 9, e.g. has correct precision in offset and origin. result_data[i] += (origin + offset); } else From 1117284be7f72a22ac841af8f1a91ec853adc900 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 5 Jan 2024 23:43:58 +0000 Subject: [PATCH 032/289] fix overflow --- src/Functions/DateTimeTransforms.h | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index dbe2b11d7b2..e9cee9616fb 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -697,7 +697,12 @@ struct ToStartOfInterval if (origin == 0) return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t / scale_multiplier), weeks); else - return ToStartOfInterval::execute(t, weeks * 7, time_zone, scale_multiplier, origin); + { + if (const auto weeks_to_days = weeks * 7; weeks_to_days / 7 == weeks) // Check if multiplication doesn't overflow Int64 value + return ToStartOfInterval::execute(t, weeks_to_days, time_zone, scale_multiplier, origin); + else + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 7 is out of bounds for type Int64", weeks); + } } }; @@ -728,11 +733,8 @@ struct ToStartOfInterval months_to_add = days < 0 ? months_to_add - 1 : months_to_add; months_to_add += years * 12; Int64 month_multiplier = (months_to_add / months) * months; - Int64 a = 0; - a = time_zone.addMonths(time_zone.toDate(origin), month_multiplier); - // a += time_zone.toTime(origin); - return a - time_zone.toDate(origin); + return time_zone.addMonths(time_zone.toDate(origin), month_multiplier) - time_zone.toDate(origin); } } }; @@ -757,7 +759,12 @@ struct ToStartOfInterval if (origin == 0) return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t / scale_multiplier), quarters); else - return ToStartOfInterval::execute(t, quarters * 3, time_zone, scale_multiplier, origin); + { + if (const auto quarters_to_months = quarters * 3; quarters_to_months / 3 == quarters) // Check if multiplication doesn't overflow Int64 value + return ToStartOfInterval::execute(t, quarters_to_months, time_zone, scale_multiplier, origin); + else + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 3 is out of bounds for type Int64", quarters); + } } }; @@ -782,8 +789,10 @@ struct ToStartOfInterval return time_zone.toStartOfYearInterval(time_zone.toDayNum(t / scale_multiplier), years); else { - auto a = ToStartOfInterval::execute(t, years * 12, time_zone, scale_multiplier, origin); - return a; + if (const auto years_to_months = years * 12; years_to_months / 12 == years) // Check if multiplication doesn't overflow Int64 value + return ToStartOfInterval::execute(t, years_to_months, time_zone, scale_multiplier, origin); + else + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 12 is out of bounds for type Int64", years); } } }; From 07f031ec8a26d0ee7081d725fe59b312101bcae8 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:34:22 +0100 Subject: [PATCH 033/289] fix fuzzer --- src/Functions/DateTimeTransforms.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index e9cee9616fb..dd843daed8c 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -20,6 +20,7 @@ namespace DB { +static Int64 Int64_max_value = std::numeric_limits::max(); static constexpr auto microsecond_multiplier = 1000000; static constexpr auto millisecond_multiplier = 1000; @@ -698,8 +699,8 @@ struct ToStartOfInterval return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t / scale_multiplier), weeks); else { - if (const auto weeks_to_days = weeks * 7; weeks_to_days / 7 == weeks) // Check if multiplication doesn't overflow Int64 value - return ToStartOfInterval::execute(t, weeks_to_days, time_zone, scale_multiplier, origin); + if (weeks < Int64_max_value / 7) // Check if multiplication doesn't overflow Int64 value + return ToStartOfInterval::execute(t, weeks * 7, time_zone, scale_multiplier, origin); else throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 7 is out of bounds for type Int64", weeks); } @@ -760,8 +761,8 @@ struct ToStartOfInterval return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t / scale_multiplier), quarters); else { - if (const auto quarters_to_months = quarters * 3; quarters_to_months / 3 == quarters) // Check if multiplication doesn't overflow Int64 value - return ToStartOfInterval::execute(t, quarters_to_months, time_zone, scale_multiplier, origin); + if (quarters < Int64_max_value / 3) // Check if multiplication doesn't overflow Int64 value + return ToStartOfInterval::execute(t, quarters * 3, time_zone, scale_multiplier, origin); else throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 3 is out of bounds for type Int64", quarters); } @@ -789,8 +790,8 @@ struct ToStartOfInterval return time_zone.toStartOfYearInterval(time_zone.toDayNum(t / scale_multiplier), years); else { - if (const auto years_to_months = years * 12; years_to_months / 12 == years) // Check if multiplication doesn't overflow Int64 value - return ToStartOfInterval::execute(t, years_to_months, time_zone, scale_multiplier, origin); + if (years < Int64_max_value / 12) // Check if multiplication doesn't overflow Int64 value + return ToStartOfInterval::execute(t, years * 12, time_zone, scale_multiplier, origin); else throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 12 is out of bounds for type Int64", years); } From 26561c6bdd22085d6fe8537fa37cbf12de573efd Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 12 Jan 2024 17:47:17 +0100 Subject: [PATCH 034/289] fix due to #58557 --- .../02916_to_start_of_interval_with_origin.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference index 969e2726902..552323be1a5 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference @@ -24,7 +24,7 @@ Time and origin as DateTime64(9) 2023-10-09 10:11:11.123456789 2023-10-09 10:11:12.123456789 2023-10-09 10:11:12.987456789 -2023-10-09 10:11:12.987653789 +2023-10-09 10:11:12.987654789 2023-10-09 10:11:12.987654321 Time and origin as DateTime64(3) 2023-02-01 09:08:07.123 From 85a35dce28d3a367cf306c2e95edb41a3484a9c8 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 12 Jan 2024 17:22:16 +0000 Subject: [PATCH 035/289] fix tests --- .../0_stateless/02207_subseconds_intervals.reference | 6 +++--- .../02956_fix_to_start_of_milli_microsecond.reference | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/02207_subseconds_intervals.reference b/tests/queries/0_stateless/02207_subseconds_intervals.reference index b0edbda5e76..6cde773c3c4 100644 --- a/tests/queries/0_stateless/02207_subseconds_intervals.reference +++ b/tests/queries/0_stateless/02207_subseconds_intervals.reference @@ -10,14 +10,14 @@ test intervals - test microseconds 1980-12-12 12:12:12.123456 1980-12-12 12:12:12.123400 -1980-12-12 12:12:12.12345600 -1980-12-12 12:12:12.12345600 +1980-12-12 12:12:12.12345700 +1980-12-12 12:12:12.12345700 1930-12-12 12:12:12.123456 1930-12-12 12:12:12.123400 1930-12-12 12:12:12.12345600 2220-12-12 12:12:12.123456 2220-12-12 12:12:12.123400 -2220-12-12 12:12:12.12345600 +2220-12-12 12:12:12.12345700 - test milliseconds 1980-12-12 12:12:12.123 1980-12-12 12:12:12.120 diff --git a/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference b/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference index d3a002c4fd4..dff0c2a9585 100644 --- a/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference +++ b/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference @@ -1,4 +1,4 @@ -2023-10-09 10:11:12.001 -2023-10-09 10:11:12.001 -2023-10-09 10:11:12.000 -2023-10-09 10:11:12.000 +2023-10-09 10:11:12.001000 +2023-10-09 10:11:12.001000 +2023-10-09 10:11:12.000000 +2023-10-09 10:11:12.000000 From 9aa0fa11f843bf82432f93d497ec9a68dc756db6 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 10 Apr 2024 14:01:53 +0200 Subject: [PATCH 036/289] IntervalKind -> IntervalKind::Kind --- src/Functions/DateTimeTransforms.h | 6 +-- src/Functions/toStartOfInterval.cpp | 70 ++++++++++++++--------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 8d70dbea685..20dc1bc21f2 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -703,7 +703,7 @@ struct ToStartOfInterval else { if (weeks < Int64_max_value / 7) // Check if multiplication doesn't overflow Int64 value - return ToStartOfInterval::execute(t, weeks * 7, time_zone, scale_multiplier, origin); + return ToStartOfInterval::execute(t, weeks * 7, time_zone, scale_multiplier, origin); else throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 7 is out of bounds for type Int64", weeks); } @@ -765,7 +765,7 @@ struct ToStartOfInterval else { if (quarters < Int64_max_value / 3) // Check if multiplication doesn't overflow Int64 value - return ToStartOfInterval::execute(t, quarters * 3, time_zone, scale_multiplier, origin); + return ToStartOfInterval::execute(t, quarters * 3, time_zone, scale_multiplier, origin); else throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 3 is out of bounds for type Int64", quarters); } @@ -794,7 +794,7 @@ struct ToStartOfInterval else { if (years < Int64_max_value / 12) // Check if multiplication doesn't overflow Int64 value - return ToStartOfInterval::execute(t, years * 12, time_zone, scale_multiplier, origin); + return ToStartOfInterval::execute(t, years * 12, time_zone, scale_multiplier, origin); else throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 12 is out of bounds for type Int64", years); } diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index bdf947977b6..ab8dfef58ca 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -192,11 +192,11 @@ public: UInt32 scale = 0; if (isDateTime64(arguments[0].type)) scale = assert_cast(*arguments[0].type.get()).getScale(); - if (interval_type->getKind() == IntervalKind::Nanosecond) + if (interval_type->getKind() == IntervalKind::Kind::Nanosecond) scale = (9 > scale) ? 9 : scale; - else if (interval_type->getKind() == IntervalKind::Microsecond) + else if (interval_type->getKind() == IntervalKind::Kind::Microsecond) scale = (6 > scale) ? 6 : scale; - else if (interval_type->getKind() == IntervalKind::Millisecond) + else if (interval_type->getKind() == IntervalKind::Kind::Millisecond) scale = (3 > scale) ? 3 : scale; const size_t time_zone_arg_num = (overload == Overload::Default) ? 2 : 3; @@ -272,15 +272,15 @@ private: { switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) { - case IntervalKind::Nanosecond: - case IntervalKind::Microsecond: - case IntervalKind::Millisecond: + case IntervalKind::Kind::Nanosecond: + case IntervalKind::Kind::Microsecond: + case IntervalKind::Kind::Millisecond: if (isDate(time_data_type) || isDateTime(time_data_type)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal interval kind for argument data type {}", isDate(time_data_type) ? "Date" : "DateTime"); break; - case IntervalKind::Second: - case IntervalKind::Minute: - case IntervalKind::Hour: + case IntervalKind::Kind::Second: + case IntervalKind::Kind::Minute: + case IntervalKind::Kind::Hour: if (isDate(time_data_type)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal interval kind for argument data type Date"); break; @@ -299,28 +299,28 @@ private: switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) { - case IntervalKind::Nanosecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); - case IntervalKind::Microsecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); - case IntervalKind::Millisecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); - case IntervalKind::Second: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); - case IntervalKind::Minute: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); - case IntervalKind::Hour: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); - case IntervalKind::Day: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); - case IntervalKind::Week: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); - case IntervalKind::Month: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); - case IntervalKind::Quarter: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); - case IntervalKind::Year: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + case IntervalKind::Kind::Nanosecond: + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + case IntervalKind::Kind::Microsecond: + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + case IntervalKind::Kind::Millisecond: + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + case IntervalKind::Kind::Second: + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + case IntervalKind::Kind::Minute: + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + case IntervalKind::Kind::Hour: + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + case IntervalKind::Kind::Day: + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + case IntervalKind::Kind::Week: + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + case IntervalKind::Kind::Month: + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + case IntervalKind::Kind::Quarter: + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + case IntervalKind::Kind::Year: + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); } std::unreachable(); @@ -331,11 +331,11 @@ private: { switch (unit) { - case IntervalKind::Millisecond: + case IntervalKind::Kind::Millisecond: return 1'000; - case IntervalKind::Microsecond: + case IntervalKind::Kind::Microsecond: return 1'000'000; - case IntervalKind::Nanosecond: + case IntervalKind::Kind::Nanosecond: return 1'000'000'000; default: return 1; @@ -424,7 +424,7 @@ private: if (isDate(result_type)) /// The result should be a date and the calculations were as datetime. result_data[i] += (origin + offset) / SECONDS_PER_DAY; - else if (unit == IntervalKind::Week || unit == IntervalKind::Month || unit == IntervalKind::Quarter || unit == IntervalKind::Year) + else if (unit == IntervalKind::Kind::Week || unit == IntervalKind::Kind::Month || unit == IntervalKind::Kind::Quarter || unit == IntervalKind::Kind::Year) { if (isDateTime64(result_type)) /// We need to have the right scale for offset, origin already has the right scale. offset *= scale_endtime; From 6109da248f9a4a3d26e0ca8c41225129202e9688 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 10 Apr 2024 14:29:13 +0200 Subject: [PATCH 037/289] fix test --- .../02956_fix_to_start_of_milli_microsecond.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference b/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference index b005ce6dfb0..95a05a24981 100644 --- a/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference +++ b/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference @@ -2,6 +2,6 @@ 2023-10-09 10:11:12.001000 2023-10-09 10:11:12.000000 2023-10-09 10:11:12.000000 -2023-10-09 00:00:00.000000 -2023-10-09 00:00:00.000 +2023-10-09 00:00:00.000000000 +2023-10-09 00:00:00.000000000 2023-10-09 00:00:00 From 938c888b131d15b311c8368337d36a52ad2b0a02 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 11 Apr 2024 17:01:01 +0200 Subject: [PATCH 038/289] Reload CI to remove build error (empty commit) From a959663e977361af3896f010f9aeadaa8b64c323 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 23 May 2024 16:43:18 +0200 Subject: [PATCH 039/289] Update toStartOfInterval.cpp --- src/Functions/toStartOfInterval.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 749aa9f8800..a4f870613d3 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -241,14 +241,14 @@ private: if (isDate(time_column_type)) { - const auto * time_column_vec = checkAndGetColumn(time_column_col); + const auto * time_column_vec = checkAndGetColumn(time_column_col); if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDateTime(time_column_type)) { - const auto * time_column_vec = checkAndGetColumn(&time_column_col); + const auto * time_column_vec = checkAndGetColumn(&time_column_col); if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } From 1f26281493d66f15408d8459c198170e71901f68 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 27 May 2024 15:43:14 +0200 Subject: [PATCH 040/289] build fix --- src/Functions/toStartOfInterval.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index a4f870613d3..c45501aa905 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -241,20 +241,20 @@ private: if (isDate(time_column_type)) { - const auto * time_column_vec = checkAndGetColumn(time_column_col); + const auto * time_column_vec = checkAndGetColumn(&time_column_col); if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDateTime(time_column_type)) { - const auto * time_column_vec = checkAndGetColumn(&time_column_col); + const auto * time_column_vec = checkAndGetColumn(&time_column_col); if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDateTime64(time_column_type)) { - const auto * time_column_vec = checkAndGetColumn(time_column_col); + const auto * time_column_vec = checkAndGetColumn(&time_column_col); auto scale = assert_cast(time_column_type).getScale(); if (time_column_vec) From 9a7888ba516420c00978901bbe5b85cafe6e7bdf Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 5 Jun 2024 16:39:54 +0200 Subject: [PATCH 041/289] fix tests --- src/Functions/toStartOfInterval.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index c45501aa905..b98d78171ae 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -376,7 +376,7 @@ private: /// If we have a time argument that has bigger scale than the interval can contain and interval is not default, we need /// to return a value with bigger precision and thus we should multiply result on the scale difference. result_data[i] = 0; - result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_endtime)) * scale_diff; + result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_endtime)); } } else From 63b0d13a62c84245a3efe6e3cd9cf08cd2a588a8 Mon Sep 17 00:00:00 2001 From: yariks5s Date: Thu, 6 Jun 2024 16:15:35 +0000 Subject: [PATCH 042/289] trying to fix tests --- src/Functions/toStartOfInterval.cpp | 2 -- .../02207_subseconds_intervals.reference | 16 ++++++++-------- ...6_fix_to_start_of_milli_microsecond.reference | 12 ++++++------ 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index b98d78171ae..e358addf972 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -192,8 +192,6 @@ public: case ResultType::DateTime64: { UInt32 scale = 0; - if (isDateTime64(arguments[0].type)) - scale = assert_cast(*arguments[0].type.get()).getScale(); if (interval_type->getKind() == IntervalKind::Kind::Nanosecond) scale = (9 > scale) ? 9 : scale; else if (interval_type->getKind() == IntervalKind::Kind::Microsecond) diff --git a/tests/queries/0_stateless/02207_subseconds_intervals.reference b/tests/queries/0_stateless/02207_subseconds_intervals.reference index 6cde773c3c4..7e6d64b6b9f 100644 --- a/tests/queries/0_stateless/02207_subseconds_intervals.reference +++ b/tests/queries/0_stateless/02207_subseconds_intervals.reference @@ -10,25 +10,25 @@ test intervals - test microseconds 1980-12-12 12:12:12.123456 1980-12-12 12:12:12.123400 -1980-12-12 12:12:12.12345700 -1980-12-12 12:12:12.12345700 +1980-12-12 12:12:12.123457 +1980-12-12 12:12:12.123457 1930-12-12 12:12:12.123456 1930-12-12 12:12:12.123400 -1930-12-12 12:12:12.12345600 +1930-12-12 12:12:12.123456 2220-12-12 12:12:12.123456 2220-12-12 12:12:12.123400 -2220-12-12 12:12:12.12345700 +2220-12-12 12:12:12.123457 - test milliseconds 1980-12-12 12:12:12.123 1980-12-12 12:12:12.120 -1980-12-12 12:12:12.123000 -1980-12-12 12:12:12.123000 +1980-12-12 12:12:12.123 +1980-12-12 12:12:12.123 1930-12-12 12:12:12.123 1930-12-12 12:12:12.120 -1930-12-12 12:12:12.123000 +1930-12-12 12:12:12.123 2220-12-12 12:12:12.123 2220-12-12 12:12:12.120 -2220-12-12 12:12:12.123000 +2220-12-12 12:12:12.123 test add[...]seconds() - test nanoseconds 1980-12-12 12:12:12.123456790 diff --git a/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference b/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference index 95a05a24981..413c79828c7 100644 --- a/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference +++ b/tests/queries/0_stateless/02956_fix_to_start_of_milli_microsecond.reference @@ -1,7 +1,7 @@ -2023-10-09 10:11:12.001000 -2023-10-09 10:11:12.001000 -2023-10-09 10:11:12.000000 -2023-10-09 10:11:12.000000 -2023-10-09 00:00:00.000000000 -2023-10-09 00:00:00.000000000 +2023-10-09 10:11:12.001 +2023-10-09 10:11:12.001 +2023-10-09 10:11:12.000 +2023-10-09 10:11:12.000 +2023-10-09 00:00:00.000000 +2023-10-09 00:00:00.000 2023-10-09 00:00:00 From c5ba8eb477fb168d38dd1493b2b20a121a3ed1fd Mon Sep 17 00:00:00 2001 From: yariks5s Date: Thu, 6 Jun 2024 16:46:38 +0000 Subject: [PATCH 043/289] fix for origin --- src/Functions/toStartOfInterval.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index e358addf972..cc5ffd56976 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -192,6 +192,8 @@ public: case ResultType::DateTime64: { UInt32 scale = 0; + if (isDateTime64(arguments[0].type) && overload == Overload::Origin) + scale = assert_cast(*arguments[0].type.get()).getScale(); if (interval_type->getKind() == IntervalKind::Kind::Nanosecond) scale = (9 > scale) ? 9 : scale; else if (interval_type->getKind() == IntervalKind::Kind::Microsecond) From f9b70ea77a3b3059d9c784e2fe4b90000e75d3d4 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sat, 22 Jun 2024 09:30:16 +0200 Subject: [PATCH 044/289] Add JSONCompactWithProgressRowOutputFormat --- src/Formats/registerFormats.cpp | 2 + ...JSONCompactWithProgressRowOutputFormat.cpp | 125 ++++++++++++++++++ .../JSONCompactWithProgressRowOutputFormat.h | 53 ++++++++ 3 files changed, 180 insertions(+) create mode 100644 src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp create mode 100644 src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h diff --git a/src/Formats/registerFormats.cpp b/src/Formats/registerFormats.cpp index 57ca1bb49c8..770b747fafd 100644 --- a/src/Formats/registerFormats.cpp +++ b/src/Formats/registerFormats.cpp @@ -95,6 +95,7 @@ void registerOutputFormatMarkdown(FormatFactory & factory); void registerOutputFormatPostgreSQLWire(FormatFactory & factory); void registerOutputFormatPrometheus(FormatFactory & factory); void registerOutputFormatSQLInsert(FormatFactory & factory); +void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory); /// Input only formats. @@ -242,6 +243,7 @@ void registerFormats() registerOutputFormatCapnProto(factory); registerOutputFormatPrometheus(factory); registerOutputFormatSQLInsert(factory); + registerOutputFormatJSONCompactWithProgress(factory); registerInputFormatRegexp(factory); registerInputFormatJSONAsString(factory); diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp new file mode 100644 index 00000000000..78cf5b9a003 --- /dev/null +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -0,0 +1,125 @@ +#include +#include +#include + +#include + +#include + + + +namespace DB +{ + +JSONCompactWithProgressRowOutputFormat::JSONCompactWithProgressRowOutputFormat( + WriteBuffer & out_, + const Block & header, + const FormatSettings & settings_, + bool yield_strings_) + : JSONRowOutputFormat(out_, header, settings_, yield_strings_) +{ +} + +void JSONCompactWithProgressRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) +{ + JSONUtils::writeFieldFromColumn(column, serialization, row_num, yield_strings, settings, *ostr); + ++field_number; + LOG_DEBUG(getLogger("JSONCompactWithProgressRowOutputFormat"), "Field number: {}", field_number); +} + +void JSONCompactWithProgressRowOutputFormat::writeFieldDelimiter() +{ + JSONUtils::writeFieldCompactDelimiter(*ostr); +} + +void JSONCompactWithProgressRowOutputFormat::writeRowStartDelimiter() +{ + if (has_progress) + writeProgress(); + JSONUtils::writeCompactArrayStart(*ostr, 2); +} + +void JSONCompactWithProgressRowOutputFormat::writeRowEndDelimiter() +{ + JSONUtils::writeCompactArrayEnd(*ostr); + field_number = 0; + ++row_count; +} + +void JSONCompactWithProgressRowOutputFormat::writeBeforeTotals() +{ + JSONUtils::writeFieldDelimiter(*ostr, 2); + JSONUtils::writeCompactArrayStart(*ostr, 1, "totals"); +} + +void JSONCompactWithProgressRowOutputFormat::writeTotals(const Columns & columns, size_t row_num) +{ + JSONUtils::writeCompactColumns(columns, serializations, row_num, yield_strings, settings, *ostr); +} + +void JSONCompactWithProgressRowOutputFormat::writeAfterTotals() +{ + JSONUtils::writeCompactArrayEnd(*ostr); +} + +void JSONCompactWithProgressRowOutputFormat::writeExtremesElement(const char * title, const Columns & columns, size_t row_num) +{ + JSONUtils::writeCompactArrayStart(*ostr, 2, title); + JSONUtils::writeCompactColumns(columns, serializations, row_num, yield_strings, settings, *ostr); + JSONUtils::writeCompactArrayEnd(*ostr); +} + +void JSONCompactWithProgressRowOutputFormat::onProgress(const Progress & value) +{ + LOG_DEBUG(getLogger("JSONCompactWithProgressRowOutputFormat"), "onProgress: {}", value.read_rows); + + progress.incrementPiecewiseAtomically(value); + String progress_line; + WriteBufferFromString buf(progress_line); + writeCString("{\"progress\":", buf); + progress.writeJSON(buf); + writeCString("}\n", buf); + buf.finalize(); + std::lock_guard lock(progress_lines_mutex); + progress_lines.emplace_back(std::move(progress_line)); + has_progress = true; +} + + +void JSONCompactWithProgressRowOutputFormat::flush() +{ + if (has_progress) + writeProgress(); + JSONRowOutputFormat::flush(); +} + +void JSONCompactWithProgressRowOutputFormat::writeSuffix() +{ + if (has_progress) + writeProgress(); + JSONRowOutputFormat::writeSuffix(); +} + +void JSONCompactWithProgressRowOutputFormat::writeProgress() +{ + std::lock_guard lock(progress_lines_mutex); + for (const auto & progress_line : progress_lines) + writeString(progress_line, *ostr); + progress_lines.clear(); + has_progress = false; +} + +void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory) +{ + factory.registerOutputFormat("JSONCompactWithProgress", []( + WriteBuffer & buf, + const Block & sample, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings, false); + }); + + factory.markOutputFormatSupportsParallelFormatting("JSONCompactWithProgress"); +} + +} diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h new file mode 100644 index 00000000000..4bc10d41f19 --- /dev/null +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +struct FormatSettings; + +/** The stream for outputting data in the JSONCompact- formats. + */ +class JSONCompactWithProgressRowOutputFormat final : public JSONRowOutputFormat +{ +public: + JSONCompactWithProgressRowOutputFormat( + WriteBuffer & out_, + const Block & header, + const FormatSettings & settings_, + bool yield_strings_); + + String getName() const override { return "JSONCompactWithProgressRowOutputFormat"; } + + void onProgress(const Progress & value) override; + void flush() override; + +private: + void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; + void writeFieldDelimiter() override; + void writeRowStartDelimiter() override; + void writeRowEndDelimiter() override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; + void writeAfterTotals() override; + void writeExtremesElement(const char * title, const Columns & columns, size_t row_num) override; + void writeTotals(const Columns & columns, size_t row_num) override; + + void writeProgress(); + void writeSuffix() override; + + Progress progress; + std::vector progress_lines; + std::mutex progress_lines_mutex; + /// To not lock mutex and check progress_lines every row, + /// we will use atomic flag that progress_lines is not empty. + std::atomic_bool has_progress = false; +}; + +} From 20860ed8b04f6588de12b4e59baf932f02f90d27 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 23 Jun 2024 14:34:54 +0200 Subject: [PATCH 045/289] Enable onProgress calls to JSONCompactWithProgressRowOutputFormat --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 78cf5b9a003..2cc7d99f8e7 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -118,8 +118,6 @@ void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory) { return std::make_shared(buf, sample, format_settings, false); }); - - factory.markOutputFormatSupportsParallelFormatting("JSONCompactWithProgress"); } } From e1c60c4e40a03db459e71dd2b54b082d5205654d Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 24 Jun 2024 10:11:46 +0200 Subject: [PATCH 046/289] Remove debug output --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 2cc7d99f8e7..39532fb76fb 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -4,9 +4,6 @@ #include -#include - - namespace DB { @@ -24,7 +21,6 @@ void JSONCompactWithProgressRowOutputFormat::writeField(const IColumn & column, { JSONUtils::writeFieldFromColumn(column, serialization, row_num, yield_strings, settings, *ostr); ++field_number; - LOG_DEBUG(getLogger("JSONCompactWithProgressRowOutputFormat"), "Field number: {}", field_number); } void JSONCompactWithProgressRowOutputFormat::writeFieldDelimiter() @@ -71,8 +67,6 @@ void JSONCompactWithProgressRowOutputFormat::writeExtremesElement(const char * t void JSONCompactWithProgressRowOutputFormat::onProgress(const Progress & value) { - LOG_DEBUG(getLogger("JSONCompactWithProgressRowOutputFormat"), "onProgress: {}", value.read_rows); - progress.incrementPiecewiseAtomically(value); String progress_line; WriteBufferFromString buf(progress_line); From 4e350ff44a4dbdecb1c044fb10df131f6c31ae3d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 4 Jul 2024 18:48:41 +0200 Subject: [PATCH 047/289] don't allow explicit uuids and rmt args in replicated db --- src/Core/Settings.h | 3 +- src/Databases/DatabaseReplicated.cpp | 12 ----- src/Interpreters/InterpreterCreateQuery.cpp | 20 ++++++++ ...tractZooKeeperPathFromReplicatedTableDef.h | 2 +- .../MergeTree/registerStorageMergeTree.cpp | 46 ++++++++++++++----- tests/config/users.d/database_replicated.xml | 1 + .../configs/settings.xml | 1 + .../configs/settings2.xml | 1 + .../02858_explicit_uuid_and_zk_path.reference | 10 ++++ .../02858_explicit_uuid_and_zk_path.sh | 41 +++++++++++++++++ 10 files changed, 112 insertions(+), 25 deletions(-) create mode 100644 tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference create mode 100755 tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d84e5b149f6..c162c75b6a0 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -731,7 +731,8 @@ class IColumn; M(UInt64, max_distributed_depth, 5, "Maximum distributed query depth", 0) \ M(Bool, database_replicated_always_detach_permanently, false, "Execute DETACH TABLE as DETACH TABLE PERMANENTLY if database engine is Replicated", 0) \ M(Bool, database_replicated_allow_only_replicated_engine, false, "Allow to create only Replicated tables in database with engine Replicated", 0) \ - M(Bool, database_replicated_allow_replicated_engine_arguments, true, "Allow to create only Replicated tables in database with engine Replicated with explicit arguments", 0) \ + M(UInt64, database_replicated_allow_replicated_engine_arguments, 0, "0 - Don't allow to explicitly specify ZooKeeper path and replica name for *MergeTree tables in Replicated databases. 1 - Allow. 2 - Allow, but ignore the specified path and use default one instead.", 0) \ + M(UInt64, database_replicated_allow_explicit_uuid, 0, "0 - Don't allow to explicitly specify UUIDs for tables in Replicated databases. 1 - Allow. 2 - Allow, but ignore the specified UUID and generate a random one instead.", 0) \ M(Bool, cloud_mode, false, "Only available in ClickHouse Cloud", 0) \ M(UInt64, cloud_mode_engine, 1, "Only available in ClickHouse Cloud", 0) \ M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result, one of: 'none', 'throw', 'null_status_on_timeout', 'never_throw', 'none_only_active', 'throw_only_active', 'null_status_on_timeout_only_active'", 0) \ diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 4ca9afc49eb..507d1a9affc 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -774,18 +774,6 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_ bool maybe_replica_macros = info.expanded_other; bool enable_functional_tests_helper = getContext()->getConfigRef().has("_functional_tests_helper_database_replicated_replace_args_macros"); - if (!enable_functional_tests_helper) - { - if (query_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments) - LOG_WARNING(log, "It's not recommended to explicitly specify zookeeper_path and replica_name in ReplicatedMergeTree arguments"); - else - throw Exception(ErrorCodes::INCORRECT_QUERY, - "It's not allowed to specify explicit zookeeper_path and replica_name " - "for ReplicatedMergeTree arguments in Replicated database. If you really want to " - "specify them explicitly, enable setting " - "database_replicated_allow_replicated_engine_arguments."); - } - if (maybe_shard_macros && maybe_replica_macros) return; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index a990eb651ce..28827a1fe37 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1055,6 +1055,26 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data bool from_path = create.attach_from_path.has_value(); bool is_on_cluster = getContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; + if (database->getEngineName() == "Replicated" && create.uuid != UUIDHelpers::Nil && !is_replicated_database_internal) + { + if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 0) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "It's not allowed to explicitly specify UUIDs for tables in Replicated databases, " + "see database_replicated_allow_explicit_uuid"); + } + else if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 0) + { + LOG_WARNING(&Poco::Logger::get("InterpreterCreateQuery"), "It's not recommended to explicitly specify UUIDs for tables in Replicated databases"); + } + else // if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 2) + { + UUID old_uuid = create.uuid; + create.generateRandomUUID(/*always_generate_new_uuid*/ true); + LOG_INFO(&Poco::Logger::get("InterpreterCreateQuery"), "Replaced a user-provided UUID ({}) with a random one ({}) " + "to make sure it's unique", old_uuid, create.uuid); + } + } + if (is_replicated_database_internal && !internal) { if (create.uuid == UUIDHelpers::Nil) diff --git a/src/Storages/MergeTree/extractZooKeeperPathFromReplicatedTableDef.h b/src/Storages/MergeTree/extractZooKeeperPathFromReplicatedTableDef.h index 5ef5e1db62e..212dc048868 100644 --- a/src/Storages/MergeTree/extractZooKeeperPathFromReplicatedTableDef.h +++ b/src/Storages/MergeTree/extractZooKeeperPathFromReplicatedTableDef.h @@ -14,6 +14,6 @@ using ContextPtr = std::shared_ptr; /// Extracts a zookeeper path from a specified CREATE TABLE query. /// The function checks the table engine and if it is Replicated*MergeTree then it takes the first argument and expands macros in it. /// Returns std::nullopt if the specified CREATE query doesn't describe a Replicated table or its arguments can't be evaluated. -std::optional extractZooKeeperPathFromReplicatedTableDef(const ASTCreateQuery & create_query, const ContextPtr & context); +std::optional extractZooKeeperPathFromReplicatedTableDef(const ASTCreateQuery & create_query, const ContextPtr & local_context); } diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index d234103e52b..66f7b2114bf 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -185,7 +186,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( const String & engine_name, ASTs & engine_args, LoadingStrictnessLevel mode, - const ContextPtr & context, + const ContextPtr & local_context, String & zookeeper_path, String & replica_name, RenamingRestrictions & renaming_restrictions) @@ -202,11 +203,11 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( { /// Allow expressions in engine arguments. /// In new syntax argument can be literal or identifier or array/tuple of identifiers. - evaluateEngineArgs(engine_args, context); + evaluateEngineArgs(engine_args, local_context); } - bool is_on_cluster = context->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; - bool is_replicated_database = context->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY && + bool is_on_cluster = local_context->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; + bool is_replicated_database = local_context->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY && DatabaseCatalog::instance().getDatabase(table_id.database_name)->getEngineName() == "Replicated"; /// Allow implicit {uuid} macros only for zookeeper_path in ON CLUSTER queries @@ -226,10 +227,10 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( /// We did unfold it in previous versions to make moving table from Atomic to Ordinary database work correctly, /// but now it's not allowed (and it was the only reason to unfold {uuid} macro). info.table_id.uuid = UUIDHelpers::Nil; - zookeeper_path = context->getMacros()->expand(zookeeper_path, info); + zookeeper_path = local_context->getMacros()->expand(zookeeper_path, info); info.level = 0; - replica_name = context->getMacros()->expand(replica_name, info); + replica_name = local_context->getMacros()->expand(replica_name, info); } ast_zk_path->value = zookeeper_path; @@ -247,11 +248,11 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( } if (!allow_uuid_macro) info.table_id.uuid = UUIDHelpers::Nil; - zookeeper_path = context->getMacros()->expand(zookeeper_path, info); + zookeeper_path = local_context->getMacros()->expand(zookeeper_path, info); info.level = 0; info.table_id.uuid = UUIDHelpers::Nil; - replica_name = context->getMacros()->expand(replica_name, info); + replica_name = local_context->getMacros()->expand(replica_name, info); /// We do not allow renaming table with these macros in metadata, because zookeeper_path will be broken after RENAME TABLE. /// NOTE: it may happen if table was created by older version of ClickHouse (< 20.10) and macros was not unfolded on table creation @@ -268,9 +269,24 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( bool has_arguments = (arg_num + 2 <= arg_cnt); bool has_valid_arguments = has_arguments && engine_args[arg_num]->as() && engine_args[arg_num + 1]->as(); + const auto & server_settings = local_context->getServerSettings(); if (has_valid_arguments) { + if (is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 0) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "It's not allowed to specify explicit zookeeper_path and replica_name " + "for ReplicatedMergeTree arguments in Replicated database. If you really want to " + "specify them explicitly, enable setting " + "database_replicated_allow_replicated_engine_arguments."); + } + else if (is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 1) + { + LOG_WARNING(&Poco::Logger::get("registerStorageMergeTree"), "It's not recommended to explicitly specify " + "zookeeper_path and replica_name in ReplicatedMergeTree arguments"); + } + /// Get path and name from engine arguments auto * ast_zk_path = engine_args[arg_num]->as(); if (ast_zk_path && ast_zk_path->value.getType() == Field::Types::String) @@ -284,6 +300,15 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Replica name must be a string literal{}", verbose_help_message); + + if (is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 2) + { + LOG_INFO(&Poco::Logger::get("registerStorageMergeTree"), "Replacing user-provided ZooKeeper path and replica name ({}, {}) " + "with default arguments", zookeeper_path, replica_name); + engine_args[arg_num]->as()->value = zookeeper_path = server_settings.default_replica_path; + engine_args[arg_num + 1]->as()->value = replica_name = server_settings.default_replica_name; + } + expand_macro(ast_zk_path, ast_replica_name); } else if (is_extended_storage_def @@ -293,7 +318,6 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( { /// Try use default values if arguments are not specified. /// Note: {uuid} macro works for ON CLUSTER queries when database engine is Atomic. - const auto & server_settings = context->getServerSettings(); zookeeper_path = server_settings.default_replica_path; /// TODO maybe use hostname if {replica} is not defined? replica_name = server_settings.default_replica_name; @@ -318,7 +342,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( } /// Extracts a zookeeper path from a specified CREATE TABLE query. -std::optional extractZooKeeperPathFromReplicatedTableDef(const ASTCreateQuery & query, const ContextPtr & context) +std::optional extractZooKeeperPathFromReplicatedTableDef(const ASTCreateQuery & query, const ContextPtr & local_context) { if (!query.storage || !query.storage->engine) return {}; @@ -342,7 +366,7 @@ std::optional extractZooKeeperPathFromReplicatedTableDef(const ASTCreate try { - extractZooKeeperPathAndReplicaNameFromEngineArgs(query, table_id, engine_name, engine_args, mode, context, + extractZooKeeperPathAndReplicaNameFromEngineArgs(query, table_id, engine_name, engine_args, mode, local_context, zookeeper_path, replica_name, renaming_restrictions); } catch (Exception & e) diff --git a/tests/config/users.d/database_replicated.xml b/tests/config/users.d/database_replicated.xml index c049c3559fc..1c2cf2ac22b 100644 --- a/tests/config/users.d/database_replicated.xml +++ b/tests/config/users.d/database_replicated.xml @@ -6,6 +6,7 @@ 120 1 1 + 3 diff --git a/tests/integration/test_replicated_database/configs/settings.xml b/tests/integration/test_replicated_database/configs/settings.xml index c637fe8eead..4b1122b1b5f 100644 --- a/tests/integration/test_replicated_database/configs/settings.xml +++ b/tests/integration/test_replicated_database/configs/settings.xml @@ -5,6 +5,7 @@ 1 0 0 + 3 diff --git a/tests/integration/test_replicated_database/configs/settings2.xml b/tests/integration/test_replicated_database/configs/settings2.xml index dad5740a8ae..9a3038871e1 100644 --- a/tests/integration/test_replicated_database/configs/settings2.xml +++ b/tests/integration/test_replicated_database/configs/settings2.xml @@ -5,6 +5,7 @@ 1 0 0 + 3 0 diff --git a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference new file mode 100644 index 00000000000..1d3a90daccd --- /dev/null +++ b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference @@ -0,0 +1,10 @@ +2 +3 +m1 +m2 +rmt1 +rmt2 +02858000-1000-4000-8000-0000000000 +0 +CREATE TABLE rdb_default.rmt1\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/test/default/auto_{shard}\', \'1auto_{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 +CREATE TABLE rdb_default.rmt2\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{uuid}/{shard}\', \'{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh new file mode 100755 index 00000000000..f2c2f47cd0d --- /dev/null +++ b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +db=$CLICKHOUSE_DATABASE +if [[ $($CLICKHOUSE_CLIENT -q "SELECT engine = 'Replicated' FROM system.databases WHERE name='$CLICKHOUSE_DATABASE'") != 1 ]]; then + $CLICKHOUSE_CLIENT -q "CREATE DATABASE rdb_$CLICKHOUSE_DATABASE ENGINE=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', '1', '1')" + db="rdb_$CLICKHOUSE_DATABASE" +fi + +$CLICKHOUSE_CLIENT --database_replicated_allow_explicit_uuid=0 -q "CREATE TABLE $db.m0 +UUID '02858000-1000-4000-8000-000000000000' (n int) ENGINE=Memory" 2>&1| grep -Fac "database_replicated_allow_explicit_uuid" + +$CLICKHOUSE_CLIENT --database_replicated_allow_explicit_uuid=1 -q "CREATE TABLE $db.m1 +UUID '02858000-1000-4000-8000-0000000000$(($RANDOM % 10))$(($RANDOM % 10))' (n int) ENGINE=Memory" + +$CLICKHOUSE_CLIENT --database_replicated_allow_explicit_uuid=2 -q "CREATE TABLE $db.m2 +UUID '02858000-1000-4000-8000-000000000002' (n int) ENGINE=Memory" + + +$CLICKHOUSE_CLIENT --database_replicated_allow_replicated_engine_arguments=0 -q "CREATE TABLE $db.rmt0 (n int) +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_DATABASE', '1') ORDER BY n" 2>&1| grep -Fac "database_replicated_allow_replicated_engine_arguments" + +$CLICKHOUSE_CLIENT --database_replicated_allow_replicated_engine_arguments=1 -q "CREATE TABLE $db.rmt1 (n int) +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_DATABASE', '1') ORDER BY n" + +$CLICKHOUSE_CLIENT --database_replicated_allow_replicated_engine_arguments=2 -q "CREATE TABLE $db.rmt2 (n int) +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_DATABASE', '1') ORDER BY n" + + +$CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database='$db' ORDER BY name" + +$CLICKHOUSE_CLIENT -q "SELECT substring(toString(uuid) as s, 1, length(s) - 2) FROM system.tables WHERE database='$db' and name='m1'" +$CLICKHOUSE_CLIENT -q "SELECT toString(uuid) LIKE '02858000%' FROM system.tables WHERE database='$db' and name='m2'" + +$CLICKHOUSE_CLIENT -q "SHOW CREATE $db.rmt1" +$CLICKHOUSE_CLIENT -q "SHOW CREATE $db.rmt2" + +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS rdb_$CLICKHOUSE_DATABASE" From aa9591419d3f0cb1575ddd2cf7b82f2cfc397e2b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 4 Jul 2024 19:15:06 +0200 Subject: [PATCH 048/289] fix --- .../0_stateless/02858_explicit_uuid_and_zk_path.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh index f2c2f47cd0d..1b5d1a06a9b 100755 --- a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh +++ b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) db=$CLICKHOUSE_DATABASE if [[ $($CLICKHOUSE_CLIENT -q "SELECT engine = 'Replicated' FROM system.databases WHERE name='$CLICKHOUSE_DATABASE'") != 1 ]]; then - $CLICKHOUSE_CLIENT -q "CREATE DATABASE rdb_$CLICKHOUSE_DATABASE ENGINE=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', '1', '1')" + $CLICKHOUSE_CLIENT -q "CREATE DATABASE rdb_$CLICKHOUSE_DATABASE ENGINE=Replicated('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rdb', '1', '1')" db="rdb_$CLICKHOUSE_DATABASE" fi @@ -21,13 +21,13 @@ UUID '02858000-1000-4000-8000-000000000002' (n int) ENGINE=Memory" $CLICKHOUSE_CLIENT --database_replicated_allow_replicated_engine_arguments=0 -q "CREATE TABLE $db.rmt0 (n int) -ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_DATABASE', '1') ORDER BY n" 2>&1| grep -Fac "database_replicated_allow_replicated_engine_arguments" +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') ORDER BY n" 2>&1| grep -Fac "database_replicated_allow_replicated_engine_arguments" $CLICKHOUSE_CLIENT --database_replicated_allow_replicated_engine_arguments=1 -q "CREATE TABLE $db.rmt1 (n int) -ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_DATABASE', '1') ORDER BY n" +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') ORDER BY n" $CLICKHOUSE_CLIENT --database_replicated_allow_replicated_engine_arguments=2 -q "CREATE TABLE $db.rmt2 (n int) -ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_DATABASE', '1') ORDER BY n" +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') ORDER BY n" $CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database='$db' ORDER BY name" From 613636bb1b324acf7500c8a760abf5aea764418b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 4 Jul 2024 20:49:10 +0200 Subject: [PATCH 049/289] fix --- src/Core/SettingsChangesHistory.cpp | 2 ++ src/Databases/DatabaseReplicated.cpp | 3 ++- src/Interpreters/InterpreterCreateQuery.cpp | 6 +++--- .../MergeTree/registerStorageMergeTree.cpp | 2 +- .../01148_zookeeper_path_macros_unfolding.sql | 2 ++ ...atabase_replicated_no_arguments_for_rmt.sh | 2 +- .../02858_explicit_uuid_and_zk_path.reference | 2 +- .../02858_explicit_uuid_and_zk_path.sh | 21 +++++++++++-------- 8 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 6af6b4b15aa..6c730940c87 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -63,6 +63,8 @@ static std::initializer_listgetSettingsRef().database_replicated_allow_replicated_engine_arguments == 0) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Explicit zookeeper_path and replica_name are specified in ReplicatedMergeTree arguments. " "If you really want to specify it explicitly, then you should use some macros " "to distinguish different shards and replicas"); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index c32a32be5cc..2862a907bec 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1062,15 +1062,15 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data throw Exception(ErrorCodes::BAD_ARGUMENTS, "It's not allowed to explicitly specify UUIDs for tables in Replicated databases, " "see database_replicated_allow_explicit_uuid"); } - else if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 0) + else if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 1) { LOG_WARNING(&Poco::Logger::get("InterpreterCreateQuery"), "It's not recommended to explicitly specify UUIDs for tables in Replicated databases"); } - else // if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 2) + else if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 2) { UUID old_uuid = create.uuid; create.generateRandomUUID(/*always_generate_new_uuid*/ true); - LOG_INFO(&Poco::Logger::get("InterpreterCreateQuery"), "Replaced a user-provided UUID ({}) with a random one ({}) " + LOG_WARNING(&Poco::Logger::get("InterpreterCreateQuery"), "Replaced a user-provided UUID ({}) with a random one ({}) " "to make sure it's unique", old_uuid, create.uuid); } } diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 66f7b2114bf..d27c786d422 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -303,7 +303,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( if (is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 2) { - LOG_INFO(&Poco::Logger::get("registerStorageMergeTree"), "Replacing user-provided ZooKeeper path and replica name ({}, {}) " + LOG_WARNING(&Poco::Logger::get("registerStorageMergeTree"), "Replacing user-provided ZooKeeper path and replica name ({}, {}) " "with default arguments", zookeeper_path, replica_name); engine_args[arg_num]->as()->value = zookeeper_path = server_settings.default_replica_path; engine_args[arg_num + 1]->as()->value = replica_name = server_settings.default_replica_name; diff --git a/tests/queries/0_stateless/01148_zookeeper_path_macros_unfolding.sql b/tests/queries/0_stateless/01148_zookeeper_path_macros_unfolding.sql index a585ef1c324..c689542e4c3 100644 --- a/tests/queries/0_stateless/01148_zookeeper_path_macros_unfolding.sql +++ b/tests/queries/0_stateless/01148_zookeeper_path_macros_unfolding.sql @@ -7,6 +7,8 @@ DROP TABLE IF EXISTS rmt1; DROP TABLE IF EXISTS rmt2; DROP TABLE IF EXISTS rmt3; +SET database_replicated_allow_replicated_engine_arguments=1; + CREATE TABLE rmt (n UInt64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/test_01148/{shard}/{database}/{table}', '{replica}') ORDER BY n; SHOW CREATE TABLE rmt; RENAME TABLE rmt TO rmt1; diff --git a/tests/queries/0_stateless/02514_database_replicated_no_arguments_for_rmt.sh b/tests/queries/0_stateless/02514_database_replicated_no_arguments_for_rmt.sh index a0f228e6af4..7b600ade9a0 100755 --- a/tests/queries/0_stateless/02514_database_replicated_no_arguments_for_rmt.sh +++ b/tests/queries/0_stateless/02514_database_replicated_no_arguments_for_rmt.sh @@ -15,7 +15,7 @@ ${CLICKHOUSE_CLIENT} -q "GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}_db.* TO us ${CLICKHOUSE_CLIENT} -q "GRANT TABLE ENGINE ON ReplicatedMergeTree TO user_${CLICKHOUSE_DATABASE}" ${CLICKHOUSE_CLIENT} -q "CREATE DATABASE ${CLICKHOUSE_DATABASE}_db engine = Replicated('/clickhouse/databases/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}_db', '{shard}', '{replica}')" ${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt_ok (x UInt32) engine = ReplicatedMergeTree order by x;" -${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt_fail (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/root/{shard}', '{replica}') order by x; -- { serverError 80 }" +${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt_fail (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/root/{shard}', '{replica}') order by x; -- { serverError 36 }" ${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${CLICKHOUSE_DATABASE}_db" ${CLICKHOUSE_CLIENT} -q "DROP USER user_${CLICKHOUSE_DATABASE}" diff --git a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference index 1d3a90daccd..1241d9714f9 100644 --- a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference +++ b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference @@ -6,5 +6,5 @@ rmt1 rmt2 02858000-1000-4000-8000-0000000000 0 -CREATE TABLE rdb_default.rmt1\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/test/default/auto_{shard}\', \'1auto_{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 +CREATE TABLE rdb_default.rmt1\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/test/02858_explicit_uuid_and_zk_path_default/rmt/auto_{shard}\', \'1auto_{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 CREATE TABLE rdb_default.rmt2\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{uuid}/{shard}\', \'{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh index 1b5d1a06a9b..b968c02a07a 100755 --- a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh +++ b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh @@ -1,33 +1,36 @@ #!/usr/bin/env bash +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=error + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh + db=$CLICKHOUSE_DATABASE if [[ $($CLICKHOUSE_CLIENT -q "SELECT engine = 'Replicated' FROM system.databases WHERE name='$CLICKHOUSE_DATABASE'") != 1 ]]; then $CLICKHOUSE_CLIENT -q "CREATE DATABASE rdb_$CLICKHOUSE_DATABASE ENGINE=Replicated('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rdb', '1', '1')" db="rdb_$CLICKHOUSE_DATABASE" fi -$CLICKHOUSE_CLIENT --database_replicated_allow_explicit_uuid=0 -q "CREATE TABLE $db.m0 +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allow_explicit_uuid=0 -q "CREATE TABLE $db.m0 UUID '02858000-1000-4000-8000-000000000000' (n int) ENGINE=Memory" 2>&1| grep -Fac "database_replicated_allow_explicit_uuid" -$CLICKHOUSE_CLIENT --database_replicated_allow_explicit_uuid=1 -q "CREATE TABLE $db.m1 +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allow_explicit_uuid=1 -q "CREATE TABLE $db.m1 UUID '02858000-1000-4000-8000-0000000000$(($RANDOM % 10))$(($RANDOM % 10))' (n int) ENGINE=Memory" -$CLICKHOUSE_CLIENT --database_replicated_allow_explicit_uuid=2 -q "CREATE TABLE $db.m2 +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allow_explicit_uuid=2 -q "CREATE TABLE $db.m2 UUID '02858000-1000-4000-8000-000000000002' (n int) ENGINE=Memory" -$CLICKHOUSE_CLIENT --database_replicated_allow_replicated_engine_arguments=0 -q "CREATE TABLE $db.rmt0 (n int) -ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') ORDER BY n" 2>&1| grep -Fac "database_replicated_allow_replicated_engine_arguments" +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allow_replicated_engine_arguments=0 -q "CREATE TABLE $db.rmt0 (n int) +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt', '1') ORDER BY n" 2>&1| grep -Fac "database_replicated_allow_replicated_engine_arguments" -$CLICKHOUSE_CLIENT --database_replicated_allow_replicated_engine_arguments=1 -q "CREATE TABLE $db.rmt1 (n int) -ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') ORDER BY n" +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allow_replicated_engine_arguments=1 -q "CREATE TABLE $db.rmt1 (n int) +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt', '1') ORDER BY n" -$CLICKHOUSE_CLIENT --database_replicated_allow_replicated_engine_arguments=2 -q "CREATE TABLE $db.rmt2 (n int) -ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') ORDER BY n" +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allow_replicated_engine_arguments=2 -q "CREATE TABLE $db.rmt2 (n int) +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt', '1') ORDER BY n" $CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database='$db' ORDER BY name" From 0e6fd2251e9048bbdb8535c3af0971fb0c670350 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 5 Jul 2024 17:15:04 +0200 Subject: [PATCH 050/289] fix --- .../0_stateless/02858_explicit_uuid_and_zk_path.reference | 2 +- .../queries/0_stateless/02858_explicit_uuid_and_zk_path.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference index 1241d9714f9..cb6d0135adf 100644 --- a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference +++ b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference @@ -6,5 +6,5 @@ rmt1 rmt2 02858000-1000-4000-8000-0000000000 0 -CREATE TABLE rdb_default.rmt1\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/test/02858_explicit_uuid_and_zk_path_default/rmt/auto_{shard}\', \'1auto_{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 +CREATE TABLE rdb_default.rmt1\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/test/02858_explicit_uuid_and_zk_path_default/rmt/{shard}\', \'_{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 CREATE TABLE rdb_default.rmt2\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{uuid}/{shard}\', \'{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh index b968c02a07a..a959446c822 100755 --- a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh +++ b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh @@ -24,13 +24,13 @@ UUID '02858000-1000-4000-8000-000000000002' (n int) ENGINE=Memory" $CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allow_replicated_engine_arguments=0 -q "CREATE TABLE $db.rmt0 (n int) -ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt', '1') ORDER BY n" 2>&1| grep -Fac "database_replicated_allow_replicated_engine_arguments" +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt/{shard}', '_{replica}') ORDER BY n" 2>&1| grep -Fac "database_replicated_allow_replicated_engine_arguments" $CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allow_replicated_engine_arguments=1 -q "CREATE TABLE $db.rmt1 (n int) -ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt', '1') ORDER BY n" +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt/{shard}', '_{replica}') ORDER BY n" $CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allow_replicated_engine_arguments=2 -q "CREATE TABLE $db.rmt2 (n int) -ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt', '1') ORDER BY n" +ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt/{shard}', '_{replica}') ORDER BY n" $CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database='$db' ORDER BY name" From e7105a3faa50ba012e6cff1e9e26e62fce6d75ba Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 5 Jul 2024 20:51:30 +0200 Subject: [PATCH 051/289] fix --- src/Backups/RestorerFromBackup.cpp | 6 +++++- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- .../0_stateless/02858_explicit_uuid_and_zk_path.reference | 2 +- .../queries/0_stateless/02858_explicit_uuid_and_zk_path.sh | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 454a0468e9f..63ca272db0b 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -903,11 +903,15 @@ void RestorerFromBackup::createTable(const QualifiedTableName & table_name) table_info.database = DatabaseCatalog::instance().getDatabase(table_name.database); DatabasePtr database = table_info.database; + auto query_context = Context::createCopy(context); + query_context->setSetting("database_replicated_allow_explicit_uuid", 3); + query_context->setSetting("database_replicated_allow_replicated_engine_arguments", 3); + /// Execute CREATE TABLE query (we call IDatabase::createTableRestoredFromBackup() to allow the database to do some /// database-specific things). database->createTableRestoredFromBackup( create_table_query, - context, + query_context, restore_coordination, std::chrono::duration_cast(create_table_timeout).count()); } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 2862a907bec..89f503f5fb1 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1055,7 +1055,7 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data bool from_path = create.attach_from_path.has_value(); bool is_on_cluster = getContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; - if (database->getEngineName() == "Replicated" && create.uuid != UUIDHelpers::Nil && !is_replicated_database_internal) + if (database->getEngineName() == "Replicated" && create.uuid != UUIDHelpers::Nil && !is_replicated_database_internal && !create.attach) { if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 0) { diff --git a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference index cb6d0135adf..6444e10bb48 100644 --- a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference +++ b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference @@ -4,7 +4,7 @@ m1 m2 rmt1 rmt2 -02858000-1000-4000-8000-0000000000 +02858000-1000-4000-8000-000000000 0 CREATE TABLE rdb_default.rmt1\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/test/02858_explicit_uuid_and_zk_path_default/rmt/{shard}\', \'_{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 CREATE TABLE rdb_default.rmt2\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{uuid}/{shard}\', \'{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh index a959446c822..b011aed613a 100755 --- a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh +++ b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh @@ -17,7 +17,7 @@ $CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allo UUID '02858000-1000-4000-8000-000000000000' (n int) ENGINE=Memory" 2>&1| grep -Fac "database_replicated_allow_explicit_uuid" $CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allow_explicit_uuid=1 -q "CREATE TABLE $db.m1 -UUID '02858000-1000-4000-8000-0000000000$(($RANDOM % 10))$(($RANDOM % 10))' (n int) ENGINE=Memory" +UUID '02858000-1000-4000-8000-000000000$(($RANDOM % 10))$(($RANDOM % 10))$(($RANDOM % 10))' (n int) ENGINE=Memory" $CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --database_replicated_allow_explicit_uuid=2 -q "CREATE TABLE $db.m2 UUID '02858000-1000-4000-8000-000000000002' (n int) ENGINE=Memory" @@ -35,7 +35,7 @@ ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt/{shard}' $CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database='$db' ORDER BY name" -$CLICKHOUSE_CLIENT -q "SELECT substring(toString(uuid) as s, 1, length(s) - 2) FROM system.tables WHERE database='$db' and name='m1'" +$CLICKHOUSE_CLIENT -q "SELECT substring(toString(uuid) as s, 1, length(s) - 3) FROM system.tables WHERE database='$db' and name='m1'" $CLICKHOUSE_CLIENT -q "SELECT toString(uuid) LIKE '02858000%' FROM system.tables WHERE database='$db' and name='m2'" $CLICKHOUSE_CLIENT -q "SHOW CREATE $db.rmt1" From 88736a0d74b068f271ee21c69760128f82adbb91 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 7 Jul 2024 21:12:26 +0200 Subject: [PATCH 052/289] Update JSONCompactWithProgressRowOutputFormat to print JSON on each row --- src/Formats/JSONUtils.cpp | 64 +++++++++++++++++++ src/Formats/JSONUtils.h | 10 +++ ...JSONCompactWithProgressRowOutputFormat.cpp | 39 ++++++++++- .../JSONCompactWithProgressRowOutputFormat.h | 4 ++ 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/src/Formats/JSONUtils.cpp b/src/Formats/JSONUtils.cpp index f0985f4a6b7..16083441f3a 100644 --- a/src/Formats/JSONUtils.cpp +++ b/src/Formats/JSONUtils.cpp @@ -483,6 +483,33 @@ namespace JSONUtils writeArrayEnd(out, 1); } + + void writeCompactMetadata(const Names & names, const DataTypes & types, const FormatSettings & settings, WriteBuffer & out) + { + writeCompactArrayStart(out, 0, "meta"); + + for (size_t i = 0; i < names.size(); ++i) + { + writeCompactObjectStart(out); + writeTitle("name", out, 0, ""); + + /// The field names are pre-escaped to be put into JSON string literal. + writeChar('"', out); + writeString(names[i], out); + writeChar('"', out); + + writeFieldCompactDelimiter(out); + writeTitle("type", out, 0, ""); + writeJSONString(types[i]->getName(), out, settings); + writeCompactObjectEnd(out); + + if (i + 1 < names.size()) + writeFieldCompactDelimiter(out); + } + + writeCompactArrayEnd(out); + } + void writeAdditionalInfo( size_t rows, size_t rows_before_limit, @@ -523,6 +550,43 @@ namespace JSONUtils } } + void writeCompactAdditionalInfo( + size_t rows, + size_t rows_before_limit, + bool applied_limit, + const Stopwatch & watch, + const Progress & progress, + bool write_statistics, + WriteBuffer & out) + { + writeCompactObjectStart(out, 0, "statistics"); + writeTitle("rows", out, 0, ""); + writeIntText(rows, out); + writeFieldCompactDelimiter(out); + + if (applied_limit) + { + writeTitle("rows_before_limit_at_least", out, 0, ""); + writeIntText(rows_before_limit, out); + writeFieldCompactDelimiter(out); + } + + if (write_statistics) + { + writeTitle("elapsed", out, 0, ""); + writeText(watch.elapsedSeconds(), out); + writeFieldCompactDelimiter(out); + + writeTitle("rows_read", out, 0, ""); + writeText(progress.read_rows.load(), out); + writeFieldCompactDelimiter(out); + + writeTitle("bytes_read", out, 0, ""); + writeText(progress.read_bytes.load(), out); + } + writeCompactObjectEnd(out); + } + void writeException(const String & exception_message, WriteBuffer & out, const FormatSettings & settings, size_t indent) { writeTitle("exception", out, indent, " "); diff --git a/src/Formats/JSONUtils.h b/src/Formats/JSONUtils.h index 7ee111c1285..718b9b2f610 100644 --- a/src/Formats/JSONUtils.h +++ b/src/Formats/JSONUtils.h @@ -99,6 +99,7 @@ namespace JSONUtils WriteBuffer & out); void writeMetadata(const Names & names, const DataTypes & types, const FormatSettings & settings, WriteBuffer & out); + void writeCompactMetadata(const Names & names, const DataTypes & types, const FormatSettings & settings, WriteBuffer & out); void writeAdditionalInfo( size_t rows, @@ -109,6 +110,15 @@ namespace JSONUtils bool write_statistics, WriteBuffer & out); + void writeCompactAdditionalInfo( + size_t rows, + size_t rows_before_limit, + bool applied_limit, + const Stopwatch & watch, + const Progress & progress, + bool write_statistics, + WriteBuffer & out); + void writeException(const String & exception_message, WriteBuffer & out, const FormatSettings & settings, size_t indent = 0); void skipColon(ReadBuffer & in); diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 39532fb76fb..7a19b56d0ca 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -17,6 +17,14 @@ JSONCompactWithProgressRowOutputFormat::JSONCompactWithProgressRowOutputFormat( { } +void JSONCompactWithProgressRowOutputFormat::writePrefix() +{ + JSONUtils::writeCompactObjectStart(*ostr); + JSONUtils::writeCompactMetadata(names, types, settings, *ostr); + JSONUtils::writeCompactObjectEnd(*ostr); + writeCString("}\n", *ostr); +} + void JSONCompactWithProgressRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) { JSONUtils::writeFieldFromColumn(column, serialization, row_num, yield_strings, settings, *ostr); @@ -32,16 +40,22 @@ void JSONCompactWithProgressRowOutputFormat::writeRowStartDelimiter() { if (has_progress) writeProgress(); - JSONUtils::writeCompactArrayStart(*ostr, 2); + writeCString("{\"data\":", *ostr); + JSONUtils::writeCompactArrayStart(*ostr); } void JSONCompactWithProgressRowOutputFormat::writeRowEndDelimiter() { JSONUtils::writeCompactArrayEnd(*ostr); + writeCString("}\n", *ostr); field_number = 0; ++row_count; } +void JSONCompactWithProgressRowOutputFormat::writeRowBetweenDelimiter() +{ +} + void JSONCompactWithProgressRowOutputFormat::writeBeforeTotals() { JSONUtils::writeFieldDelimiter(*ostr, 2); @@ -91,7 +105,6 @@ void JSONCompactWithProgressRowOutputFormat::writeSuffix() { if (has_progress) writeProgress(); - JSONRowOutputFormat::writeSuffix(); } void JSONCompactWithProgressRowOutputFormat::writeProgress() @@ -103,6 +116,28 @@ void JSONCompactWithProgressRowOutputFormat::writeProgress() has_progress = false; } +void JSONCompactWithProgressRowOutputFormat::finalizeImpl() +{ + JSONUtils::writeCompactAdditionalInfo( + row_count, + statistics.rows_before_limit, + statistics.applied_limit, + statistics.watch, + statistics.progress, + settings.write_statistics && exception_message.empty(), + *ostr); + + exception_message = "Test exception message."; + if (!exception_message.empty()) + { + writeCString("\n", *ostr); + JSONUtils::writeCompactObjectStart(*ostr); + JSONUtils::writeException(exception_message, *ostr, settings, 0); + JSONUtils::writeCompactObjectEnd(*ostr); + } + ostr->next(); +} + void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory) { factory.registerOutputFormat("JSONCompactWithProgress", []( diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h index 4bc10d41f19..0c7e6b295e8 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h @@ -32,6 +32,7 @@ private: void writeFieldDelimiter() override; void writeRowStartDelimiter() override; void writeRowEndDelimiter() override; + void writeRowBetweenDelimiter() override; bool supportTotals() const override { return true; } bool supportExtremes() const override { return true; } void writeBeforeTotals() override; @@ -40,7 +41,10 @@ private: void writeTotals(const Columns & columns, size_t row_num) override; void writeProgress(); + void writePrefix() override; void writeSuffix() override; + void finalizeImpl() override; + Progress progress; std::vector progress_lines; From 2ae9490a681d3d522dd30e96b9f36b2d638ffd10 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 7 Jul 2024 21:22:02 +0200 Subject: [PATCH 053/289] Fix progress rows and bytes read --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 5 ++--- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.h | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 7a19b56d0ca..d72a6556c2b 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -81,11 +81,11 @@ void JSONCompactWithProgressRowOutputFormat::writeExtremesElement(const char * t void JSONCompactWithProgressRowOutputFormat::onProgress(const Progress & value) { - progress.incrementPiecewiseAtomically(value); + statistics.progress.incrementPiecewiseAtomically(value); String progress_line; WriteBufferFromString buf(progress_line); writeCString("{\"progress\":", buf); - progress.writeJSON(buf); + statistics.progress.writeJSON(buf); writeCString("}\n", buf); buf.finalize(); std::lock_guard lock(progress_lines_mutex); @@ -127,7 +127,6 @@ void JSONCompactWithProgressRowOutputFormat::finalizeImpl() settings.write_statistics && exception_message.empty(), *ostr); - exception_message = "Test exception message."; if (!exception_message.empty()) { writeCString("\n", *ostr); diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h index 0c7e6b295e8..669e342c583 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h @@ -46,7 +46,6 @@ private: void finalizeImpl() override; - Progress progress; std::vector progress_lines; std::mutex progress_lines_mutex; /// To not lock mutex and check progress_lines every row, From 2469647609c53ac968d7fba60867c78ecebe522c Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 7 Jul 2024 21:30:36 +0200 Subject: [PATCH 054/289] Fix statistics section --- src/Formats/JSONUtils.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Formats/JSONUtils.cpp b/src/Formats/JSONUtils.cpp index 16083441f3a..0918658cf6f 100644 --- a/src/Formats/JSONUtils.cpp +++ b/src/Formats/JSONUtils.cpp @@ -559,6 +559,7 @@ namespace JSONUtils bool write_statistics, WriteBuffer & out) { + writeCompactObjectStart(out); writeCompactObjectStart(out, 0, "statistics"); writeTitle("rows", out, 0, ""); writeIntText(rows, out); @@ -585,6 +586,7 @@ namespace JSONUtils writeText(progress.read_bytes.load(), out); } writeCompactObjectEnd(out); + writeCompactObjectEnd(out); } void writeException(const String & exception_message, WriteBuffer & out, const FormatSettings & settings, size_t indent) From 00e47d64b0c12023d926b383c6dea327e9806fc7 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 8 Jul 2024 09:02:07 +0200 Subject: [PATCH 055/289] Add JSONCompactWithProgressStrings format --- .../Impl/JSONCompactWithProgressRowOutputFormat.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index d72a6556c2b..91608cd5050 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -146,6 +146,14 @@ void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory) { return std::make_shared(buf, sample, format_settings, false); }); + + factory.registerOutputFormat("JSONCompactWithProgressStrings", []( + WriteBuffer & buf, + const Block & sample, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings, true); + }); } } From 58d34e306cec5b41ac372934e43639c2df24e54e Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 8 Jul 2024 09:08:03 +0200 Subject: [PATCH 056/289] Add documentation on JSONCompactWithProgress --- docs/en/interfaces/formats.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index ffdd7e2ca25..6161cddefdd 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -38,6 +38,7 @@ The supported formats are: | [JSONCompact](#jsoncompact) | ✔ | ✔ | | [JSONCompactStrings](#jsoncompactstrings) | ✗ | ✔ | | [JSONCompactColumns](#jsoncompactcolumns) | ✔ | ✔ | +| [JSONCompactWithProgress](#jsoncompactwithprogress) | ✗ | ✔ | | [JSONEachRow](#jsoneachrow) | ✔ | ✔ | | [PrettyJSONEachRow](#prettyjsoneachrow) | ✗ | ✔ | | [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ | @@ -925,6 +926,23 @@ Example: Columns that are not present in the block will be filled with default values (you can use [input_format_defaults_for_omitted_fields](/docs/en/operations/settings/settings-formats.md/#input_format_defaults_for_omitted_fields) setting here) +## JSONCompactWithProgress (#jsoncompactwithprogress) + +In this format, ClickHouse outputs each row as a separated, newline-delimited JSON Object. + +Each row is either a metadata object, data object, progress information or statistics object. + +Example: + +```json +{"meta": [{"name":"id", "type":"UInt32"}, {"name":"name", "type":"String"}]}} +{"progress":{"read_rows":"8","read_bytes":"168","written_rows":"0","written_bytes":"0","total_rows_to_read":"2","result_rows":"0","result_bytes":"0","elapsed_ns":"0"}} +{"data":["1", "John Doe"]} +{"data":["2", "Joe Doe"]} +{"statistics": {"rows":2, "rows_before_limit_at_least":8, "elapsed":0.001995, "rows_read":8, "bytes_read":168}} +``` + + ## JSONEachRow {#jsoneachrow} In this format, ClickHouse outputs each row as a separated, newline-delimited JSON Object. From 78cb02503f05d1c1c84dbb17fd51700d74f697df Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 8 Jul 2024 09:22:07 +0200 Subject: [PATCH 057/289] Remove extra space --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 91608cd5050..6be2d092305 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -111,7 +111,7 @@ void JSONCompactWithProgressRowOutputFormat::writeProgress() { std::lock_guard lock(progress_lines_mutex); for (const auto & progress_line : progress_lines) - writeString(progress_line, *ostr); + writeString(progress_line, *ostr); progress_lines.clear(); has_progress = false; } From 7fbcaeaae2bddc07898fd4e2eb7aa9331e2d0df5 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 8 Jul 2024 09:29:14 +0200 Subject: [PATCH 058/289] Remove wrong comment --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h index 669e342c583..dc7a70229e7 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h @@ -11,8 +11,6 @@ namespace DB struct FormatSettings; -/** The stream for outputting data in the JSONCompact- formats. - */ class JSONCompactWithProgressRowOutputFormat final : public JSONRowOutputFormat { public: From a3360d0e63c08132ecd73423d2dd9fc576b54589 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 8 Jul 2024 13:53:24 +0200 Subject: [PATCH 059/289] Fix code style --- ...JSONCompactWithProgressRowOutputFormat.cpp | 29 +++++++------------ .../JSONCompactWithProgressRowOutputFormat.h | 6 +--- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 6be2d092305..9603ce0265d 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -1,6 +1,6 @@ -#include #include #include +#include #include @@ -9,10 +9,7 @@ namespace DB { JSONCompactWithProgressRowOutputFormat::JSONCompactWithProgressRowOutputFormat( - WriteBuffer & out_, - const Block & header, - const FormatSettings & settings_, - bool yield_strings_) + WriteBuffer & out_, const Block & header, const FormatSettings & settings_, bool yield_strings_) : JSONRowOutputFormat(out_, header, settings_, yield_strings_) { } @@ -139,21 +136,15 @@ void JSONCompactWithProgressRowOutputFormat::finalizeImpl() void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory) { - factory.registerOutputFormat("JSONCompactWithProgress", []( - WriteBuffer & buf, - const Block & sample, - const FormatSettings & format_settings) - { - return std::make_shared(buf, sample, format_settings, false); - }); + factory.registerOutputFormat( + "JSONCompactWithProgress", + [](WriteBuffer & buf, const Block & sample, const FormatSettings & format_settings) + { return std::make_shared(buf, sample, format_settings, false); }); - factory.registerOutputFormat("JSONCompactWithProgressStrings", []( - WriteBuffer & buf, - const Block & sample, - const FormatSettings & format_settings) - { - return std::make_shared(buf, sample, format_settings, true); - }); + factory.registerOutputFormat( + "JSONCompactWithProgressStrings", + [](WriteBuffer & buf, const Block & sample, const FormatSettings & format_settings) + { return std::make_shared(buf, sample, format_settings, true); }); } } diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h index dc7a70229e7..1c21914d8cb 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h @@ -14,11 +14,7 @@ struct FormatSettings; class JSONCompactWithProgressRowOutputFormat final : public JSONRowOutputFormat { public: - JSONCompactWithProgressRowOutputFormat( - WriteBuffer & out_, - const Block & header, - const FormatSettings & settings_, - bool yield_strings_); + JSONCompactWithProgressRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & settings_, bool yield_strings_); String getName() const override { return "JSONCompactWithProgressRowOutputFormat"; } From 6cb361413eff6f870eb2bd43876d82d1c2c5e882 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 9 Jul 2024 16:45:15 +0200 Subject: [PATCH 060/289] fix --- src/Databases/DatabaseReplicated.cpp | 3 +++ tests/integration/test_disk_over_web_server/test.py | 3 ++- .../00510_materizlized_view_and_deduplication_zookeeper.sql | 2 ++ tests/queries/0_stateless/00609_mv_index_in_in.sql | 1 + tests/queries/0_stateless/00738_lock_for_inner_table.sh | 2 ++ tests/queries/0_stateless/01153_attach_mv_uuid.sql | 1 + .../0_stateless/02858_explicit_uuid_and_zk_path.reference | 4 ++-- tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh | 4 ++-- .../0_stateless/02888_replicated_merge_tree_creation.sh | 1 + 9 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index bc9b2537d1e..25d4a3eb704 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -1008,6 +1008,9 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep query_context->setSetting("allow_create_index_without_type", 1); query_context->setSetting("allow_experimental_s3queue", 1); + query_context->setSetting("database_replicated_allow_explicit_uuid", 3); + query_context->setSetting("database_replicated_allow_replicated_engine_arguments", 3); + auto txn = std::make_shared(current_zookeeper, zookeeper_path, false, ""); query_context->initZooKeeperMetadataTransaction(txn); return query_context; diff --git a/tests/integration/test_disk_over_web_server/test.py b/tests/integration/test_disk_over_web_server/test.py index 891ee8f00f5..ec0bef23731 100644 --- a/tests/integration/test_disk_over_web_server/test.py +++ b/tests/integration/test_disk_over_web_server/test.py @@ -311,7 +311,8 @@ def test_replicated_database(cluster): SETTINGS storage_policy = 'web'; """.format( uuids[0] - ) + ), + settings={"database_replicated_allow_explicit_uuid": 3}, ) node2 = cluster.instances["node2"] diff --git a/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql b/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql index d3c4da86b41..5d2f8192d04 100644 --- a/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql +++ b/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql @@ -6,6 +6,8 @@ DROP TABLE IF EXISTS without_deduplication; DROP TABLE IF EXISTS with_deduplication_mv; DROP TABLE IF EXISTS without_deduplication_mv; +SET database_replicated_allow_explicit_uuid=3; +SET database_replicated_allow_replicated_engine_arguments=3; CREATE TABLE with_deduplication(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/{database}/test_00510/with_deduplication', 'r1') ORDER BY x; CREATE TABLE without_deduplication(x UInt32) diff --git a/tests/queries/0_stateless/00609_mv_index_in_in.sql b/tests/queries/0_stateless/00609_mv_index_in_in.sql index bd9f35350c1..89fc85651ad 100644 --- a/tests/queries/0_stateless/00609_mv_index_in_in.sql +++ b/tests/queries/0_stateless/00609_mv_index_in_in.sql @@ -6,6 +6,7 @@ DROP TABLE IF EXISTS test_mv_00609; create table test_00609 (a Int8) engine=Memory; insert into test_00609 values (1); +set database_replicated_allow_explicit_uuid=3; set allow_deprecated_syntax_for_merge_tree=1; create materialized view test_mv_00609 uuid '00000609-1000-4000-8000-000000000001' Engine=MergeTree(date, (a), 8192) populate as select a, toDate('2000-01-01') date from test_00609; diff --git a/tests/queries/0_stateless/00738_lock_for_inner_table.sh b/tests/queries/0_stateless/00738_lock_for_inner_table.sh index b62a639d8f4..9bc84dd1063 100755 --- a/tests/queries/0_stateless/00738_lock_for_inner_table.sh +++ b/tests/queries/0_stateless/00738_lock_for_inner_table.sh @@ -7,6 +7,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +CLICKHOUSE_CLIENT="${CLICKHOUSE_CLIENT} --database_replicated_allow_explicit_uuid 3" + # there are some issues with Atomic database, let's generate it uniq # otherwise flaky check will not pass. uuid=$(${CLICKHOUSE_CLIENT} --query "SELECT reinterpretAsUUID(currentDatabase())") diff --git a/tests/queries/0_stateless/01153_attach_mv_uuid.sql b/tests/queries/0_stateless/01153_attach_mv_uuid.sql index 00cce8a1de4..6b167253d8f 100644 --- a/tests/queries/0_stateless/01153_attach_mv_uuid.sql +++ b/tests/queries/0_stateless/01153_attach_mv_uuid.sql @@ -14,6 +14,7 @@ INSERT INTO src VALUES (3), (4); SELECT * FROM mv ORDER BY n; DROP TABLE mv SYNC; +SET database_replicated_allow_explicit_uuid=3; SET show_table_uuid_in_table_create_query_if_not_nil=1; CREATE TABLE ".inner_id.e15f3ab5-6cae-4df3-b879-f40deafd82c2" (n Int32, n2 Int64) ENGINE = MergeTree PARTITION BY n % 10 ORDER BY n; ATTACH MATERIALIZED VIEW mv UUID 'e15f3ab5-6cae-4df3-b879-f40deafd82c2' (n Int32, n2 Int64) ENGINE = MergeTree PARTITION BY n % 10 ORDER BY n AS SELECT n, n * n AS n2 FROM src; diff --git a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference index 6444e10bb48..874494fb061 100644 --- a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference +++ b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.reference @@ -6,5 +6,5 @@ rmt1 rmt2 02858000-1000-4000-8000-000000000 0 -CREATE TABLE rdb_default.rmt1\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/test/02858_explicit_uuid_and_zk_path_default/rmt/{shard}\', \'_{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 -CREATE TABLE rdb_default.rmt2\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{uuid}/{shard}\', \'{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rmt1\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/test/02858_explicit_uuid_and_zk_path_default/rmt/{shard}\', \'_{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rmt2\n(\n `n` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{uuid}/{shard}\', \'{replica}\')\nORDER BY n\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh index b011aed613a..81a9cef02ff 100755 --- a/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh +++ b/tests/queries/0_stateless/02858_explicit_uuid_and_zk_path.sh @@ -38,7 +38,7 @@ $CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database='$db' ORDER $CLICKHOUSE_CLIENT -q "SELECT substring(toString(uuid) as s, 1, length(s) - 3) FROM system.tables WHERE database='$db' and name='m1'" $CLICKHOUSE_CLIENT -q "SELECT toString(uuid) LIKE '02858000%' FROM system.tables WHERE database='$db' and name='m2'" -$CLICKHOUSE_CLIENT -q "SHOW CREATE $db.rmt1" -$CLICKHOUSE_CLIENT -q "SHOW CREATE $db.rmt2" +$CLICKHOUSE_CLIENT -q "SHOW CREATE $db.rmt1" | sed "s/$db/default/g" +$CLICKHOUSE_CLIENT -q "SHOW CREATE $db.rmt2" | sed "s/$db/default/g" $CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS rdb_$CLICKHOUSE_DATABASE" diff --git a/tests/queries/0_stateless/02888_replicated_merge_tree_creation.sh b/tests/queries/0_stateless/02888_replicated_merge_tree_creation.sh index b9603e75d2e..466f0d01a7f 100755 --- a/tests/queries/0_stateless/02888_replicated_merge_tree_creation.sh +++ b/tests/queries/0_stateless/02888_replicated_merge_tree_creation.sh @@ -5,6 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh +CLICKHOUSE_CLIENT="${CLICKHOUSE_CLIENT} --database_replicated_allow_explicit_uuid 3 --database_replicated_allow_replicated_engine_arguments 3" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS test_exception_replicated SYNC" From 97f4ec2adbf614842e9b16badb69a8d5c642abe0 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 17 Jul 2024 16:59:35 +0200 Subject: [PATCH 061/289] Read cgroup memory usage in async metrics thread --- programs/keeper/Keeper.cpp | 26 ++-- programs/server/Server.cpp | 34 ++--- src/Common/AsynchronousMetrics.cpp | 13 +- src/Common/AsynchronousMetrics.h | 5 +- src/Common/CgroupsMemoryUsageObserver.cpp | 127 ++++-------------- src/Common/CgroupsMemoryUsageObserver.h | 19 +-- src/Common/MemoryTracker.cpp | 38 ++---- src/Common/MemoryTracker.h | 4 +- .../KeeperAsynchronousMetrics.cpp | 2 +- src/Interpreters/Context.cpp | 13 ++ src/Interpreters/Context.h | 4 + .../ServerAsynchronousMetrics.cpp | 2 +- 12 files changed, 112 insertions(+), 175 deletions(-) diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 44c2daa33ad..3f6020ad48c 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -399,6 +399,18 @@ try registerDisks(/*global_skip_access_check=*/false); + auto cgroups_memory_observer_wait_time = config().getUInt64("keeper_server.cgroups_memory_observer_wait_time", 15); + try + { + auto cgroups_reader = createCgroupsReader(); + global_context->setCgroupsReader(createCgroupsReader()); + } + catch (...) + { + if (cgroups_memory_observer_wait_time != 0) + tryLogCurrentException(log, "Failed to create cgroups reader"); + } + /// This object will periodically calculate some metrics. KeeperAsynchronousMetrics async_metrics( global_context, @@ -622,21 +634,19 @@ try main_config_reloader->start(); std::optional cgroups_memory_usage_observer; - try + if (cgroups_memory_observer_wait_time != 0) { - auto wait_time = config().getUInt64("keeper_server.cgroups_memory_observer_wait_time", 15); - if (wait_time != 0) + auto cgroups_reader = global_context->getCgroupsReader(); + if (cgroups_reader) { - cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time)); + cgroups_memory_usage_observer.emplace(std::chrono::seconds(cgroups_memory_observer_wait_time), global_context->getCgroupsReader()); /// Not calling cgroups_memory_usage_observer->setLimits() here (as for the normal ClickHouse server) because Keeper controls /// its memory usage by other means (via setting 'max_memory_usage_soft_limit'). cgroups_memory_usage_observer->setOnMemoryAmountAvailableChangedFn([&]() { main_config_reloader->reload(); }); cgroups_memory_usage_observer->startThread(); } - } - catch (Exception &) - { - tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization"); + else + LOG_ERROR(log, "Disabling cgroup memory observer because of an error during initialization of cgroups reader"); } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 053ddaf8d8b..c52b1e037ec 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -897,6 +897,17 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); + try + { + auto cgroups_reader = createCgroupsReader(); + global_context->setCgroupsReader(createCgroupsReader()); + } + catch (...) + { + if (server_settings.cgroups_memory_usage_observer_wait_time != 0) + tryLogCurrentException(log, "Failed to create cgroups reader"); + } + /// This object will periodically calculate some metrics. ServerAsynchronousMetrics async_metrics( global_context, @@ -1456,15 +1467,13 @@ try } std::optional cgroups_memory_usage_observer; - try + if (auto wait_time = server_settings.cgroups_memory_usage_observer_wait_time; wait_time != 0) { - auto wait_time = server_settings.cgroups_memory_usage_observer_wait_time; - if (wait_time != 0) - cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time)); - } - catch (Exception &) - { - tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization"); + auto cgroups_reader = global_context->getCgroupsReader(); + if (cgroups_reader) + cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time), std::move(cgroups_reader)); + else + LOG_ERROR(log, "Disabling cgroup memory observer because of an error during initialization of cgroups reader"); } std::string cert_path = config().getString("openSSL.server.certificateFile", ""); @@ -1532,15 +1541,6 @@ try total_memory_tracker.setDescription("(total)"); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); - if (cgroups_memory_usage_observer) - { - double hard_limit_ratio = new_server_settings.cgroup_memory_watcher_hard_limit_ratio; - double soft_limit_ratio = new_server_settings.cgroup_memory_watcher_soft_limit_ratio; - cgroups_memory_usage_observer->setMemoryUsageLimits( - static_cast(max_server_memory_usage * hard_limit_ratio), - static_cast(max_server_memory_usage * soft_limit_ratio)); - } - size_t merges_mutations_memory_usage_soft_limit = new_server_settings.merges_mutations_memory_usage_soft_limit; size_t default_merges_mutations_server_memory_usage = static_cast(current_physical_server_memory * new_server_settings.merges_mutations_memory_usage_to_ram_ratio); diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index 6309f6079f6..0953ad88697 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -57,10 +57,12 @@ static std::unique_ptr openFileIfExists(const std::stri AsynchronousMetrics::AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_) + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + std::shared_ptr cgroups_reader_) : update_period(update_period_seconds) , log(getLogger("AsynchronousMetrics")) , protocol_server_metrics_func(protocol_server_metrics_func_) + , cgroups_reader(std::move(cgroups_reader_)) { #if defined(OS_LINUX) openFileIfExists("/proc/meminfo", meminfo); @@ -669,6 +671,13 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) free_memory_in_allocator_arenas = je_malloc_pdirty * getPageSize(); #endif + if (cgroups_reader != nullptr) + { + rss = cgroups_reader->readMemoryUsage(); + new_values["CgroupsMemoryUsage"] = { rss, + "The amount of physical memory used by the server process, reported by cgroups." }; + } + Int64 difference = rss - amount; /// Log only if difference is high. This is for convenience. The threshold is arbitrary. @@ -681,7 +690,7 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) ReadableSize(rss), ReadableSize(difference)); - MemoryTracker::setRSS(rss, free_memory_in_allocator_arenas); + MemoryTracker::setRSS(rss, /*has_free_memory_in_allocator_arenas_=*/free_memory_in_allocator_arenas > 0); } } diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index 10a972d2458..0b110f41fc3 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -68,7 +69,8 @@ public: AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_); + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + std::shared_ptr cgroups_reader_); virtual ~AsynchronousMetrics(); @@ -91,6 +93,7 @@ private: virtual void logImpl(AsynchronousMetricValues &) {} ProtocolServerMetricsFunc protocol_server_metrics_func; + std::shared_ptr cgroups_reader; std::unique_ptr thread; diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp index 02bde0d80b7..b12845df098 100644 --- a/src/Common/CgroupsMemoryUsageObserver.cpp +++ b/src/Common/CgroupsMemoryUsageObserver.cpp @@ -17,13 +17,6 @@ #include #include -#include "config.h" -#if USE_JEMALLOC -# include -#define STRINGIFY_HELPER(x) #x -#define STRINGIFY(x) STRINGIFY_HELPER(x) -#endif - using namespace DB; namespace fs = std::filesystem; @@ -155,15 +148,21 @@ std::optional getCgroupsV1Path() return {default_cgroups_mount / "memory"}; } -std::pair getCgroupsPath() +enum class CgroupsVersion : uint8_t +{ + V1, + V2 +}; + +std::pair getCgroupsPath() { auto v2_path = getCgroupsV2Path(); if (v2_path.has_value()) - return {*v2_path, CgroupsMemoryUsageObserver::CgroupsVersion::V2}; + return {*v2_path, CgroupsVersion::V2}; auto v1_path = getCgroupsV1Path(); if (v1_path.has_value()) - return {*v1_path, CgroupsMemoryUsageObserver::CgroupsVersion::V1}; + return {*v1_path, CgroupsVersion::V1}; throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot find cgroups v1 or v2 current memory file"); } @@ -173,22 +172,29 @@ std::pair getCgroupsPat namespace DB { -CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_) - : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_) +std::shared_ptr createCgroupsReader() { const auto [cgroup_path, version] = getCgroupsPath(); + LOG_INFO( + getLogger("CgroupsReader"), + "Will create cgroup reader from '{}' (cgroups version: {})", + cgroup_path, + (version == CgroupsVersion::V1) ? "v1" : "v2"); if (version == CgroupsVersion::V2) - cgroup_reader = std::make_unique(cgroup_path); + return std::make_shared(cgroup_path); else - cgroup_reader = std::make_unique(cgroup_path); + { + chassert(version == CgroupsVersion::V1); + return std::make_shared(cgroup_path); + } - LOG_INFO( - log, - "Will read the current memory usage from '{}' (cgroups version: {}), wait time is {} sec", - cgroup_path, - (version == CgroupsVersion::V1) ? "v1" : "v2", - wait_time.count()); +} + +CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_, std::shared_ptr cgroups_reader_) + : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_), cgroups_reader(std::move(cgroups_reader_)) +{ + cgroups_reader = createCgroupsReader(); } CgroupsMemoryUsageObserver::~CgroupsMemoryUsageObserver() @@ -196,58 +202,6 @@ CgroupsMemoryUsageObserver::~CgroupsMemoryUsageObserver() stopThread(); } -void CgroupsMemoryUsageObserver::setMemoryUsageLimits(uint64_t hard_limit_, uint64_t soft_limit_) -{ - std::lock_guard limit_lock(limit_mutex); - - if (hard_limit_ == hard_limit && soft_limit_ == soft_limit) - return; - - hard_limit = hard_limit_; - soft_limit = soft_limit_; - - on_hard_limit = [this, hard_limit_](bool up) - { - if (up) - { - LOG_WARNING(log, "Exceeded hard memory limit ({})", ReadableSize(hard_limit_)); - - /// Update current usage in memory tracker. Also reset free_memory_in_allocator_arenas to zero though we don't know if they are - /// really zero. Trying to avoid OOM ... - MemoryTracker::setRSS(hard_limit_, 0); - } - else - { - LOG_INFO(log, "Dropped below hard memory limit ({})", ReadableSize(hard_limit_)); - } - }; - - on_soft_limit = [this, soft_limit_](bool up) - { - if (up) - { - LOG_WARNING(log, "Exceeded soft memory limit ({})", ReadableSize(soft_limit_)); - -# if USE_JEMALLOC - LOG_INFO(log, "Purging jemalloc arenas"); - mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); -# endif - /// Reset current usage in memory tracker. Expect zero for free_memory_in_allocator_arenas as we just purged them. - uint64_t memory_usage = cgroup_reader->readMemoryUsage(); - LOG_TRACE(log, "Read current memory usage {} bytes ({}) from cgroups", memory_usage, ReadableSize(memory_usage)); - MemoryTracker::setRSS(memory_usage, 0); - - LOG_INFO(log, "Purged jemalloc arenas. Current memory usage is {}", ReadableSize(memory_usage)); - } - else - { - LOG_INFO(log, "Dropped below soft memory limit ({})", ReadableSize(soft_limit_)); - } - }; - - LOG_INFO(log, "Set new limits, soft limit: {}, hard limit: {}", ReadableSize(soft_limit_), ReadableSize(hard_limit_)); -} - void CgroupsMemoryUsageObserver::setOnMemoryAmountAvailableChangedFn(OnMemoryAmountAvailableChangedFn on_memory_amount_available_changed_) { std::lock_guard memory_amount_available_changed_lock(memory_amount_available_changed_mutex); @@ -301,35 +255,6 @@ void CgroupsMemoryUsageObserver::runThread() std::lock_guard memory_amount_available_changed_lock(memory_amount_available_changed_mutex); on_memory_amount_available_changed(); } - - std::lock_guard limit_lock(limit_mutex); - if (soft_limit > 0 && hard_limit > 0) - { - uint64_t memory_usage = cgroup_reader->readMemoryUsage(); - LOG_TRACE(log, "Read current memory usage {} bytes ({}) from cgroups", memory_usage, ReadableSize(memory_usage)); - if (memory_usage > hard_limit) - { - if (last_memory_usage <= hard_limit) - on_hard_limit(true); - } - else - { - if (last_memory_usage > hard_limit) - on_hard_limit(false); - } - - if (memory_usage > soft_limit) - { - if (last_memory_usage <= soft_limit) - on_soft_limit(true); - } - else - { - if (last_memory_usage > soft_limit) - on_soft_limit(false); - } - last_memory_usage = memory_usage; - } } catch (...) { diff --git a/src/Common/CgroupsMemoryUsageObserver.h b/src/Common/CgroupsMemoryUsageObserver.h index b848a2bff3c..078307a6fa0 100644 --- a/src/Common/CgroupsMemoryUsageObserver.h +++ b/src/Common/CgroupsMemoryUsageObserver.h @@ -16,6 +16,8 @@ struct ICgroupsReader virtual uint64_t readMemoryUsage() = 0; }; +std::shared_ptr createCgroupsReader(); + /// Does two things: /// 1. Periodically reads the memory usage of the process from Linux cgroups. /// You can specify soft or hard memory limits: @@ -35,19 +37,11 @@ struct ICgroupsReader class CgroupsMemoryUsageObserver { public: - using OnMemoryLimitFn = std::function; using OnMemoryAmountAvailableChangedFn = std::function; - enum class CgroupsVersion : uint8_t - { - V1, - V2 - }; - - explicit CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_); + explicit CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_, std::shared_ptr cgroups_reader_); ~CgroupsMemoryUsageObserver(); - void setMemoryUsageLimits(uint64_t hard_limit_, uint64_t soft_limit_); void setOnMemoryAmountAvailableChangedFn(OnMemoryAmountAvailableChangedFn on_memory_amount_available_changed_); void startThread(); @@ -58,22 +52,17 @@ private: const std::chrono::seconds wait_time; std::mutex limit_mutex; - size_t hard_limit TSA_GUARDED_BY(limit_mutex) = 0; - size_t soft_limit TSA_GUARDED_BY(limit_mutex) = 0; - OnMemoryLimitFn on_hard_limit TSA_GUARDED_BY(limit_mutex); - OnMemoryLimitFn on_soft_limit TSA_GUARDED_BY(limit_mutex); std::mutex memory_amount_available_changed_mutex; OnMemoryAmountAvailableChangedFn on_memory_amount_available_changed TSA_GUARDED_BY(memory_amount_available_changed_mutex); - uint64_t last_memory_usage = 0; /// how much memory does the process use uint64_t last_available_memory_amount; /// how much memory can the process use void stopThread(); void runThread(); - std::unique_ptr cgroup_reader; + std::shared_ptr cgroups_reader; std::mutex thread_mutex; std::condition_variable cond; diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 28cfa98666a..eaf34d87ec5 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -26,7 +26,6 @@ #endif #include -#include #include #include #include @@ -123,7 +122,7 @@ static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; MemoryTracker total_memory_tracker(nullptr, VariableContext::Global); MemoryTracker background_memory_tracker(&total_memory_tracker, VariableContext::User, false); -std::atomic MemoryTracker::free_memory_in_allocator_arenas; +std::atomic MemoryTracker::has_free_memory_in_allocator_arenas; MemoryTracker::MemoryTracker(VariableContext level_) : parent(&total_memory_tracker), level(level_) {} MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_) : parent(parent_), level(level_) {} @@ -204,7 +203,7 @@ void MemoryTracker::debugLogBigAllocationWithoutCheck(Int64 size [[maybe_unused] LOG_TEST(getLogger("MemoryTracker"), "Too big allocation ({} bytes) without checking memory limits, " "it may lead to OOM. Stack trace: {}", size, StackTrace().toString()); #else - return; /// Avoid trash logging in release builds + /// Avoid trash logging in release builds #endif } @@ -294,33 +293,18 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed } } - Int64 limit_to_check = current_hard_limit; - #if USE_JEMALLOC - if (level == VariableContext::Global && allow_use_jemalloc_memory.load(std::memory_order_relaxed)) + if (level == VariableContext::Global && will_be > soft_limit.load(std::memory_order_relaxed) + && has_free_memory_in_allocator_arenas.exchange(false)) { - /// Jemalloc arenas may keep some extra memory. - /// This memory was substucted from RSS to decrease memory drift. - /// In case memory is close to limit, try to pugre the arenas. - /// This is needed to avoid OOM, because some allocations are directly done with mmap. - Int64 current_free_memory_in_allocator_arenas = free_memory_in_allocator_arenas.load(std::memory_order_relaxed); - - if (current_free_memory_in_allocator_arenas > 0 && current_hard_limit && current_free_memory_in_allocator_arenas + will_be > current_hard_limit) - { - if (free_memory_in_allocator_arenas.exchange(-current_free_memory_in_allocator_arenas) > 0) - { - Stopwatch watch; - mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); - ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); - ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, watch.elapsedMicroseconds()); - } - } - - limit_to_check += abs(current_free_memory_in_allocator_arenas); + Stopwatch watch; + mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); + ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); + ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, watch.elapsedMicroseconds()); } #endif - if (unlikely(current_hard_limit && will_be > limit_to_check)) + if (unlikely(current_hard_limit && will_be > current_hard_limit)) { if (memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded) { @@ -526,11 +510,11 @@ void MemoryTracker::reset() } -void MemoryTracker::setRSS(Int64 rss_, Int64 free_memory_in_allocator_arenas_) +void MemoryTracker::setRSS(Int64 rss_, bool has_free_memory_in_allocator_arenas_) { Int64 new_amount = rss_; total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); - free_memory_in_allocator_arenas.store(free_memory_in_allocator_arenas_, std::memory_order_relaxed); + has_free_memory_in_allocator_arenas.store(has_free_memory_in_allocator_arenas_, std::memory_order_relaxed); auto metric_loaded = total_memory_tracker.metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index fd32b631774..48d02fd1fc6 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -59,7 +59,7 @@ private: std::atomic profiler_limit {0}; std::atomic_bool allow_use_jemalloc_memory {true}; - static std::atomic free_memory_in_allocator_arenas; + static std::atomic has_free_memory_in_allocator_arenas; Int64 profiler_step = 0; @@ -252,7 +252,7 @@ public: /// Reset current counter to an RSS value. /// Jemalloc may have pre-allocated arenas, they are accounted in RSS. /// We can free this arenas in case of exception to avoid OOM. - static void setRSS(Int64 rss_, Int64 free_memory_in_allocator_arenas_); + static void setRSS(Int64 rss_, bool has_free_memory_in_allocator_arenas_); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage(); diff --git a/src/Coordination/KeeperAsynchronousMetrics.cpp b/src/Coordination/KeeperAsynchronousMetrics.cpp index 86166ffe31b..3e404b7152b 100644 --- a/src/Coordination/KeeperAsynchronousMetrics.cpp +++ b/src/Coordination/KeeperAsynchronousMetrics.cpp @@ -115,7 +115,7 @@ void updateKeeperInformation(KeeperDispatcher & keeper_dispatcher, AsynchronousM KeeperAsynchronousMetrics::KeeperAsynchronousMetrics( ContextPtr context_, unsigned update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_) - : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_), context(std::move(context_)) + : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, context_->getCgroupsReader()), context(std::move(context_)) { } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 2602afd8b78..771b8e9e558 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -405,6 +406,8 @@ struct ContextSharedPart : boost::noncopyable std::unique_ptr cluster_discovery TSA_GUARDED_BY(clusters_mutex); size_t clusters_version TSA_GUARDED_BY(clusters_mutex) = 0; + std::shared_ptr cgroups_reader; + /// No lock required for async_insert_queue modified only during initialization std::shared_ptr async_insert_queue; @@ -5635,6 +5638,16 @@ const ServerSettings & Context::getServerSettings() const return shared->server_settings; } +void Context::setCgroupsReader(std::shared_ptr cgroups_reader_) +{ + shared->cgroups_reader = std::move(cgroups_reader_); +} + +std::shared_ptr Context::getCgroupsReader() const +{ + return shared->cgroups_reader; +} + uint64_t HTTPContext::getMaxHstsAge() const { return context->getSettingsRef().hsts_max_age; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 284cac50769..f183a72e8e2 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -150,6 +150,7 @@ class ServerType; template class MergeTreeBackgroundExecutor; class AsyncLoader; +struct ICgroupsReader; struct TemporaryTableHolder; using TemporaryTablesMapping = std::map>; @@ -1344,6 +1345,9 @@ public: const ServerSettings & getServerSettings() const; + void setCgroupsReader(std::shared_ptr cgroups_reader_); + std::shared_ptr getCgroupsReader() const; + private: std::shared_ptr getSettingsConstraintsAndCurrentProfilesWithLock() const; diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 872a9f864df..6ee0168bede 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -57,7 +57,7 @@ ServerAsynchronousMetrics::ServerAsynchronousMetrics( unsigned heavy_metrics_update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_) : WithContext(global_context_) - , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_) + , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, getContext()->getCgroupsReader()) , heavy_metric_update_period(heavy_metrics_update_period_seconds) { /// sanity check From 21009577d867d493a6ceda48987c060498774f94 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 18 Jul 2024 09:01:08 +0200 Subject: [PATCH 062/289] Dedicated memory background thread --- programs/server/Server.cpp | 39 ++++++++++++++++++++++++--- src/CMakeLists.txt | 2 +- src/Common/AsynchronousMetrics.cpp | 43 ------------------------------ src/Common/Jemalloc.cpp | 16 ----------- src/Common/Jemalloc.h | 29 ++++++++++++++++++++ src/Common/MemoryTracker.cpp | 33 +++++++++-------------- src/Common/MemoryTracker.h | 1 - 7 files changed, 78 insertions(+), 85 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index c52b1e037ec..ca46338d1c1 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -447,9 +447,12 @@ void checkForUsersNotInMainConfig( } } +namespace +{ + /// Unused in other builds #if defined(OS_LINUX) -static String readLine(const String & path) +String readLine(const String & path) { ReadBufferFromFile in(path); String contents; @@ -457,7 +460,7 @@ static String readLine(const String & path) return contents; } -static int readNumber(const String & path) +int readNumber(const String & path) { ReadBufferFromFile in(path); int result; @@ -467,7 +470,7 @@ static int readNumber(const String & path) #endif -static void sanityChecks(Server & server) +void sanityChecks(Server & server) { std::string data_path = getCanonicalPath(server.config().getString("path", DBMS_DEFAULT_PATH)); std::string logs_path = server.config().getString("logger.log", ""); @@ -588,6 +591,31 @@ static void sanityChecks(Server & server) } } +[[noreturn]] void backgroundMemoryThread() +{ + std::mutex mutex; + std::condition_variable cv; + + std::unique_lock lock(mutex); + while (true) + { + cv.wait_for(lock, std::chrono::microseconds(200)); + uint64_t epoch = 0; + mallctl("epoch", nullptr, nullptr, &epoch, sizeof(epoch)); + auto maybe_resident = getJemallocValue("stats.resident"); + if (!maybe_resident.has_value()) + continue; + + Int64 resident = *maybe_resident; + //LOG_INFO(getLogger("JEmalloc"), "Resident {}", ReadableSize(resident)); + MemoryTracker::setRSS(resident, false); + if (resident > total_memory_tracker.getHardLimit()) + purgeJemallocArenas(); + } +} + +} + void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, ContextMutablePtr context, Poco::Logger * log) { try @@ -877,6 +905,11 @@ try total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size); } + ThreadFromGlobalPool background_memory_thread([] + { + backgroundMemoryThread(); + }); + Poco::ThreadPool server_pool( /* minCapacity */3, /* maxCapacity */server_settings.max_connections, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d985595154c..bfa41eacea1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -174,7 +174,7 @@ add_library (clickhouse_new_delete STATIC Common/new_delete.cpp) target_link_libraries (clickhouse_new_delete PRIVATE clickhouse_common_io) if (TARGET ch_contrib::jemalloc) target_link_libraries (clickhouse_new_delete PRIVATE ch_contrib::jemalloc) - target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::jemalloc) + target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::jemalloc) target_link_libraries (clickhouse_storages_system PRIVATE ch_contrib::jemalloc) endif() diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index 0953ad88697..b3e53c29d4a 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -649,49 +649,6 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) "The amount of virtual memory mapped for the use of stack and for the allocated memory, in bytes." " It is unspecified whether it includes the per-thread stacks and most of the allocated memory, that is allocated with the 'mmap' system call." " This metric exists only for completeness reasons. I recommend to use the `MemoryResident` metric for monitoring."}; - - /// We must update the value of total_memory_tracker periodically. - /// Otherwise it might be calculated incorrectly - it can include a "drift" of memory amount. - /// See https://github.com/ClickHouse/ClickHouse/issues/10293 - { - Int64 amount = total_memory_tracker.get(); - Int64 peak = total_memory_tracker.getPeak(); - Int64 rss = data.resident; - Int64 free_memory_in_allocator_arenas = 0; - -#if USE_JEMALLOC - /// According to jemalloc man, pdirty is: - /// - /// Number of pages within unused extents that are potentially - /// dirty, and for which madvise() or similar has not been called. - /// - /// So they will be subtracted from RSS to make accounting more - /// accurate, since those pages are not really RSS but a memory - /// that can be used at anytime via jemalloc. - free_memory_in_allocator_arenas = je_malloc_pdirty * getPageSize(); -#endif - - if (cgroups_reader != nullptr) - { - rss = cgroups_reader->readMemoryUsage(); - new_values["CgroupsMemoryUsage"] = { rss, - "The amount of physical memory used by the server process, reported by cgroups." }; - } - - Int64 difference = rss - amount; - - /// Log only if difference is high. This is for convenience. The threshold is arbitrary. - if (difference >= 1048576 || difference <= -1048576) - LOG_TRACE(log, - "MemoryTracking: was {}, peak {}, free memory in arenas {}, will set to {} (RSS), difference: {}", - ReadableSize(amount), - ReadableSize(peak), - ReadableSize(free_memory_in_allocator_arenas), - ReadableSize(rss), - ReadableSize(difference)); - - MemoryTracker::setRSS(rss, /*has_free_memory_in_allocator_arenas_=*/free_memory_in_allocator_arenas > 0); - } } { diff --git a/src/Common/Jemalloc.cpp b/src/Common/Jemalloc.cpp index d7cc246db6a..d8ff9268cca 100644 --- a/src/Common/Jemalloc.cpp +++ b/src/Common/Jemalloc.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #define STRINGIFY_HELPER(x) #x #define STRINGIFY(x) STRINGIFY_HELPER(x) @@ -26,7 +25,6 @@ namespace ErrorCodes void purgeJemallocArenas() { - LOG_TRACE(getLogger("SystemJemalloc"), "Purging unused memory"); Stopwatch watch; mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); @@ -46,20 +44,6 @@ void checkJemallocProfilingEnabled() "set: MALLOC_CONF=background_thread:true,prof:true"); } -template -void setJemallocValue(const char * name, T value) -{ - T old_value; - size_t old_value_size = sizeof(T); - if (mallctl(name, &old_value, &old_value_size, reinterpret_cast(&value), sizeof(T))) - { - LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name); - return; - } - - LOG_INFO(getLogger("Jemalloc"), "Value for {} set to {} (from {})", name, value, old_value); -} - void setJemallocProfileActive(bool value) { checkJemallocProfilingEnabled(); diff --git a/src/Common/Jemalloc.h b/src/Common/Jemalloc.h index 499a906fd3d..0c533711f78 100644 --- a/src/Common/Jemalloc.h +++ b/src/Common/Jemalloc.h @@ -5,6 +5,8 @@ #if USE_JEMALLOC #include +#include +#include namespace DB { @@ -21,6 +23,33 @@ void setJemallocBackgroundThreads(bool enabled); void setJemallocMaxBackgroundThreads(size_t max_threads); +template +void setJemallocValue(const char * name, T value) +{ + T old_value; + size_t old_value_size = sizeof(T); + if (mallctl(name, &old_value, &old_value_size, reinterpret_cast(&value), sizeof(T))) + { + LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name); + return; + } + + LOG_INFO(getLogger("Jemalloc"), "Value for {} set to {} (from {})", name, value, old_value); +} + +template +std::optional getJemallocValue(const char * name) +{ + T value; + size_t value_size = sizeof(T); + if (mallctl(name, &value, &value_size, nullptr, 0)) + { + LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name); + return std::nullopt; + } + return value; +} + } #endif diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index eaf34d87ec5..a541eeeb25f 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -20,9 +20,6 @@ #if USE_JEMALLOC # include -#define STRINGIFY_HELPER(x) #x -#define STRINGIFY(x) STRINGIFY_HELPER(x) - #endif #include @@ -126,11 +123,11 @@ std::atomic MemoryTracker::has_free_memory_in_allocator_arenas; MemoryTracker::MemoryTracker(VariableContext level_) : parent(&total_memory_tracker), level(level_) {} MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_) : parent(parent_), level(level_) {} + MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_, bool log_peak_memory_usage_in_destructor_) - : parent(parent_) - , log_peak_memory_usage_in_destructor(log_peak_memory_usage_in_destructor_) - , level(level_) -{} + : parent(parent_), log_peak_memory_usage_in_destructor(log_peak_memory_usage_in_destructor_), level(level_) +{ +} MemoryTracker::~MemoryTracker() { @@ -200,8 +197,12 @@ void MemoryTracker::debugLogBigAllocationWithoutCheck(Int64 size [[maybe_unused] return; MemoryTrackerBlockerInThread blocker(VariableContext::Global); - LOG_TEST(getLogger("MemoryTracker"), "Too big allocation ({} bytes) without checking memory limits, " - "it may lead to OOM. Stack trace: {}", size, StackTrace().toString()); + LOG_TEST( + getLogger("MemoryTracker"), + "Too big allocation ({} bytes) without checking memory limits, " + "it may lead to OOM. Stack trace: {}", + size, + StackTrace().toString()); #else /// Avoid trash logging in release builds #endif @@ -293,17 +294,6 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed } } -#if USE_JEMALLOC - if (level == VariableContext::Global && will_be > soft_limit.load(std::memory_order_relaxed) - && has_free_memory_in_allocator_arenas.exchange(false)) - { - Stopwatch watch; - mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); - ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); - ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, watch.elapsedMicroseconds()); - } -#endif - if (unlikely(current_hard_limit && will_be > current_hard_limit)) { if (memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded) @@ -513,7 +503,8 @@ void MemoryTracker::reset() void MemoryTracker::setRSS(Int64 rss_, bool has_free_memory_in_allocator_arenas_) { Int64 new_amount = rss_; - total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); + if (rss_) + total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); has_free_memory_in_allocator_arenas.store(has_free_memory_in_allocator_arenas_, std::memory_order_relaxed); auto metric_loaded = total_memory_tracker.metric.load(std::memory_order_relaxed); diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 48d02fd1fc6..257ed7d0629 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include From 9a43183eb39dc056186432e6478e050db5045ecf Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 18 Jul 2024 10:31:24 +0200 Subject: [PATCH 063/289] Finish background memory thread --- programs/keeper/Keeper.cpp | 30 ++++------ programs/server/Server.cpp | 59 ++++--------------- src/Common/AsynchronousMetrics.cpp | 19 ++---- src/Common/AsynchronousMetrics.h | 3 +- src/Common/CgroupsMemoryUsageObserver.cpp | 8 +-- src/Common/CgroupsMemoryUsageObserver.h | 24 +++----- src/Common/Jemalloc.h | 42 +++++++++---- src/Common/MemoryTracker.cpp | 10 +--- src/Common/MemoryTracker.h | 13 +--- src/Common/MemoryWorker.cpp | 49 +++++++++++++++ src/Common/MemoryWorker.h | 34 +++++++++++ .../KeeperAsynchronousMetrics.cpp | 2 +- src/Core/ServerSettings.h | 1 + src/Interpreters/Context.cpp | 12 ---- src/Interpreters/Context.h | 3 - .../ServerAsynchronousMetrics.cpp | 2 +- .../System/StorageSystemServerSettings.cpp | 1 - tests/integration/test_memory_limit/test.py | 1 - 18 files changed, 158 insertions(+), 155 deletions(-) create mode 100644 src/Common/MemoryWorker.cpp create mode 100644 src/Common/MemoryWorker.h diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 3f6020ad48c..87f126a0046 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -371,6 +372,8 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); + MemoryWorker memory_worker(config().getUInt64("memory_worker_period_ms", 100)); + static ServerErrorHandler error_handler; Poco::ErrorHandler::set(&error_handler); @@ -399,18 +402,6 @@ try registerDisks(/*global_skip_access_check=*/false); - auto cgroups_memory_observer_wait_time = config().getUInt64("keeper_server.cgroups_memory_observer_wait_time", 15); - try - { - auto cgroups_reader = createCgroupsReader(); - global_context->setCgroupsReader(createCgroupsReader()); - } - catch (...) - { - if (cgroups_memory_observer_wait_time != 0) - tryLogCurrentException(log, "Failed to create cgroups reader"); - } - /// This object will periodically calculate some metrics. KeeperAsynchronousMetrics async_metrics( global_context, @@ -634,21 +625,22 @@ try main_config_reloader->start(); std::optional cgroups_memory_usage_observer; - if (cgroups_memory_observer_wait_time != 0) + try { - auto cgroups_reader = global_context->getCgroupsReader(); - if (cgroups_reader) + auto wait_time = config().getUInt64("keeper_server.cgroups_memory_observer_wait_time", 15); + if (wait_time != 0) { - cgroups_memory_usage_observer.emplace(std::chrono::seconds(cgroups_memory_observer_wait_time), global_context->getCgroupsReader()); + cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time)); /// Not calling cgroups_memory_usage_observer->setLimits() here (as for the normal ClickHouse server) because Keeper controls /// its memory usage by other means (via setting 'max_memory_usage_soft_limit'). cgroups_memory_usage_observer->setOnMemoryAmountAvailableChangedFn([&]() { main_config_reloader->reload(); }); cgroups_memory_usage_observer->startThread(); } - else - LOG_ERROR(log, "Disabling cgroup memory observer because of an error during initialization of cgroups reader"); } - + catch (Exception &) + { + tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization"); + } LOG_INFO(log, "Ready for connections."); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index ca46338d1c1..ae445477c3a 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -24,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -110,6 +110,8 @@ #include #include +#include + #include "config.h" #include @@ -591,29 +593,6 @@ void sanityChecks(Server & server) } } -[[noreturn]] void backgroundMemoryThread() -{ - std::mutex mutex; - std::condition_variable cv; - - std::unique_lock lock(mutex); - while (true) - { - cv.wait_for(lock, std::chrono::microseconds(200)); - uint64_t epoch = 0; - mallctl("epoch", nullptr, nullptr, &epoch, sizeof(epoch)); - auto maybe_resident = getJemallocValue("stats.resident"); - if (!maybe_resident.has_value()) - continue; - - Int64 resident = *maybe_resident; - //LOG_INFO(getLogger("JEmalloc"), "Resident {}", ReadableSize(resident)); - MemoryTracker::setRSS(resident, false); - if (resident > total_memory_tracker.getHardLimit()) - purgeJemallocArenas(); - } -} - } void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, ContextMutablePtr context, Poco::Logger * log) @@ -905,11 +884,6 @@ try total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size); } - ThreadFromGlobalPool background_memory_thread([] - { - backgroundMemoryThread(); - }); - Poco::ThreadPool server_pool( /* minCapacity */3, /* maxCapacity */server_settings.max_connections, @@ -930,16 +904,7 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); - try - { - auto cgroups_reader = createCgroupsReader(); - global_context->setCgroupsReader(createCgroupsReader()); - } - catch (...) - { - if (server_settings.cgroups_memory_usage_observer_wait_time != 0) - tryLogCurrentException(log, "Failed to create cgroups reader"); - } + MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms); /// This object will periodically calculate some metrics. ServerAsynchronousMetrics async_metrics( @@ -1500,13 +1465,15 @@ try } std::optional cgroups_memory_usage_observer; - if (auto wait_time = server_settings.cgroups_memory_usage_observer_wait_time; wait_time != 0) + try { - auto cgroups_reader = global_context->getCgroupsReader(); - if (cgroups_reader) - cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time), std::move(cgroups_reader)); - else - LOG_ERROR(log, "Disabling cgroup memory observer because of an error during initialization of cgroups reader"); + auto wait_time = server_settings.cgroups_memory_usage_observer_wait_time; + if (wait_time != 0) + cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time)); + } + catch (Exception &) + { + tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization"); } std::string cert_path = config().getString("openSSL.server.certificateFile", ""); @@ -1602,8 +1569,6 @@ try background_memory_tracker.setDescription("(background)"); background_memory_tracker.setMetric(CurrentMetrics::MergesMutationsMemoryTracking); - total_memory_tracker.setAllowUseJemallocMemory(new_server_settings.allow_use_jemalloc_memory); - auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker(); total_memory_tracker.setOvercommitTracker(global_overcommit_tracker); diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index b3e53c29d4a..a5c9875188b 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -57,12 +58,10 @@ static std::unique_ptr openFileIfExists(const std::stri AsynchronousMetrics::AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_, - std::shared_ptr cgroups_reader_) + const ProtocolServerMetricsFunc & protocol_server_metrics_func_) : update_period(update_period_seconds) , log(getLogger("AsynchronousMetrics")) , protocol_server_metrics_func(protocol_server_metrics_func_) - , cgroups_reader(std::move(cgroups_reader_)) { #if defined(OS_LINUX) openFileIfExists("/proc/meminfo", meminfo); @@ -378,23 +377,13 @@ void AsynchronousMetrics::run() namespace { -uint64_t updateJemallocEpoch() -{ - uint64_t value = 0; - size_t size = sizeof(value); - mallctl("epoch", &value, &size, &value, size); - return value; -} - template Value saveJemallocMetricImpl( AsynchronousMetricValues & values, const std::string & jemalloc_full_name, const std::string & clickhouse_full_name) { - Value value{}; - size_t size = sizeof(value); - mallctl(jemalloc_full_name.c_str(), &value, &size, nullptr, 0); + auto value = getJemallocValue(jemalloc_full_name.c_str()); values[clickhouse_full_name] = AsynchronousMetricValue(value, "An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html"); return value; } @@ -604,7 +593,7 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) // 'epoch' is a special mallctl -- it updates the statistics. Without it, all // the following calls will return stale values. It increments and returns // the current epoch number, which might be useful to log as a sanity check. - auto epoch = updateJemallocEpoch(); + auto epoch = getJemallocValue("epoch"); new_values["jemalloc.epoch"] = { epoch, "An internal incremental update number of the statistics of jemalloc (Jason Evans' memory allocator), used in all other `jemalloc` metrics." }; // Collect the statistics themselves. diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index 0b110f41fc3..bc379d4e92b 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -69,8 +69,7 @@ public: AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_, - std::shared_ptr cgroups_reader_); + const ProtocolServerMetricsFunc & protocol_server_metrics_func_); virtual ~AsynchronousMetrics(); diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp index b12845df098..ab7ca69ca04 100644 --- a/src/Common/CgroupsMemoryUsageObserver.cpp +++ b/src/Common/CgroupsMemoryUsageObserver.cpp @@ -191,11 +191,9 @@ std::shared_ptr createCgroupsReader() } -CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_, std::shared_ptr cgroups_reader_) - : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_), cgroups_reader(std::move(cgroups_reader_)) -{ - cgroups_reader = createCgroupsReader(); -} +CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_) + : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_), cgroups_reader(createCgroupsReader()) +{} CgroupsMemoryUsageObserver::~CgroupsMemoryUsageObserver() { diff --git a/src/Common/CgroupsMemoryUsageObserver.h b/src/Common/CgroupsMemoryUsageObserver.h index 078307a6fa0..33e0f167a59 100644 --- a/src/Common/CgroupsMemoryUsageObserver.h +++ b/src/Common/CgroupsMemoryUsageObserver.h @@ -18,28 +18,20 @@ struct ICgroupsReader std::shared_ptr createCgroupsReader(); -/// Does two things: -/// 1. Periodically reads the memory usage of the process from Linux cgroups. -/// You can specify soft or hard memory limits: -/// - When the soft memory limit is hit, drop jemalloc cache. -/// - When the hard memory limit is hit, update MemoryTracking metric to throw memory exceptions faster. -/// The goal of this is to avoid that the process hits the maximum allowed memory limit at which there is a good -/// chance that the Limux OOM killer terminates it. All of this is done is because internal memory tracking in -/// ClickHouse can unfortunately under-estimate the actually used memory. -/// 2. Periodically reads the the maximum memory available to the process (which can change due to cgroups settings). -/// You can specify a callback to react on changes. The callback typically reloads the configuration, i.e. Server -/// or Keeper configuration file. This reloads settings 'max_server_memory_usage' (Server) and 'max_memory_usage_soft_limit' -/// (Keeper) from which various other internal limits are calculated, including the soft and hard limits for (1.). -/// The goal of this is to provide elasticity when the container is scaled-up/scaled-down. The mechanism (polling -/// cgroups) is quite implicit, unfortunately there is currently no better way to communicate memory threshold changes -/// to the database. +/// Periodically reads the the maximum memory available to the process (which can change due to cgroups settings). +/// You can specify a callback to react on changes. The callback typically reloads the configuration, i.e. Server +/// or Keeper configuration file. This reloads settings 'max_server_memory_usage' (Server) and 'max_memory_usage_soft_limit' +/// (Keeper) from which various other internal limits are calculated, including the soft and hard limits for (1.). +/// The goal of this is to provide elasticity when the container is scaled-up/scaled-down. The mechanism (polling +/// cgroups) is quite implicit, unfortunately there is currently no better way to communicate memory threshold changes +/// to the database. #if defined(OS_LINUX) class CgroupsMemoryUsageObserver { public: using OnMemoryAmountAvailableChangedFn = std::function; - explicit CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_, std::shared_ptr cgroups_reader_); + explicit CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_); ~CgroupsMemoryUsageObserver(); void setOnMemoryAmountAvailableChangedFn(OnMemoryAmountAvailableChangedFn on_memory_amount_available_changed_); diff --git a/src/Common/Jemalloc.h b/src/Common/Jemalloc.h index 0c533711f78..629d039b483 100644 --- a/src/Common/Jemalloc.h +++ b/src/Common/Jemalloc.h @@ -28,28 +28,46 @@ void setJemallocValue(const char * name, T value) { T old_value; size_t old_value_size = sizeof(T); - if (mallctl(name, &old_value, &old_value_size, reinterpret_cast(&value), sizeof(T))) - { - LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name); - return; - } - + mallctl(name, &old_value, &old_value_size, reinterpret_cast(&value), sizeof(T)); LOG_INFO(getLogger("Jemalloc"), "Value for {} set to {} (from {})", name, value, old_value); } template -std::optional getJemallocValue(const char * name) +T getJemallocValue(const char * name) { T value; size_t value_size = sizeof(T); - if (mallctl(name, &value, &value_size, nullptr, 0)) - { - LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name); - return std::nullopt; - } + mallctl(name, &value, &value_size, nullptr, 0); return value; } +template +struct JemallocMibCache +{ + explicit JemallocMibCache(const char * name) + { + mallctlnametomib(name, mib, &mib_length); + } + + void setValue(T value) + { + mallctlbymib(mib, mib_length, nullptr, nullptr, reinterpret_cast(&value), sizeof(T)); + } + + T getValue() + { + T value; + size_t value_size = sizeof(T); + mallctlbymib(mib, mib_length, &value, &value_size, nullptr, 0); + return value; + } + +private: + static constexpr size_t max_mib_length = 4; + size_t mib[max_mib_length]; + size_t mib_length = max_mib_length; +}; + } #endif diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index a541eeeb25f..e237c3a0d33 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -108,8 +108,6 @@ void AllocationTrace::onFreeImpl(void * ptr, size_t size) const namespace ProfileEvents { extern const Event QueryMemoryLimitExceeded; - extern const Event MemoryAllocatorPurge; - extern const Event MemoryAllocatorPurgeTimeMicroseconds; } using namespace std::chrono_literals; @@ -119,8 +117,6 @@ static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; MemoryTracker total_memory_tracker(nullptr, VariableContext::Global); MemoryTracker background_memory_tracker(&total_memory_tracker, VariableContext::User, false); -std::atomic MemoryTracker::has_free_memory_in_allocator_arenas; - MemoryTracker::MemoryTracker(VariableContext level_) : parent(&total_memory_tracker), level(level_) {} MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_) : parent(parent_), level(level_) {} @@ -500,12 +496,10 @@ void MemoryTracker::reset() } -void MemoryTracker::setRSS(Int64 rss_, bool has_free_memory_in_allocator_arenas_) +void MemoryTracker::setRSS(Int64 rss_) { Int64 new_amount = rss_; - if (rss_) - total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); - has_free_memory_in_allocator_arenas.store(has_free_memory_in_allocator_arenas_, std::memory_order_relaxed); + total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); auto metric_loaded = total_memory_tracker.metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 257ed7d0629..4085bb321ed 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -56,9 +56,6 @@ private: std::atomic soft_limit {0}; std::atomic hard_limit {0}; std::atomic profiler_limit {0}; - std::atomic_bool allow_use_jemalloc_memory {true}; - - static std::atomic has_free_memory_in_allocator_arenas; Int64 profiler_step = 0; @@ -153,14 +150,6 @@ public: { return soft_limit.load(std::memory_order_relaxed); } - void setAllowUseJemallocMemory(bool value) - { - allow_use_jemalloc_memory.store(value, std::memory_order_relaxed); - } - bool getAllowUseJemallocMmemory() const - { - return allow_use_jemalloc_memory.load(std::memory_order_relaxed); - } /** Set limit if it was not set. * Otherwise, set limit to new value, if new value is greater than previous limit. @@ -251,7 +240,7 @@ public: /// Reset current counter to an RSS value. /// Jemalloc may have pre-allocated arenas, they are accounted in RSS. /// We can free this arenas in case of exception to avoid OOM. - static void setRSS(Int64 rss_, bool has_free_memory_in_allocator_arenas_); + static void setRSS(Int64 rss_); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage(); diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp new file mode 100644 index 00000000000..dce47b83667 --- /dev/null +++ b/src/Common/MemoryWorker.cpp @@ -0,0 +1,49 @@ +#include "Common/ThreadPool.h" +#include + +#include +#include + +namespace DB +{ + +#if USE_JEMALLOC +MemoryWorker::MemoryWorker(uint64_t period_ms_) + : period_ms(period_ms_) +{ + background_thread = ThreadFromGlobalPool([this] { backgroundThread(); }); +} + +MemoryWorker::~MemoryWorker() +{ + { + std::unique_lock lock(mutex); + shutdown = true; + } + cv.notify_all(); + + if (background_thread.joinable()) + background_thread.join(); +} + +void MemoryWorker::backgroundThread() +{ + JemallocMibCache epoch_mib("epoch"); + JemallocMibCache resident_mib("stats.resident"); + std::unique_lock lock(mutex); + while (true) + { + cv.wait_for(lock, period_ms, [this] { return shutdown; }); + if (shutdown) + return; + + epoch_mib.setValue(0); + Int64 resident = resident_mib.getValue(); + MemoryTracker::setRSS(resident); + if (resident > total_memory_tracker.getHardLimit()) + purgeJemallocArenas(); + } +} +#endif + +} diff --git a/src/Common/MemoryWorker.h b/src/Common/MemoryWorker.h new file mode 100644 index 00000000000..8048194d9cd --- /dev/null +++ b/src/Common/MemoryWorker.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include "config.h" + +namespace DB +{ + +#if USE_JEMALLOC +class MemoryWorker +{ +public: + explicit MemoryWorker(uint64_t period_ms_); + + ~MemoryWorker(); +private: + void backgroundThread(); + + ThreadFromGlobalPool background_thread; + + std::mutex mutex; + std::condition_variable cv; + bool shutdown = false; + + std::chrono::milliseconds period_ms; +}; +#else +class MemoryWorker +{ +}; +#endif + +} diff --git a/src/Coordination/KeeperAsynchronousMetrics.cpp b/src/Coordination/KeeperAsynchronousMetrics.cpp index 3e404b7152b..86166ffe31b 100644 --- a/src/Coordination/KeeperAsynchronousMetrics.cpp +++ b/src/Coordination/KeeperAsynchronousMetrics.cpp @@ -115,7 +115,7 @@ void updateKeeperInformation(KeeperDispatcher & keeper_dispatcher, AsynchronousM KeeperAsynchronousMetrics::KeeperAsynchronousMetrics( ContextPtr context_, unsigned update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_) - : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, context_->getCgroupsReader()), context(std::move(context_)) + : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_), context(std::move(context_)) { } diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 28b32a6e6a5..aaea0388239 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -157,6 +157,7 @@ namespace DB M(Bool, prepare_system_log_tables_on_startup, false, "If true, ClickHouse creates all configured `system.*_log` tables before the startup. It can be helpful if some startup scripts depend on these tables.", 0) \ M(Double, gwp_asan_force_sample_probability, 0.0003, "Probability that an allocation from specific places will be sampled by GWP Asan (i.e. PODArray allocations)", 0) \ M(UInt64, config_reload_interval_ms, 2000, "How often clickhouse will reload config and check for new changes", 0) \ + M(UInt64, memory_worker_period_ms, 100, "Period of background memory worker which corrects memory tracker memory usages and cleans up unused pages during higher memory usage.", 0) \ /// If you add a setting which can be updated at runtime, please update 'changeable_settings' map in StorageSystemServerSettings.cpp diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 6a24e049998..f70ccfd77be 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -406,8 +406,6 @@ struct ContextSharedPart : boost::noncopyable std::unique_ptr cluster_discovery TSA_GUARDED_BY(clusters_mutex); size_t clusters_version TSA_GUARDED_BY(clusters_mutex) = 0; - std::shared_ptr cgroups_reader; - /// No lock required for async_insert_queue modified only during initialization std::shared_ptr async_insert_queue; @@ -5631,16 +5629,6 @@ const ServerSettings & Context::getServerSettings() const return shared->server_settings; } -void Context::setCgroupsReader(std::shared_ptr cgroups_reader_) -{ - shared->cgroups_reader = std::move(cgroups_reader_); -} - -std::shared_ptr Context::getCgroupsReader() const -{ - return shared->cgroups_reader; -} - uint64_t HTTPContext::getMaxHstsAge() const { return context->getSettingsRef().hsts_max_age; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 33b742d20ad..0e3c0591c12 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1344,9 +1344,6 @@ public: const ServerSettings & getServerSettings() const; - void setCgroupsReader(std::shared_ptr cgroups_reader_); - std::shared_ptr getCgroupsReader() const; - private: std::shared_ptr getSettingsConstraintsAndCurrentProfilesWithLock() const; diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 6ee0168bede..872a9f864df 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -57,7 +57,7 @@ ServerAsynchronousMetrics::ServerAsynchronousMetrics( unsigned heavy_metrics_update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_) : WithContext(global_context_) - , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, getContext()->getCgroupsReader()) + , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_) , heavy_metric_update_period(heavy_metrics_update_period_seconds) { /// sanity check diff --git a/src/Storages/System/StorageSystemServerSettings.cpp b/src/Storages/System/StorageSystemServerSettings.cpp index d242b6de4ec..ee99c472620 100644 --- a/src/Storages/System/StorageSystemServerSettings.cpp +++ b/src/Storages/System/StorageSystemServerSettings.cpp @@ -63,7 +63,6 @@ void StorageSystemServerSettings::fillData(MutableColumns & res_columns, Context /// current setting values, one needs to ask the components directly. std::unordered_map> changeable_settings = { {"max_server_memory_usage", {std::to_string(total_memory_tracker.getHardLimit()), ChangeableWithoutRestart::Yes}}, - {"allow_use_jemalloc_memory", {std::to_string(total_memory_tracker.getAllowUseJemallocMmemory()), ChangeableWithoutRestart::Yes}}, {"max_table_size_to_drop", {std::to_string(context->getMaxTableSizeToDrop()), ChangeableWithoutRestart::Yes}}, {"max_partition_size_to_drop", {std::to_string(context->getMaxPartitionSizeToDrop()), ChangeableWithoutRestart::Yes}}, diff --git a/tests/integration/test_memory_limit/test.py b/tests/integration/test_memory_limit/test.py index 6d6745711da..db68a38c1b1 100644 --- a/tests/integration/test_memory_limit/test.py +++ b/tests/integration/test_memory_limit/test.py @@ -13,7 +13,6 @@ node = cluster.add_instance( "configs/async_metrics_no.xml", ], mem_limit="4g", - env_variables={"MALLOC_CONF": "dirty_decay_ms:0"}, ) From 9ec1fd1ab769b2f6c6ad713b4e747a89bde48b78 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 18 Jul 2024 11:29:03 +0200 Subject: [PATCH 064/289] Fix non-jemalloc builds --- src/Common/MemoryWorker.cpp | 2 +- src/Common/MemoryWorker.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index dce47b83667..e5ebbe3b979 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -1,8 +1,8 @@ -#include "Common/ThreadPool.h" #include #include #include +#include namespace DB { diff --git a/src/Common/MemoryWorker.h b/src/Common/MemoryWorker.h index 8048194d9cd..5f02fd0b1d0 100644 --- a/src/Common/MemoryWorker.h +++ b/src/Common/MemoryWorker.h @@ -28,6 +28,8 @@ private: #else class MemoryWorker { +public: + explicit MemoryWorker(uint64_t /*period_ms_*/) {} }; #endif From 05c7dc582a48d738bed82bcb24d3a7619fec8bc9 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 18 Jul 2024 13:40:03 +0200 Subject: [PATCH 065/289] Add some comments --- src/Common/Jemalloc.h | 3 +++ src/Common/MemoryWorker.h | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/Common/Jemalloc.h b/src/Common/Jemalloc.h index 629d039b483..dfa265c5e59 100644 --- a/src/Common/Jemalloc.h +++ b/src/Common/Jemalloc.h @@ -41,6 +41,9 @@ T getJemallocValue(const char * name) return value; } +/// Each mallctl call consists of string name lookup which can be expensive. +/// This can be avoided by translating name to "Management Information Base" (MIB) +/// and using it in mallctlbymib calls template struct JemallocMibCache { diff --git a/src/Common/MemoryWorker.h b/src/Common/MemoryWorker.h index 5f02fd0b1d0..6c0a578aa61 100644 --- a/src/Common/MemoryWorker.h +++ b/src/Common/MemoryWorker.h @@ -8,6 +8,12 @@ namespace DB { #if USE_JEMALLOC +/// Correct MemoryTracker based on stats.resident read from jemalloc. +/// This requires jemalloc built with --enable-stats which we use. +/// The worker spawns a background thread which moves the jemalloc epoch (updates internal stats), +/// and fetches the current stats.resident whose value is sent to global MemoryTracker. +/// Additionally, if the current memory usage is higher than global hard limit, +/// jemalloc's dirty pages are forcefully purged. class MemoryWorker { public: From 7d66f400b25a5bf0f2f43f06c6ed432d1c572fb6 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 18 Jul 2024 16:51:49 +0200 Subject: [PATCH 066/289] Better --- src/Common/Jemalloc.h | 5 +++++ src/Common/MemoryTracker.cpp | 13 ++++++++----- src/Common/MemoryTracker.h | 8 ++++---- src/Common/MemoryWorker.cpp | 27 +++++++++++++++++++++++++-- src/Common/ProfileEvents.cpp | 3 +++ 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/Common/Jemalloc.h b/src/Common/Jemalloc.h index dfa265c5e59..22a94a44eba 100644 --- a/src/Common/Jemalloc.h +++ b/src/Common/Jemalloc.h @@ -65,6 +65,11 @@ struct JemallocMibCache return value; } + void run() + { + mallctlbymib(mib, mib_length, nullptr, nullptr, nullptr, 0); + } + private: static constexpr size_t max_mib_length = 4; size_t mib[max_mib_length]; diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index e237c3a0d33..49a3a6ef7ef 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -242,6 +242,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed * So, we allow over-allocations. */ Int64 will_be = size ? size + amount.fetch_add(size, std::memory_order_relaxed) : amount.load(std::memory_order_relaxed); + Int64 will_be_rss = size + rss.load(std::memory_order_relaxed); auto metric_loaded = metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end() && size) @@ -290,7 +291,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed } } - if (unlikely(current_hard_limit && will_be > current_hard_limit)) + if (unlikely(current_hard_limit && (will_be > current_hard_limit || will_be_rss > current_hard_limit))) { if (memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded) { @@ -310,12 +311,13 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed throw DB::Exception( DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Memory limit{}{} exceeded: " - "would use {} (attempt to allocate chunk of {} bytes), maximum: {}." + "would use {} (attempt to allocate chunk of {} bytes), current RSS {}, maximum: {}." "{}{}", description ? " " : "", description ? description : "", formatReadableSizeWithBinarySuffix(will_be), size, + formatReadableSizeWithBinarySuffix(rss.load(std::memory_order_relaxed)), formatReadableSizeWithBinarySuffix(current_hard_limit), overcommit_result == OvercommitResult::NONE ? "" : " OvercommitTracker decision: ", toDescription(overcommit_result)); @@ -496,17 +498,18 @@ void MemoryTracker::reset() } -void MemoryTracker::setRSS(Int64 rss_) +void MemoryTracker::updateValues(Int64 rss_, Int64 allocated_) { - Int64 new_amount = rss_; + Int64 new_amount = allocated_; total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); + total_memory_tracker.rss.store(rss_, std::memory_order_relaxed); auto metric_loaded = total_memory_tracker.metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) CurrentMetrics::set(metric_loaded, new_amount); bool log_memory_usage = true; - total_memory_tracker.updatePeak(rss_, log_memory_usage); + total_memory_tracker.updatePeak(new_amount, log_memory_usage); } diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 4085bb321ed..add8bcb43d2 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -57,6 +57,8 @@ private: std::atomic hard_limit {0}; std::atomic profiler_limit {0}; + std::atomic rss{0}; + Int64 profiler_step = 0; /// To test exception safety of calling code, memory tracker throws an exception on each memory allocation with specified probability. @@ -237,10 +239,8 @@ public: /// Reset the accumulated data. void reset(); - /// Reset current counter to an RSS value. - /// Jemalloc may have pre-allocated arenas, they are accounted in RSS. - /// We can free this arenas in case of exception to avoid OOM. - static void setRSS(Int64 rss_); + /// update values based on external information (e.g. jemalloc's stat) + static void updateValues(Int64 rss_, Int64 allocated_); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage(); diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index e5ebbe3b979..ae488a47b67 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -3,11 +3,23 @@ #include #include #include +#include + +namespace ProfileEvents +{ + extern const Event MemoryAllocatorPurge; + extern const Event MemoryAllocatorPurgeTimeMicroseconds; + extern const Event MemoryWorkerRun; + extern const Event MemoryWorkerRunElapsedMicroseconds; +} namespace DB { #if USE_JEMALLOC +#define STRINGIFY_HELPER(x) #x +#define STRINGIFY(x) STRINGIFY_HELPER(x) + MemoryWorker::MemoryWorker(uint64_t period_ms_) : period_ms(period_ms_) { @@ -30,6 +42,8 @@ void MemoryWorker::backgroundThread() { JemallocMibCache epoch_mib("epoch"); JemallocMibCache resident_mib("stats.resident"); + JemallocMibCache allocated_mib("stats.allocated"); + JemallocMibCache purge_mib("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge"); std::unique_lock lock(mutex); while (true) { @@ -37,11 +51,20 @@ void MemoryWorker::backgroundThread() if (shutdown) return; + Stopwatch total_watch; epoch_mib.setValue(0); Int64 resident = resident_mib.getValue(); - MemoryTracker::setRSS(resident); if (resident > total_memory_tracker.getHardLimit()) - purgeJemallocArenas(); + { + Stopwatch purge_watch; + purge_mib.run(); + ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); + ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, purge_watch.elapsedMicroseconds()); + } + + MemoryTracker::updateValues(resident, allocated_mib.getValue()); + ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); + ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); } } #endif diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 871ba7cab8b..d85c21fcded 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -778,6 +778,9 @@ The server successfully detected this situation and will download merged part fr M(GWPAsanAllocateSuccess, "Number of successful allocations done by GWPAsan") \ M(GWPAsanAllocateFailed, "Number of failed allocations done by GWPAsan (i.e. filled pool)") \ M(GWPAsanFree, "Number of free operations done by GWPAsan") \ + \ + M(MemoryWorkerRun, "Number of runs done by MemoryWorker in background") \ + M(MemoryWorkerRunElapsedMicroseconds, "Total time spent by MemoryWorker for background work") \ #ifdef APPLY_FOR_EXTERNAL_EVENTS From 2147a96475717f0af53dd62f487c011a5b9b933a Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Sun, 21 Jul 2024 11:32:57 +0200 Subject: [PATCH 067/289] Better --- src/Common/AsynchronousMetrics.cpp | 4 +++ src/Common/MemoryTracker.cpp | 31 +++++++++++++++---- src/Common/MemoryTracker.h | 2 +- src/Common/MemoryWorker.cpp | 7 ++++- src/Coordination/KeeperDispatcher.cpp | 8 ++++- .../configs/keeper_config2.xml | 2 +- .../configs/keeper_config3.xml | 2 +- 7 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index a5c9875188b..dc2f687004b 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -638,6 +638,10 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) "The amount of virtual memory mapped for the use of stack and for the allocated memory, in bytes." " It is unspecified whether it includes the per-thread stacks and most of the allocated memory, that is allocated with the 'mmap' system call." " This metric exists only for completeness reasons. I recommend to use the `MemoryResident` metric for monitoring."}; + +#if !USE_JEMALLOC + MemoryTracker::updateValues(data.resident, data.resident, /*force_update=*/true); +#endif } { diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 49a3a6ef7ef..07d6ba98745 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -221,6 +221,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed { /// For global memory tracker always update memory usage. amount.fetch_add(size, std::memory_order_relaxed); + rss.fetch_add(size, std::memory_order_relaxed); auto metric_loaded = metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) @@ -242,7 +243,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed * So, we allow over-allocations. */ Int64 will_be = size ? size + amount.fetch_add(size, std::memory_order_relaxed) : amount.load(std::memory_order_relaxed); - Int64 will_be_rss = size + rss.load(std::memory_order_relaxed); + Int64 will_be_rss = size ? size + rss.fetch_add(size, std::memory_order_relaxed) : rss.load(std::memory_order_relaxed); auto metric_loaded = metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end() && size) @@ -269,6 +270,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed { /// Revert amount.fetch_sub(size, std::memory_order_relaxed); + rss.fetch_sub(size, std::memory_order_relaxed); /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global); @@ -291,7 +293,8 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed } } - if (unlikely(current_hard_limit && (will_be > current_hard_limit || will_be_rss > current_hard_limit))) + if (unlikely( + current_hard_limit && (will_be > current_hard_limit || (level == VariableContext::Global && will_be_rss > current_hard_limit)))) { if (memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded) { @@ -303,6 +306,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed { /// Revert amount.fetch_sub(size, std::memory_order_relaxed); + rss.fetch_sub(size, std::memory_order_relaxed); /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global); @@ -411,6 +415,7 @@ AllocationTrace MemoryTracker::free(Int64 size, double _sample_probability) { /// For global memory tracker always update memory usage. amount.fetch_sub(size, std::memory_order_relaxed); + rss.fetch_sub(size, std::memory_order_relaxed); auto metric_loaded = metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) CurrentMetrics::sub(metric_loaded, size); @@ -424,7 +429,12 @@ AllocationTrace MemoryTracker::free(Int64 size, double _sample_probability) } Int64 accounted_size = size; - if (level == VariableContext::Thread || level == VariableContext::Global) + if (level == VariableContext::Global) + { + amount.fetch_sub(accounted_size, std::memory_order_relaxed); + rss.fetch_sub(accounted_size, std::memory_order_relaxed); + } + else if (level == VariableContext::Thread) { /// Could become negative if memory allocated in this thread is freed in another one amount.fetch_sub(accounted_size, std::memory_order_relaxed); @@ -498,12 +508,21 @@ void MemoryTracker::reset() } -void MemoryTracker::updateValues(Int64 rss_, Int64 allocated_) +void MemoryTracker::updateValues(Int64 rss_, Int64 allocated_, bool force_update) { - Int64 new_amount = allocated_; - total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); total_memory_tracker.rss.store(rss_, std::memory_order_relaxed); + if (likely(!force_update && total_memory_tracker.amount.load(std::memory_order_relaxed) >= 0)) + return; + + Int64 new_amount = allocated_; + LOG_INFO( + getLogger("MemoryTracker"), + "Correcting the value of global memory tracker from {} to {}", + ReadableSize(total_memory_tracker.amount.load(std::memory_order_relaxed)), + ReadableSize(allocated_)); + total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); + auto metric_loaded = total_memory_tracker.metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) CurrentMetrics::set(metric_loaded, new_amount); diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index add8bcb43d2..4913be9781f 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -240,7 +240,7 @@ public: void reset(); /// update values based on external information (e.g. jemalloc's stat) - static void updateValues(Int64 rss_, Int64 allocated_); + static void updateValues(Int64 rss_, Int64 allocated_, bool force_update); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage(); diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index ae488a47b67..23cd90178ff 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -44,6 +44,7 @@ void MemoryWorker::backgroundThread() JemallocMibCache resident_mib("stats.resident"); JemallocMibCache allocated_mib("stats.allocated"); JemallocMibCache purge_mib("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge"); + bool first_run = false; std::unique_lock lock(mutex); while (true) { @@ -62,9 +63,13 @@ void MemoryWorker::backgroundThread() ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, purge_watch.elapsedMicroseconds()); } - MemoryTracker::updateValues(resident, allocated_mib.getValue()); + /// force update the allocated stat from jemalloc for the first run to cover the allocations we missed + /// during initialization + MemoryTracker::updateValues(resident, allocated_mib.getValue(), first_run); ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); + + first_run = false; } } #endif diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 8c7e6405153..332662117d8 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -148,7 +148,13 @@ void KeeperDispatcher::requestThread() Int64 mem_soft_limit = keeper_context->getKeeperMemorySoftLimit(); if (configuration_and_settings->standalone_keeper && isExceedingMemorySoftLimit() && checkIfRequestIncreaseMem(request.request)) { - LOG_WARNING(log, "Processing requests refused because of max_memory_usage_soft_limit {}, the total used memory is {}, request type is {}", ReadableSize(mem_soft_limit), ReadableSize(total_memory_tracker.get()), request.request->getOpNum()); + LOG_WARNING( + log, + "Processing requests refused because of max_memory_usage_soft_limit {}, the total used memory is {}, request type " + "is {}", + ReadableSize(mem_soft_limit), + ReadableSize(total_memory_tracker.get()), + request.request->getOpNum()); addErrorResponses({request}, Coordination::Error::ZCONNECTIONLOSS); continue; } diff --git a/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config2.xml b/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config2.xml index 25ececea3e8..e71b93379d0 100644 --- a/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config2.xml +++ b/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config2.xml @@ -16,7 +16,7 @@ az-zoo2 1 - 20000000 + 200000000 10000 diff --git a/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config3.xml b/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config3.xml index 81e343b77c9..cf4a4686f2c 100644 --- a/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config3.xml +++ b/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config3.xml @@ -13,7 +13,7 @@ 2181 3 - 20000000 + 200000000 10000 From 1c3f7d0fd0fbf27692ff29d8309382eae7b7a598 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 22 Jul 2024 14:58:00 +0200 Subject: [PATCH 068/289] Small fix --- programs/server/Server.cpp | 4 ++-- src/Common/MemoryWorker.cpp | 15 ++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 04480f0bfe9..5691d82e216 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -904,8 +904,6 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); - MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms); - /// This object will periodically calculate some metrics. ServerAsynchronousMetrics async_metrics( global_context, @@ -1198,6 +1196,8 @@ try FailPointInjection::enableFromGlobalConfig(config()); + MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms); + int default_oom_score = 0; #if !defined(NDEBUG) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index 23cd90178ff..2b945a30d3d 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -2,8 +2,9 @@ #include #include -#include #include +#include +#include namespace ProfileEvents { @@ -23,6 +24,7 @@ namespace DB MemoryWorker::MemoryWorker(uint64_t period_ms_) : period_ms(period_ms_) { + LOG_INFO(getLogger("MemoryWorker"), "Starting background memory thread with period of {}ms", period_ms.count()); background_thread = ThreadFromGlobalPool([this] { backgroundThread(); }); } @@ -42,9 +44,10 @@ void MemoryWorker::backgroundThread() { JemallocMibCache epoch_mib("epoch"); JemallocMibCache resident_mib("stats.resident"); + JemallocMibCache active_mib("stats.active"); JemallocMibCache allocated_mib("stats.allocated"); JemallocMibCache purge_mib("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge"); - bool first_run = false; + bool first_run = true; std::unique_lock lock(mutex); while (true) { @@ -55,6 +58,11 @@ void MemoryWorker::backgroundThread() Stopwatch total_watch; epoch_mib.setValue(0); Int64 resident = resident_mib.getValue(); + + /// force update the allocated stat from jemalloc for the first run to cover the allocations we missed + /// during initialization + MemoryTracker::updateValues(resident, allocated_mib.getValue(), first_run); + if (resident > total_memory_tracker.getHardLimit()) { Stopwatch purge_watch; @@ -63,9 +71,6 @@ void MemoryWorker::backgroundThread() ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, purge_watch.elapsedMicroseconds()); } - /// force update the allocated stat from jemalloc for the first run to cover the allocations we missed - /// during initialization - MemoryTracker::updateValues(resident, allocated_mib.getValue(), first_run); ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); From d78cfd030fa8364456ac5283a6a1469703c53b40 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 22 Jul 2024 21:47:46 +0200 Subject: [PATCH 069/289] Use cgroups as source --- programs/keeper/Keeper.cpp | 8 +- programs/server/Server.cpp | 9 +- src/Common/AsynchronousMetrics.cpp | 32 +- src/Common/AsynchronousMetrics.h | 10 +- src/Common/CgroupsMemoryUsageObserver.cpp | 168 +---------- src/Common/CgroupsMemoryUsageObserver.h | 12 - src/Common/MemoryTracker.cpp | 9 +- src/Common/MemoryTracker.h | 3 +- src/Common/MemoryWorker.cpp | 280 ++++++++++++++++-- src/Common/MemoryWorker.h | 49 ++- .../KeeperAsynchronousMetrics.cpp | 9 +- src/Coordination/KeeperAsynchronousMetrics.h | 8 +- src/Core/ServerSettings.h | 2 +- .../ServerAsynchronousMetrics.cpp | 6 +- src/Interpreters/ServerAsynchronousMetrics.h | 5 +- 15 files changed, 367 insertions(+), 243 deletions(-) diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index b10d3f34623..d308e741311 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -376,7 +376,8 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); - MemoryWorker memory_worker(config().getUInt64("memory_worker_period_ms", 100)); + MemoryWorker memory_worker(config().getUInt64("memory_worker_period_ms", 0)); + memory_worker.start(); static ServerErrorHandler error_handler; Poco::ErrorHandler::set(&error_handler); @@ -419,8 +420,9 @@ try for (const auto & server : *servers) metrics.emplace_back(ProtocolServerMetrics{server.getPortName(), server.currentThreads()}); return metrics; - } - ); + }, + /*update_jemalloc_epoch_=*/memory_worker.getSource() != MemoryWorker::MemoryUsageSource::Jemalloc, + /*update_rss_=*/memory_worker.getSource() == MemoryWorker::MemoryUsageSource::None); std::vector listen_hosts = DB::getMultipleValuesFromConfig(config(), "", "listen_host"); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 5691d82e216..1fc1df1494c 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -904,6 +904,8 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); + MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms); + /// This object will periodically calculate some metrics. ServerAsynchronousMetrics async_metrics( global_context, @@ -922,8 +924,9 @@ try for (const auto & server : servers) metrics.emplace_back(ProtocolServerMetrics{server.getPortName(), server.currentThreads()}); return metrics; - } - ); + }, + /*update_jemalloc_epoch_=*/memory_worker.getSource() != MemoryWorker::MemoryUsageSource::Jemalloc, + /*update_rss_=*/memory_worker.getSource() == MemoryWorker::MemoryUsageSource::None); /// NOTE: global context should be destroyed *before* GlobalThreadPool::shutdown() /// Otherwise GlobalThreadPool::shutdown() will hang, since Context holds some threads. @@ -1196,7 +1199,7 @@ try FailPointInjection::enableFromGlobalConfig(config()); - MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms); + memory_worker.start(); int default_oom_score = 0; diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index dc2f687004b..53b8e13eaaa 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -58,10 +58,14 @@ static std::unique_ptr openFileIfExists(const std::stri AsynchronousMetrics::AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_) + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_) : update_period(update_period_seconds) , log(getLogger("AsynchronousMetrics")) , protocol_server_metrics_func(protocol_server_metrics_func_) + , update_jemalloc_epoch(update_jemalloc_epoch_) + , update_rss(update_rss_) { #if defined(OS_LINUX) openFileIfExists("/proc/meminfo", meminfo); @@ -377,6 +381,14 @@ void AsynchronousMetrics::run() namespace { +uint64_t updateJemallocEpoch() +{ + uint64_t value = 0; + size_t size = sizeof(value); + mallctl("epoch", &value, &size, &value, size); + return value; +} + template Value saveJemallocMetricImpl( AsynchronousMetricValues & values, @@ -593,8 +605,11 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) // 'epoch' is a special mallctl -- it updates the statistics. Without it, all // the following calls will return stale values. It increments and returns // the current epoch number, which might be useful to log as a sanity check. - auto epoch = getJemallocValue("epoch"); - new_values["jemalloc.epoch"] = { epoch, "An internal incremental update number of the statistics of jemalloc (Jason Evans' memory allocator), used in all other `jemalloc` metrics." }; + auto epoch = update_jemalloc_epoch ? updateJemallocEpoch() : getJemallocValue("epoch"); + new_values["jemalloc.epoch"] + = {epoch, + "An internal incremental update number of the statistics of jemalloc (Jason Evans' memory allocator), used in all other " + "`jemalloc` metrics."}; // Collect the statistics themselves. saveJemallocMetric(new_values, "allocated"); @@ -607,10 +622,10 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) saveJemallocMetric(new_values, "background_thread.num_threads"); saveJemallocMetric(new_values, "background_thread.num_runs"); saveJemallocMetric(new_values, "background_thread.run_intervals"); - saveJemallocProf(new_values, "active"); + saveJemallocProf(new_values, "active"); saveAllArenasMetric(new_values, "pactive"); - [[maybe_unused]] size_t je_malloc_pdirty = saveAllArenasMetric(new_values, "pdirty"); - [[maybe_unused]] size_t je_malloc_pmuzzy = saveAllArenasMetric(new_values, "pmuzzy"); + saveAllArenasMetric(new_values, "pdirty"); + saveAllArenasMetric(new_values, "pmuzzy"); saveAllArenasMetric(new_values, "dirty_purged"); saveAllArenasMetric(new_values, "muzzy_purged"); #endif @@ -639,9 +654,8 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) " It is unspecified whether it includes the per-thread stacks and most of the allocated memory, that is allocated with the 'mmap' system call." " This metric exists only for completeness reasons. I recommend to use the `MemoryResident` metric for monitoring."}; -#if !USE_JEMALLOC - MemoryTracker::updateValues(data.resident, data.resident, /*force_update=*/true); -#endif + if (update_rss) + MemoryTracker::updateRSS(data.resident); } { diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index bc379d4e92b..eb6ede7a558 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -7,10 +7,8 @@ #include #include -#include #include #include -#include #include #include #include @@ -69,7 +67,9 @@ public: AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_); + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_); virtual ~AsynchronousMetrics(); @@ -92,7 +92,6 @@ private: virtual void logImpl(AsynchronousMetricValues &) {} ProtocolServerMetricsFunc protocol_server_metrics_func; - std::shared_ptr cgroups_reader; std::unique_ptr thread; @@ -113,6 +112,9 @@ private: MemoryStatisticsOS memory_stat TSA_GUARDED_BY(data_mutex); #endif + const bool update_jemalloc_epoch; + const bool update_rss; + #if defined(OS_LINUX) std::optional meminfo TSA_GUARDED_BY(data_mutex); std::optional loadavg TSA_GUARDED_BY(data_mutex); diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp index ab7ca69ca04..afeac1808b2 100644 --- a/src/Common/CgroupsMemoryUsageObserver.cpp +++ b/src/Common/CgroupsMemoryUsageObserver.cpp @@ -13,12 +13,8 @@ #include #include -#include -#include -#include using namespace DB; -namespace fs = std::filesystem; namespace DB { @@ -29,170 +25,8 @@ extern const int FILE_DOESNT_EXIST; extern const int INCORRECT_DATA; } -} - -namespace -{ - -/// Format is -/// kernel 5 -/// rss 15 -/// [...] -uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, const std::string & key) -{ - while (!buf.eof()) - { - std::string current_key; - readStringUntilWhitespace(current_key, buf); - if (current_key != key) - { - std::string dummy; - readStringUntilNewlineInto(dummy, buf); - buf.ignore(); - continue; - } - - assertChar(' ', buf); - uint64_t value = 0; - readIntText(value, buf); - return value; - } - - throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot find '{}' in '{}'", key, buf.getFileName()); -} - -struct CgroupsV1Reader : ICgroupsReader -{ - explicit CgroupsV1Reader(const fs::path & stat_file_dir) : buf(stat_file_dir / "memory.stat") { } - - uint64_t readMemoryUsage() override - { - std::lock_guard lock(mutex); - buf.rewind(); - return readMetricFromStatFile(buf, "rss"); - } - -private: - std::mutex mutex; - ReadBufferFromFile buf TSA_GUARDED_BY(mutex); -}; - -struct CgroupsV2Reader : ICgroupsReader -{ - explicit CgroupsV2Reader(const fs::path & stat_file_dir) - : current_buf(stat_file_dir / "memory.current"), stat_buf(stat_file_dir / "memory.stat") - { - } - - uint64_t readMemoryUsage() override - { - std::lock_guard lock(mutex); - current_buf.rewind(); - stat_buf.rewind(); - - int64_t mem_usage = 0; - /// memory.current contains a single number - /// the reason why we subtract it described here: https://github.com/ClickHouse/ClickHouse/issues/64652#issuecomment-2149630667 - readIntText(mem_usage, current_buf); - mem_usage -= readMetricFromStatFile(stat_buf, "inactive_file"); - chassert(mem_usage >= 0, "Negative memory usage"); - return mem_usage; - } - -private: - std::mutex mutex; - ReadBufferFromFile current_buf TSA_GUARDED_BY(mutex); - ReadBufferFromFile stat_buf TSA_GUARDED_BY(mutex); -}; - -/// Caveats: -/// - All of the logic in this file assumes that the current process is the only process in the -/// containing cgroup (or more precisely: the only process with significant memory consumption). -/// If this is not the case, then other processe's memory consumption may affect the internal -/// memory tracker ... -/// - Cgroups v1 and v2 allow nested cgroup hierarchies. As v1 is deprecated for over half a -/// decade and will go away at some point, hierarchical detection is only implemented for v2. -/// - I did not test what happens if a host has v1 and v2 simultaneously enabled. I believe such -/// systems existed only for a short transition period. - -std::optional getCgroupsV2Path() -{ - if (!cgroupsV2Enabled()) - return {}; - - if (!cgroupsV2MemoryControllerEnabled()) - return {}; - - fs::path current_cgroup = cgroupV2PathOfProcess(); - if (current_cgroup.empty()) - return {}; - - /// Return the bottom-most nested current memory file. If there is no such file at the current - /// level, try again at the parent level as memory settings are inherited. - while (current_cgroup != default_cgroups_mount.parent_path()) - { - const auto current_path = current_cgroup / "memory.current"; - const auto stat_path = current_cgroup / "memory.stat"; - if (fs::exists(current_path) && fs::exists(stat_path)) - return {current_cgroup}; - current_cgroup = current_cgroup.parent_path(); - } - return {}; -} - -std::optional getCgroupsV1Path() -{ - auto path = default_cgroups_mount / "memory/memory.stat"; - if (!fs::exists(path)) - return {}; - return {default_cgroups_mount / "memory"}; -} - -enum class CgroupsVersion : uint8_t -{ - V1, - V2 -}; - -std::pair getCgroupsPath() -{ - auto v2_path = getCgroupsV2Path(); - if (v2_path.has_value()) - return {*v2_path, CgroupsVersion::V2}; - - auto v1_path = getCgroupsV1Path(); - if (v1_path.has_value()) - return {*v1_path, CgroupsVersion::V1}; - - throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot find cgroups v1 or v2 current memory file"); -} - -} - -namespace DB -{ - -std::shared_ptr createCgroupsReader() -{ - const auto [cgroup_path, version] = getCgroupsPath(); - LOG_INFO( - getLogger("CgroupsReader"), - "Will create cgroup reader from '{}' (cgroups version: {})", - cgroup_path, - (version == CgroupsVersion::V1) ? "v1" : "v2"); - - if (version == CgroupsVersion::V2) - return std::make_shared(cgroup_path); - else - { - chassert(version == CgroupsVersion::V1); - return std::make_shared(cgroup_path); - } - -} - CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_) - : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_), cgroups_reader(createCgroupsReader()) + : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_) {} CgroupsMemoryUsageObserver::~CgroupsMemoryUsageObserver() diff --git a/src/Common/CgroupsMemoryUsageObserver.h b/src/Common/CgroupsMemoryUsageObserver.h index 33e0f167a59..3de83d6b437 100644 --- a/src/Common/CgroupsMemoryUsageObserver.h +++ b/src/Common/CgroupsMemoryUsageObserver.h @@ -3,21 +3,11 @@ #include #include -#include #include namespace DB { -struct ICgroupsReader -{ - virtual ~ICgroupsReader() = default; - - virtual uint64_t readMemoryUsage() = 0; -}; - -std::shared_ptr createCgroupsReader(); - /// Periodically reads the the maximum memory available to the process (which can change due to cgroups settings). /// You can specify a callback to react on changes. The callback typically reloads the configuration, i.e. Server /// or Keeper configuration file. This reloads settings 'max_server_memory_usage' (Server) and 'max_memory_usage_soft_limit' @@ -54,8 +44,6 @@ private: void runThread(); - std::shared_ptr cgroups_reader; - std::mutex thread_mutex; std::condition_variable cond; ThreadFromGlobalPool thread; diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 07d6ba98745..0ffae89ffa6 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -508,13 +508,13 @@ void MemoryTracker::reset() } -void MemoryTracker::updateValues(Int64 rss_, Int64 allocated_, bool force_update) +void MemoryTracker::updateRSS(Int64 rss_) { total_memory_tracker.rss.store(rss_, std::memory_order_relaxed); +} - if (likely(!force_update && total_memory_tracker.amount.load(std::memory_order_relaxed) >= 0)) - return; - +void MemoryTracker::updateAllocated(Int64 allocated_) +{ Int64 new_amount = allocated_; LOG_INFO( getLogger("MemoryTracker"), @@ -531,7 +531,6 @@ void MemoryTracker::updateValues(Int64 rss_, Int64 allocated_, bool force_update total_memory_tracker.updatePeak(new_amount, log_memory_usage); } - void MemoryTracker::setSoftLimit(Int64 value) { soft_limit.store(value, std::memory_order_relaxed); diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 4913be9781f..d2db8489f19 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -240,7 +240,8 @@ public: void reset(); /// update values based on external information (e.g. jemalloc's stat) - static void updateValues(Int64 rss_, Int64 allocated_, bool force_update); + static void updateRSS(Int64 rss_); + static void updateAllocated(Int64 allocated_); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage(); diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index 2b945a30d3d..42e797a80d6 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -1,11 +1,21 @@ #include +#include +#include +#include +#include #include #include #include #include #include +#include +#include +#include + +namespace fs = std::filesystem; + namespace ProfileEvents { extern const Event MemoryAllocatorPurge; @@ -17,14 +27,227 @@ namespace ProfileEvents namespace DB { -#if USE_JEMALLOC -#define STRINGIFY_HELPER(x) #x -#define STRINGIFY(x) STRINGIFY_HELPER(x) - -MemoryWorker::MemoryWorker(uint64_t period_ms_) - : period_ms(period_ms_) +namespace ErrorCodes { - LOG_INFO(getLogger("MemoryWorker"), "Starting background memory thread with period of {}ms", period_ms.count()); + extern const int FILE_DOESNT_EXIST; + extern const int INCORRECT_DATA; +} + +#if defined(OS_LINUX) +struct ICgroupsReader +{ + virtual ~ICgroupsReader() = default; + + virtual uint64_t readMemoryUsage() = 0; +}; + +namespace +{ + +/// Format is +/// kernel 5 +/// rss 15 +/// [...] +uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, const std::string & key) +{ + while (!buf.eof()) + { + std::string current_key; + readStringUntilWhitespace(current_key, buf); + if (current_key != key) + { + std::string dummy; + readStringUntilNewlineInto(dummy, buf); + buf.ignore(); + continue; + } + + assertChar(' ', buf); + uint64_t value = 0; + readIntText(value, buf); + return value; + } + + LOG_ERROR(getLogger("CgroupsReader"), "Cannot find '{}' in '{}'", key, buf.getFileName()); + return 0; +} + +struct CgroupsV1Reader : ICgroupsReader +{ + explicit CgroupsV1Reader(const fs::path & stat_file_dir) : buf(stat_file_dir / "memory.stat") { } + + uint64_t readMemoryUsage() override + { + std::lock_guard lock(mutex); + buf.rewind(); + return readMetricFromStatFile(buf, "rss"); + } + +private: + std::mutex mutex; + ReadBufferFromFile buf TSA_GUARDED_BY(mutex); +}; + +struct CgroupsV2Reader : ICgroupsReader +{ + explicit CgroupsV2Reader(const fs::path & stat_file_dir) : stat_buf(stat_file_dir / "memory.stat") { } + + uint64_t readMemoryUsage() override + { + std::lock_guard lock(mutex); + stat_buf.rewind(); + return readMetricFromStatFile(stat_buf, "anon"); + } + +private: + std::mutex mutex; + ReadBufferFromFile stat_buf TSA_GUARDED_BY(mutex); +}; + +/// Caveats: +/// - All of the logic in this file assumes that the current process is the only process in the +/// containing cgroup (or more precisely: the only process with significant memory consumption). +/// If this is not the case, then other processe's memory consumption may affect the internal +/// memory tracker ... +/// - Cgroups v1 and v2 allow nested cgroup hierarchies. As v1 is deprecated for over half a +/// decade and will go away at some point, hierarchical detection is only implemented for v2. +/// - I did not test what happens if a host has v1 and v2 simultaneously enabled. I believe such +/// systems existed only for a short transition period. + +std::optional getCgroupsV2Path() +{ + if (!cgroupsV2Enabled()) + return {}; + + if (!cgroupsV2MemoryControllerEnabled()) + return {}; + + fs::path current_cgroup = cgroupV2PathOfProcess(); + if (current_cgroup.empty()) + return {}; + + /// Return the bottom-most nested current memory file. If there is no such file at the current + /// level, try again at the parent level as memory settings are inherited. + while (current_cgroup != default_cgroups_mount.parent_path()) + { + const auto current_path = current_cgroup / "memory.current"; + const auto stat_path = current_cgroup / "memory.stat"; + if (fs::exists(current_path) && fs::exists(stat_path)) + return {current_cgroup}; + current_cgroup = current_cgroup.parent_path(); + } + return {}; +} + +std::optional getCgroupsV1Path() +{ + auto path = default_cgroups_mount / "memory/memory.stat"; + if (!fs::exists(path)) + return {}; + return {default_cgroups_mount / "memory"}; +} + +enum class CgroupsVersion : uint8_t +{ + V1, + V2 +}; + +std::pair getCgroupsPath() +{ + auto v2_path = getCgroupsV2Path(); + if (v2_path.has_value()) + return {*v2_path, CgroupsVersion::V2}; + + auto v1_path = getCgroupsV1Path(); + if (v1_path.has_value()) + return {*v1_path, CgroupsVersion::V1}; + + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot find cgroups v1 or v2 current memory file"); +} + +std::shared_ptr createCgroupsReader() +{ + const auto [cgroup_path, version] = getCgroupsPath(); + LOG_INFO( + getLogger("CgroupsReader"), + "Will create cgroup reader from '{}' (cgroups version: {})", + cgroup_path, + (version == CgroupsVersion::V1) ? "v1" : "v2"); + + if (version == CgroupsVersion::V2) + return std::make_shared(cgroup_path); + else + { + chassert(version == CgroupsVersion::V1); + return std::make_shared(cgroup_path); + } + +} +#endif + +constexpr uint64_t cgroups_memory_usage_tick_ms{50}; +constexpr uint64_t jemalloc_memory_usage_tick_ms{100}; + +std::string_view sourceToString(MemoryWorker::MemoryUsageSource source) +{ + switch (source) + { + case MemoryWorker::MemoryUsageSource::Cgroups: return "Cgroups"; + case MemoryWorker::MemoryUsageSource::Jemalloc: return "Jemalloc"; + case MemoryWorker::MemoryUsageSource::None: return "None"; + } +} + +} + +/// We try to pick the best possible supported source for reading memory usage. +/// Supported sources in order of priority +/// - reading from cgroups' pseudo-files (fastest and most accurate) +/// - reading jemalloc's resident stat (doesn't take into account allocations that didn't use jemalloc) +/// Also, different tick rates are used because not all options are equally fast +MemoryWorker::MemoryWorker(uint64_t period_ms_) + : log(getLogger("MemoryWorker")) + , period_ms(period_ms_) +{ +#if defined(OS_LINUX) + try + { + cgroups_reader = createCgroupsReader(); + source = MemoryUsageSource::Cgroups; + if (period_ms == 0) + period_ms = cgroups_memory_usage_tick_ms; + + return; + } + catch (...) + { + tryLogCurrentException(log, "Cannot use cgroups reader"); + } +#endif + +#if USE_JEMALLOC + source = MemoryUsageSource::Jemalloc; + if (period_ms == 0) + period_ms = jemalloc_memory_usage_tick_ms; +#endif +} + +MemoryWorker::MemoryUsageSource MemoryWorker::getSource() +{ + return source; +} + +void MemoryWorker::start() +{ + if (source == MemoryUsageSource::None) + return; + + LOG_INFO( + getLogger("MemoryWorker"), + "Starting background memory thread with period of {}ms, using {} as source", + period_ms, + sourceToString(source)); background_thread = ThreadFromGlobalPool([this] { backgroundThread(); }); } @@ -40,29 +263,39 @@ MemoryWorker::~MemoryWorker() background_thread.join(); } +uint64_t MemoryWorker::getMemoryUsage() +{ + switch (source) + { + case MemoryUsageSource::Cgroups: + return cgroups_reader->readMemoryUsage(); + case MemoryUsageSource::Jemalloc: + return resident_mib.getValue(); + case MemoryUsageSource::None: + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Trying to fetch memory usage while no memory source can be used"); + } +} + void MemoryWorker::backgroundThread() { - JemallocMibCache epoch_mib("epoch"); - JemallocMibCache resident_mib("stats.resident"); - JemallocMibCache active_mib("stats.active"); - JemallocMibCache allocated_mib("stats.allocated"); - JemallocMibCache purge_mib("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge"); + std::chrono::milliseconds chrono_period_ms{period_ms}; bool first_run = true; std::unique_lock lock(mutex); while (true) { - cv.wait_for(lock, period_ms, [this] { return shutdown; }); + cv.wait_for(lock, chrono_period_ms, [this] { return shutdown; }); if (shutdown) return; Stopwatch total_watch; - epoch_mib.setValue(0); - Int64 resident = resident_mib.getValue(); - /// force update the allocated stat from jemalloc for the first run to cover the allocations we missed - /// during initialization - MemoryTracker::updateValues(resident, allocated_mib.getValue(), first_run); + if (source == MemoryUsageSource::Jemalloc) + epoch_mib.setValue(0); + Int64 resident = getMemoryUsage(); + MemoryTracker::updateRSS(resident); + +#if USE_JEMALLOC if (resident > total_memory_tracker.getHardLimit()) { Stopwatch purge_watch; @@ -71,12 +304,19 @@ void MemoryWorker::backgroundThread() ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, purge_watch.elapsedMicroseconds()); } + if (unlikely(first_run || total_memory_tracker.get() < 0)) + { + if (source != MemoryUsageSource::Jemalloc) + epoch_mib.setValue(0); + + MemoryTracker::updateAllocated(allocated_mib.getValue()); + } +#endif + ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); - first_run = false; } } -#endif } diff --git a/src/Common/MemoryWorker.h b/src/Common/MemoryWorker.h index 6c0a578aa61..6fde93d63ad 100644 --- a/src/Common/MemoryWorker.h +++ b/src/Common/MemoryWorker.h @@ -1,13 +1,14 @@ #pragma once +#include #include - -#include "config.h" +#include namespace DB { -#if USE_JEMALLOC +struct ICgroupsReader; + /// Correct MemoryTracker based on stats.resident read from jemalloc. /// This requires jemalloc built with --enable-stats which we use. /// The worker spawns a background thread which moves the jemalloc epoch (updates internal stats), @@ -19,8 +20,21 @@ class MemoryWorker public: explicit MemoryWorker(uint64_t period_ms_); + enum class MemoryUsageSource : uint8_t + { + None, + Cgroups, + Jemalloc + }; + + MemoryUsageSource getSource(); + + void start(); + ~MemoryWorker(); private: + uint64_t getMemoryUsage(); + void backgroundThread(); ThreadFromGlobalPool background_thread; @@ -29,14 +43,27 @@ private: std::condition_variable cv; bool shutdown = false; - std::chrono::milliseconds period_ms; -}; -#else -class MemoryWorker -{ -public: - explicit MemoryWorker(uint64_t /*period_ms_*/) {} -}; + LoggerPtr log; + + uint64_t period_ms; + + MemoryUsageSource source{MemoryUsageSource::None}; + +#if defined(OS_LINUX) + std::shared_ptr cgroups_reader; #endif +#if USE_JEMALLOC + JemallocMibCache epoch_mib{"epoch"}; + JemallocMibCache resident_mib{"stats.resident"}; + JemallocMibCache allocated_mib{"stats.allocated"}; + +#define STRINGIFY_HELPER(x) #x +#define STRINGIFY(x) STRINGIFY_HELPER(x) + JemallocMibCache purge_mib{"arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge"}; +#undef STRINGIFY +#undef STRINGIFY_HELPER +#endif +}; + } diff --git a/src/Coordination/KeeperAsynchronousMetrics.cpp b/src/Coordination/KeeperAsynchronousMetrics.cpp index 86166ffe31b..157858f3c44 100644 --- a/src/Coordination/KeeperAsynchronousMetrics.cpp +++ b/src/Coordination/KeeperAsynchronousMetrics.cpp @@ -114,8 +114,13 @@ void updateKeeperInformation(KeeperDispatcher & keeper_dispatcher, AsynchronousM } KeeperAsynchronousMetrics::KeeperAsynchronousMetrics( - ContextPtr context_, unsigned update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_) - : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_), context(std::move(context_)) + ContextPtr context_, + unsigned update_period_seconds, + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_) + : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, update_jemalloc_epoch_, update_rss_) + , context(std::move(context_)) { } diff --git a/src/Coordination/KeeperAsynchronousMetrics.h b/src/Coordination/KeeperAsynchronousMetrics.h index ec0e60cbb6e..a2ab7cab756 100644 --- a/src/Coordination/KeeperAsynchronousMetrics.h +++ b/src/Coordination/KeeperAsynchronousMetrics.h @@ -13,9 +13,13 @@ class KeeperAsynchronousMetrics : public AsynchronousMetrics { public: KeeperAsynchronousMetrics( - ContextPtr context_, unsigned update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_); - ~KeeperAsynchronousMetrics() override; + ContextPtr context_, + unsigned update_period_seconds, + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_); + ~KeeperAsynchronousMetrics() override; private: ContextPtr context; diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index aaea0388239..ea5a3f19638 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -157,7 +157,7 @@ namespace DB M(Bool, prepare_system_log_tables_on_startup, false, "If true, ClickHouse creates all configured `system.*_log` tables before the startup. It can be helpful if some startup scripts depend on these tables.", 0) \ M(Double, gwp_asan_force_sample_probability, 0.0003, "Probability that an allocation from specific places will be sampled by GWP Asan (i.e. PODArray allocations)", 0) \ M(UInt64, config_reload_interval_ms, 2000, "How often clickhouse will reload config and check for new changes", 0) \ - M(UInt64, memory_worker_period_ms, 100, "Period of background memory worker which corrects memory tracker memory usages and cleans up unused pages during higher memory usage.", 0) \ + M(UInt64, memory_worker_period_ms, 0, "Tick period of background memory worker which corrects memory tracker memory usages and cleans up unused pages during higher memory usage. If set to 0, default value will be used depending on the memory usage source", 0) \ /// If you add a setting which can be updated at runtime, please update 'changeable_settings' map in StorageSystemServerSettings.cpp diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 872a9f864df..079029695c9 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -55,9 +55,11 @@ ServerAsynchronousMetrics::ServerAsynchronousMetrics( ContextPtr global_context_, unsigned update_period_seconds, unsigned heavy_metrics_update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_) + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_) : WithContext(global_context_) - , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_) + , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, update_jemalloc_epoch_, update_rss_) , heavy_metric_update_period(heavy_metrics_update_period_seconds) { /// sanity check diff --git a/src/Interpreters/ServerAsynchronousMetrics.h b/src/Interpreters/ServerAsynchronousMetrics.h index e3c83dc748e..5fab419a32b 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.h +++ b/src/Interpreters/ServerAsynchronousMetrics.h @@ -14,7 +14,10 @@ public: ContextPtr global_context_, unsigned update_period_seconds, unsigned heavy_metrics_update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_); + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_); + ~ServerAsynchronousMetrics() override; private: From 5a1b96ac8453f73de5e891e3a9f235fd96270b50 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 23 Jul 2024 10:52:14 +0200 Subject: [PATCH 070/289] Style fix --- src/Common/CgroupsMemoryUsageObserver.cpp | 6 ------ src/Common/MemoryWorker.cpp | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp index afeac1808b2..16d5d1cccde 100644 --- a/src/Common/CgroupsMemoryUsageObserver.cpp +++ b/src/Common/CgroupsMemoryUsageObserver.cpp @@ -19,12 +19,6 @@ using namespace DB; namespace DB { -namespace ErrorCodes -{ -extern const int FILE_DOESNT_EXIST; -extern const int INCORRECT_DATA; -} - CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_) : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_) {} diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index 42e797a80d6..ddc3fd783f4 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -30,7 +30,7 @@ namespace DB namespace ErrorCodes { extern const int FILE_DOESNT_EXIST; - extern const int INCORRECT_DATA; + extern const int LOGICAL_ERROR; } #if defined(OS_LINUX) From 5b51a35e015336227dcc5b25445fefbc3da3059c Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 23 Jul 2024 11:31:34 +0200 Subject: [PATCH 071/289] Add unused variables --- src/Common/AsynchronousMetrics.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index eb6ede7a558..fedba4e55be 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -112,8 +112,8 @@ private: MemoryStatisticsOS memory_stat TSA_GUARDED_BY(data_mutex); #endif - const bool update_jemalloc_epoch; - const bool update_rss; + [[maybe_unused]] const bool update_jemalloc_epoch; + [[maybe_unused]] const bool update_rss; #if defined(OS_LINUX) std::optional meminfo TSA_GUARDED_BY(data_mutex); From ade79cfd7a8465bf6e332f3b2ca6143ba652251a Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 23 Jul 2024 11:50:56 +0200 Subject: [PATCH 072/289] More fixes --- src/Common/MemoryWorker.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index ddc3fd783f4..8169bc7d177 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -186,9 +186,6 @@ std::shared_ptr createCgroupsReader() } #endif -constexpr uint64_t cgroups_memory_usage_tick_ms{50}; -constexpr uint64_t jemalloc_memory_usage_tick_ms{100}; - std::string_view sourceToString(MemoryWorker::MemoryUsageSource source) { switch (source) @@ -213,6 +210,8 @@ MemoryWorker::MemoryWorker(uint64_t period_ms_) #if defined(OS_LINUX) try { + static constexpr uint64_t cgroups_memory_usage_tick_ms{50}; + cgroups_reader = createCgroupsReader(); source = MemoryUsageSource::Cgroups; if (period_ms == 0) @@ -227,6 +226,8 @@ MemoryWorker::MemoryWorker(uint64_t period_ms_) #endif #if USE_JEMALLOC + static constexpr uint64_t jemalloc_memory_usage_tick_ms{100}; + source = MemoryUsageSource::Jemalloc; if (period_ms == 0) period_ms = jemalloc_memory_usage_tick_ms; @@ -270,7 +271,11 @@ uint64_t MemoryWorker::getMemoryUsage() case MemoryUsageSource::Cgroups: return cgroups_reader->readMemoryUsage(); case MemoryUsageSource::Jemalloc: +#if USE_JEMALLOC return resident_mib.getValue(); +#else + return 0; +#endif case MemoryUsageSource::None: throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Trying to fetch memory usage while no memory source can be used"); } @@ -279,7 +284,7 @@ uint64_t MemoryWorker::getMemoryUsage() void MemoryWorker::backgroundThread() { std::chrono::milliseconds chrono_period_ms{period_ms}; - bool first_run = true; + [[maybe_unused]] bool first_run = true; std::unique_lock lock(mutex); while (true) { @@ -289,8 +294,10 @@ void MemoryWorker::backgroundThread() Stopwatch total_watch; +#if USE_JEMALLOC if (source == MemoryUsageSource::Jemalloc) epoch_mib.setValue(0); +#endif Int64 resident = getMemoryUsage(); MemoryTracker::updateRSS(resident); From 0631ff93778801768edde14c5fd5ed22701f6ec3 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Tue, 23 Jul 2024 12:14:54 +0100 Subject: [PATCH 073/289] Add jsoncompactwithprogress to aspell-dict.txt --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 3967179f078..2e403f44e4e 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1855,6 +1855,7 @@ jsoncompactstrings jsoncompactstringseachrow jsoncompactstringseachrowwithnames jsoncompactstringseachrowwithnamesandtypes +jsoncompactwithprogress jsoneachrow jsoneachrowwithprogress jsonobjecteachrow From 53a55221845802e67f9586af615fcb5ed84b20eb Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Tue, 23 Jul 2024 12:19:06 +0100 Subject: [PATCH 074/289] Fix example output in the documentation --- docs/en/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 6161cddefdd..8795b71e0ac 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -935,7 +935,7 @@ Each row is either a metadata object, data object, progress information or stati Example: ```json -{"meta": [{"name":"id", "type":"UInt32"}, {"name":"name", "type":"String"}]}} +{"meta": [{"name":"id", "type":"UInt32"}, {"name":"name", "type":"String"}]} {"progress":{"read_rows":"8","read_bytes":"168","written_rows":"0","written_bytes":"0","total_rows_to_read":"2","result_rows":"0","result_bytes":"0","elapsed_ns":"0"}} {"data":["1", "John Doe"]} {"data":["2", "Joe Doe"]} From a4a0611c1c0b40ea6b99d7e5cf3972fda201bb9d Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Tue, 23 Jul 2024 12:29:40 +0100 Subject: [PATCH 075/289] Fix docs spelling --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 2e403f44e4e..c64cfcb0968 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -406,6 +406,7 @@ JSONCompactStrings JSONCompactStringsEachRow JSONCompactStringsEachRowWithNames JSONCompactStringsEachRowWithNamesAndTypes +JSONCompactWithProgress JSONEachRow JSONEachRowWithProgress JSONExtract From ef4ad01d7d57942a79b46ad05c10160d557ff75f Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Tue, 23 Jul 2024 15:03:01 +0100 Subject: [PATCH 076/289] Fix totals formatting --- .../Impl/JSONCompactWithProgressRowOutputFormat.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 9603ce0265d..ddf071324c6 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -55,8 +55,8 @@ void JSONCompactWithProgressRowOutputFormat::writeRowBetweenDelimiter() void JSONCompactWithProgressRowOutputFormat::writeBeforeTotals() { - JSONUtils::writeFieldDelimiter(*ostr, 2); - JSONUtils::writeCompactArrayStart(*ostr, 1, "totals"); + JSONUtils::writeCompactObjectStart(*ostr); + JSONUtils::writeCompactArrayStart(*ostr, 0, "totals"); } void JSONCompactWithProgressRowOutputFormat::writeTotals(const Columns & columns, size_t row_num) @@ -67,6 +67,8 @@ void JSONCompactWithProgressRowOutputFormat::writeTotals(const Columns & columns void JSONCompactWithProgressRowOutputFormat::writeAfterTotals() { JSONUtils::writeCompactArrayEnd(*ostr); + JSONUtils::writeCompactObjectEnd(*ostr); + writeCString("}\n", *ostr); } void JSONCompactWithProgressRowOutputFormat::writeExtremesElement(const char * title, const Columns & columns, size_t row_num) @@ -131,6 +133,7 @@ void JSONCompactWithProgressRowOutputFormat::finalizeImpl() JSONUtils::writeException(exception_message, *ostr, settings, 0); JSONUtils::writeCompactObjectEnd(*ostr); } + writeCString("\n", *ostr); ostr->next(); } From f3710e9a29e4b866bfd55886c18ebad954a48e42 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Tue, 23 Jul 2024 15:04:05 +0100 Subject: [PATCH 077/289] Add stateless test for JSONCompactWithProgress output format --- ...03174_json_compact_with_progress.reference | 14 +++++++++++++ .../03174_json_compact_with_progress.sh | 20 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/queries/0_stateless/03174_json_compact_with_progress.reference create mode 100755 tests/queries/0_stateless/03174_json_compact_with_progress.sh diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.reference b/tests/queries/0_stateless/03174_json_compact_with_progress.reference new file mode 100644 index 00000000000..6e83db33565 --- /dev/null +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.reference @@ -0,0 +1,14 @@ +1 +{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}]}} +{"data":[1, "a"]} +{"data":[2, "b"]} +{"data":[3, "c"]} +{"progress":{"read_rows":"3","read_bytes":"33","written_rows":"0","written_bytes":"0","total_rows_to_read":"3","result_rows":"0","result_bytes":"0","elapsed_ns":"ELAPSED_NS"}} +{"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":33}} +2 +{"meta": [{"name":"name", "type":"String"}, {"name":"c", "type":"UInt64"}]}} +{"data":["a", "1"]} +{"data":["b", "1"]} +{"data":["c", "1"]} +{"totals": ["", "3"]}} +{"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":30}} diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.sh b/tests/queries/0_stateless/03174_json_compact_with_progress.sh new file mode 100755 index 00000000000..c21f7228517 --- /dev/null +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_table;" + +$CLICKHOUSE_CLIENT -q "SELECT 1;" +# Check JSONCompactWithProgress Output +$CLICKHOUSE_CLIENT -q "CREATE TABLE test_table (value UInt8, name String) ENGINE = MergeTree() ORDER BY value;" +$CLICKHOUSE_CLIENT -q "INSERT INTO test_table VALUES (1, 'a'), (2, 'b'), (3, 'c');" +$CLICKHOUSE_CLIENT -q "SELECT * FROM test_table FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' + +$CLICKHOUSE_CLIENT -q "SELECT 2;" +# Check Totals +$CLICKHOUSE_CLIENT -q "SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_table;" From 04d80ec2763a0677f32e9f6190cf2bcda0ebf8b7 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 23 Jul 2024 16:13:07 +0200 Subject: [PATCH 078/289] Fix non-linux build --- src/Common/MemoryWorker.cpp | 4 ++-- src/Common/MemoryWorker.h | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index 8169bc7d177..c576772d303 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -33,7 +33,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -#if defined(OS_LINUX) struct ICgroupsReader { virtual ~ICgroupsReader() = default; @@ -44,6 +43,7 @@ struct ICgroupsReader namespace { +#if defined(OS_LINUX) /// Format is /// kernel 5 /// rss 15 @@ -269,7 +269,7 @@ uint64_t MemoryWorker::getMemoryUsage() switch (source) { case MemoryUsageSource::Cgroups: - return cgroups_reader->readMemoryUsage(); + return cgroups_reader != nullptr ? cgroups_reader->readMemoryUsage() : 0; case MemoryUsageSource::Jemalloc: #if USE_JEMALLOC return resident_mib.getValue(); diff --git a/src/Common/MemoryWorker.h b/src/Common/MemoryWorker.h index 6fde93d63ad..b1b0495bf14 100644 --- a/src/Common/MemoryWorker.h +++ b/src/Common/MemoryWorker.h @@ -49,9 +49,7 @@ private: MemoryUsageSource source{MemoryUsageSource::None}; -#if defined(OS_LINUX) std::shared_ptr cgroups_reader; -#endif #if USE_JEMALLOC JemallocMibCache epoch_mib{"epoch"}; From ec5459a60d0be15e7bc100bd257e2b5c65baf594 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 24 Jul 2024 09:15:34 +0200 Subject: [PATCH 079/289] Update allocated with resident if no jemalloc --- src/Common/MemoryWorker.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index c576772d303..75d0e7c32d8 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -310,15 +310,19 @@ void MemoryWorker::backgroundThread() ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, purge_watch.elapsedMicroseconds()); } +#endif if (unlikely(first_run || total_memory_tracker.get() < 0)) { +#if USE_JEMALLOC if (source != MemoryUsageSource::Jemalloc) epoch_mib.setValue(0); MemoryTracker::updateAllocated(allocated_mib.getValue()); - } +#elif defined(OS_LINUX) + MemoryTracker::updateAllocated(resident); #endif + } ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); From e2e4c8ee0f8fcf1d6ef4d566c2fa4f9ee2123a56 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 24 Jul 2024 10:21:09 +0200 Subject: [PATCH 080/289] Better --- src/Common/MemoryWorker.cpp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index 84ccffb8e90..e148f7f8f49 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -65,11 +65,25 @@ Metrics readAllMetricsFromStatFile(ReadBufferFromFile & buf) return metrics; } -uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, const std::string & key) +uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, std::string_view key) { - const auto all_metrics = readAllMetricsFromStatFile(buf); - if (const auto it = all_metrics.find(key); it != all_metrics.end()) - return it->second; + while (!buf.eof()) + { + std::string current_key; + readStringUntilWhitespace(current_key, buf); + if (current_key != key) + { + std::string dummy; + readStringUntilNewlineInto(dummy, buf); + buf.ignore(); + continue; + } + + assertChar(' ', buf); + uint64_t value = 0; + readIntText(value, buf); + return value; + } LOG_ERROR(getLogger("CgroupsReader"), "Cannot find '{}' in '{}'", key, buf.getFileName()); return 0; } From 4712b79960d9da21741aa504cd508a5902e569fe Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Wed, 24 Jul 2024 16:41:37 +0800 Subject: [PATCH 081/289] Add min_max as statistics type --- src/Storages/Statistics/Statistics.cpp | 8 +- src/Storages/Statistics/StatisticsMinMax.cpp | 91 +++++++++++++++++++ src/Storages/Statistics/StatisticsMinMax.h | 31 +++++++ src/Storages/StatisticsDescription.cpp | 8 +- src/Storages/StatisticsDescription.h | 1 + ...2864_statistics_count_min_sketch.reference | 14 --- .../02864_statistics_estimation.reference | 25 +++++ ...ch.sql => 02864_statistics_estimation.sql} | 41 ++++++++- 8 files changed, 198 insertions(+), 21 deletions(-) create mode 100644 src/Storages/Statistics/StatisticsMinMax.cpp create mode 100644 src/Storages/Statistics/StatisticsMinMax.h delete mode 100644 tests/queries/0_stateless/02864_statistics_count_min_sketch.reference create mode 100644 tests/queries/0_stateless/02864_statistics_estimation.reference rename tests/queries/0_stateless/{02864_statistics_count_min_sketch.sql => 02864_statistics_estimation.sql} (57%) diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp index ade3326288a..5227f3d235e 100644 --- a/src/Storages/Statistics/Statistics.cpp +++ b/src/Storages/Statistics/Statistics.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -103,6 +104,8 @@ Float64 ColumnStatistics::estimateLess(const Field & val) const { if (stats.contains(StatisticsType::TDigest)) return stats.at(StatisticsType::TDigest)->estimateLess(val); + if (stats.contains(StatisticsType::MinMax)) + return stats.at(StatisticsType::MinMax)->estimateLess(val); return rows * ConditionSelectivityEstimator::default_normal_cond_factor; } @@ -204,6 +207,9 @@ void MergeTreeStatisticsFactory::registerValidator(StatisticsType stats_type, Va MergeTreeStatisticsFactory::MergeTreeStatisticsFactory() { + registerValidator(StatisticsType::MinMax, minMaxValidator); + registerCreator(StatisticsType::MinMax, minMaxCreator); + registerValidator(StatisticsType::TDigest, tdigestValidator); registerCreator(StatisticsType::TDigest, tdigestCreator); @@ -240,7 +246,7 @@ ColumnStatisticsPtr MergeTreeStatisticsFactory::get(const ColumnStatisticsDescri { auto it = creators.find(type); if (it == creators.end()) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistic type '{}'. Available types: 'tdigest' 'uniq' and 'count_min'", type); + throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistic type '{}'. Available types: 'min_max', 'tdigest' 'uniq' and 'count_min'", type); auto stat_ptr = (it->second)(desc, stats.data_type); column_stat->stats[type] = stat_ptr; } diff --git a/src/Storages/Statistics/StatisticsMinMax.cpp b/src/Storages/Statistics/StatisticsMinMax.cpp new file mode 100644 index 00000000000..8138ea9fa87 --- /dev/null +++ b/src/Storages/Statistics/StatisticsMinMax.cpp @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ +extern const int ILLEGAL_STATISTICS; +} + +StatisticsMinMax::StatisticsMinMax(const SingleStatisticsDescription & stat_, const DataTypePtr & data_type_) + : IStatistics(stat_) + , min(std::numeric_limits::max()) + , max(std::numeric_limits::min()) + , row_count(0) + , data_type(data_type_) +{ +} + +Float64 StatisticsMinMax::estimateLess(const Field & val) const +{ + Field val_converted = convertFieldToType(val, *data_type); + if (val_converted.isNull()) + return 0; + + auto val_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); + + if (val_float < min) + return 0; + + if (val_float > max) + return row_count; + + if (max == min) + return row_count; + + return ((val_float - min) / (max - min)) * row_count; +} + +void StatisticsMinMax::update(const ColumnPtr & column) +{ + for (size_t row = 0; row < column->size(); ++row) + { + if (column->isNullAt(row)) + continue; + + auto data = column->getFloat64(row); + min = std::min(data, min); + max = std::max(data, max); + } + row_count += column->size(); +} + +void StatisticsMinMax::serialize(WriteBuffer & buf) +{ + writeIntBinary(row_count, buf); + writeFloatBinary(min, buf); + writeFloatBinary(max, buf); +} + +void StatisticsMinMax::deserialize(ReadBuffer & buf) +{ + readIntBinary(row_count, buf); + readFloatBinary(min, buf); + readFloatBinary(max, buf); +} + + +void minMaxValidator(const SingleStatisticsDescription &, DataTypePtr data_type) +{ + data_type = removeNullable(data_type); + data_type = removeLowCardinalityAndNullable(data_type); + if (!data_type->isValueRepresentedByNumber()) + throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'min_max' do not support type {}", data_type->getName()); +} + +StatisticsPtr minMaxCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) +{ + return std::make_shared(stat, data_type); +} + +} diff --git a/src/Storages/Statistics/StatisticsMinMax.h b/src/Storages/Statistics/StatisticsMinMax.h new file mode 100644 index 00000000000..913dcccb798 --- /dev/null +++ b/src/Storages/Statistics/StatisticsMinMax.h @@ -0,0 +1,31 @@ +#include + + +namespace DB +{ + +class StatisticsMinMax : public IStatistics +{ +public: + StatisticsMinMax(const SingleStatisticsDescription & stat_, const DataTypePtr & data_type_); + ~StatisticsMinMax() override = default; + + Float64 estimateLess(const Field & val) const override; + + void update(const ColumnPtr & column) override; + + void serialize(WriteBuffer & buf) override; + void deserialize(ReadBuffer & buf) override; + +private: + Float64 min; + Float64 max; + Float64 row_count; + + DataTypePtr data_type; +}; + +void minMaxValidator(const SingleStatisticsDescription &, DataTypePtr data_type); +StatisticsPtr minMaxCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type); + +} diff --git a/src/Storages/StatisticsDescription.cpp b/src/Storages/StatisticsDescription.cpp index 9c5fd3604b2..0d99c36259f 100644 --- a/src/Storages/StatisticsDescription.cpp +++ b/src/Storages/StatisticsDescription.cpp @@ -51,7 +51,9 @@ static StatisticsType stringToStatisticsType(String type) return StatisticsType::Uniq; if (type == "count_min") return StatisticsType::CountMinSketch; - throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are 'tdigest', 'uniq' and 'count_min'.", type); + if (type == "min_max") + return StatisticsType::MinMax; + throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are 'min_max', 'tdigest', 'uniq' and 'count_min'.", type); } String SingleStatisticsDescription::getTypeName() const @@ -64,8 +66,10 @@ String SingleStatisticsDescription::getTypeName() const return "Uniq"; case StatisticsType::CountMinSketch: return "count_min"; + case StatisticsType::MinMax: + return "min_max"; default: - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are 'tdigest', 'uniq' and 'count_min'.", type); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are 'min_max', 'tdigest', 'uniq' and 'count_min'.", type); } } diff --git a/src/Storages/StatisticsDescription.h b/src/Storages/StatisticsDescription.h index 03b8fb0d583..1aa51ed03dd 100644 --- a/src/Storages/StatisticsDescription.h +++ b/src/Storages/StatisticsDescription.h @@ -14,6 +14,7 @@ enum class StatisticsType : UInt8 TDigest = 0, Uniq = 1, CountMinSketch = 2, + MinMax = 3, Max = 63, }; diff --git a/tests/queries/0_stateless/02864_statistics_count_min_sketch.reference b/tests/queries/0_stateless/02864_statistics_count_min_sketch.reference deleted file mode 100644 index 02c41656a36..00000000000 --- a/tests/queries/0_stateless/02864_statistics_count_min_sketch.reference +++ /dev/null @@ -1,14 +0,0 @@ -CREATE TABLE default.tab\n(\n `a` String,\n `b` UInt64,\n `c` Int64,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 -Test statistics count_min: - Prewhere info - Prewhere filter - Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed) -Test statistics multi-types: - Prewhere info - Prewhere filter - Prewhere filter column: and(equals(a, \'0\'), less(c, -90), greater(b, 900)) (removed) - Prewhere info - Prewhere filter - Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0)) (removed) -Test LowCardinality and Nullable data type: -tab2 diff --git a/tests/queries/0_stateless/02864_statistics_estimation.reference b/tests/queries/0_stateless/02864_statistics_estimation.reference new file mode 100644 index 00000000000..a1db6b8dc96 --- /dev/null +++ b/tests/queries/0_stateless/02864_statistics_estimation.reference @@ -0,0 +1,25 @@ +CREATE TABLE default.tab\n(\n `a` String,\n `b` UInt64,\n `c` Int64,\n `d` DateTime64(3),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 +Test statistics min_max: + Prewhere info + Prewhere filter + Prewhere filter column: and(greater(d, _CAST(9998_DateTime(\'UTC\'), \'DateTime(\'UTC\')\')), less(c, -1), greater(b, 0)) (removed) +Test statistics count_min: + Prewhere info + Prewhere filter + Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed) +Test statistics multi-types: + Prewhere info + Prewhere filter + Prewhere filter column: and(equals(a, \'0\'), less(c, -90), greater(b, 900)) (removed) + Prewhere info + Prewhere filter + Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0)) (removed) +Test statistics min_max and tdigest: + Prewhere info + Prewhere filter + Prewhere filter column: and(less(b, 10), less(c, 0)) (removed) + Prewhere info + Prewhere filter + Prewhere filter column: and(less(b, 10), less(c, 0)) (removed) +Test LowCardinality and Nullable data type: +tab2 diff --git a/tests/queries/0_stateless/02864_statistics_count_min_sketch.sql b/tests/queries/0_stateless/02864_statistics_estimation.sql similarity index 57% rename from tests/queries/0_stateless/02864_statistics_count_min_sketch.sql rename to tests/queries/0_stateless/02864_statistics_estimation.sql index c730aa7b4a7..f3d085b837b 100644 --- a/tests/queries/0_stateless/02864_statistics_count_min_sketch.sql +++ b/tests/queries/0_stateless/02864_statistics_estimation.sql @@ -12,13 +12,28 @@ CREATE TABLE tab a String, b UInt64, c Int64, + d DateTime64, pk String, ) Engine = MergeTree() ORDER BY pk SETTINGS min_bytes_for_wide_part = 0; SHOW CREATE TABLE tab; -INSERT INTO tab select toString(number % 10000), number % 1000, -(number % 100), generateUUIDv4() FROM system.numbers LIMIT 10000; +INSERT INTO tab select toString(number % 10000), number % 1000, -(number % 100), toDateTime(number, 'UTC'), generateUUIDv4() FROM system.numbers LIMIT 10000; + +SELECT 'Test statistics min_max:'; + +ALTER TABLE tab ADD STATISTICS b TYPE min_max; +ALTER TABLE tab ADD STATISTICS c TYPE min_max; +ALTER TABLE tab ADD STATISTICS d TYPE min_max; +ALTER TABLE tab MATERIALIZE STATISTICS b, c, d; + +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b > 0/*10000*/ and c < -1/*9990*/ and d > toDateTime(9998, 'UTC')/*1*/) +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; + +ALTER TABLE tab DROP STATISTICS b, c, d; + SELECT 'Test statistics count_min:'; @@ -28,7 +43,7 @@ ALTER TABLE tab ADD STATISTICS c TYPE count_min; ALTER TABLE tab MATERIALIZE STATISTICS a, b, c; SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) xx +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; ALTER TABLE tab DROP STATISTICS a, b, c; @@ -39,7 +54,8 @@ SELECT 'Test statistics multi-types:'; ALTER TABLE tab ADD STATISTICS a TYPE count_min; ALTER TABLE tab ADD STATISTICS b TYPE count_min, uniq, tdigest; ALTER TABLE tab ADD STATISTICS c TYPE count_min, uniq, tdigest; -ALTER TABLE tab MATERIALIZE STATISTICS a, b, c; +ALTER TABLE tab ADD STATISTICS d TYPE count_min, uniq, tdigest; +ALTER TABLE tab MATERIALIZE STATISTICS a, b, c, d; SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/) @@ -49,7 +65,24 @@ SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; -ALTER TABLE tab DROP STATISTICS a, b, c; +ALTER TABLE tab DROP STATISTICS a, b, c, d; + + +SELECT 'Test statistics min_max and tdigest:'; + +ALTER TABLE tab ADD STATISTICS b TYPE min_max; +ALTER TABLE tab MATERIALIZE STATISTICS b; +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*5000*/ and b < 10/*100*/) +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; + +ALTER TABLE tab ADD STATISTICS b TYPE tdigest; +ALTER TABLE tab MATERIALIZE STATISTICS b; +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*5000*/ and b < 10/*100*/) +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +ALTER TABLE tab DROP STATISTICS b; + DROP TABLE IF EXISTS tab SYNC; From 40fc0ca5743a6b1dcc464268a107ba206ca375ab Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Wed, 24 Jul 2024 17:43:27 +0800 Subject: [PATCH 082/289] Uniq statistics supports more datatypes and refactor logical when there is no statistics. --- .../ConditionSelectivityEstimator.cpp | 41 ++++++------------ .../ConditionSelectivityEstimator.h | 10 ++--- src/Storages/Statistics/Statistics.cpp | 26 ++++++------ src/Storages/Statistics/Statistics.h | 11 +++-- src/Storages/Statistics/StatisticsMinMax.cpp | 8 ++-- src/Storages/Statistics/StatisticsTDigest.cpp | 42 ++++++++++--------- src/Storages/Statistics/StatisticsTDigest.h | 9 ++-- src/Storages/Statistics/StatisticsUniq.cpp | 2 +- src/Storages/Statistics/StatisticsUniq.h | 4 +- .../02864_statistics_estimation.reference | 9 +++- .../02864_statistics_estimation.sql | 21 ++++++++-- 11 files changed, 94 insertions(+), 89 deletions(-) diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp index 57dff958b9a..86970a3ac46 100644 --- a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp +++ b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp @@ -19,7 +19,7 @@ void ConditionSelectivityEstimator::ColumnSelectivityEstimator::merge(String par Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateLess(const Field & val, Float64 rows) const { if (part_statistics.empty()) - return default_normal_cond_factor * rows; + return default_cond_range_factor * rows; Float64 result = 0; Float64 part_rows = 0; for (const auto & [key, estimator] : part_statistics) @@ -38,15 +38,8 @@ Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateGreat Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateEqual(const Field & val, Float64 rows) const { if (part_statistics.empty()) - { - auto float_val = StatisticsUtils::tryConvertToFloat64(val); - if (!float_val) - return default_unknown_cond_factor * rows; - else if (float_val.value() < - threshold || float_val.value() > threshold) - return default_normal_cond_factor * rows; - else - return default_good_cond_factor * rows; - } + return default_cond_equal_factor * rows; + Float64 result = 0; Float64 partial_cnt = 0; for (const auto & [key, estimator] : part_statistics) @@ -149,30 +142,22 @@ Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode auto [op, val] = extractBinaryOp(node, col); + if (dummy) + { + if (op == "equals") + return default_cond_equal_factor * total_rows; + else if (op == "less" || op == "lessOrEquals" || op == "greater" || op == "greaterOrEquals") + return default_cond_range_factor * total_rows; + else + return default_unknown_cond_factor * total_rows; + } + if (op == "equals") - { - if (dummy) - { - auto float_val = StatisticsUtils::tryConvertToFloat64(val); - if (!float_val || (float_val < - threshold || float_val > threshold)) - return default_normal_cond_factor * total_rows; - else - return default_good_cond_factor * total_rows; - } return estimator.estimateEqual(val, total_rows); - } else if (op == "less" || op == "lessOrEquals") - { - if (dummy) - return default_normal_cond_factor * total_rows; return estimator.estimateLess(val, total_rows); - } else if (op == "greater" || op == "greaterOrEquals") - { - if (dummy) - return default_normal_cond_factor * total_rows; return estimator.estimateGreater(val, total_rows); - } else return default_unknown_cond_factor * total_rows; } diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.h b/src/Storages/Statistics/ConditionSelectivityEstimator.h index ce7fdd12e92..269ee9ac6cb 100644 --- a/src/Storages/Statistics/ConditionSelectivityEstimator.h +++ b/src/Storages/Statistics/ConditionSelectivityEstimator.h @@ -38,12 +38,10 @@ private: std::pair extractBinaryOp(const RPNBuilderTreeNode & node, const String & column_name) const; - static constexpr auto default_good_cond_factor = 0.1; - static constexpr auto default_normal_cond_factor = 0.5; - static constexpr auto default_unknown_cond_factor = 1.0; - /// Conditions like "x = N" are considered good if abs(N) > threshold. - /// This is used to assume that condition is likely to have good selectivity. - static constexpr auto threshold = 2; + /// Used to estimate the selectivity of a condition when there is no statistics. + static constexpr auto default_cond_range_factor = 0.5; + static constexpr auto default_cond_equal_factor = 0.01; + static constexpr auto default_unknown_cond_factor = 1; UInt64 total_rows = 0; std::map column_estimators; diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp index 5227f3d235e..3513638ea9e 100644 --- a/src/Storages/Statistics/Statistics.cpp +++ b/src/Storages/Statistics/Statistics.cpp @@ -51,13 +51,6 @@ std::optional StatisticsUtils::tryConvertToFloat64(const Field & field) } } -std::optional StatisticsUtils::tryConvertToString(const DB::Field & field) -{ - if (field.getType() == Field::Types::String) - return field.get(); - return {}; -} - IStatistics::IStatistics(const SingleStatisticsDescription & stat_) : stat(stat_) { @@ -106,7 +99,7 @@ Float64 ColumnStatistics::estimateLess(const Field & val) const return stats.at(StatisticsType::TDigest)->estimateLess(val); if (stats.contains(StatisticsType::MinMax)) return stats.at(StatisticsType::MinMax)->estimateLess(val); - return rows * ConditionSelectivityEstimator::default_normal_cond_factor; + return rows * ConditionSelectivityEstimator::default_cond_range_factor; } Float64 ColumnStatistics::estimateGreater(const Field & val) const @@ -116,8 +109,7 @@ Float64 ColumnStatistics::estimateGreater(const Field & val) const Float64 ColumnStatistics::estimateEqual(const Field & val) const { - auto float_val = StatisticsUtils::tryConvertToFloat64(val); - if (float_val.has_value() && stats.contains(StatisticsType::Uniq) && stats.contains(StatisticsType::TDigest)) + if (stats_desc.data_type->isValueRepresentedByNumber() && stats.contains(StatisticsType::Uniq) && stats.contains(StatisticsType::TDigest)) { /// 2048 is the default number of buckets in TDigest. In this case, TDigest stores exactly one value (with many rows) for every bucket. if (stats.at(StatisticsType::Uniq)->estimateCardinality() < 2048) @@ -127,10 +119,16 @@ Float64 ColumnStatistics::estimateEqual(const Field & val) const if (stats.contains(StatisticsType::CountMinSketch)) return stats.at(StatisticsType::CountMinSketch)->estimateEqual(val); #endif - if (!float_val.has_value() && (float_val < - ConditionSelectivityEstimator::threshold || float_val > ConditionSelectivityEstimator::threshold)) - return rows * ConditionSelectivityEstimator::default_normal_cond_factor; - else - return rows * ConditionSelectivityEstimator::default_good_cond_factor; + if (stats.contains(StatisticsType::Uniq)) + { + auto cardinality = stats.at(StatisticsType::Uniq)->estimateCardinality(); + if (cardinality == 0) + return 0; + /// Assume that the value is uniformly distributed among the unique values. + return static_cast(1) / stats.at(StatisticsType::Uniq)->estimateCardinality(); + } + + return rows * ConditionSelectivityEstimator::default_cond_equal_factor; } /// ------------------------------------- diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h index 16f0c67eabd..f47a707663c 100644 --- a/src/Storages/Statistics/Statistics.h +++ b/src/Storages/Statistics/Statistics.h @@ -19,7 +19,6 @@ struct StatisticsUtils { /// Returns std::nullopt if input Field cannot be converted to a concrete value static std::optional tryConvertToFloat64(const Field & field); - static std::optional tryConvertToString(const Field & field); }; /// Statistics describe properties of the values in the column, @@ -32,11 +31,6 @@ public: explicit IStatistics(const SingleStatisticsDescription & stat_); virtual ~IStatistics() = default; - virtual void update(const ColumnPtr & column) = 0; - - virtual void serialize(WriteBuffer & buf) = 0; - virtual void deserialize(ReadBuffer & buf) = 0; - /// Estimate the cardinality of the column. /// Throws if the statistics object is not able to do a meaningful estimation. virtual UInt64 estimateCardinality() const; @@ -46,6 +40,11 @@ public: virtual Float64 estimateEqual(const Field & val) const; /// cardinality of val in the column virtual Float64 estimateLess(const Field & val) const; /// summarized cardinality of values < val in the column + virtual void update(const ColumnPtr & column) = 0; + + virtual void serialize(WriteBuffer & buf) = 0; + virtual void deserialize(ReadBuffer & buf) = 0; + protected: SingleStatisticsDescription stat; }; diff --git a/src/Storages/Statistics/StatisticsMinMax.cpp b/src/Storages/Statistics/StatisticsMinMax.cpp index 8138ea9fa87..083e8634841 100644 --- a/src/Storages/Statistics/StatisticsMinMax.cpp +++ b/src/Storages/Statistics/StatisticsMinMax.cpp @@ -32,18 +32,18 @@ Float64 StatisticsMinMax::estimateLess(const Field & val) const if (val_converted.isNull()) return 0; - auto val_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); + auto val_as_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); - if (val_float < min) + if (val_as_float < min) return 0; - if (val_float > max) + if (val_as_float > max) return row_count; if (max == min) return row_count; - return ((val_float - min) / (max - min)) * row_count; + return ((val_as_float - min) / (max - min)) * row_count; } void StatisticsMinMax::update(const ColumnPtr & column) diff --git a/src/Storages/Statistics/StatisticsTDigest.cpp b/src/Storages/Statistics/StatisticsTDigest.cpp index 66150e00fdb..3544f5cdea3 100644 --- a/src/Storages/Statistics/StatisticsTDigest.cpp +++ b/src/Storages/Statistics/StatisticsTDigest.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include namespace DB { @@ -10,24 +12,20 @@ extern const int ILLEGAL_STATISTICS; extern const int LOGICAL_ERROR; } -StatisticsTDigest::StatisticsTDigest(const SingleStatisticsDescription & stat_) - : IStatistics(stat_) +StatisticsTDigest::StatisticsTDigest(const SingleStatisticsDescription & stat_, DataTypePtr data_type_) + : IStatistics(stat_), data_type(data_type_) { } void StatisticsTDigest::update(const ColumnPtr & column) { - size_t rows = column->size(); - for (size_t row = 0; row < rows; ++row) + for (size_t row = 0; row < column->size(); ++row) { - Field field; - column->get(row, field); - - if (field.isNull()) + if (column->isNullAt(row)) continue; - if (auto field_as_float = StatisticsUtils::tryConvertToFloat64(field)) - t_digest.add(*field_as_float, 1); + auto data = column->getFloat64(row); + t_digest.add(data, 1); } } @@ -43,18 +41,22 @@ void StatisticsTDigest::deserialize(ReadBuffer & buf) Float64 StatisticsTDigest::estimateLess(const Field & val) const { - auto val_as_float = StatisticsUtils::tryConvertToFloat64(val); - if (val_as_float) - return t_digest.getCountLessThan(*val_as_float); - throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'tdigest' does not support estimating value of type {}", val.getTypeName()); + Field val_converted = convertFieldToType(val, *data_type); + if (val_converted.isNull()) + return 0; + + auto val_as_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); + return t_digest.getCountLessThan(val_as_float); } Float64 StatisticsTDigest::estimateEqual(const Field & val) const { - auto val_as_float = StatisticsUtils::tryConvertToFloat64(val); - if (val_as_float) - return t_digest.getCountEqual(*val_as_float); - throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'tdigest' does not support estimating value of type {}", val.getTypeName()); + Field val_converted = convertFieldToType(val, *data_type); + if (val_converted.isNull()) + return 0; + + auto val_as_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); + return t_digest.getCountEqual(val_as_float); } void tdigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type) @@ -65,9 +67,9 @@ void tdigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'tdigest' do not support type {}", data_type->getName()); } -StatisticsPtr tdigestCreator(const SingleStatisticsDescription & stat, DataTypePtr) +StatisticsPtr tdigestCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) { - return std::make_shared(stat); + return std::make_shared(stat, data_type); } } diff --git a/src/Storages/Statistics/StatisticsTDigest.h b/src/Storages/Statistics/StatisticsTDigest.h index 614973e5d8b..f4de6cc41e0 100644 --- a/src/Storages/Statistics/StatisticsTDigest.h +++ b/src/Storages/Statistics/StatisticsTDigest.h @@ -9,18 +9,19 @@ namespace DB class StatisticsTDigest : public IStatistics { public: - explicit StatisticsTDigest(const SingleStatisticsDescription & stat_); + explicit StatisticsTDigest(const SingleStatisticsDescription & stat_, DataTypePtr data_type_); + + Float64 estimateLess(const Field & val) const override; + Float64 estimateEqual(const Field & val) const override; void update(const ColumnPtr & column) override; void serialize(WriteBuffer & buf) override; void deserialize(ReadBuffer & buf) override; - Float64 estimateLess(const Field & val) const override; - Float64 estimateEqual(const Field & val) const override; - private: QuantileTDigest t_digest; + DataTypePtr data_type; }; void tdigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type); diff --git a/src/Storages/Statistics/StatisticsUniq.cpp b/src/Storages/Statistics/StatisticsUniq.cpp index 8f60ffcf0b5..aace45a14b1 100644 --- a/src/Storages/Statistics/StatisticsUniq.cpp +++ b/src/Storages/Statistics/StatisticsUniq.cpp @@ -56,7 +56,7 @@ void uniqValidator(const SingleStatisticsDescription &, DataTypePtr data_type) { data_type = removeNullable(data_type); data_type = removeLowCardinalityAndNullable(data_type); - if (!data_type->isValueRepresentedByNumber()) + if (!data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type)) throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'uniq' do not support type {}", data_type->getName()); } diff --git a/src/Storages/Statistics/StatisticsUniq.h b/src/Storages/Statistics/StatisticsUniq.h index faabde8d47c..81162b0a5b9 100644 --- a/src/Storages/Statistics/StatisticsUniq.h +++ b/src/Storages/Statistics/StatisticsUniq.h @@ -13,13 +13,13 @@ public: StatisticsUniq(const SingleStatisticsDescription & stat_, const DataTypePtr & data_type); ~StatisticsUniq() override; + UInt64 estimateCardinality() const override; + void update(const ColumnPtr & column) override; void serialize(WriteBuffer & buf) override; void deserialize(ReadBuffer & buf) override; - UInt64 estimateCardinality() const override; - private: std::unique_ptr arena; AggregateFunctionPtr collector; diff --git a/tests/queries/0_stateless/02864_statistics_estimation.reference b/tests/queries/0_stateless/02864_statistics_estimation.reference index a1db6b8dc96..64f264017d8 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.reference +++ b/tests/queries/0_stateless/02864_statistics_estimation.reference @@ -14,12 +14,19 @@ Test statistics multi-types: Prewhere info Prewhere filter Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0)) (removed) -Test statistics min_max and tdigest: +Test estimating range condition: Prewhere info Prewhere filter Prewhere filter column: and(less(b, 10), less(c, 0)) (removed) Prewhere info Prewhere filter Prewhere filter column: and(less(b, 10), less(c, 0)) (removed) +Test estimating equals condition: + Prewhere info + Prewhere filter + Prewhere filter column: and(equals(a, \'0\'), equals(b, 10)) (removed) + Prewhere info + Prewhere filter + Prewhere filter column: and(equals(a, \'0\'), equals(b, 10)) (removed) Test LowCardinality and Nullable data type: tab2 diff --git a/tests/queries/0_stateless/02864_statistics_estimation.sql b/tests/queries/0_stateless/02864_statistics_estimation.sql index f3d085b837b..0ea9eb38e60 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.sql +++ b/tests/queries/0_stateless/02864_statistics_estimation.sql @@ -68,7 +68,7 @@ WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; ALTER TABLE tab DROP STATISTICS a, b, c, d; -SELECT 'Test statistics min_max and tdigest:'; +SELECT 'Test estimating range condition:'; ALTER TABLE tab ADD STATISTICS b TYPE min_max; ALTER TABLE tab MATERIALIZE STATISTICS b; @@ -84,6 +84,21 @@ WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; ALTER TABLE tab DROP STATISTICS b; +SELECT 'Test estimating equals condition:'; + +ALTER TABLE tab ADD STATISTICS a TYPE uniq; +ALTER TABLE tab MATERIALIZE STATISTICS a; +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b = 10/*100*/ and a = '0'/*1*/) +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; + +ALTER TABLE tab ADD STATISTICS a TYPE count_min; +ALTER TABLE tab MATERIALIZE STATISTICS a; +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b = 10/*100*/ and a = '0'/*1*/) +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +ALTER TABLE tab DROP STATISTICS a; + DROP TABLE IF EXISTS tab SYNC; @@ -93,8 +108,8 @@ SET allow_suspicious_low_cardinality_types=1; CREATE TABLE tab2 ( a LowCardinality(Int64) STATISTICS(count_min), - b Nullable(Int64) STATISTICS(count_min), - c LowCardinality(Nullable(Int64)) STATISTICS(count_min), + b Nullable(Int64) STATISTICS(min_max, count_min), + c LowCardinality(Nullable(Int64)) STATISTICS(min_max, count_min), pk String, ) Engine = MergeTree() ORDER BY pk; From 5ae356e6df457e24220f0f63f4a4984334471202 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Wed, 24 Jul 2024 17:54:48 +0800 Subject: [PATCH 083/289] Add document for min_max statistics --- docs/en/engines/table-engines/mergetree-family/mergetree.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 7ffbd9a5bae..8e8d5e55772 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -991,6 +991,9 @@ They can be used for prewhere optimization only if we enable `set allow_statisti #### Available Types of Column Statistics {#available-types-of-column-statistics} +- `min_max` + + min_max statistics allows to estimate selectivity of range condition for numeric columns. - `TDigest` [TDigest](https://github.com/tdunning/t-digest) sketches which allow to compute approximate percentiles (e.g. the 90th percentile) for numeric columns. From 3f7cd09d168ea13e4ec281c8f3943c0aebe9f920 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Wed, 24 Jul 2024 18:02:26 +0800 Subject: [PATCH 084/289] Remove useless function tryConvertToFloat64 --- src/Storages/Statistics/Statistics.cpp | 22 ---------------------- src/Storages/Statistics/Statistics.h | 6 ------ 2 files changed, 28 deletions(-) diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp index 3513638ea9e..0f0f1de4552 100644 --- a/src/Storages/Statistics/Statistics.cpp +++ b/src/Storages/Statistics/Statistics.cpp @@ -28,28 +28,6 @@ enum StatisticsFileVersion : UInt16 V0 = 0, }; -std::optional StatisticsUtils::tryConvertToFloat64(const Field & field) -{ - switch (field.getType()) - { - case Field::Types::Int64: - return field.get(); - case Field::Types::UInt64: - return field.get(); - case Field::Types::Float64: - return field.get(); - case Field::Types::Int128: - return field.get(); - case Field::Types::UInt128: - return field.get(); - case Field::Types::Int256: - return field.get(); - case Field::Types::UInt256: - return field.get(); - default: - return {}; - } -} IStatistics::IStatistics(const SingleStatisticsDescription & stat_) : stat(stat_) diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h index f47a707663c..3eeb5cf92a8 100644 --- a/src/Storages/Statistics/Statistics.h +++ b/src/Storages/Statistics/Statistics.h @@ -15,12 +15,6 @@ constexpr auto STATS_FILE_PREFIX = "statistics_"; constexpr auto STATS_FILE_SUFFIX = ".stats"; -struct StatisticsUtils -{ - /// Returns std::nullopt if input Field cannot be converted to a concrete value - static std::optional tryConvertToFloat64(const Field & field); -}; - /// Statistics describe properties of the values in the column, /// e.g. how many unique values exist, /// what are the N most frequent values, From 118e329e23b51673e7d62bb3ffa8a34a8ecdb330 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Wed, 24 Jul 2024 13:09:47 +0100 Subject: [PATCH 085/289] Fix flaky test --- ...03174_json_compact_with_progress.reference | 19 ++++++++++--------- .../03174_json_compact_with_progress.sh | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.reference b/tests/queries/0_stateless/03174_json_compact_with_progress.reference index 6e83db33565..b735c871fbd 100644 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.reference +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.reference @@ -1,14 +1,15 @@ 1 -{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}]}} -{"data":[1, "a"]} -{"data":[2, "b"]} -{"data":[3, "c"]} +{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}, {"name":"sleep(0.1)", "type":"UInt8"}]}} {"progress":{"read_rows":"3","read_bytes":"33","written_rows":"0","written_bytes":"0","total_rows_to_read":"3","result_rows":"0","result_bytes":"0","elapsed_ns":"ELAPSED_NS"}} +{"data":[1, "a", 0]} +{"data":[2, "b", 0]} +{"data":[3, "c", 0]} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":33}} 2 -{"meta": [{"name":"name", "type":"String"}, {"name":"c", "type":"UInt64"}]}} -{"data":["a", "1"]} -{"data":["b", "1"]} -{"data":["c", "1"]} -{"totals": ["", "3"]}} +{"meta": [{"name":"name", "type":"String"}, {"name":"count()", "type":"UInt64"}, {"name":"c", "type":"UInt8"}]}} +{"progress":{"read_rows":"3","read_bytes":"30","written_rows":"0","written_bytes":"0","total_rows_to_read":"3","result_rows":"0","result_bytes":"0","elapsed_ns":"ELAPSED_NS"}} +{"data":["a", "1", 0]} +{"data":["b", "1", 0]} +{"data":["c", "1", 0]} +{"totals": ["", "3", 0]}} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":30}} diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.sh b/tests/queries/0_stateless/03174_json_compact_with_progress.sh index c21f7228517..383668de858 100755 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.sh +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.sh @@ -11,10 +11,10 @@ $CLICKHOUSE_CLIENT -q "SELECT 1;" # Check JSONCompactWithProgress Output $CLICKHOUSE_CLIENT -q "CREATE TABLE test_table (value UInt8, name String) ENGINE = MergeTree() ORDER BY value;" $CLICKHOUSE_CLIENT -q "INSERT INTO test_table VALUES (1, 'a'), (2, 'b'), (3, 'c');" -$CLICKHOUSE_CLIENT -q "SELECT * FROM test_table FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' +$CLICKHOUSE_CLIENT -q "SELECT *, sleep(0.1) FROM test_table FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' $CLICKHOUSE_CLIENT -q "SELECT 2;" # Check Totals -$CLICKHOUSE_CLIENT -q "SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' +$CLICKHOUSE_CLIENT -q "SELECT name, count(), sleep(0.1) AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_table;" From b0ec8c92fcc09857345acb4e4f627827556d17cd Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Wed, 24 Jul 2024 17:49:33 +0100 Subject: [PATCH 086/289] Fix totals json object --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index ddf071324c6..0814e5f45d6 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -68,7 +68,7 @@ void JSONCompactWithProgressRowOutputFormat::writeAfterTotals() { JSONUtils::writeCompactArrayEnd(*ostr); JSONUtils::writeCompactObjectEnd(*ostr); - writeCString("}\n", *ostr); + writeCString("\n", *ostr); } void JSONCompactWithProgressRowOutputFormat::writeExtremesElement(const char * title, const Columns & columns, size_t row_num) From caaf14a0a91bd4779cc292dd674761a47148d24d Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Wed, 24 Jul 2024 17:56:14 +0100 Subject: [PATCH 087/289] Fix flaky test by ignoring progress lines when comparing output --- ...03174_json_compact_with_progress.reference | 20 +++++++++---------- .../03174_json_compact_with_progress.sh | 4 ++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.reference b/tests/queries/0_stateless/03174_json_compact_with_progress.reference index b735c871fbd..b45f296e5c0 100644 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.reference +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.reference @@ -1,15 +1,13 @@ 1 -{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}, {"name":"sleep(0.1)", "type":"UInt8"}]}} -{"progress":{"read_rows":"3","read_bytes":"33","written_rows":"0","written_bytes":"0","total_rows_to_read":"3","result_rows":"0","result_bytes":"0","elapsed_ns":"ELAPSED_NS"}} -{"data":[1, "a", 0]} -{"data":[2, "b", 0]} -{"data":[3, "c", 0]} +{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}]}} +{"data":[1, "a"]} +{"data":[2, "b"]} +{"data":[3, "c"]} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":33}} 2 -{"meta": [{"name":"name", "type":"String"}, {"name":"count()", "type":"UInt64"}, {"name":"c", "type":"UInt8"}]}} -{"progress":{"read_rows":"3","read_bytes":"30","written_rows":"0","written_bytes":"0","total_rows_to_read":"3","result_rows":"0","result_bytes":"0","elapsed_ns":"ELAPSED_NS"}} -{"data":["a", "1", 0]} -{"data":["b", "1", 0]} -{"data":["c", "1", 0]} -{"totals": ["", "3", 0]}} +{"meta": [{"name":"name", "type":"String"}, {"name":"c", "type":"UInt64"}]}} +{"data":["a", "1"]} +{"data":["b", "1"]} +{"data":["c", "1"]} +{"totals": ["", "3"]} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":30}} diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.sh b/tests/queries/0_stateless/03174_json_compact_with_progress.sh index 383668de858..b440dbe2891 100755 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.sh +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.sh @@ -11,10 +11,10 @@ $CLICKHOUSE_CLIENT -q "SELECT 1;" # Check JSONCompactWithProgress Output $CLICKHOUSE_CLIENT -q "CREATE TABLE test_table (value UInt8, name String) ENGINE = MergeTree() ORDER BY value;" $CLICKHOUSE_CLIENT -q "INSERT INTO test_table VALUES (1, 'a'), (2, 'b'), (3, 'c');" -$CLICKHOUSE_CLIENT -q "SELECT *, sleep(0.1) FROM test_table FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' +$CLICKHOUSE_CLIENT -q "SELECT * FROM test_table FORMAT JSONCompactWithProgress settings max_block_size=2;" | grep -v --text "progress" | sed -E 's/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' $CLICKHOUSE_CLIENT -q "SELECT 2;" # Check Totals -$CLICKHOUSE_CLIENT -q "SELECT name, count(), sleep(0.1) AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' +$CLICKHOUSE_CLIENT -q "SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | grep -v --text "progress" | sed -E 's/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_table;" From f449c2fea0487abbe262c10fc5af9a99df1bc822 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 25 Jul 2024 08:45:08 +0200 Subject: [PATCH 088/289] Fix --- src/Common/MemoryTracker.h | 5 +++++ src/Common/MemoryWorker.cpp | 8 +++----- src/Coordination/KeeperDispatcher.cpp | 3 ++- src/Coordination/KeeperServer.cpp | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index d2db8489f19..f15465a20c1 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -120,6 +120,11 @@ public: return amount.load(std::memory_order_relaxed); } + Int64 getRSS() const + { + return rss.load(std::memory_order_relaxed); + } + // Merges and mutations may pass memory ownership to other threads thus in the end of execution // MemoryTracker for background task may have a non-zero counter. // This method is intended to fix the counter inside of background_memory_tracker. diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index e148f7f8f49..1b869ed9d6b 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -312,7 +312,7 @@ uint64_t MemoryWorker::getMemoryUsage() void MemoryWorker::backgroundThread() { std::chrono::milliseconds chrono_period_ms{period_ms}; - bool first_run = true; + [[maybe_unused]] bool first_run = true; std::unique_lock lock(mutex); while (true) { @@ -340,17 +340,15 @@ void MemoryWorker::backgroundThread() } #endif +#if USE_JEMALLOC if (unlikely(first_run || total_memory_tracker.get() < 0)) { -#if USE_JEMALLOC if (source != MemoryUsageSource::Jemalloc) epoch_mib.setValue(0); MemoryTracker::updateAllocated(allocated_mib.getValue()); -#elif defined(OS_LINUX) - MemoryTracker::updateAllocated(resident); -#endif } +#endif ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 4c2ccb8db64..893bb8e6082 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -150,10 +150,11 @@ void KeeperDispatcher::requestThread() { LOG_WARNING( log, - "Processing requests refused because of max_memory_usage_soft_limit {}, the total used memory is {}, request type " + "Processing requests refused because of max_memory_usage_soft_limit {}, the total allocated memory is {}, RSS is {}, request type " "is {}", ReadableSize(mem_soft_limit), ReadableSize(total_memory_tracker.get()), + ReadableSize(total_memory_tracker.getRSS()), request.request->getOpNum()); addErrorResponses({request}, Coordination::Error::ZCONNECTIONLOSS); continue; diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index d40e5ef2e50..ad9e8d32caa 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -599,7 +599,7 @@ bool KeeperServer::isLeaderAlive() const bool KeeperServer::isExceedingMemorySoftLimit() const { Int64 mem_soft_limit = keeper_context->getKeeperMemorySoftLimit(); - return mem_soft_limit > 0 && total_memory_tracker.get() >= mem_soft_limit; + return mem_soft_limit > 0 && std::max(total_memory_tracker.get(), total_memory_tracker.getRSS()) >= mem_soft_limit; } /// TODO test whether taking failed peer in count From b9703ad9242f7492de6f6d10e13ac89a74231210 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Thu, 25 Jul 2024 15:14:12 +0800 Subject: [PATCH 089/289] Fix logical error when estimating --- src/Storages/Statistics/Statistics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp index 0f0f1de4552..17ff8deb720 100644 --- a/src/Storages/Statistics/Statistics.cpp +++ b/src/Storages/Statistics/Statistics.cpp @@ -103,7 +103,7 @@ Float64 ColumnStatistics::estimateEqual(const Field & val) const if (cardinality == 0) return 0; /// Assume that the value is uniformly distributed among the unique values. - return static_cast(1) / stats.at(StatisticsType::Uniq)->estimateCardinality(); + return static_cast(1) / stats.at(StatisticsType::Uniq)->estimateCardinality() * rows; } return rows * ConditionSelectivityEstimator::default_cond_equal_factor; From 63ccbcbdfc7aa012e1879b11f9f6f4a23f9a1a0c Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Thu, 25 Jul 2024 13:22:47 +0100 Subject: [PATCH 090/289] Remove test tags --- tests/queries/0_stateless/03174_json_compact_with_progress.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.sh b/tests/queries/0_stateless/03174_json_compact_with_progress.sh index b440dbe2891..8f524cdff76 100755 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.sh +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-parallel CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 3adbc4cd334a05560ca2397ac5e1a14d8c580167 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Wed, 31 Jul 2024 15:10:29 +0800 Subject: [PATCH 091/289] Fix code style --- src/Storages/Statistics/StatisticsMinMax.h | 2 ++ src/Storages/Statistics/StatisticsTDigest.cpp | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Storages/Statistics/StatisticsMinMax.h b/src/Storages/Statistics/StatisticsMinMax.h index 913dcccb798..3e08e6b5768 100644 --- a/src/Storages/Statistics/StatisticsMinMax.h +++ b/src/Storages/Statistics/StatisticsMinMax.h @@ -1,3 +1,5 @@ +#pragma once + #include diff --git a/src/Storages/Statistics/StatisticsTDigest.cpp b/src/Storages/Statistics/StatisticsTDigest.cpp index 3544f5cdea3..25cee4ac8e8 100644 --- a/src/Storages/Statistics/StatisticsTDigest.cpp +++ b/src/Storages/Statistics/StatisticsTDigest.cpp @@ -9,7 +9,6 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_STATISTICS; -extern const int LOGICAL_ERROR; } StatisticsTDigest::StatisticsTDigest(const SingleStatisticsDescription & stat_, DataTypePtr data_type_) @@ -44,7 +43,7 @@ Float64 StatisticsTDigest::estimateLess(const Field & val) const Field val_converted = convertFieldToType(val, *data_type); if (val_converted.isNull()) return 0; - + auto val_as_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); return t_digest.getCountLessThan(val_as_float); } @@ -54,7 +53,7 @@ Float64 StatisticsTDigest::estimateEqual(const Field & val) const Field val_converted = convertFieldToType(val, *data_type); if (val_converted.isNull()) return 0; - + auto val_as_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); return t_digest.getCountEqual(val_as_float); } From 0f2024306deeb9b67b631391afc9d2c70d622ffc Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sat, 3 Aug 2024 17:03:27 +0800 Subject: [PATCH 092/289] optimize array() with single arguments and map() with double arguments --- src/Functions/array/array.cpp | 28 +++++++++++++++------ src/Functions/map.cpp | 47 +++++++++++++++++++++++------------ 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index 03b51808799..202aa83216b 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -46,8 +46,10 @@ public: size_t num_elements = arguments.size(); if (num_elements == 0) + { /// We should return constant empty array. return result_type->createColumnConstWithDefaultValue(input_rows_count); + } const DataTypePtr & elem_type = static_cast(*result_type).getNestedType(); @@ -59,7 +61,6 @@ public: Columns columns_holder(num_elements); ColumnRawPtrs column_ptrs(num_elements); - for (size_t i = 0; i < num_elements; ++i) { const auto & arg = arguments[i]; @@ -76,22 +77,33 @@ public: } /// Create and fill the result array. - auto out = ColumnArray::create(elem_type->createColumn()); IColumn & out_data = out->getData(); IColumn::Offsets & out_offsets = out->getOffsets(); out_data.reserve(input_rows_count * num_elements); - out_offsets.resize(input_rows_count); + out_offsets.resize_exact(input_rows_count); IColumn::Offset current_offset = 0; - for (size_t i = 0; i < input_rows_count; ++i) + if (num_elements == 1) { - for (size_t j = 0; j < num_elements; ++j) - out_data.insertFrom(*column_ptrs[j], i); + for (size_t i = 0; i < input_rows_count; ++i) + { + ++current_offset; + out_offsets[i] = current_offset; + } + out_data.insertManyFrom(*column_ptrs[0], 0, input_rows_count); + } + else + { + for (size_t i = 0; i < input_rows_count; ++i) + { + for (size_t j = 0; j < num_elements; ++j) + out_data.insertFrom(*column_ptrs[j], i); - current_offset += num_elements; - out_offsets[i] = current_offset; + current_offset += num_elements; + out_offsets[i] = current_offset; + } } return out; diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 66cd10a3f0b..b869a475af0 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -1,14 +1,15 @@ -#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include +#include #include +#include #include @@ -123,24 +124,38 @@ public: MutableColumnPtr keys_data = key_type->createColumn(); MutableColumnPtr values_data = value_type->createColumn(); - MutableColumnPtr offsets = DataTypeNumber().createColumn(); + MutableColumnPtr offsets = DataTypeUInt64().createColumn(); size_t total_elements = input_rows_count * num_elements / 2; keys_data->reserve(total_elements); values_data->reserve(total_elements); - offsets->reserve(input_rows_count); + auto & offsets_data = assert_cast(*offsets).getData(); + offsets_data.resize_exact(input_rows_count); IColumn::Offset current_offset = 0; - for (size_t i = 0; i < input_rows_count; ++i) + if (num_elements == 2) { - for (size_t j = 0; j < num_elements; j += 2) + for (size_t i = 0; i < input_rows_count; ++i) { - keys_data->insertFrom(*column_ptrs[j], i); - values_data->insertFrom(*column_ptrs[j + 1], i); + ++current_offset; + offsets_data[i] = current_offset; } + keys_data->insertManyFrom(*column_ptrs[0], 0, input_rows_count); + values_data->insertManyFrom(*column_ptrs[1], 0, input_rows_count); + } + else + { + for (size_t i = 0; i < input_rows_count; ++i) + { + for (size_t j = 0; j < num_elements; j += 2) + { + keys_data->insertFrom(*column_ptrs[j], i); + values_data->insertFrom(*column_ptrs[j + 1], i); + } - current_offset += num_elements / 2; - offsets->insert(current_offset); + current_offset += num_elements / 2; + offsets_data[i] = current_offset; + } } auto nested_column = ColumnArray::create( From e5d4eaa4580f5921a472174d805c852e8b057165 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sat, 3 Aug 2024 21:08:21 +0800 Subject: [PATCH 093/289] fix bugs --- src/Functions/array/array.cpp | 2 +- src/Functions/map.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index 202aa83216b..a15fe86a232 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -92,7 +92,7 @@ public: ++current_offset; out_offsets[i] = current_offset; } - out_data.insertManyFrom(*column_ptrs[0], 0, input_rows_count); + out_data.insertRangeFrom(*column_ptrs[0], 0, input_rows_count); } else { diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index b869a475af0..ebfa9d4eab4 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -140,8 +140,8 @@ public: ++current_offset; offsets_data[i] = current_offset; } - keys_data->insertManyFrom(*column_ptrs[0], 0, input_rows_count); - values_data->insertManyFrom(*column_ptrs[1], 0, input_rows_count); + keys_data->insertRangeFrom(*column_ptrs[0], 0, input_rows_count); + values_data->insertRangeFrom(*column_ptrs[1], 0, input_rows_count); } else { From a498259ec8c08d4f37f77c8a6870f519bc295801 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sun, 4 Aug 2024 13:32:17 +0800 Subject: [PATCH 094/289] optimize array with mutiple arguments --- src/Functions/array/array.cpp | 212 +++++++++++++++++++++++++++++----- 1 file changed, 184 insertions(+), 28 deletions(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index a15fe86a232..b25505e9ac1 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -1,10 +1,14 @@ -#include -#include +#include +#include +#include #include #include -#include -#include +#include +#include #include +#include +#include "Columns/ColumnFixedString.h" +#include "Columns/ColumnNullable.h" namespace DB @@ -43,7 +47,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { - size_t num_elements = arguments.size(); + const size_t num_elements = arguments.size(); if (num_elements == 0) { @@ -81,42 +85,194 @@ public: IColumn & out_data = out->getData(); IColumn::Offsets & out_offsets = out->getOffsets(); - out_data.reserve(input_rows_count * num_elements); + /// Fill out_offsets out_offsets.resize_exact(input_rows_count); - IColumn::Offset current_offset = 0; + for (size_t i = 0; i < input_rows_count; ++i) + { + current_offset += num_elements; + out_offsets[i] = current_offset; + } + + /// Fill out_data + out_data.reserve(input_rows_count * num_elements); if (num_elements == 1) - { - for (size_t i = 0; i < input_rows_count; ++i) - { - ++current_offset; - out_offsets[i] = current_offset; - } out_data.insertRangeFrom(*column_ptrs[0], 0, input_rows_count); - } else - { - for (size_t i = 0; i < input_rows_count; ++i) - { - for (size_t j = 0; j < num_elements; ++j) - out_data.insertFrom(*column_ptrs[j], i); - - current_offset += num_elements; - out_offsets[i] = current_offset; - } - } - + executeAny(column_ptrs, out_data, input_rows_count); return out; } - private: + bool executeAny(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + return false // NOLINT + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeString(columns, out_data, input_rows_count) + || executeNullable(columns, out_data, input_rows_count) || executeTuple(columns, out_data, input_rows_count) + || executeFixedString(columns, out_data, input_rows_count) || executeGeneric(columns, out_data, input_rows_count); + } + + bool executeGeneric(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + for (size_t i = 0; i < input_rows_count; ++i) + for (const auto * column : columns) + out_data.insertFrom(*column, i); + return true; + } + + template + bool executeNumber(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + using Container = ColumnVectorOrDecimal::Container; + std::vector containers(columns.size(), nullptr); + for (size_t i = 0; i < columns.size(); ++i) + { + const ColumnVectorOrDecimal * concrete_column = checkAndGetColumn>(columns[i]); + if (!concrete_column) + return false; + + containers[i] = &concrete_column->getData(); + } + + ColumnVectorOrDecimal & concrete_out_data = assert_cast &>(out_data); + Container & out_container = concrete_out_data.getData(); + out_container.resize_exact(columns.size() * input_rows_count); + + for (size_t row_i = 0; row_i < input_rows_count; ++row_i) + { + const size_t base = row_i * columns.size(); + for (size_t col_i = 0; col_i < columns.size(); ++col_i) + out_container[base + col_i] = (*containers[col_i])[row_i]; + } + return true; + } + + bool executeString(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + size_t total_bytes = 0; + std::vector concrete_columns(columns.size(), nullptr); + for (size_t i = 0; i < columns.size(); ++i) + { + const ColumnString * concrete_column = checkAndGetColumn(columns[i]); + if (!concrete_column) + return false; + + total_bytes += concrete_column->getChars().size(); + concrete_columns[i] = concrete_column; + } + + ColumnString & concreate_out_data = assert_cast(out_data); + auto & out_chars = concreate_out_data.getChars(); + auto & out_offsets = concreate_out_data.getOffsets(); + out_chars.resize_exact(total_bytes); + out_offsets.resize_exact(input_rows_count * columns.size()); + + size_t curr_out_offset = 0; + for (size_t row_i = 0; row_i < input_rows_count; ++row_i) + { + const size_t base = row_i * columns.size(); + for (size_t col_i = 0; col_i < columns.size(); ++col_i) + { + StringRef ref = concrete_columns[col_i]->getDataAt(row_i); + memcpySmallAllowReadWriteOverflow15(&out_chars[curr_out_offset], ref.data, ref.size); + out_chars[curr_out_offset + ref.size] = 0; + + curr_out_offset += ref.size + 1; + out_offsets[base + col_i] = curr_out_offset; + } + } + return true; + } + + bool executeFixedString(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + size_t total_bytes = 0; + std::vector concrete_columns(columns.size(), nullptr); + for (size_t i = 0; i < columns.size(); ++i) + { + const ColumnFixedString * concrete_column = checkAndGetColumn(columns[i]); + if (!concrete_column) + return false; + + total_bytes += concrete_column->getChars().size(); + concrete_columns[i] = concrete_column; + } + + ColumnFixedString & concreate_out_data = assert_cast(out_data); + auto & out_chars = concreate_out_data.getChars(); + out_chars.resize_exact(total_bytes); + + const size_t n = concreate_out_data.getN(); + size_t curr_out_offset = 0; + for (size_t row_i = 0; row_i < input_rows_count; ++row_i) + { + for (size_t col_i = 0; col_i < columns.size(); ++col_i) + { + StringRef ref = concrete_columns[col_i]->getDataAt(row_i); + memcpySmallAllowReadWriteOverflow15(&out_chars[curr_out_offset], ref.data, n); + curr_out_offset += n; + } + } + return true; + } + + bool executeNullable(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + ColumnRawPtrs null_maps(columns.size(), nullptr); + ColumnRawPtrs nested_columns(columns.size(), nullptr); + for (size_t i = 0; i < columns.size(); ++i) + { + const ColumnNullable * concrete_column = checkAndGetColumn(columns[i]); + if (!concrete_column) + return false; + + null_maps[i] = &concrete_column->getNullMapColumn(); + nested_columns[i] = &concrete_column->getNestedColumn(); + } + + ColumnNullable & concreate_out_data = assert_cast(out_data); + auto & out_null_map = concreate_out_data.getNullMapColumn(); + auto & out_nested_column = concreate_out_data.getNestedColumn(); + executeAny(null_maps, out_null_map, input_rows_count); + executeAny(nested_columns, out_nested_column, input_rows_count); + return true; + } + + bool executeTuple(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + ColumnTuple * concreate_out_data = typeid_cast(&out_data); + if (!concreate_out_data) + return false; + + const size_t tuple_size = concreate_out_data->tupleSize(); + for (size_t i = 0; i < tuple_size; ++i) + { + ColumnRawPtrs elem_columns(columns.size(), nullptr); + for (size_t j = 0; j < columns.size(); ++j) + { + const ColumnTuple * concrete_column = assert_cast(columns[j]); + elem_columns[j] = &concrete_column->getColumn(i); + } + executeAny(elem_columns, concreate_out_data->getColumn(i), input_rows_count); + } + return true; + } + String getName() const override { return name; } - bool addField(DataTypePtr type_res, const Field & f, Array & arr) const; - bool use_variant_as_common_type = false; }; From 00a9c30363e16aa1893320c35052ad551b803394 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sun, 4 Aug 2024 16:16:51 +0800 Subject: [PATCH 095/289] optimize map function --- src/Functions/map.cpp | 44 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index ebfa9d4eab4..994a03374c9 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -32,11 +32,18 @@ class FunctionMap : public IFunction public: static constexpr auto name = "map"; - explicit FunctionMap(bool use_variant_as_common_type_) : use_variant_as_common_type(use_variant_as_common_type_) {} + explicit FunctionMap(ContextPtr context_) + : context(std::move(context_)) + , use_variant_as_common_type( + context->getSettingsRef().allow_experimental_variant_type && context->getSettingsRef().use_variant_as_common_type) + , function_array(FunctionFactory::instance().get("array", context)) + , function_map_from_arrays(FunctionFactory::instance().get("mapFromArrays", context)) + { + } static FunctionPtr create(ContextPtr context) { - return std::make_shared(context->getSettingsRef().allow_experimental_variant_type && context->getSettingsRef().use_variant_as_common_type); + return std::make_shared(std::move(context)); } String getName() const override @@ -97,7 +104,36 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { size_t num_elements = arguments.size(); + if (num_elements == 0) + return result_type->createColumnConstWithDefaultValue(input_rows_count); + ColumnsWithTypeAndName key_args; + ColumnsWithTypeAndName value_args; + for (size_t i = 0; i < num_elements; i += 2) + { + key_args.emplace_back(arguments[i]); + value_args.emplace_back(arguments[i+1]); + } + + const auto & result_type_map = static_cast(*result_type); + const DataTypePtr & key_type = result_type_map.getKeyType(); + const DataTypePtr & value_type = result_type_map.getValueType(); + const DataTypePtr & key_array_type = std::make_shared(key_type); + const DataTypePtr & value_array_type = std::make_shared(value_type); + + /// key_array = array(args[0], args[2]...) + ColumnPtr key_array = function_array->build(key_args)->execute(key_args, key_array_type, input_rows_count); + /// value_array = array(args[1], args[3]...) + ColumnPtr value_array = function_array->build(value_args)->execute(value_args, value_array_type, input_rows_count); + + /// result = mapFromArrays(key_array, value_array) + ColumnsWithTypeAndName map_args{{key_array, key_array_type, ""}, {value_array, value_array_type, ""}}; + return function_map_from_arrays->build(map_args)->execute(map_args, result_type, input_rows_count); + } + /* + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + size_t num_elements = arguments.size(); if (num_elements == 0) return result_type->createColumnConstWithDefaultValue(input_rows_count); @@ -164,9 +200,13 @@ public: return ColumnMap::create(nested_column); } + */ private: + ContextPtr context; bool use_variant_as_common_type = false; + FunctionOverloadResolverPtr function_array; + FunctionOverloadResolverPtr function_map_from_arrays; }; /// mapFromArrays(keys, values) is a function that allows you to make key-value pair from a pair of arrays From 122c9794a65cb6d0579ec41f6fb3b4848e5c989b Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sun, 4 Aug 2024 17:04:04 +0800 Subject: [PATCH 096/289] remove useless codes --- src/Functions/map.cpp | 71 ------------------------------------------- 1 file changed, 71 deletions(-) diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 994a03374c9..4b1e41af796 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -130,77 +130,6 @@ public: ColumnsWithTypeAndName map_args{{key_array, key_array_type, ""}, {value_array, value_array_type, ""}}; return function_map_from_arrays->build(map_args)->execute(map_args, result_type, input_rows_count); } - /* - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override - { - size_t num_elements = arguments.size(); - if (num_elements == 0) - return result_type->createColumnConstWithDefaultValue(input_rows_count); - - const auto & result_type_map = static_cast(*result_type); - const DataTypePtr & key_type = result_type_map.getKeyType(); - const DataTypePtr & value_type = result_type_map.getValueType(); - - Columns columns_holder(num_elements); - ColumnRawPtrs column_ptrs(num_elements); - - for (size_t i = 0; i < num_elements; ++i) - { - const auto & arg = arguments[i]; - const auto to_type = i % 2 == 0 ? key_type : value_type; - - ColumnPtr preprocessed_column = castColumn(arg, to_type); - preprocessed_column = preprocessed_column->convertToFullColumnIfConst(); - - columns_holder[i] = std::move(preprocessed_column); - column_ptrs[i] = columns_holder[i].get(); - } - - /// Create and fill the result map. - - MutableColumnPtr keys_data = key_type->createColumn(); - MutableColumnPtr values_data = value_type->createColumn(); - MutableColumnPtr offsets = DataTypeUInt64().createColumn(); - - size_t total_elements = input_rows_count * num_elements / 2; - keys_data->reserve(total_elements); - values_data->reserve(total_elements); - auto & offsets_data = assert_cast(*offsets).getData(); - offsets_data.resize_exact(input_rows_count); - - IColumn::Offset current_offset = 0; - if (num_elements == 2) - { - for (size_t i = 0; i < input_rows_count; ++i) - { - ++current_offset; - offsets_data[i] = current_offset; - } - keys_data->insertRangeFrom(*column_ptrs[0], 0, input_rows_count); - values_data->insertRangeFrom(*column_ptrs[1], 0, input_rows_count); - } - else - { - for (size_t i = 0; i < input_rows_count; ++i) - { - for (size_t j = 0; j < num_elements; j += 2) - { - keys_data->insertFrom(*column_ptrs[j], i); - values_data->insertFrom(*column_ptrs[j + 1], i); - } - - current_offset += num_elements / 2; - offsets_data[i] = current_offset; - } - } - - auto nested_column = ColumnArray::create( - ColumnTuple::create(Columns{std::move(keys_data), std::move(values_data)}), - std::move(offsets)); - - return ColumnMap::create(nested_column); - } - */ private: ContextPtr context; From d1305d9fadf3ea2ad2393ee5f13a676110bf6ef7 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Mon, 5 Aug 2024 11:26:57 +0800 Subject: [PATCH 097/289] Some fixups and split tests --- .../mergetree-family/mergetree.md | 5 ++- src/Storages/Statistics/Statistics.cpp | 24 ++++++------ .../Statistics/StatisticsCountMinSketch.cpp | 4 +- .../Statistics/StatisticsCountMinSketch.h | 4 +- src/Storages/Statistics/StatisticsMinMax.cpp | 17 ++++----- src/Storages/Statistics/StatisticsMinMax.h | 11 +++--- src/Storages/Statistics/StatisticsTDigest.cpp | 4 +- src/Storages/Statistics/StatisticsTDigest.h | 4 +- src/Storages/Statistics/StatisticsUniq.cpp | 4 +- src/Storages/Statistics/StatisticsUniq.h | 4 +- src/Storages/StatisticsDescription.cpp | 8 ++-- ...atistics_create_materialize_drop.reference | 6 +++ ...864_statistics_create_materialize_drop.sql | 37 +++++++++++++++++++ .../02864_statistics_estimation.reference | 6 +-- .../02864_statistics_estimation.sql | 28 +++----------- 15 files changed, 94 insertions(+), 72 deletions(-) create mode 100644 tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference create mode 100644 tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 8e8d5e55772..42968097799 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -991,9 +991,10 @@ They can be used for prewhere optimization only if we enable `set allow_statisti #### Available Types of Column Statistics {#available-types-of-column-statistics} -- `min_max` +- `MinMax` + + MinMax statistics allows to estimate selectivity of range condition for numeric columns. - min_max statistics allows to estimate selectivity of range condition for numeric columns. - `TDigest` [TDigest](https://github.com/tdunning/t-digest) sketches which allow to compute approximate percentiles (e.g. the 90th percentile) for numeric columns. diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp index 17ff8deb720..42b742419f0 100644 --- a/src/Storages/Statistics/Statistics.cpp +++ b/src/Storages/Statistics/Statistics.cpp @@ -4,9 +4,9 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -99,11 +99,11 @@ Float64 ColumnStatistics::estimateEqual(const Field & val) const #endif if (stats.contains(StatisticsType::Uniq)) { - auto cardinality = stats.at(StatisticsType::Uniq)->estimateCardinality(); + UInt64 cardinality = stats.at(StatisticsType::Uniq)->estimateCardinality(); if (cardinality == 0) return 0; /// Assume that the value is uniformly distributed among the unique values. - return static_cast(1) / stats.at(StatisticsType::Uniq)->estimateCardinality() * rows; + return 1.0 / cardinality * rows; } return rows * ConditionSelectivityEstimator::default_cond_equal_factor; @@ -183,18 +183,18 @@ void MergeTreeStatisticsFactory::registerValidator(StatisticsType stats_type, Va MergeTreeStatisticsFactory::MergeTreeStatisticsFactory() { - registerValidator(StatisticsType::MinMax, minMaxValidator); - registerCreator(StatisticsType::MinMax, minMaxCreator); + registerValidator(StatisticsType::MinMax, minMaxStatisticsValidator); + registerCreator(StatisticsType::MinMax, minMaxStatisticsCreator); - registerValidator(StatisticsType::TDigest, tdigestValidator); - registerCreator(StatisticsType::TDigest, tdigestCreator); + registerValidator(StatisticsType::TDigest, tdigestStatisticsValidator); + registerCreator(StatisticsType::TDigest, tdigestStatisticsCreator); - registerValidator(StatisticsType::Uniq, uniqValidator); - registerCreator(StatisticsType::Uniq, uniqCreator); + registerValidator(StatisticsType::Uniq, uniqStatisticsValidator); + registerCreator(StatisticsType::Uniq, uniqStatisticsCreator); #if USE_DATASKETCHES - registerValidator(StatisticsType::CountMinSketch, countMinSketchValidator); - registerCreator(StatisticsType::CountMinSketch, countMinSketchCreator); + registerValidator(StatisticsType::CountMinSketch, countMinSketchStatisticsValidator); + registerCreator(StatisticsType::CountMinSketch, countMinSketchStatisticsCreator); #endif } @@ -222,7 +222,7 @@ ColumnStatisticsPtr MergeTreeStatisticsFactory::get(const ColumnStatisticsDescri { auto it = creators.find(type); if (it == creators.end()) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistic type '{}'. Available types: 'min_max', 'tdigest' 'uniq' and 'count_min'", type); + throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistic type '{}'. Available types: 'count_min', 'minmax', 'tdigest' and 'uniq'", type); auto stat_ptr = (it->second)(desc, stats.data_type); column_stat->stats[type] = stat_ptr; } diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.cpp b/src/Storages/Statistics/StatisticsCountMinSketch.cpp index e69bbc1515b..50d3b6e515c 100644 --- a/src/Storages/Statistics/StatisticsCountMinSketch.cpp +++ b/src/Storages/Statistics/StatisticsCountMinSketch.cpp @@ -84,7 +84,7 @@ void StatisticsCountMinSketch::deserialize(ReadBuffer & buf) } -void countMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type) +void countMinSketchStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type) { data_type = removeNullable(data_type); data_type = removeLowCardinalityAndNullable(data_type); @@ -92,7 +92,7 @@ void countMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr da throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'count_min' does not support type {}", data_type->getName()); } -StatisticsPtr countMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) +StatisticsPtr countMinSketchStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) { return std::make_shared(stat, data_type); } diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.h b/src/Storages/Statistics/StatisticsCountMinSketch.h index 6c8b74f8c35..d10bc78a88e 100644 --- a/src/Storages/Statistics/StatisticsCountMinSketch.h +++ b/src/Storages/Statistics/StatisticsCountMinSketch.h @@ -31,8 +31,8 @@ private: }; -void countMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type); -StatisticsPtr countMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr); +void countMinSketchStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type); +StatisticsPtr countMinSketchStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr); } diff --git a/src/Storages/Statistics/StatisticsMinMax.cpp b/src/Storages/Statistics/StatisticsMinMax.cpp index 083e8634841..f689f23d75d 100644 --- a/src/Storages/Statistics/StatisticsMinMax.cpp +++ b/src/Storages/Statistics/StatisticsMinMax.cpp @@ -19,9 +19,6 @@ extern const int ILLEGAL_STATISTICS; StatisticsMinMax::StatisticsMinMax(const SingleStatisticsDescription & stat_, const DataTypePtr & data_type_) : IStatistics(stat_) - , min(std::numeric_limits::max()) - , max(std::numeric_limits::min()) - , row_count(0) , data_type(data_type_) { } @@ -41,7 +38,7 @@ Float64 StatisticsMinMax::estimateLess(const Field & val) const return row_count; if (max == min) - return row_count; + return (val_as_float < max) ? 0 : row_count; return ((val_as_float - min) / (max - min)) * row_count; } @@ -53,9 +50,9 @@ void StatisticsMinMax::update(const ColumnPtr & column) if (column->isNullAt(row)) continue; - auto data = column->getFloat64(row); - min = std::min(data, min); - max = std::max(data, max); + auto value = column->getFloat64(row); + min = std::min(value, min); + max = std::max(value, max); } row_count += column->size(); } @@ -75,15 +72,15 @@ void StatisticsMinMax::deserialize(ReadBuffer & buf) } -void minMaxValidator(const SingleStatisticsDescription &, DataTypePtr data_type) +void minMaxStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type) { data_type = removeNullable(data_type); data_type = removeLowCardinalityAndNullable(data_type); if (!data_type->isValueRepresentedByNumber()) - throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'min_max' do not support type {}", data_type->getName()); + throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'minmax' do not support type {}", data_type->getName()); } -StatisticsPtr minMaxCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) +StatisticsPtr minMaxStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) { return std::make_shared(stat, data_type); } diff --git a/src/Storages/Statistics/StatisticsMinMax.h b/src/Storages/Statistics/StatisticsMinMax.h index 3e08e6b5768..4a67504350b 100644 --- a/src/Storages/Statistics/StatisticsMinMax.h +++ b/src/Storages/Statistics/StatisticsMinMax.h @@ -10,7 +10,6 @@ class StatisticsMinMax : public IStatistics { public: StatisticsMinMax(const SingleStatisticsDescription & stat_, const DataTypePtr & data_type_); - ~StatisticsMinMax() override = default; Float64 estimateLess(const Field & val) const override; @@ -20,14 +19,14 @@ public: void deserialize(ReadBuffer & buf) override; private: - Float64 min; - Float64 max; - Float64 row_count; + Float64 min = std::numeric_limits::max(); + Float64 max = std::numeric_limits::min(); + UInt64 row_count = 0; DataTypePtr data_type; }; -void minMaxValidator(const SingleStatisticsDescription &, DataTypePtr data_type); -StatisticsPtr minMaxCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type); +void minMaxStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type); +StatisticsPtr minMaxStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type); } diff --git a/src/Storages/Statistics/StatisticsTDigest.cpp b/src/Storages/Statistics/StatisticsTDigest.cpp index 25cee4ac8e8..d418a761e7c 100644 --- a/src/Storages/Statistics/StatisticsTDigest.cpp +++ b/src/Storages/Statistics/StatisticsTDigest.cpp @@ -58,7 +58,7 @@ Float64 StatisticsTDigest::estimateEqual(const Field & val) const return t_digest.getCountEqual(val_as_float); } -void tdigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type) +void tdigestStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type) { data_type = removeNullable(data_type); data_type = removeLowCardinalityAndNullable(data_type); @@ -66,7 +66,7 @@ void tdigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'tdigest' do not support type {}", data_type->getName()); } -StatisticsPtr tdigestCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) +StatisticsPtr tdigestStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) { return std::make_shared(stat, data_type); } diff --git a/src/Storages/Statistics/StatisticsTDigest.h b/src/Storages/Statistics/StatisticsTDigest.h index f4de6cc41e0..b83132c5e0d 100644 --- a/src/Storages/Statistics/StatisticsTDigest.h +++ b/src/Storages/Statistics/StatisticsTDigest.h @@ -24,7 +24,7 @@ private: DataTypePtr data_type; }; -void tdigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type); -StatisticsPtr tdigestCreator(const SingleStatisticsDescription & stat, DataTypePtr); +void tdigestStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type); +StatisticsPtr tdigestStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr); } diff --git a/src/Storages/Statistics/StatisticsUniq.cpp b/src/Storages/Statistics/StatisticsUniq.cpp index aace45a14b1..628ba98aba3 100644 --- a/src/Storages/Statistics/StatisticsUniq.cpp +++ b/src/Storages/Statistics/StatisticsUniq.cpp @@ -52,7 +52,7 @@ UInt64 StatisticsUniq::estimateCardinality() const return column->getUInt(0); } -void uniqValidator(const SingleStatisticsDescription &, DataTypePtr data_type) +void uniqStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type) { data_type = removeNullable(data_type); data_type = removeLowCardinalityAndNullable(data_type); @@ -60,7 +60,7 @@ void uniqValidator(const SingleStatisticsDescription &, DataTypePtr data_type) throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'uniq' do not support type {}", data_type->getName()); } -StatisticsPtr uniqCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) +StatisticsPtr uniqStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) { return std::make_shared(stat, data_type); } diff --git a/src/Storages/Statistics/StatisticsUniq.h b/src/Storages/Statistics/StatisticsUniq.h index 81162b0a5b9..219e2ffbc55 100644 --- a/src/Storages/Statistics/StatisticsUniq.h +++ b/src/Storages/Statistics/StatisticsUniq.h @@ -27,7 +27,7 @@ private: }; -void uniqValidator(const SingleStatisticsDescription &, DataTypePtr data_type); -StatisticsPtr uniqCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type); +void uniqStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type); +StatisticsPtr uniqStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type); } diff --git a/src/Storages/StatisticsDescription.cpp b/src/Storages/StatisticsDescription.cpp index c54752dd42f..f3ce9deb836 100644 --- a/src/Storages/StatisticsDescription.cpp +++ b/src/Storages/StatisticsDescription.cpp @@ -51,9 +51,9 @@ static StatisticsType stringToStatisticsType(String type) return StatisticsType::Uniq; if (type == "count_min") return StatisticsType::CountMinSketch; - if (type == "min_max") + if (type == "minmax") return StatisticsType::MinMax; - throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are 'min_max', 'tdigest', 'uniq' and 'count_min'.", type); + throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are 'count_min', 'minmax', 'tdigest' and 'uniq'.", type); } String SingleStatisticsDescription::getTypeName() const @@ -67,9 +67,9 @@ String SingleStatisticsDescription::getTypeName() const case StatisticsType::CountMinSketch: return "count_min"; case StatisticsType::MinMax: - return "min_max"; + return "minmax"; default: - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are 'min_max', 'tdigest', 'uniq' and 'count_min'.", type); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are 'count_min', 'minmax', 'tdigest' and 'uniq'.", type); } } diff --git a/tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference new file mode 100644 index 00000000000..5aff1ac63d4 --- /dev/null +++ b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference @@ -0,0 +1,6 @@ +Test create statistics: +CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64) STATISTICS(uniq, count_min, minmax),\n `b` Nullable(Int64),\n `c` LowCardinality(Nullable(Int64)) STATISTICS(count_min, minmax),\n `d` DateTime STATISTICS(tdigest, uniq, count_min, minmax),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 +tab +Test materialize and drop statistics: +CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64) STATISTICS(uniq, count_min, minmax),\n `b` Nullable(Int64) STATISTICS(tdigest, uniq, count_min, minmax),\n `c` LowCardinality(Nullable(Int64)) STATISTICS(count_min, minmax),\n `d` DateTime STATISTICS(tdigest, uniq, count_min, minmax),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 +CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64),\n `b` Nullable(Int64),\n `c` LowCardinality(Nullable(Int64)),\n `d` DateTime,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql new file mode 100644 index 00000000000..442079ab947 --- /dev/null +++ b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql @@ -0,0 +1,37 @@ +-- Tags: no-fasttest + +DROP TABLE IF EXISTS tab SYNC; + +SET allow_experimental_statistics = 1; +SET allow_statistics_optimize = 1; +SET allow_suspicious_low_cardinality_types=1; +SET mutations_sync = 2; + + +SELECT 'Test create statistics:'; + +CREATE TABLE tab +( + a LowCardinality(Int64) STATISTICS(count_min, minmax, uniq), + b Nullable(Int64), + c LowCardinality(Nullable(Int64)) STATISTICS(minmax, count_min), + d DateTime STATISTICS(count_min, minmax, tdigest, uniq), + pk String, +) Engine = MergeTree() ORDER BY pk; + +SHOW CREATE TABLE tab; + +SELECT name FROM system.tables WHERE name = 'tab' AND database = currentDatabase(); +INSERT INTO tab select number, number, number, toDateTime(number, 'UTC'), generateUUIDv4() FROM system.numbers LIMIT 10000; + + +SELECT 'Test materialize and drop statistics:'; + +ALTER TABLE tab ADD STATISTICS b TYPE count_min, minmax, tdigest, uniq; +ALTER TABLE tab MATERIALIZE STATISTICS b; +SHOW CREATE TABLE tab; + +ALTER TABLE tab DROP STATISTICS a, b, c, d; +SHOW CREATE TABLE tab; + +DROP TABLE IF EXISTS tab SYNC; diff --git a/tests/queries/0_stateless/02864_statistics_estimation.reference b/tests/queries/0_stateless/02864_statistics_estimation.reference index 64f264017d8..a5225c6338f 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.reference +++ b/tests/queries/0_stateless/02864_statistics_estimation.reference @@ -1,5 +1,5 @@ -CREATE TABLE default.tab\n(\n `a` String,\n `b` UInt64,\n `c` Int64,\n `d` DateTime64(3),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 -Test statistics min_max: +CREATE TABLE default.tab\n(\n `a` String,\n `b` UInt64,\n `c` Int64,\n `d` DateTime,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 +Test statistics minmax: Prewhere info Prewhere filter Prewhere filter column: and(greater(d, _CAST(9998_DateTime(\'UTC\'), \'DateTime(\'UTC\')\')), less(c, -1), greater(b, 0)) (removed) @@ -28,5 +28,3 @@ Test estimating equals condition: Prewhere info Prewhere filter Prewhere filter column: and(equals(a, \'0\'), equals(b, 10)) (removed) -Test LowCardinality and Nullable data type: -tab2 diff --git a/tests/queries/0_stateless/02864_statistics_estimation.sql b/tests/queries/0_stateless/02864_statistics_estimation.sql index 0ea9eb38e60..117e24bd732 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.sql +++ b/tests/queries/0_stateless/02864_statistics_estimation.sql @@ -12,7 +12,7 @@ CREATE TABLE tab a String, b UInt64, c Int64, - d DateTime64, + d DateTime, pk String, ) Engine = MergeTree() ORDER BY pk SETTINGS min_bytes_for_wide_part = 0; @@ -21,11 +21,11 @@ SHOW CREATE TABLE tab; INSERT INTO tab select toString(number % 10000), number % 1000, -(number % 100), toDateTime(number, 'UTC'), generateUUIDv4() FROM system.numbers LIMIT 10000; -SELECT 'Test statistics min_max:'; +SELECT 'Test statistics minmax:'; -ALTER TABLE tab ADD STATISTICS b TYPE min_max; -ALTER TABLE tab ADD STATISTICS c TYPE min_max; -ALTER TABLE tab ADD STATISTICS d TYPE min_max; +ALTER TABLE tab ADD STATISTICS b TYPE minmax; +ALTER TABLE tab ADD STATISTICS c TYPE minmax; +ALTER TABLE tab ADD STATISTICS d TYPE minmax; ALTER TABLE tab MATERIALIZE STATISTICS b, c, d; SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') @@ -70,7 +70,7 @@ ALTER TABLE tab DROP STATISTICS a, b, c, d; SELECT 'Test estimating range condition:'; -ALTER TABLE tab ADD STATISTICS b TYPE min_max; +ALTER TABLE tab ADD STATISTICS b TYPE minmax; ALTER TABLE tab MATERIALIZE STATISTICS b; SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*5000*/ and b < 10/*100*/) @@ -100,19 +100,3 @@ WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; ALTER TABLE tab DROP STATISTICS a; DROP TABLE IF EXISTS tab SYNC; - - -SELECT 'Test LowCardinality and Nullable data type:'; -DROP TABLE IF EXISTS tab2 SYNC; -SET allow_suspicious_low_cardinality_types=1; -CREATE TABLE tab2 -( - a LowCardinality(Int64) STATISTICS(count_min), - b Nullable(Int64) STATISTICS(min_max, count_min), - c LowCardinality(Nullable(Int64)) STATISTICS(min_max, count_min), - pk String, -) Engine = MergeTree() ORDER BY pk; - -select name from system.tables where name = 'tab2' and database = currentDatabase(); - -DROP TABLE IF EXISTS tab2 SYNC; From 045551f5098e36822c69cc2bb5852d7d16cfe41b Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Mon, 5 Aug 2024 11:33:58 +0800 Subject: [PATCH 098/289] fix bugs --- src/Functions/map.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index f49d66c0329..d9643087b11 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -157,6 +157,7 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { From 7765ff6d5227da58eecc426287c98c333c75402d Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 5 Aug 2024 06:46:04 +0000 Subject: [PATCH 099/289] Minor fixups --- .../mergetree-family/mergetree.md | 2 +- src/Storages/Statistics/Statistics.cpp | 22 ++++---- src/Storages/Statistics/Statistics.h | 12 ++-- .../Statistics/StatisticsCountMinSketch.cpp | 52 +++++++++--------- .../Statistics/StatisticsCountMinSketch.h | 11 ++-- src/Storages/Statistics/StatisticsMinMax.cpp | 55 +++++++++---------- src/Storages/Statistics/StatisticsMinMax.h | 11 ++-- src/Storages/Statistics/StatisticsTDigest.cpp | 11 ++-- src/Storages/Statistics/StatisticsTDigest.h | 13 +++-- src/Storages/Statistics/StatisticsUniq.cpp | 10 ++-- src/Storages/Statistics/StatisticsUniq.h | 12 ++-- 11 files changed, 107 insertions(+), 104 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 42968097799..c2438ac593e 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -993,7 +993,7 @@ They can be used for prewhere optimization only if we enable `set allow_statisti - `MinMax` - MinMax statistics allows to estimate selectivity of range condition for numeric columns. + The minimum and maximum column value which allows to estimate the selectivity of range filters on numeric columns. - `TDigest` diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp index 42b742419f0..1f159211a93 100644 --- a/src/Storages/Statistics/Statistics.cpp +++ b/src/Storages/Statistics/Statistics.cpp @@ -62,14 +62,17 @@ Float64 IStatistics::estimateLess(const Field & /*val*/) const } /// ------------------------------------- -/// Implementation of the estimation: -/// Note: Each statistics object supports certain types predicates natively, e.g. -/// - TDigest: '< X' (less-than predicates) -/// - Count-min sketches: '= X' (equal predicates) -/// - Uniq (HyperLogLog): 'count distinct(*)' (column cardinality) -/// If multiple statistics objects are available per column, it is sometimes also possible to combine them in a clever way. -/// For that reason, all estimation are performed in a central place (here), and we don't simply pass the predicate to the first statistics -/// object that supports it natively. +/// Notes: +/// - Statistics object usually only support estimation for certain types of predicates, e.g. +/// - TDigest: '< X' (less-than predicates) +/// - Count-min sketches: '= X' (equal predicates) +/// - Uniq (HyperLogLog): 'count distinct(*)' (column cardinality) +/// +/// If multiple statistics objects in a column support estimating a predicate, we want to try statistics in order of descending accuracy +/// (e.g. MinMax statistics are simpler than TDigest statistics and thus worse for estimating 'less' predicates). +/// +/// Sometimes, it is possible to combine multiple statistics in a clever way. For that reason, all estimation are performed in a central +/// place (here), and we don't simply pass the predicate to the first statistics object that supports it natively. Float64 ColumnStatistics::estimateLess(const Field & val) const { @@ -102,8 +105,7 @@ Float64 ColumnStatistics::estimateEqual(const Field & val) const UInt64 cardinality = stats.at(StatisticsType::Uniq)->estimateCardinality(); if (cardinality == 0) return 0; - /// Assume that the value is uniformly distributed among the unique values. - return 1.0 / cardinality * rows; + return 1.0 / cardinality * rows; /// assume uniform distribution } return rows * ConditionSelectivityEstimator::default_cond_equal_factor; diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h index 3eeb5cf92a8..1425be89d47 100644 --- a/src/Storages/Statistics/Statistics.h +++ b/src/Storages/Statistics/Statistics.h @@ -25,20 +25,20 @@ public: explicit IStatistics(const SingleStatisticsDescription & stat_); virtual ~IStatistics() = default; + virtual void update(const ColumnPtr & column) = 0; + + virtual void serialize(WriteBuffer & buf) = 0; + virtual void deserialize(ReadBuffer & buf) = 0; + /// Estimate the cardinality of the column. /// Throws if the statistics object is not able to do a meaningful estimation. virtual UInt64 estimateCardinality() const; /// Per-value estimations. - /// Throws if the statistics object is not able to do a meaningful estimation. + /// Throws a LOGICAL_ERROR if the statistics object is not able to do a meaningful estimation. virtual Float64 estimateEqual(const Field & val) const; /// cardinality of val in the column virtual Float64 estimateLess(const Field & val) const; /// summarized cardinality of values < val in the column - virtual void update(const ColumnPtr & column) = 0; - - virtual void serialize(WriteBuffer & buf) = 0; - virtual void deserialize(ReadBuffer & buf) = 0; - protected: SingleStatisticsDescription stat; }; diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.cpp b/src/Storages/Statistics/StatisticsCountMinSketch.cpp index 50d3b6e515c..c0ea7a84ded 100644 --- a/src/Storages/Statistics/StatisticsCountMinSketch.cpp +++ b/src/Storages/Statistics/StatisticsCountMinSketch.cpp @@ -1,4 +1,3 @@ - #include #include #include @@ -25,34 +24,13 @@ extern const int ILLEGAL_STATISTICS; static constexpr auto num_hashes = 7uz; static constexpr auto num_buckets = 2718uz; -StatisticsCountMinSketch::StatisticsCountMinSketch(const SingleStatisticsDescription & stat_, DataTypePtr data_type_) - : IStatistics(stat_) +StatisticsCountMinSketch::StatisticsCountMinSketch(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type_) + : IStatistics(statistics_description) , sketch(num_hashes, num_buckets) , data_type(data_type_) { } -Float64 StatisticsCountMinSketch::estimateEqual(const Field & val) const -{ - /// Try to convert field to data_type. Converting string to proper data types such as: number, date, datetime, IPv4, Decimal etc. - /// Return null if val larger than the range of data_type - /// - /// For example: if data_type is Int32: - /// 1. For 1.0, 1, '1', return Field(1) - /// 2. For 1.1, max_value_int64, return null - Field val_converted = convertFieldToType(val, *data_type); - if (val_converted.isNull()) - return 0; - - if (data_type->isValueRepresentedByNumber()) - return sketch.get_estimate(&val_converted, data_type->getSizeOfValueInMemory()); - - if (isStringOrFixedString(data_type)) - return sketch.get_estimate(val.get()); - - throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'count_min' does not support estimate data type of {}", data_type->getName()); -} - void StatisticsCountMinSketch::update(const ColumnPtr & column) { for (size_t row = 0; row < column->size(); ++row) @@ -83,8 +61,28 @@ void StatisticsCountMinSketch::deserialize(ReadBuffer & buf) sketch = Sketch::deserialize(bytes.data(), size); } +Float64 StatisticsCountMinSketch::estimateEqual(const Field & val) const +{ + /// Try to convert field to data_type. Converting string to proper data types such as: number, date, datetime, IPv4, Decimal etc. + /// Return null if val larger than the range of data_type + /// + /// For example: if data_type is Int32: + /// 1. For 1.0, 1, '1', return Field(1) + /// 2. For 1.1, max_value_int64, return null + Field val_converted = convertFieldToType(val, *data_type); + if (val_converted.isNull()) + return 0; -void countMinSketchStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type) + if (data_type->isValueRepresentedByNumber()) + return sketch.get_estimate(&val_converted, data_type->getSizeOfValueInMemory()); + + if (isStringOrFixedString(data_type)) + return sketch.get_estimate(val.get()); + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'count_min' does not support estimate data type of {}", data_type->getName()); +} + +void countMinSketchStatisticsValidator(const SingleStatisticsDescription & /*statistics_description*/, DataTypePtr data_type) { data_type = removeNullable(data_type); data_type = removeLowCardinalityAndNullable(data_type); @@ -92,9 +90,9 @@ void countMinSketchStatisticsValidator(const SingleStatisticsDescription &, Data throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'count_min' does not support type {}", data_type->getName()); } -StatisticsPtr countMinSketchStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) +StatisticsPtr countMinSketchStatisticsCreator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type) { - return std::make_shared(stat, data_type); + return std::make_shared(statistics_description, data_type); } } diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.h b/src/Storages/Statistics/StatisticsCountMinSketch.h index d10bc78a88e..452483c8773 100644 --- a/src/Storages/Statistics/StatisticsCountMinSketch.h +++ b/src/Storages/Statistics/StatisticsCountMinSketch.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "config.h" @@ -14,15 +15,15 @@ namespace DB class StatisticsCountMinSketch : public IStatistics { public: - StatisticsCountMinSketch(const SingleStatisticsDescription & stat_, DataTypePtr data_type_); - - Float64 estimateEqual(const Field & val) const override; + StatisticsCountMinSketch(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type_); void update(const ColumnPtr & column) override; void serialize(WriteBuffer & buf) override; void deserialize(ReadBuffer & buf) override; + Float64 estimateEqual(const Field & val) const override; + private: using Sketch = datasketches::count_min_sketch; Sketch sketch; @@ -31,8 +32,8 @@ private: }; -void countMinSketchStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type); -StatisticsPtr countMinSketchStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr); +void countMinSketchStatisticsValidator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type); +StatisticsPtr countMinSketchStatisticsCreator(const SingleStatisticsDescription & statistics_description, DataTypePtr); } diff --git a/src/Storages/Statistics/StatisticsMinMax.cpp b/src/Storages/Statistics/StatisticsMinMax.cpp index f689f23d75d..11a0704dae7 100644 --- a/src/Storages/Statistics/StatisticsMinMax.cpp +++ b/src/Storages/Statistics/StatisticsMinMax.cpp @@ -1,10 +1,10 @@ #include -#include -#include -#include #include +#include +#include #include #include +#include #include @@ -17,32 +17,12 @@ namespace ErrorCodes extern const int ILLEGAL_STATISTICS; } -StatisticsMinMax::StatisticsMinMax(const SingleStatisticsDescription & stat_, const DataTypePtr & data_type_) - : IStatistics(stat_) +StatisticsMinMax::StatisticsMinMax(const SingleStatisticsDescription & statistics_description, const DataTypePtr & data_type_) + : IStatistics(statistics_description) , data_type(data_type_) { } -Float64 StatisticsMinMax::estimateLess(const Field & val) const -{ - Field val_converted = convertFieldToType(val, *data_type); - if (val_converted.isNull()) - return 0; - - auto val_as_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); - - if (val_as_float < min) - return 0; - - if (val_as_float > max) - return row_count; - - if (max == min) - return (val_as_float < max) ? 0 : row_count; - - return ((val_as_float - min) / (max - min)) * row_count; -} - void StatisticsMinMax::update(const ColumnPtr & column) { for (size_t row = 0; row < column->size(); ++row) @@ -71,8 +51,27 @@ void StatisticsMinMax::deserialize(ReadBuffer & buf) readFloatBinary(max, buf); } +Float64 StatisticsMinMax::estimateLess(const Field & val) const +{ + Field val_converted = convertFieldToType(val, *data_type); + if (val_converted.isNull()) + return 0; -void minMaxStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type) + auto val_as_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); + + if (val_as_float < min) + return 0; + + if (val_as_float > max) + return row_count; + + if (min == max) + return (val_as_float != max) ? 0 : row_count; + + return ((val_as_float - min) / (max - min)) * row_count; +} + +void minMaxStatisticsValidator(const SingleStatisticsDescription & /*statistics_description*/, DataTypePtr data_type) { data_type = removeNullable(data_type); data_type = removeLowCardinalityAndNullable(data_type); @@ -80,9 +79,9 @@ void minMaxStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'minmax' do not support type {}", data_type->getName()); } -StatisticsPtr minMaxStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) +StatisticsPtr minMaxStatisticsCreator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type) { - return std::make_shared(stat, data_type); + return std::make_shared(statistics_description, data_type); } } diff --git a/src/Storages/Statistics/StatisticsMinMax.h b/src/Storages/Statistics/StatisticsMinMax.h index 4a67504350b..524494b520b 100644 --- a/src/Storages/Statistics/StatisticsMinMax.h +++ b/src/Storages/Statistics/StatisticsMinMax.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -9,15 +10,15 @@ namespace DB class StatisticsMinMax : public IStatistics { public: - StatisticsMinMax(const SingleStatisticsDescription & stat_, const DataTypePtr & data_type_); - - Float64 estimateLess(const Field & val) const override; + StatisticsMinMax(const SingleStatisticsDescription & statistics_description, const DataTypePtr & data_type_); void update(const ColumnPtr & column) override; void serialize(WriteBuffer & buf) override; void deserialize(ReadBuffer & buf) override; + Float64 estimateLess(const Field & val) const override; + private: Float64 min = std::numeric_limits::max(); Float64 max = std::numeric_limits::min(); @@ -26,7 +27,7 @@ private: DataTypePtr data_type; }; -void minMaxStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type); -StatisticsPtr minMaxStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type); +void minMaxStatisticsValidator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type); +StatisticsPtr minMaxStatisticsCreator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type); } diff --git a/src/Storages/Statistics/StatisticsTDigest.cpp b/src/Storages/Statistics/StatisticsTDigest.cpp index d418a761e7c..cfd626c8ea8 100644 --- a/src/Storages/Statistics/StatisticsTDigest.cpp +++ b/src/Storages/Statistics/StatisticsTDigest.cpp @@ -11,8 +11,9 @@ namespace ErrorCodes extern const int ILLEGAL_STATISTICS; } -StatisticsTDigest::StatisticsTDigest(const SingleStatisticsDescription & stat_, DataTypePtr data_type_) - : IStatistics(stat_), data_type(data_type_) +StatisticsTDigest::StatisticsTDigest(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type_) + : IStatistics(statistics_description) + , data_type(data_type_) { } @@ -58,7 +59,7 @@ Float64 StatisticsTDigest::estimateEqual(const Field & val) const return t_digest.getCountEqual(val_as_float); } -void tdigestStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type) +void tdigestStatisticsValidator(const SingleStatisticsDescription & /*statistics_description*/, DataTypePtr data_type) { data_type = removeNullable(data_type); data_type = removeLowCardinalityAndNullable(data_type); @@ -66,9 +67,9 @@ void tdigestStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'tdigest' do not support type {}", data_type->getName()); } -StatisticsPtr tdigestStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) +StatisticsPtr tdigestStatisticsCreator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type) { - return std::make_shared(stat, data_type); + return std::make_shared(statistics_description, data_type); } } diff --git a/src/Storages/Statistics/StatisticsTDigest.h b/src/Storages/Statistics/StatisticsTDigest.h index b83132c5e0d..ad2a03836aa 100644 --- a/src/Storages/Statistics/StatisticsTDigest.h +++ b/src/Storages/Statistics/StatisticsTDigest.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB { @@ -9,22 +10,22 @@ namespace DB class StatisticsTDigest : public IStatistics { public: - explicit StatisticsTDigest(const SingleStatisticsDescription & stat_, DataTypePtr data_type_); - - Float64 estimateLess(const Field & val) const override; - Float64 estimateEqual(const Field & val) const override; + explicit StatisticsTDigest(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type_); void update(const ColumnPtr & column) override; void serialize(WriteBuffer & buf) override; void deserialize(ReadBuffer & buf) override; + Float64 estimateLess(const Field & val) const override; + Float64 estimateEqual(const Field & val) const override; + private: QuantileTDigest t_digest; DataTypePtr data_type; }; -void tdigestStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type); -StatisticsPtr tdigestStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr); +void tdigestStatisticsValidator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type); +StatisticsPtr tdigestStatisticsCreator(const SingleStatisticsDescription & statistics_description, DataTypePtr); } diff --git a/src/Storages/Statistics/StatisticsUniq.cpp b/src/Storages/Statistics/StatisticsUniq.cpp index 628ba98aba3..e895157635f 100644 --- a/src/Storages/Statistics/StatisticsUniq.cpp +++ b/src/Storages/Statistics/StatisticsUniq.cpp @@ -11,8 +11,8 @@ namespace ErrorCodes extern const int ILLEGAL_STATISTICS; } -StatisticsUniq::StatisticsUniq(const SingleStatisticsDescription & stat_, const DataTypePtr & data_type) - : IStatistics(stat_) +StatisticsUniq::StatisticsUniq(const SingleStatisticsDescription & statistics_description, const DataTypePtr & data_type) + : IStatistics(statistics_description) { arena = std::make_unique(); AggregateFunctionProperties properties; @@ -52,7 +52,7 @@ UInt64 StatisticsUniq::estimateCardinality() const return column->getUInt(0); } -void uniqStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type) +void uniqStatisticsValidator(const SingleStatisticsDescription & /*statistics_description*/, DataTypePtr data_type) { data_type = removeNullable(data_type); data_type = removeLowCardinalityAndNullable(data_type); @@ -60,9 +60,9 @@ void uniqStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr da throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'uniq' do not support type {}", data_type->getName()); } -StatisticsPtr uniqStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type) +StatisticsPtr uniqStatisticsCreator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type) { - return std::make_shared(stat, data_type); + return std::make_shared(statistics_description, data_type); } } diff --git a/src/Storages/Statistics/StatisticsUniq.h b/src/Storages/Statistics/StatisticsUniq.h index 219e2ffbc55..b3000e4895a 100644 --- a/src/Storages/Statistics/StatisticsUniq.h +++ b/src/Storages/Statistics/StatisticsUniq.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -10,24 +11,23 @@ namespace DB class StatisticsUniq : public IStatistics { public: - StatisticsUniq(const SingleStatisticsDescription & stat_, const DataTypePtr & data_type); + StatisticsUniq(const SingleStatisticsDescription & statistics_description, const DataTypePtr & data_type); ~StatisticsUniq() override; - UInt64 estimateCardinality() const override; - void update(const ColumnPtr & column) override; void serialize(WriteBuffer & buf) override; void deserialize(ReadBuffer & buf) override; + UInt64 estimateCardinality() const override; + private: std::unique_ptr arena; AggregateFunctionPtr collector; AggregateDataPtr data; - }; -void uniqStatisticsValidator(const SingleStatisticsDescription &, DataTypePtr data_type); -StatisticsPtr uniqStatisticsCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type); +void uniqStatisticsValidator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type); +StatisticsPtr uniqStatisticsCreator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type); } From c61225ede47e37bd773bd249804739f915feefd7 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Mon, 5 Aug 2024 18:37:36 +0800 Subject: [PATCH 100/289] Rewrite tests --- ...atistics_create_materialize_drop.reference | 7 +- ...864_statistics_create_materialize_drop.sql | 16 ++-- .../02864_statistics_estimation.reference | 34 +++---- .../02864_statistics_estimation.sql | 92 ++++++++----------- 4 files changed, 62 insertions(+), 87 deletions(-) diff --git a/tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference index 5aff1ac63d4..4ca2c5e5f9b 100644 --- a/tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference +++ b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference @@ -1,6 +1,5 @@ Test create statistics: -CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64) STATISTICS(uniq, count_min, minmax),\n `b` Nullable(Int64),\n `c` LowCardinality(Nullable(Int64)) STATISTICS(count_min, minmax),\n `d` DateTime STATISTICS(tdigest, uniq, count_min, minmax),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 -tab +CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64) STATISTICS(tdigest, uniq, count_min, minmax),\n `b` LowCardinality(Nullable(String)) STATISTICS(uniq, count_min),\n `c` LowCardinality(Nullable(Int64)) STATISTICS(tdigest, uniq, count_min, minmax),\n `d` DateTime STATISTICS(tdigest, uniq, count_min, minmax),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 Test materialize and drop statistics: -CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64) STATISTICS(uniq, count_min, minmax),\n `b` Nullable(Int64) STATISTICS(tdigest, uniq, count_min, minmax),\n `c` LowCardinality(Nullable(Int64)) STATISTICS(count_min, minmax),\n `d` DateTime STATISTICS(tdigest, uniq, count_min, minmax),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 -CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64),\n `b` Nullable(Int64),\n `c` LowCardinality(Nullable(Int64)),\n `d` DateTime,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 +CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64),\n `b` LowCardinality(Nullable(String)) STATISTICS(uniq, count_min),\n `c` LowCardinality(Nullable(Int64)),\n `d` DateTime,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 +CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64),\n `b` LowCardinality(Nullable(String)),\n `c` LowCardinality(Nullable(Int64)),\n `d` DateTime,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql index 442079ab947..90a57c99624 100644 --- a/tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql +++ b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql @@ -12,26 +12,24 @@ SELECT 'Test create statistics:'; CREATE TABLE tab ( - a LowCardinality(Int64) STATISTICS(count_min, minmax, uniq), - b Nullable(Int64), - c LowCardinality(Nullable(Int64)) STATISTICS(minmax, count_min), + a LowCardinality(Int64) STATISTICS(count_min, minmax, tdigest, uniq), + b LowCardinality(Nullable(String)) STATISTICS(count_min, uniq), + c LowCardinality(Nullable(Int64)) STATISTICS(count_min, minmax, tdigest, uniq), d DateTime STATISTICS(count_min, minmax, tdigest, uniq), pk String, ) Engine = MergeTree() ORDER BY pk; +INSERT INTO tab select number, number, number, toDateTime(number), generateUUIDv4() FROM system.numbers LIMIT 10000; SHOW CREATE TABLE tab; -SELECT name FROM system.tables WHERE name = 'tab' AND database = currentDatabase(); -INSERT INTO tab select number, number, number, toDateTime(number, 'UTC'), generateUUIDv4() FROM system.numbers LIMIT 10000; - SELECT 'Test materialize and drop statistics:'; - -ALTER TABLE tab ADD STATISTICS b TYPE count_min, minmax, tdigest, uniq; +ALTER TABLE tab DROP STATISTICS a, b, c, d; +ALTER TABLE tab ADD STATISTICS b TYPE count_min, uniq; ALTER TABLE tab MATERIALIZE STATISTICS b; SHOW CREATE TABLE tab; -ALTER TABLE tab DROP STATISTICS a, b, c, d; +ALTER TABLE tab DROP STATISTICS b; SHOW CREATE TABLE tab; DROP TABLE IF EXISTS tab SYNC; diff --git a/tests/queries/0_stateless/02864_statistics_estimation.reference b/tests/queries/0_stateless/02864_statistics_estimation.reference index a5225c6338f..f59f94013da 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.reference +++ b/tests/queries/0_stateless/02864_statistics_estimation.reference @@ -1,30 +1,24 @@ CREATE TABLE default.tab\n(\n `a` String,\n `b` UInt64,\n `c` Int64,\n `d` DateTime,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 -Test statistics minmax: - Prewhere info - Prewhere filter - Prewhere filter column: and(greater(d, _CAST(9998_DateTime(\'UTC\'), \'DateTime(\'UTC\')\')), less(c, -1), greater(b, 0)) (removed) Test statistics count_min: Prewhere info Prewhere filter Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed) +Test statistics minmax: + Prewhere info + Prewhere filter + Prewhere filter column: and(greater(d, _CAST(9998, \'DateTime\')), less(c, -1), greater(b, 0)) (removed) +Test statistics tdigest: + Prewhere info + Prewhere filter + Prewhere filter column: and(greater(d, _CAST(9998, \'DateTime\')), less(c, -1), greater(b, 0)) (removed) +Test statistics uniq: + Prewhere info + Prewhere filter + Prewhere filter column: and(equals(a, \'0\'), equals(d, _CAST(1, \'DateTime\')), equals(b, 0), equals(c, 0)) (removed) Test statistics multi-types: Prewhere info Prewhere filter - Prewhere filter column: and(equals(a, \'0\'), less(c, -90), greater(b, 900)) (removed) + Prewhere filter column: and(equals(a, \'0\'), equals(d, _CAST(1, \'DateTime\')), less(c, -90), greater(b, 900)) (removed) Prewhere info Prewhere filter - Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0)) (removed) -Test estimating range condition: - Prewhere info - Prewhere filter - Prewhere filter column: and(less(b, 10), less(c, 0)) (removed) - Prewhere info - Prewhere filter - Prewhere filter column: and(less(b, 10), less(c, 0)) (removed) -Test estimating equals condition: - Prewhere info - Prewhere filter - Prewhere filter column: and(equals(a, \'0\'), equals(b, 10)) (removed) - Prewhere info - Prewhere filter - Prewhere filter column: and(equals(a, \'0\'), equals(b, 10)) (removed) + Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0), greater(d, _CAST(1, \'DateTime\'))) (removed) diff --git a/tests/queries/0_stateless/02864_statistics_estimation.sql b/tests/queries/0_stateless/02864_statistics_estimation.sql index 117e24bd732..65b111c285b 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.sql +++ b/tests/queries/0_stateless/02864_statistics_estimation.sql @@ -19,84 +19,68 @@ SETTINGS min_bytes_for_wide_part = 0; SHOW CREATE TABLE tab; -INSERT INTO tab select toString(number % 10000), number % 1000, -(number % 100), toDateTime(number, 'UTC'), generateUUIDv4() FROM system.numbers LIMIT 10000; +INSERT INTO tab select toString(number % 10000), number % 1000, -(number % 100), cast(number, 'DateTime'), generateUUIDv4() FROM system.numbers LIMIT 10000; + + +SELECT 'Test statistics count_min:'; + +ALTER TABLE tab ADD STATISTICS a, b, c TYPE count_min; +ALTER TABLE tab MATERIALIZE STATISTICS a, b, c; +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +ALTER TABLE tab DROP STATISTICS a, b, c; + SELECT 'Test statistics minmax:'; -ALTER TABLE tab ADD STATISTICS b TYPE minmax; -ALTER TABLE tab ADD STATISTICS c TYPE minmax; -ALTER TABLE tab ADD STATISTICS d TYPE minmax; +ALTER TABLE tab ADD STATISTICS b, c, d TYPE minmax; ALTER TABLE tab MATERIALIZE STATISTICS b, c, d; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b > 0/*10000*/ and c < -1/*9990*/ and d > toDateTime(9998, 'UTC')/*1*/) +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b > 0/*10000*/ and c < -1/*9990*/ and d > cast(9998, 'DateTime')/*1*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; ALTER TABLE tab DROP STATISTICS b, c, d; -SELECT 'Test statistics count_min:'; +SELECT 'Test statistics tdigest:'; -ALTER TABLE tab ADD STATISTICS a TYPE count_min; -ALTER TABLE tab ADD STATISTICS b TYPE count_min; -ALTER TABLE tab ADD STATISTICS c TYPE count_min; -ALTER TABLE tab MATERIALIZE STATISTICS a, b, c; - -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) +ALTER TABLE tab ADD STATISTICS b, c, d TYPE tdigest; +ALTER TABLE tab MATERIALIZE STATISTICS b, c, d; +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b > 0/*10000*/ and c < -1/*9990*/ and d > cast(9998, 'DateTime')/*1*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +ALTER TABLE tab DROP STATISTICS b, c, d; -ALTER TABLE tab DROP STATISTICS a, b, c; + +SELECT 'Test statistics uniq:'; + +ALTER TABLE tab ADD STATISTICS a, b, c, d TYPE uniq; +ALTER TABLE tab MATERIALIZE STATISTICS a, b, c, d; +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = cast(1, 'DateTime')/*100*/ and c = 0/*1000*/ and b = 0/*100*/ and a = '0'/*100*/) +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +ALTER TABLE tab DROP STATISTICS a, b, c, d; SELECT 'Test statistics multi-types:'; -ALTER TABLE tab ADD STATISTICS a TYPE count_min; -ALTER TABLE tab ADD STATISTICS b TYPE count_min, uniq, tdigest; -ALTER TABLE tab ADD STATISTICS c TYPE count_min, uniq, tdigest; -ALTER TABLE tab ADD STATISTICS d TYPE count_min, uniq, tdigest; +ALTER TABLE tab ADD STATISTICS a TYPE count_min, uniq; +ALTER TABLE tab ADD STATISTICS b TYPE count_min, minmax, uniq, tdigest; +ALTER TABLE tab ADD STATISTICS c TYPE count_min, minmax, uniq, tdigest; +ALTER TABLE tab ADD STATISTICS d TYPE count_min, minmax, uniq, tdigest; ALTER TABLE tab MATERIALIZE STATISTICS a, b, c, d; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/) +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = cast(1, 'DateTime')/*1*/ and c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/) +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d > cast(1, 'DateTime')/*9999*/ and c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; ALTER TABLE tab DROP STATISTICS a, b, c, d; -SELECT 'Test estimating range condition:'; - -ALTER TABLE tab ADD STATISTICS b TYPE minmax; -ALTER TABLE tab MATERIALIZE STATISTICS b; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*5000*/ and b < 10/*100*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; - -ALTER TABLE tab ADD STATISTICS b TYPE tdigest; -ALTER TABLE tab MATERIALIZE STATISTICS b; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*5000*/ and b < 10/*100*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; -ALTER TABLE tab DROP STATISTICS b; - - -SELECT 'Test estimating equals condition:'; - -ALTER TABLE tab ADD STATISTICS a TYPE uniq; -ALTER TABLE tab MATERIALIZE STATISTICS a; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b = 10/*100*/ and a = '0'/*1*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; - -ALTER TABLE tab ADD STATISTICS a TYPE count_min; -ALTER TABLE tab MATERIALIZE STATISTICS a; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b = 10/*100*/ and a = '0'/*1*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; -ALTER TABLE tab DROP STATISTICS a; - DROP TABLE IF EXISTS tab SYNC; From a36424fc8c99fd5c20b52d80a10d232c8a61451c Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Mon, 5 Aug 2024 18:59:09 +0800 Subject: [PATCH 101/289] Add supported data types to documents --- .../table-engines/mergetree-family/mergetree.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index c2438ac593e..8fd60c36f94 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -1007,6 +1007,19 @@ They can be used for prewhere optimization only if we enable `set allow_statisti [Count-min](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch) sketches which provide an approximate count of the frequency of each value in a column. +#### Supported Data Types {#supported-data-types} + +| Data Type / Statistics | count_min | minmax | tdigest | uniq | +|-------------------------|-----------|--------|---------|------| +| Integer types | ✔ | ✔ | ✔ | ✔ | +| Floating-point numbers | ✔ | ✔ | ✔ | ✔ | +| Boolean | ✔ | ✔ | ✔ | ✔ | +| Strings | ✔ | ✗ | ✗ | ✔ | +| Dates | ✔ | ✔ | ✔ | ✔ | + +You can also add `LowCardinality` and `Nullable` modifiers to the data types. + + ## Column-level Settings {#column-level-settings} Certain MergeTree settings can be override at column level: From 4fa30da1189a3ce8139dad6bce5da9264019d3b3 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Tue, 6 Aug 2024 09:56:38 +0800 Subject: [PATCH 102/289] Fix docs --- .../mergetree-family/mergetree.md | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 8fd60c36f94..80e5bf694d4 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -994,30 +994,36 @@ They can be used for prewhere optimization only if we enable `set allow_statisti - `MinMax` The minimum and maximum column value which allows to estimate the selectivity of range filters on numeric columns. + Supported data types: (U)Int*, Float*, Decimal(*), Boolean and Date*. - `TDigest` [TDigest](https://github.com/tdunning/t-digest) sketches which allow to compute approximate percentiles (e.g. the 90th percentile) for numeric columns. + Supported data types: (U)Int*, Float*, Decimal(*), Boolean and Date*. - `Uniq` [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) sketches which provide an estimation how many distinct values a column contains. + Supported data types: (U)Int*, Float*, Decimal(*), Boolean, Date* and (Fixed)String. - `count_min` [Count-min](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch) sketches which provide an approximate count of the frequency of each value in a column. + Supported data types: (U)Int*, Float*, Decimal(*), Boolean, Date* and (Fixed)String. -#### Supported Data Types {#supported-data-types} +Note that all statistics types support `LowCardinality` and `Nullable` modifiers to data types. -| Data Type / Statistics | count_min | minmax | tdigest | uniq | -|-------------------------|-----------|--------|---------|------| -| Integer types | ✔ | ✔ | ✔ | ✔ | -| Floating-point numbers | ✔ | ✔ | ✔ | ✔ | -| Boolean | ✔ | ✔ | ✔ | ✔ | -| Strings | ✔ | ✗ | ✗ | ✔ | -| Dates | ✔ | ✔ | ✔ | ✔ | +#### Supported operations of Column Statistics {#supported-operations-of-column-statistics} + +| | Equals | Range | +|-----------|---------|-------| +| count_min | ✔ | ✔ | +| MinMax | ✔ | ✔ | +| TDigest | ✔ | ✔ | +| Uniq | ✔ | ✗ | + +Please note that operation range represents >, >=, <, <=. -You can also add `LowCardinality` and `Nullable` modifiers to the data types. ## Column-level Settings {#column-level-settings} From 46da03030cd6ae115edb7d4b9054704a8312283c Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Tue, 6 Aug 2024 10:00:34 +0800 Subject: [PATCH 103/289] Add test for implicitly type conversion --- .../table-engines/mergetree-family/mergetree.md | 10 +++++++--- .../0_stateless/02864_statistics_estimation.sql | 6 ++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 80e5bf694d4..caf72e39467 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -994,22 +994,26 @@ They can be used for prewhere optimization only if we enable `set allow_statisti - `MinMax` The minimum and maximum column value which allows to estimate the selectivity of range filters on numeric columns. + Supported data types: (U)Int*, Float*, Decimal(*), Boolean and Date*. - `TDigest` [TDigest](https://github.com/tdunning/t-digest) sketches which allow to compute approximate percentiles (e.g. the 90th percentile) for numeric columns. + Supported data types: (U)Int*, Float*, Decimal(*), Boolean and Date*. - `Uniq` [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) sketches which provide an estimation how many distinct values a column contains. - Supported data types: (U)Int*, Float*, Decimal(*), Boolean, Date* and (Fixed)String. + + Supported data types: (U)Int*, Float*, Decimal*, Boolean, Date* and (Fixed)String. - `count_min` [Count-min](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch) sketches which provide an approximate count of the frequency of each value in a column. - Supported data types: (U)Int*, Float*, Decimal(*), Boolean, Date* and (Fixed)String. + + Supported data types: (U)Int*, Float*, Decimal*, Boolean, Date* and (Fixed)String. Note that all statistics types support `LowCardinality` and `Nullable` modifiers to data types. @@ -1022,7 +1026,7 @@ Note that all statistics types support `LowCardinality` and `Nullable` modifiers | TDigest | ✔ | ✔ | | Uniq | ✔ | ✗ | -Please note that operation range represents >, >=, <, <=. +Please note that operation `Range` represents >, >=, < or <=. diff --git a/tests/queries/0_stateless/02864_statistics_estimation.sql b/tests/queries/0_stateless/02864_statistics_estimation.sql index 65b111c285b..4a221686069 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.sql +++ b/tests/queries/0_stateless/02864_statistics_estimation.sql @@ -80,6 +80,12 @@ SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTi FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d > cast(1, 'DateTime')/*9999*/ and c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; + +SELECT 'Test statistics implicitly type conversion:'; +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = '2024-08-06 09:58:09'/*0*/ and c = '0'/*100*/ and b > 0/*9990*/ and a = '10000'/*0*/) +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; + ALTER TABLE tab DROP STATISTICS a, b, c, d; From 8259a9827e5f1764fffa508f8e721f643c102d15 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Tue, 6 Aug 2024 10:29:42 +0800 Subject: [PATCH 104/289] Update reference file for tests --- .../engines/table-engines/mergetree-family/mergetree.md | 9 ++++----- .../0_stateless/02864_statistics_estimation.reference | 4 ++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index caf72e39467..751b07afe91 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -1021,18 +1021,17 @@ Note that all statistics types support `LowCardinality` and `Nullable` modifiers | | Equals | Range | |-----------|---------|-------| -| count_min | ✔ | ✔ | -| MinMax | ✔ | ✔ | -| TDigest | ✔ | ✔ | +| count_min | ✔ | ✗ | +| MinMax | ✗ | ✔ | +| TDigest | ✗ | ✔ | | Uniq | ✔ | ✗ | Please note that operation `Range` represents >, >=, < or <=. - ## Column-level Settings {#column-level-settings} -Certain MergeTree settings can be override at column level: +Certain MergeTree settings can be overridden at column level: - `max_compress_block_size` — Maximum size of blocks of uncompressed data before compressing for writing to a table. - `min_compress_block_size` — Minimum size of blocks of uncompressed data required for compression when writing the next mark. diff --git a/tests/queries/0_stateless/02864_statistics_estimation.reference b/tests/queries/0_stateless/02864_statistics_estimation.reference index f59f94013da..cb274b2990e 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.reference +++ b/tests/queries/0_stateless/02864_statistics_estimation.reference @@ -22,3 +22,7 @@ Test statistics multi-types: Prewhere info Prewhere filter Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0), greater(d, _CAST(1, \'DateTime\'))) (removed) +Test statistics implicitly type conversion: + Prewhere info + Prewhere filter + Prewhere filter column: and(equals(a, \'10000\'), equals(d, \'2024-08-06 09:58:09\'), equals(c, \'0\'), greater(b, 0)) (removed) From 3769f8a46537bdc2865d13ff948e48328efb3548 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Tue, 6 Aug 2024 20:44:42 +0800 Subject: [PATCH 105/289] Fix tests --- .../0_stateless/02864_statistics_estimation.reference | 6 +++--- tests/queries/0_stateless/02864_statistics_estimation.sql | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/02864_statistics_estimation.reference b/tests/queries/0_stateless/02864_statistics_estimation.reference index cb274b2990e..fc370691830 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.reference +++ b/tests/queries/0_stateless/02864_statistics_estimation.reference @@ -14,15 +14,15 @@ Test statistics tdigest: Test statistics uniq: Prewhere info Prewhere filter - Prewhere filter column: and(equals(a, \'0\'), equals(d, _CAST(1, \'DateTime\')), equals(b, 0), equals(c, 0)) (removed) + Prewhere filter column: and(equals(d, _CAST(1, \'DateTime\')), equals(c, 0)) (removed) Test statistics multi-types: Prewhere info Prewhere filter - Prewhere filter column: and(equals(a, \'0\'), equals(d, _CAST(1, \'DateTime\')), less(c, -90), greater(b, 900)) (removed) + Prewhere filter column: and(equals(d, _CAST(1, \'DateTime\')), less(c, -90), greater(b, 900)) (removed) Prewhere info Prewhere filter Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0), greater(d, _CAST(1, \'DateTime\'))) (removed) Test statistics implicitly type conversion: Prewhere info Prewhere filter - Prewhere filter column: and(equals(a, \'10000\'), equals(d, \'2024-08-06 09:58:09\'), equals(c, \'0\'), greater(b, 0)) (removed) + Prewhere filter column: and(equals(a, \'1\'), equals(d, \'2024-08-06 09:58:09\'), equals(c, \'0\'), greater(b, 0)) (removed) diff --git a/tests/queries/0_stateless/02864_statistics_estimation.sql b/tests/queries/0_stateless/02864_statistics_estimation.sql index 4a221686069..6436cf68738 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.sql +++ b/tests/queries/0_stateless/02864_statistics_estimation.sql @@ -59,7 +59,7 @@ SELECT 'Test statistics uniq:'; ALTER TABLE tab ADD STATISTICS a, b, c, d TYPE uniq; ALTER TABLE tab MATERIALIZE STATISTICS a, b, c, d; SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = cast(1, 'DateTime')/*100*/ and c = 0/*1000*/ and b = 0/*100*/ and a = '0'/*100*/) +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = cast(1, 'DateTime')/*100*/ and c = 0/*1000*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; ALTER TABLE tab DROP STATISTICS a, b, c, d; @@ -73,7 +73,7 @@ ALTER TABLE tab ADD STATISTICS d TYPE count_min, minmax, uniq, tdigest; ALTER TABLE tab MATERIALIZE STATISTICS a, b, c, d; SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = cast(1, 'DateTime')/*1*/ and c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/) +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = cast(1, 'DateTime')/*1*/ and c < -90/*900*/ and b > 900/*990*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') @@ -83,7 +83,7 @@ WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; SELECT 'Test statistics implicitly type conversion:'; SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = '2024-08-06 09:58:09'/*0*/ and c = '0'/*100*/ and b > 0/*9990*/ and a = '10000'/*0*/) +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = '2024-08-06 09:58:09'/*0*/ and c = '0'/*100*/ and b > 0/*9990*/ and a = '1'/*1*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; ALTER TABLE tab DROP STATISTICS a, b, c, d; From 704ec8dea6b4fa583689508e4362644517321005 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Mon, 5 Aug 2024 14:55:33 -0400 Subject: [PATCH 106/289] Handle incomplete sequences at end of input --- src/IO/WriteBufferValidUTF8.cpp | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/IO/WriteBufferValidUTF8.cpp b/src/IO/WriteBufferValidUTF8.cpp index d611befac37..426f302cb02 100644 --- a/src/IO/WriteBufferValidUTF8.cpp +++ b/src/IO/WriteBufferValidUTF8.cpp @@ -149,9 +149,27 @@ void WriteBufferValidUTF8::finalizeImpl() /// Write all complete sequences from buffer. nextImpl(); - /// If unfinished sequence at end, then write replacement. - if (working_buffer.begin() != memory.data()) - putReplacement(); + /// Handle remaining bytes if we have an incomplete sequence + if (working_buffer.begin() != memory.data()) { + char * p = memory.data(); + + while (p < pos) { + UInt8 len = length_of_utf8_sequence[static_cast(*p)]; + if (p + len > pos) { + // Incomplete sequence. Skip one byte. + putReplacement(); + ++p; + } else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) { + // Valid sequence + putValid(p, len); + p += len; + } else { + // Invalid sequence, skip first byte. + putReplacement(); + ++p; + } + } + } } } From 2a1ee419b473350323c5582bcb950ab5630a07e2 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Tue, 6 Aug 2024 15:53:49 -0400 Subject: [PATCH 107/289] Fix style issue --- src/IO/WriteBufferValidUTF8.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/IO/WriteBufferValidUTF8.cpp b/src/IO/WriteBufferValidUTF8.cpp index 426f302cb02..8441b4eafa2 100644 --- a/src/IO/WriteBufferValidUTF8.cpp +++ b/src/IO/WriteBufferValidUTF8.cpp @@ -150,20 +150,27 @@ void WriteBufferValidUTF8::finalizeImpl() nextImpl(); /// Handle remaining bytes if we have an incomplete sequence - if (working_buffer.begin() != memory.data()) { + if (working_buffer.begin() != memory.data()) + { char * p = memory.data(); - while (p < pos) { + while (p < pos) + { UInt8 len = length_of_utf8_sequence[static_cast(*p)]; - if (p + len > pos) { + if (p + len > pos) + { // Incomplete sequence. Skip one byte. putReplacement(); ++p; - } else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) { + } + else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) + { // Valid sequence putValid(p, len); p += len; - } else { + } + else + { // Invalid sequence, skip first byte. putReplacement(); ++p; From f4ed3f5c6d138dc6ac7209393c60562ddf9e8d24 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Tue, 6 Aug 2024 16:14:26 -0400 Subject: [PATCH 108/289] Add test --- .../03221_incomplete-utf8-sequence.reference | 16 ++++++++++++++++ .../03221_incomplete-utf8-sequence.sql | 2 ++ 2 files changed, 18 insertions(+) create mode 100644 tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference create mode 100644 tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql diff --git a/tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference b/tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference new file mode 100644 index 00000000000..4577427251d --- /dev/null +++ b/tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference @@ -0,0 +1,16 @@ +{ + "meta": + [ + { + "name": "unhex('f0')", + "type": "String" + } + ], + + "data": + [ + ["�"] + ], + + "rows": 1 +} diff --git a/tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql b/tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql new file mode 100644 index 00000000000..ee4f25f3b4a --- /dev/null +++ b/tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql @@ -0,0 +1,2 @@ +SET output_format_write_statistics = 0; +SELECT unhex('f0') FORMAT JSONCompact; From f0f1015162ddfb7ef2e8046fa9c952c5c7e6b89a Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 7 Aug 2024 10:47:13 +0800 Subject: [PATCH 109/289] fix style --- src/Functions/array/array.cpp | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index 21b1841f3ed..f193acafe06 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -172,9 +172,9 @@ private: concrete_columns[i] = concrete_column; } - ColumnString & concreate_out_data = assert_cast(out_data); - auto & out_chars = concreate_out_data.getChars(); - auto & out_offsets = concreate_out_data.getOffsets(); + ColumnString & concrete_out_data = assert_cast(out_data); + auto & out_chars = concrete_out_data.getChars(); + auto & out_offsets = concrete_out_data.getOffsets(); out_chars.resize_exact(total_bytes); out_offsets.resize_exact(input_rows_count * columns.size()); @@ -197,7 +197,6 @@ private: bool executeFixedString(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const { - size_t total_bytes = 0; std::vector concrete_columns(columns.size(), nullptr); for (size_t i = 0; i < columns.size(); ++i) { @@ -205,15 +204,16 @@ private: if (!concrete_column) return false; - total_bytes += concrete_column->getChars().size(); concrete_columns[i] = concrete_column; } - ColumnFixedString & concreate_out_data = assert_cast(out_data); - auto & out_chars = concreate_out_data.getChars(); + ColumnFixedString & concrete_out_data = assert_cast(out_data); + auto & out_chars = concrete_out_data.getChars(); + + const size_t n = concrete_out_data.getN(); + size_t total_bytes = concrete_out_data.getN() * columns.size() * input_rows_count; out_chars.resize_exact(total_bytes); - const size_t n = concreate_out_data.getN(); size_t curr_out_offset = 0; for (size_t row_i = 0; row_i < input_rows_count; ++row_i) { @@ -241,9 +241,9 @@ private: nested_columns[i] = &concrete_column->getNestedColumn(); } - ColumnNullable & concreate_out_data = assert_cast(out_data); - auto & out_null_map = concreate_out_data.getNullMapColumn(); - auto & out_nested_column = concreate_out_data.getNestedColumn(); + ColumnNullable & concrete_out_data = assert_cast(out_data); + auto & out_null_map = concrete_out_data.getNullMapColumn(); + auto & out_nested_column = concrete_out_data.getNestedColumn(); executeAny(null_maps, out_null_map, input_rows_count); executeAny(nested_columns, out_nested_column, input_rows_count); return true; @@ -251,11 +251,11 @@ private: bool executeTuple(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const { - ColumnTuple * concreate_out_data = typeid_cast(&out_data); - if (!concreate_out_data) + ColumnTuple * concrete_out_data = typeid_cast(&out_data); + if (!concrete_out_data) return false; - const size_t tuple_size = concreate_out_data->tupleSize(); + const size_t tuple_size = concrete_out_data->tupleSize(); for (size_t i = 0; i < tuple_size; ++i) { ColumnRawPtrs elem_columns(columns.size(), nullptr); @@ -264,7 +264,7 @@ private: const ColumnTuple * concrete_column = assert_cast(columns[j]); elem_columns[j] = &concrete_column->getColumn(i); } - executeAny(elem_columns, concreate_out_data->getColumn(i), input_rows_count); + executeAny(elem_columns, concrete_out_data->getColumn(i), input_rows_count); } return true; } From aafe498b7f6045e75414c17abecad168aa0efb88 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Wed, 7 Aug 2024 10:43:49 +0800 Subject: [PATCH 110/289] Handle edge case: col_int32 > 10.6 --- src/Storages/Statistics/Statistics.cpp | 22 +++++++++++++++++++ src/Storages/Statistics/Statistics.h | 6 +++++ src/Storages/Statistics/StatisticsMinMax.cpp | 10 +++------ src/Storages/Statistics/StatisticsTDigest.cpp | 16 +++++--------- .../02864_statistics_estimation.sql | 10 +++++---- 5 files changed, 43 insertions(+), 21 deletions(-) diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp index 1f159211a93..e94d8b67aff 100644 --- a/src/Storages/Statistics/Statistics.cpp +++ b/src/Storages/Statistics/Statistics.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include @@ -10,6 +12,7 @@ #include #include #include +#include #include "config.h" /// USE_DATASKETCHES @@ -28,6 +31,25 @@ enum StatisticsFileVersion : UInt16 V0 = 0, }; +std::optional StatisticsUtils::tryConvertToFloat64(const Field & value, const DataTypePtr & value_data_type) +{ + if (value_data_type->isValueRepresentedByNumber()) + { + Field val_converted; + + /// For case val_int32 < 10.5 or val_int32 < '10.5' we should convert 10.5 to Float64. + if (isInteger(value_data_type) && (value.getType() == Field::Types::Float64 || value.getType() == Field::Types::String)) + val_converted = convertFieldToType(value, *DataTypeFactory::instance().get("Float64")); + + /// We should convert value to the real column data type and then translate it to Float64. + /// For example for expression col_date > '2024-08-07', if we directly convert '2024-08-07' to Float64, we will get null. + val_converted = convertFieldToType(value, *value_data_type); + if (val_converted.isNull()) + return {}; + return applyVisitor(FieldVisitorConvertToNumber(), val_converted); + } + return {}; +} IStatistics::IStatistics(const SingleStatisticsDescription & stat_) : stat(stat_) diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h index 1425be89d47..9f82f137f1c 100644 --- a/src/Storages/Statistics/Statistics.h +++ b/src/Storages/Statistics/Statistics.h @@ -15,6 +15,12 @@ constexpr auto STATS_FILE_PREFIX = "statistics_"; constexpr auto STATS_FILE_SUFFIX = ".stats"; +struct StatisticsUtils +{ + /// Returns std::nullopt if input Field cannot be converted to a concrete value + static std::optional tryConvertToFloat64(const Field & value, const DataTypePtr & value_data_type); +}; + /// Statistics describe properties of the values in the column, /// e.g. how many unique values exist, /// what are the N most frequent values, diff --git a/src/Storages/Statistics/StatisticsMinMax.cpp b/src/Storages/Statistics/StatisticsMinMax.cpp index 11a0704dae7..88522894427 100644 --- a/src/Storages/Statistics/StatisticsMinMax.cpp +++ b/src/Storages/Statistics/StatisticsMinMax.cpp @@ -1,10 +1,8 @@ #include -#include #include #include #include #include -#include #include @@ -53,12 +51,10 @@ void StatisticsMinMax::deserialize(ReadBuffer & buf) Float64 StatisticsMinMax::estimateLess(const Field & val) const { - Field val_converted = convertFieldToType(val, *data_type); - if (val_converted.isNull()) + auto val_as_float = StatisticsUtils::tryConvertToFloat64(val, data_type); + if (!val_as_float.has_value()) return 0; - auto val_as_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); - if (val_as_float < min) return 0; @@ -68,7 +64,7 @@ Float64 StatisticsMinMax::estimateLess(const Field & val) const if (min == max) return (val_as_float != max) ? 0 : row_count; - return ((val_as_float - min) / (max - min)) * row_count; + return ((*val_as_float - min) / (max - min)) * row_count; } void minMaxStatisticsValidator(const SingleStatisticsDescription & /*statistics_description*/, DataTypePtr data_type) diff --git a/src/Storages/Statistics/StatisticsTDigest.cpp b/src/Storages/Statistics/StatisticsTDigest.cpp index cfd626c8ea8..a7ce43d2fe1 100644 --- a/src/Storages/Statistics/StatisticsTDigest.cpp +++ b/src/Storages/Statistics/StatisticsTDigest.cpp @@ -41,22 +41,18 @@ void StatisticsTDigest::deserialize(ReadBuffer & buf) Float64 StatisticsTDigest::estimateLess(const Field & val) const { - Field val_converted = convertFieldToType(val, *data_type); - if (val_converted.isNull()) + auto val_as_float = StatisticsUtils::tryConvertToFloat64(val, data_type); + if (!val_as_float.has_value()) return 0; - - auto val_as_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); - return t_digest.getCountLessThan(val_as_float); + return t_digest.getCountLessThan(*val_as_float); } Float64 StatisticsTDigest::estimateEqual(const Field & val) const { - Field val_converted = convertFieldToType(val, *data_type); - if (val_converted.isNull()) + auto val_as_float = StatisticsUtils::tryConvertToFloat64(val, data_type); + if (!val_as_float.has_value()) return 0; - - auto val_as_float = applyVisitor(FieldVisitorConvertToNumber(), val_converted); - return t_digest.getCountEqual(val_as_float); + return t_digest.getCountEqual(*val_as_float); } void tdigestStatisticsValidator(const SingleStatisticsDescription & /*statistics_description*/, DataTypePtr data_type) diff --git a/tests/queries/0_stateless/02864_statistics_estimation.sql b/tests/queries/0_stateless/02864_statistics_estimation.sql index 6436cf68738..5eb51fe7111 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.sql +++ b/tests/queries/0_stateless/02864_statistics_estimation.sql @@ -67,9 +67,7 @@ ALTER TABLE tab DROP STATISTICS a, b, c, d; SELECT 'Test statistics multi-types:'; ALTER TABLE tab ADD STATISTICS a TYPE count_min, uniq; -ALTER TABLE tab ADD STATISTICS b TYPE count_min, minmax, uniq, tdigest; -ALTER TABLE tab ADD STATISTICS c TYPE count_min, minmax, uniq, tdigest; -ALTER TABLE tab ADD STATISTICS d TYPE count_min, minmax, uniq, tdigest; +ALTER TABLE tab ADD STATISTICS b, c, d TYPE count_min, minmax, uniq, tdigest; ALTER TABLE tab MATERIALIZE STATISTICS a, b, c, d; SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') @@ -82,10 +80,14 @@ WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; SELECT 'Test statistics implicitly type conversion:'; + SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = '2024-08-06 09:58:09'/*0*/ and c = '0'/*100*/ and b > 0/*9990*/ and a = '1'/*1*/) +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = '2024-08-06 09:58:09'/*0*/ and c = '0'/*100*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = '2024-08-06 09:58:09'/*0*/ and b > 50.1/*5000*/) +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; ALTER TABLE tab DROP STATISTICS a, b, c, d; From cfaa852af9e94bd25217bfbb690f2d6ad641f3d6 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Wed, 7 Aug 2024 11:40:01 +0800 Subject: [PATCH 111/289] Little fix --- src/Storages/Statistics/Statistics.cpp | 7 ++++--- .../0_stateless/02864_statistics_estimation.reference | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp index e94d8b67aff..691cf42aa42 100644 --- a/src/Storages/Statistics/Statistics.cpp +++ b/src/Storages/Statistics/Statistics.cpp @@ -40,10 +40,11 @@ std::optional StatisticsUtils::tryConvertToFloat64(const Field & value, /// For case val_int32 < 10.5 or val_int32 < '10.5' we should convert 10.5 to Float64. if (isInteger(value_data_type) && (value.getType() == Field::Types::Float64 || value.getType() == Field::Types::String)) val_converted = convertFieldToType(value, *DataTypeFactory::instance().get("Float64")); + else + /// We should convert value to the real column data type and then translate it to Float64. + /// For example for expression col_date > '2024-08-07', if we directly convert '2024-08-07' to Float64, we will get null. + val_converted = convertFieldToType(value, *value_data_type); - /// We should convert value to the real column data type and then translate it to Float64. - /// For example for expression col_date > '2024-08-07', if we directly convert '2024-08-07' to Float64, we will get null. - val_converted = convertFieldToType(value, *value_data_type); if (val_converted.isNull()) return {}; return applyVisitor(FieldVisitorConvertToNumber(), val_converted); diff --git a/tests/queries/0_stateless/02864_statistics_estimation.reference b/tests/queries/0_stateless/02864_statistics_estimation.reference index fc370691830..d475926df8b 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.reference +++ b/tests/queries/0_stateless/02864_statistics_estimation.reference @@ -25,4 +25,7 @@ Test statistics multi-types: Test statistics implicitly type conversion: Prewhere info Prewhere filter - Prewhere filter column: and(equals(a, \'1\'), equals(d, \'2024-08-06 09:58:09\'), equals(c, \'0\'), greater(b, 0)) (removed) + Prewhere filter column: and(equals(d, \'2024-08-06 09:58:09\'), equals(c, \'0\')) (removed) + Prewhere info + Prewhere filter + Prewhere filter column: and(equals(d, \'2024-08-06 09:58:09\'), greater(b, 50.1_Float64)) (removed) From cff7cbac94b1b6f39dd4585fca5bcbcb2f952747 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Wed, 7 Aug 2024 16:21:16 -0400 Subject: [PATCH 112/289] Style fix --- src/IO/WriteBufferValidUTF8.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/IO/WriteBufferValidUTF8.cpp b/src/IO/WriteBufferValidUTF8.cpp index 8441b4eafa2..25c7b3d4820 100644 --- a/src/IO/WriteBufferValidUTF8.cpp +++ b/src/IO/WriteBufferValidUTF8.cpp @@ -159,19 +159,19 @@ void WriteBufferValidUTF8::finalizeImpl() UInt8 len = length_of_utf8_sequence[static_cast(*p)]; if (p + len > pos) { - // Incomplete sequence. Skip one byte. + /// Incomplete sequence. Skip one byte. putReplacement(); ++p; } else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) { - // Valid sequence + /// Valid sequence putValid(p, len); p += len; } else { - // Invalid sequence, skip first byte. + /// Invalid sequence, skip first byte. putReplacement(); ++p; } From d09531e48a90db637a3e5b42a2fd9569c911ed44 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Wed, 7 Aug 2024 16:25:01 -0400 Subject: [PATCH 113/289] Rename test files --- ...equence.reference => 03221_incomplete_utf8_sequence.reference} | 0 ...plete-utf8-sequence.sql => 03221_incomplete_utf8_sequence.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{03221_incomplete-utf8-sequence.reference => 03221_incomplete_utf8_sequence.reference} (100%) rename tests/queries/0_stateless/{03221_incomplete-utf8-sequence.sql => 03221_incomplete_utf8_sequence.sql} (100%) diff --git a/tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference b/tests/queries/0_stateless/03221_incomplete_utf8_sequence.reference similarity index 100% rename from tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference rename to tests/queries/0_stateless/03221_incomplete_utf8_sequence.reference diff --git a/tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql b/tests/queries/0_stateless/03221_incomplete_utf8_sequence.sql similarity index 100% rename from tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql rename to tests/queries/0_stateless/03221_incomplete_utf8_sequence.sql From f8a14e86d8db56e8ad7046b8848c7521d235a777 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Wed, 7 Aug 2024 16:32:13 -0400 Subject: [PATCH 114/289] Add const a few places --- src/IO/WriteBufferValidUTF8.cpp | 8 ++++---- src/IO/WriteBufferValidUTF8.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/IO/WriteBufferValidUTF8.cpp b/src/IO/WriteBufferValidUTF8.cpp index 25c7b3d4820..2a86f8c2801 100644 --- a/src/IO/WriteBufferValidUTF8.cpp +++ b/src/IO/WriteBufferValidUTF8.cpp @@ -54,7 +54,7 @@ inline void WriteBufferValidUTF8::putReplacement() } -inline void WriteBufferValidUTF8::putValid(char *data, size_t len) +inline void WriteBufferValidUTF8::putValid(const char *data, size_t len) { if (len == 0) return; @@ -152,18 +152,18 @@ void WriteBufferValidUTF8::finalizeImpl() /// Handle remaining bytes if we have an incomplete sequence if (working_buffer.begin() != memory.data()) { - char * p = memory.data(); + const char * p = memory.data(); while (p < pos) { - UInt8 len = length_of_utf8_sequence[static_cast(*p)]; + UInt8 len = length_of_utf8_sequence[static_cast(*p)]; if (p + len > pos) { /// Incomplete sequence. Skip one byte. putReplacement(); ++p; } - else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) + else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) { /// Valid sequence putValid(p, len); diff --git a/src/IO/WriteBufferValidUTF8.h b/src/IO/WriteBufferValidUTF8.h index daaf0427f88..a398b8ded01 100644 --- a/src/IO/WriteBufferValidUTF8.h +++ b/src/IO/WriteBufferValidUTF8.h @@ -26,7 +26,7 @@ public: private: void putReplacement(); - void putValid(char * data, size_t len); + void putValid(const char * data, size_t len); void nextImpl() override; void finalizeImpl() override; From 5f747bc41174cb03c06288393a026d2385d57e8b Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Thu, 8 Aug 2024 11:09:59 +0800 Subject: [PATCH 115/289] Add more test for implicitly type conversion --- .../02864_statistics_estimation.reference | 26 +++++++-- .../02864_statistics_estimation.sql | 55 ++++++++++++++++--- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/tests/queries/0_stateless/02864_statistics_estimation.reference b/tests/queries/0_stateless/02864_statistics_estimation.reference index d475926df8b..077476d28c1 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.reference +++ b/tests/queries/0_stateless/02864_statistics_estimation.reference @@ -23,9 +23,23 @@ Test statistics multi-types: Prewhere filter Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0), greater(d, _CAST(1, \'DateTime\'))) (removed) Test statistics implicitly type conversion: - Prewhere info - Prewhere filter - Prewhere filter column: and(equals(d, \'2024-08-06 09:58:09\'), equals(c, \'0\')) (removed) - Prewhere info - Prewhere filter - Prewhere filter column: and(equals(d, \'2024-08-06 09:58:09\'), greater(b, 50.1_Float64)) (removed) +1 +0 +0 +0 +0 +50 +50 +0 +0 +50 +0 +0 +1 +1 +1 +0 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02864_statistics_estimation.sql b/tests/queries/0_stateless/02864_statistics_estimation.sql index 5eb51fe7111..1189de46267 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.sql +++ b/tests/queries/0_stateless/02864_statistics_estimation.sql @@ -1,6 +1,7 @@ -- Tags: no-fasttest DROP TABLE IF EXISTS tab SYNC; +DROP TABLE IF EXISTS tab2 SYNC; SET allow_experimental_statistics = 1; SET allow_statistics_optimize = 1; @@ -78,17 +79,55 @@ SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTi FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d > cast(1, 'DateTime')/*9999*/ and c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +DROP TABLE IF EXISTS tab SYNC; SELECT 'Test statistics implicitly type conversion:'; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = '2024-08-06 09:58:09'/*0*/ and c = '0'/*100*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +CREATE TABLE tab2 +( + a String, + b UInt64, + c UInt8, + d DateTime, + e Boolean, + f Float64, + g Decimal32(1), + pk String, +) Engine = MergeTree() ORDER BY pk; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = '2024-08-06 09:58:09'/*0*/ and b > 50.1/*5000*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; -ALTER TABLE tab DROP STATISTICS a, b, c, d; +ALTER TABLE tab2 ADD STATISTICS a TYPE count_min, uniq; +ALTER TABLE tab2 ADD STATISTICS b, c, d, e, f, g TYPE count_min, minmax, uniq, tdigest; +INSERT INTO tab2 select toString(number), number, number, cast(number, 'DateTime'), number % 2, number, toDecimal32(number, 1), toString(number) FROM system.numbers LIMIT 100; -DROP TABLE IF EXISTS tab SYNC; +SELECT count(*) FROM tab2 WHERE a = '0'; +SELECT count(*) FROM tab2 WHERE a = 0; -- { serverError NO_COMMON_TYPE } + +SELECT count(*) FROM tab2 WHERE b = 1.1; + +SELECT count(*) FROM tab2 WHERE c = 1.1; +SELECT count(*) FROM tab2 WHERE c = 1000; -- out of range of UInt16 + +SELECT count(*) FROM tab2 WHERE d = '2024-08-06 09:58:09'; +SELECT count(*) FROM tab2 WHERE d = '2024-08-06 09:58:0'; -- { serverError CANNOT_PARSE_DATETIME } + +SELECT count(*) FROM tab2 WHERE e = true; +SELECT count(*) FROM tab2 WHERE e = 'true'; -- { serverError TYPE_MISMATCH } +SELECT count(*) FROM tab2 WHERE e = 1; +SELECT count(*) FROM tab2 WHERE e = 2; +SELECT count(*) FROM tab2 WHERE e = 1.1; +SELECT count(*) FROM tab2 WHERE e = '1'; + +SELECT count(*) FROM tab2 WHERE f = 1.1; +SELECT count(*) FROM tab2 WHERE f = '1.1'; +SELECT count(*) FROM tab2 WHERE f = 1; +SELECT count(*) FROM tab2 WHERE f = '1'; + +SELECT count(*) FROM tab2 WHERE g = toDecimal32(1.0, 1); +SELECT count(*) FROM tab2 WHERE g = toDecimal32(1.10, 1); +SELECT count(*) FROM tab2 WHERE g = toDecimal32(1.0, 2); +SELECT count(*) FROM tab2 WHERE g = 1.0; +SELECT count(*) FROM tab2 WHERE g = 1.0; +SELECT count(*) FROM tab2 WHERE g = '1.0'; + +DROP TABLE IF EXISTS tab2 SYNC; From 21cdbe03f94bd3bdaef8b02e61b09e5e79ab380c Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Thu, 8 Aug 2024 11:21:19 +0800 Subject: [PATCH 116/289] Add more test cases --- .../queries/0_stateless/02864_statistics_estimation.reference | 2 ++ tests/queries/0_stateless/02864_statistics_estimation.sql | 3 +++ 2 files changed, 5 insertions(+) diff --git a/tests/queries/0_stateless/02864_statistics_estimation.reference b/tests/queries/0_stateless/02864_statistics_estimation.reference index 077476d28c1..ad923d91dda 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.reference +++ b/tests/queries/0_stateless/02864_statistics_estimation.reference @@ -24,6 +24,8 @@ Test statistics multi-types: Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0), greater(d, _CAST(1, \'DateTime\'))) (removed) Test statistics implicitly type conversion: 1 +1 +1 0 0 0 diff --git a/tests/queries/0_stateless/02864_statistics_estimation.sql b/tests/queries/0_stateless/02864_statistics_estimation.sql index 1189de46267..63cc2e47c97 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.sql +++ b/tests/queries/0_stateless/02864_statistics_estimation.sql @@ -103,7 +103,10 @@ INSERT INTO tab2 select toString(number), number, number, cast(number, 'DateTime SELECT count(*) FROM tab2 WHERE a = '0'; SELECT count(*) FROM tab2 WHERE a = 0; -- { serverError NO_COMMON_TYPE } +SELECT count(*) FROM tab2 WHERE b = 1; +SELECT count(*) FROM tab2 WHERE b = '1'; SELECT count(*) FROM tab2 WHERE b = 1.1; +SELECT count(*) FROM tab2 WHERE b = '1.1'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab2 WHERE c = 1.1; SELECT count(*) FROM tab2 WHERE c = 1000; -- out of range of UInt16 From 6e7587e9b3dbc791f6970fe23703c1d65aa4b7f0 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Thu, 8 Aug 2024 14:44:08 +0800 Subject: [PATCH 117/289] Fix test for old analyzer --- .../02864_statistics_estimation.reference | 10 ++++---- .../02864_statistics_estimation.sql | 25 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/queries/0_stateless/02864_statistics_estimation.reference b/tests/queries/0_stateless/02864_statistics_estimation.reference index ad923d91dda..0e6cad6528b 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.reference +++ b/tests/queries/0_stateless/02864_statistics_estimation.reference @@ -6,22 +6,22 @@ Test statistics count_min: Test statistics minmax: Prewhere info Prewhere filter - Prewhere filter column: and(greater(d, _CAST(9998, \'DateTime\')), less(c, -1), greater(b, 0)) (removed) + Prewhere filter column: and(greater(d, CAST(9998, \'DateTime\')), less(c, -1), greater(b, 0)) (removed) Test statistics tdigest: Prewhere info Prewhere filter - Prewhere filter column: and(greater(d, _CAST(9998, \'DateTime\')), less(c, -1), greater(b, 0)) (removed) + Prewhere filter column: and(greater(d, CAST(9998, \'DateTime\')), less(c, -1), greater(b, 0)) (removed) Test statistics uniq: Prewhere info Prewhere filter - Prewhere filter column: and(equals(d, _CAST(1, \'DateTime\')), equals(c, 0)) (removed) + Prewhere filter column: and(equals(d, CAST(1, \'DateTime\')), equals(c, 0)) (removed) Test statistics multi-types: Prewhere info Prewhere filter - Prewhere filter column: and(equals(d, _CAST(1, \'DateTime\')), less(c, -90), greater(b, 900)) (removed) + Prewhere filter column: and(equals(d, CAST(1, \'DateTime\')), less(c, -90), greater(b, 900)) (removed) Prewhere info Prewhere filter - Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0), greater(d, _CAST(1, \'DateTime\'))) (removed) + Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0), greater(d, CAST(1, \'DateTime\'))) (removed) Test statistics implicitly type conversion: 1 1 diff --git a/tests/queries/0_stateless/02864_statistics_estimation.sql b/tests/queries/0_stateless/02864_statistics_estimation.sql index 63cc2e47c97..94fd7d12e0a 100644 --- a/tests/queries/0_stateless/02864_statistics_estimation.sql +++ b/tests/queries/0_stateless/02864_statistics_estimation.sql @@ -6,6 +6,7 @@ DROP TABLE IF EXISTS tab2 SYNC; SET allow_experimental_statistics = 1; SET allow_statistics_optimize = 1; SET allow_suspicious_low_cardinality_types=1; +SET allow_experimental_analyzer=1; SET mutations_sync = 2; CREATE TABLE tab @@ -27,9 +28,9 @@ SELECT 'Test statistics count_min:'; ALTER TABLE tab ADD STATISTICS a, b, c TYPE count_min; ALTER TABLE tab MATERIALIZE STATISTICS a, b, c; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); ALTER TABLE tab DROP STATISTICS a, b, c; @@ -38,9 +39,9 @@ SELECT 'Test statistics minmax:'; ALTER TABLE tab ADD STATISTICS b, c, d TYPE minmax; ALTER TABLE tab MATERIALIZE STATISTICS b, c, d; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b > 0/*10000*/ and c < -1/*9990*/ and d > cast(9998, 'DateTime')/*1*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); ALTER TABLE tab DROP STATISTICS b, c, d; @@ -49,9 +50,9 @@ SELECT 'Test statistics tdigest:'; ALTER TABLE tab ADD STATISTICS b, c, d TYPE tdigest; ALTER TABLE tab MATERIALIZE STATISTICS b, c, d; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b > 0/*10000*/ and c < -1/*9990*/ and d > cast(9998, 'DateTime')/*1*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); ALTER TABLE tab DROP STATISTICS b, c, d; @@ -59,9 +60,9 @@ SELECT 'Test statistics uniq:'; ALTER TABLE tab ADD STATISTICS a, b, c, d TYPE uniq; ALTER TABLE tab MATERIALIZE STATISTICS a, b, c, d; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = cast(1, 'DateTime')/*100*/ and c = 0/*1000*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); ALTER TABLE tab DROP STATISTICS a, b, c, d; @@ -71,13 +72,13 @@ ALTER TABLE tab ADD STATISTICS a TYPE count_min, uniq; ALTER TABLE tab ADD STATISTICS b, c, d TYPE count_min, minmax, uniq, tdigest; ALTER TABLE tab MATERIALIZE STATISTICS a, b, c, d; -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = cast(1, 'DateTime')/*1*/ and c < -90/*900*/ and b > 900/*990*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); -SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') +SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d > cast(1, 'DateTime')/*9999*/ and c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'; +WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); DROP TABLE IF EXISTS tab SYNC; From cb9039e91d073942ff07a68e10336f8db38f4a79 Mon Sep 17 00:00:00 2001 From: Jiebin Sun Date: Tue, 6 Aug 2024 19:05:12 +0800 Subject: [PATCH 118/289] Add thread pool and cancellation if merge data with key This patch will add thread pool and cancellation if merge data with key. During the merge, if the data size is large, we may convert the singleLevelHash to twoLevelHash and merge in parallel. Test the patch with 2 x 80 vCPUs, Q8 and Q9 has got 10.3% and 7.6% performance improvement. Signed-off-by: Jiebin Sun --- .../AggregateFunctionUniq.h | 2 ++ src/AggregateFunctions/IAggregateFunction.h | 12 +++++++--- src/AggregateFunctions/UniqExactSet.h | 7 ++++++ src/Interpreters/Aggregator.cpp | 24 ++++++++++++------- src/Interpreters/Aggregator.h | 4 ++-- .../Transforms/AggregatingTransform.cpp | 2 +- 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionUniq.h b/src/AggregateFunctions/AggregateFunctionUniq.h index cef23f766c7..35d6e599e38 100644 --- a/src/AggregateFunctions/AggregateFunctionUniq.h +++ b/src/AggregateFunctions/AggregateFunctionUniq.h @@ -459,6 +459,8 @@ public: bool isParallelizeMergePrepareNeeded() const override { return is_parallelize_merge_prepare_needed; } + constexpr static bool parallelizeMergeWithKey() { return true; } + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { if constexpr (is_parallelize_merge_prepare_needed) diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index ee227db6d9d..f8e7051d635 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -145,6 +145,8 @@ public: virtual bool isParallelizeMergePrepareNeeded() const { return false; } + constexpr static bool parallelizeMergeWithKey() { return false; } + virtual void parallelizeMergePrepare(AggregateDataPtrs & /*places*/, ThreadPool & /*thread_pool*/, std::atomic & /*is_cancelled*/) const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "parallelizeMergePrepare() with thread pool parameter isn't implemented for {} ", getName()); @@ -169,7 +171,7 @@ public: /// Merges states (on which src places points to) with other states (on which dst places points to) of current aggregation function /// then destroy states (on which src places points to). - virtual void mergeAndDestroyBatch(AggregateDataPtr * dst_places, AggregateDataPtr * src_places, size_t size, size_t offset, Arena * arena) const = 0; + virtual void mergeAndDestroyBatch(AggregateDataPtr * dst_places, AggregateDataPtr * src_places, size_t size, size_t offset, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const = 0; /// Serializes state (to transmit it over the network, for example). virtual void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version = std::nullopt) const = 0; /// NOLINT @@ -499,11 +501,15 @@ public: static_cast(this)->merge(places[i] + place_offset, rhs[i], arena); } - void mergeAndDestroyBatch(AggregateDataPtr * dst_places, AggregateDataPtr * rhs_places, size_t size, size_t offset, Arena * arena) const override + void mergeAndDestroyBatch(AggregateDataPtr * dst_places, AggregateDataPtr * rhs_places, size_t size, size_t offset, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const override { for (size_t i = 0; i < size; ++i) { - static_cast(this)->merge(dst_places[i] + offset, rhs_places[i] + offset, arena); + if constexpr (Derived::parallelizeMergeWithKey()) + static_cast(this)->merge(dst_places[i] + offset, rhs_places[i] + offset, thread_pool, is_cancelled, arena); + else + static_cast(this)->merge(dst_places[i] + offset, rhs_places[i] + offset, arena); + static_cast(this)->destroy(rhs_places[i] + offset); } } diff --git a/src/AggregateFunctions/UniqExactSet.h b/src/AggregateFunctions/UniqExactSet.h index 2ae8c3a8386..25c6f7ac55f 100644 --- a/src/AggregateFunctions/UniqExactSet.h +++ b/src/AggregateFunctions/UniqExactSet.h @@ -101,6 +101,13 @@ public: auto merge(const UniqExactSet & other, ThreadPool * thread_pool = nullptr, std::atomic * is_cancelled = nullptr) { + /// If the size is large, we may convert the singleLevelHash to twoLevelHash and merge in parallel. + if (other.size() > 40000) + { + if (isSingleLevel()) + convertToTwoLevel(); + } + if (isSingleLevel() && other.isTwoLevel()) convertToTwoLevel(); diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index e073b7a49b6..71be3c1c30c 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -2371,7 +2371,7 @@ void NO_INLINE Aggregator::mergeDataNullKey( template void NO_INLINE Aggregator::mergeDataImpl( - Table & table_dst, Table & table_src, Arena * arena, bool use_compiled_functions [[maybe_unused]], bool prefetch) const + Table & table_dst, Table & table_src, Arena * arena, bool use_compiled_functions [[maybe_unused]], bool prefetch, ThreadPool & thread_pool, std::atomic & is_cancelled) const { if constexpr (Method::low_cardinality_optimization || Method::one_key_nullable_optimization) mergeDataNullKey(table_dst, table_src, arena); @@ -2410,7 +2410,7 @@ void NO_INLINE Aggregator::mergeDataImpl( { if (!is_aggregate_function_compiled[i]) aggregate_functions[i]->mergeAndDestroyBatch( - dst_places.data(), src_places.data(), dst_places.size(), offsets_of_aggregate_states[i], arena); + dst_places.data(), src_places.data(), dst_places.size(), offsets_of_aggregate_states[i], thread_pool, is_cancelled, arena); } return; @@ -2420,7 +2420,7 @@ void NO_INLINE Aggregator::mergeDataImpl( for (size_t i = 0; i < params.aggregates_size; ++i) { aggregate_functions[i]->mergeAndDestroyBatch( - dst_places.data(), src_places.data(), dst_places.size(), offsets_of_aggregate_states[i], arena); + dst_places.data(), src_places.data(), dst_places.size(), offsets_of_aggregate_states[i], thread_pool, is_cancelled, arena); } } @@ -2535,8 +2535,10 @@ void NO_INLINE Aggregator::mergeWithoutKeyDataImpl( template void NO_INLINE Aggregator::mergeSingleLevelDataImpl( - ManyAggregatedDataVariants & non_empty_data) const + ManyAggregatedDataVariants & non_empty_data, std::atomic & is_cancelled) const { + ThreadPool thread_pool{CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, params.max_threads}; + AggregatedDataVariantsPtr & res = non_empty_data[0]; bool no_more_keys = false; @@ -2557,13 +2559,13 @@ void NO_INLINE Aggregator::mergeSingleLevelDataImpl( if (compiled_aggregate_functions_holder) { mergeDataImpl( - getDataVariant(*res).data, getDataVariant(current).data, res->aggregates_pool, true, prefetch); + getDataVariant(*res).data, getDataVariant(current).data, res->aggregates_pool, true, prefetch, thread_pool, is_cancelled); } else #endif { mergeDataImpl( - getDataVariant(*res).data, getDataVariant(current).data, res->aggregates_pool, false, prefetch); + getDataVariant(*res).data, getDataVariant(current).data, res->aggregates_pool, false, prefetch, thread_pool, is_cancelled); } } else if (res->without_key) @@ -2589,7 +2591,7 @@ void NO_INLINE Aggregator::mergeSingleLevelDataImpl( #define M(NAME) \ template void NO_INLINE Aggregator::mergeSingleLevelDataImpl( \ - ManyAggregatedDataVariants & non_empty_data) const; + ManyAggregatedDataVariants & non_empty_data, std::atomic & is_cancelled) const; APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) #undef M @@ -2597,6 +2599,8 @@ template void NO_INLINE Aggregator::mergeBucketImpl( ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic & is_cancelled) const { + ThreadPool thread_pool{CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, params.max_threads}; + /// We merge all aggregation results to the first. AggregatedDataVariantsPtr & res = data[0]; @@ -2613,7 +2617,7 @@ void NO_INLINE Aggregator::mergeBucketImpl( if (compiled_aggregate_functions_holder) { mergeDataImpl( - getDataVariant(*res).data.impls[bucket], getDataVariant(current).data.impls[bucket], arena, true, prefetch); + getDataVariant(*res).data.impls[bucket], getDataVariant(current).data.impls[bucket], arena, true, prefetch, thread_pool, is_cancelled); } else #endif @@ -2623,7 +2627,9 @@ void NO_INLINE Aggregator::mergeBucketImpl( getDataVariant(current).data.impls[bucket], arena, false, - prefetch); + prefetch, + thread_pool, + is_cancelled); } } } diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index f4f1e9a1df3..c357da2d9de 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -455,7 +455,7 @@ private: /// Merge data from hash table `src` into `dst`. template - void mergeDataImpl(Table & table_dst, Table & table_src, Arena * arena, bool use_compiled_functions, bool prefetch) const; + void mergeDataImpl(Table & table_dst, Table & table_src, Arena * arena, bool use_compiled_functions, bool prefetch, ThreadPool & thread_pool, std::atomic & is_cancelled) const; /// Merge data from hash table `src` into `dst`, but only for keys that already exist in dst. In other cases, merge the data into `overflows`. template @@ -478,7 +478,7 @@ private: template void mergeSingleLevelDataImpl( - ManyAggregatedDataVariants & non_empty_data) const; + ManyAggregatedDataVariants & non_empty_data, std::atomic & is_cancelled) const; template using ConvertToBlockRes = std::conditional_t; diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index cdbe194cfac..5e7fba6d80c 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -495,7 +495,7 @@ private: #define M(NAME) \ else if (first->type == AggregatedDataVariants::Type::NAME) \ - params->aggregator.mergeSingleLevelDataImplNAME)::element_type>(*data); + params->aggregator.mergeSingleLevelDataImplNAME)::element_type>(*data, shared_data->is_cancelled); if (false) {} // NOLINT APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) #undef M From 6fb5051237448addf82faf50677179c551958f1b Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Wed, 21 Aug 2024 12:47:12 +0800 Subject: [PATCH 119/289] Fix conflicts --- .../Statistics/ConditionSelectivityEstimator.cpp | 2 -- src/Storages/Statistics/StatisticsMinMax.cpp | 16 ++++++++-------- src/Storages/Statistics/StatisticsMinMax.h | 4 ++-- src/Storages/Statistics/StatisticsTDigest.cpp | 6 +++--- src/Storages/Statistics/StatisticsTDigest.h | 6 +++--- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp index eadd15a8f7f..432659f51f8 100644 --- a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp +++ b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp @@ -41,8 +41,6 @@ Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateEqual { return default_cond_equal_factor * rows; } - return default_cond_equal_factor * rows; - Float64 result = 0; Float64 partial_cnt = 0; for (const auto & [key, estimator] : part_statistics) diff --git a/src/Storages/Statistics/StatisticsMinMax.cpp b/src/Storages/Statistics/StatisticsMinMax.cpp index 88522894427..aafa34bc6d8 100644 --- a/src/Storages/Statistics/StatisticsMinMax.cpp +++ b/src/Storages/Statistics/StatisticsMinMax.cpp @@ -15,8 +15,8 @@ namespace ErrorCodes extern const int ILLEGAL_STATISTICS; } -StatisticsMinMax::StatisticsMinMax(const SingleStatisticsDescription & statistics_description, const DataTypePtr & data_type_) - : IStatistics(statistics_description) +StatisticsMinMax::StatisticsMinMax(const SingleStatisticsDescription & description, const DataTypePtr & data_type_) + : IStatistics(description) , data_type(data_type_) { } @@ -67,17 +67,17 @@ Float64 StatisticsMinMax::estimateLess(const Field & val) const return ((*val_as_float - min) / (max - min)) * row_count; } -void minMaxStatisticsValidator(const SingleStatisticsDescription & /*statistics_description*/, DataTypePtr data_type) +void minMaxStatisticsValidator(const SingleStatisticsDescription & /*description*/, const DataTypePtr & data_type) { - data_type = removeNullable(data_type); - data_type = removeLowCardinalityAndNullable(data_type); + auto inner_data_type = removeNullable(data_type); + inner_data_type = removeLowCardinalityAndNullable(data_type); if (!data_type->isValueRepresentedByNumber()) - throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'minmax' do not support type {}", data_type->getName()); + throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'minmax' do not support type {}", inner_data_type->getName()); } -StatisticsPtr minMaxStatisticsCreator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type) +StatisticsPtr minMaxStatisticsCreator(const SingleStatisticsDescription & description, const DataTypePtr & data_type) { - return std::make_shared(statistics_description, data_type); + return std::make_shared(description, data_type); } } diff --git a/src/Storages/Statistics/StatisticsMinMax.h b/src/Storages/Statistics/StatisticsMinMax.h index 524494b520b..c60fa810c47 100644 --- a/src/Storages/Statistics/StatisticsMinMax.h +++ b/src/Storages/Statistics/StatisticsMinMax.h @@ -27,7 +27,7 @@ private: DataTypePtr data_type; }; -void minMaxStatisticsValidator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type); -StatisticsPtr minMaxStatisticsCreator(const SingleStatisticsDescription & statistics_description, DataTypePtr data_type); +void minMaxStatisticsValidator(const SingleStatisticsDescription & description, const DataTypePtr & data_type); +StatisticsPtr minMaxStatisticsCreator(const SingleStatisticsDescription & description, const DataTypePtr & data_type); } diff --git a/src/Storages/Statistics/StatisticsTDigest.cpp b/src/Storages/Statistics/StatisticsTDigest.cpp index b90075b10be..285b779036f 100644 --- a/src/Storages/Statistics/StatisticsTDigest.cpp +++ b/src/Storages/Statistics/StatisticsTDigest.cpp @@ -53,7 +53,7 @@ Float64 StatisticsTDigest::estimateEqual(const Field & val) const return t_digest.getCountEqual(*val_as_float); } -void tdigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type) +void tdigestStatisticsValidator(const SingleStatisticsDescription & /*description*/, const DataTypePtr & data_type) { DataTypePtr inner_data_type = removeNullable(data_type); inner_data_type = removeLowCardinalityAndNullable(inner_data_type); @@ -61,9 +61,9 @@ void tdigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'tdigest' do not support type {}", data_type->getName()); } -StatisticsPtr tdigestCreator(const SingleStatisticsDescription & stat, DataTypePtr) +StatisticsPtr tdigestStatisticsCreator(const SingleStatisticsDescription & description, const DataTypePtr & data_type) { - return std::make_shared(stat); + return std::make_shared(description, data_type); } } diff --git a/src/Storages/Statistics/StatisticsTDigest.h b/src/Storages/Statistics/StatisticsTDigest.h index 91a477c2b94..5e744fee2ce 100644 --- a/src/Storages/Statistics/StatisticsTDigest.h +++ b/src/Storages/Statistics/StatisticsTDigest.h @@ -9,7 +9,7 @@ namespace DB class StatisticsTDigest : public IStatistics { public: - explicit StatisticsTDigest(const SingleStatisticsDescription & stat_); + explicit StatisticsTDigest(const SingleStatisticsDescription & description, const DataTypePtr & data_type_); void update(const ColumnPtr & column) override; @@ -24,7 +24,7 @@ private: DataTypePtr data_type; }; -void tdigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type); -StatisticsPtr tdigestCreator(const SingleStatisticsDescription & stat, DataTypePtr); +void tdigestStatisticsValidator(const SingleStatisticsDescription & description, const DataTypePtr & data_type); +StatisticsPtr tdigestStatisticsCreator(const SingleStatisticsDescription & description, const DataTypePtr & data_type); } From fc704020e9d64e15b80f8dd3c6db8871925bca9b Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Wed, 21 Aug 2024 13:29:42 +0800 Subject: [PATCH 120/289] Update tests --- src/Storages/Statistics/StatisticsMinMax.cpp | 6 +- src/Storages/Statistics/StatisticsUniq.cpp | 2 +- .../0_stateless/02864_statistics_ddl.sql | 32 ++++ .../02864_statistics_estimation.reference | 47 ------ .../02864_statistics_estimation.sql | 137 ------------------ .../02864_statistics_predicates.reference | 23 +++ .../02864_statistics_predicates.sql | 46 +++++- 7 files changed, 100 insertions(+), 193 deletions(-) delete mode 100644 tests/queries/0_stateless/02864_statistics_estimation.reference delete mode 100644 tests/queries/0_stateless/02864_statistics_estimation.sql diff --git a/src/Storages/Statistics/StatisticsMinMax.cpp b/src/Storages/Statistics/StatisticsMinMax.cpp index aafa34bc6d8..6ae05cb0a6b 100644 --- a/src/Storages/Statistics/StatisticsMinMax.cpp +++ b/src/Storages/Statistics/StatisticsMinMax.cpp @@ -70,9 +70,9 @@ Float64 StatisticsMinMax::estimateLess(const Field & val) const void minMaxStatisticsValidator(const SingleStatisticsDescription & /*description*/, const DataTypePtr & data_type) { auto inner_data_type = removeNullable(data_type); - inner_data_type = removeLowCardinalityAndNullable(data_type); - if (!data_type->isValueRepresentedByNumber()) - throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'minmax' do not support type {}", inner_data_type->getName()); + inner_data_type = removeLowCardinalityAndNullable(inner_data_type); + if (!inner_data_type->isValueRepresentedByNumber()) + throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'minmax' do not support type {}", data_type->getName()); } StatisticsPtr minMaxStatisticsCreator(const SingleStatisticsDescription & description, const DataTypePtr & data_type) diff --git a/src/Storages/Statistics/StatisticsUniq.cpp b/src/Storages/Statistics/StatisticsUniq.cpp index 07311b5b86d..21bc5d227ef 100644 --- a/src/Storages/Statistics/StatisticsUniq.cpp +++ b/src/Storages/Statistics/StatisticsUniq.cpp @@ -56,7 +56,7 @@ void uniqStatisticsValidator(const SingleStatisticsDescription & /*description*/ { DataTypePtr inner_data_type = removeNullable(data_type); inner_data_type = removeLowCardinalityAndNullable(inner_data_type); - if (!inner_data_type->isValueRepresentedByNumber()) + if (!inner_data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type)) throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'uniq' do not support type {}", data_type->getName()); } diff --git a/tests/queries/0_stateless/02864_statistics_ddl.sql b/tests/queries/0_stateless/02864_statistics_ddl.sql index 32b56a842b7..0fc29d0c362 100644 --- a/tests/queries/0_stateless/02864_statistics_ddl.sql +++ b/tests/queries/0_stateless/02864_statistics_ddl.sql @@ -7,6 +7,7 @@ SET mutations_sync = 1; DROP TABLE IF EXISTS tab; +SET allow_experimental_statistics = 0; -- Error case: Can't create statistics when allow_experimental_statistics = 0 CREATE TABLE tab (col Float64 STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError INCORRECT_QUERY } @@ -94,6 +95,30 @@ CREATE TABLE tab (col Map(UInt64, UInt64) STATISTICS(count_min)) Engine = MergeT CREATE TABLE tab (col UUID STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } CREATE TABLE tab (col IPv6 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +-- minmax requires data_type.isValueRepresentedByInteger +-- These types work: +CREATE TABLE tab (col UInt8 STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col UInt256 STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Float32 STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Decimal32(3) STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Date STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Date32 STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col DateTime STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col DateTime64 STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Enum('hello', 'world') STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col IPv4 STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Nullable(UInt8) STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col LowCardinality(UInt8) STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col LowCardinality(Nullable(UInt8)) STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +-- These types don't work: +CREATE TABLE tab (col String STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +CREATE TABLE tab (col FixedString(1) STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +CREATE TABLE tab (col Array(Float64) STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +CREATE TABLE tab (col Tuple(Float64, Float64) STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +CREATE TABLE tab (col Map(UInt64, UInt64) STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +CREATE TABLE tab (col UUID STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +CREATE TABLE tab (col IPv6 STATISTICS(minmax)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } + -- CREATE TABLE was easy, ALTER is more fun CREATE TABLE tab @@ -173,6 +198,13 @@ ALTER TABLE tab MODIFY STATISTICS f64 TYPE count_min; ALTER TABLE tab DROP STATI -- Doesn't work: ALTER TABLE tab ADD STATISTICS a TYPE count_min; -- { serverError ILLEGAL_STATISTICS } ALTER TABLE tab MODIFY STATISTICS a TYPE count_min; -- { serverError ILLEGAL_STATISTICS } +-- minmax +-- Works: +ALTER TABLE tab ADD STATISTICS f64 TYPE minmax; ALTER TABLE tab DROP STATISTICS f64; +ALTER TABLE tab MODIFY STATISTICS f64 TYPE minmax; ALTER TABLE tab DROP STATISTICS f64; +-- Doesn't work: +ALTER TABLE tab ADD STATISTICS a TYPE minmax; -- { serverError ILLEGAL_STATISTICS } +ALTER TABLE tab MODIFY STATISTICS a TYPE minmax; -- { serverError ILLEGAL_STATISTICS } -- Any data type changes on columns with statistics are disallowed, for simplicity even if the new data type is compatible with all existing -- statistics objects (e.g. tdigest can be created on Float64 and UInt64) diff --git a/tests/queries/0_stateless/02864_statistics_estimation.reference b/tests/queries/0_stateless/02864_statistics_estimation.reference deleted file mode 100644 index 0e6cad6528b..00000000000 --- a/tests/queries/0_stateless/02864_statistics_estimation.reference +++ /dev/null @@ -1,47 +0,0 @@ -CREATE TABLE default.tab\n(\n `a` String,\n `b` UInt64,\n `c` Int64,\n `d` DateTime,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 -Test statistics count_min: - Prewhere info - Prewhere filter - Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed) -Test statistics minmax: - Prewhere info - Prewhere filter - Prewhere filter column: and(greater(d, CAST(9998, \'DateTime\')), less(c, -1), greater(b, 0)) (removed) -Test statistics tdigest: - Prewhere info - Prewhere filter - Prewhere filter column: and(greater(d, CAST(9998, \'DateTime\')), less(c, -1), greater(b, 0)) (removed) -Test statistics uniq: - Prewhere info - Prewhere filter - Prewhere filter column: and(equals(d, CAST(1, \'DateTime\')), equals(c, 0)) (removed) -Test statistics multi-types: - Prewhere info - Prewhere filter - Prewhere filter column: and(equals(d, CAST(1, \'DateTime\')), less(c, -90), greater(b, 900)) (removed) - Prewhere info - Prewhere filter - Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0), greater(d, CAST(1, \'DateTime\'))) (removed) -Test statistics implicitly type conversion: -1 -1 -1 -0 -0 -0 -0 -50 -50 -0 -0 -50 -0 -0 -1 -1 -1 -0 -1 -1 -1 -1 diff --git a/tests/queries/0_stateless/02864_statistics_estimation.sql b/tests/queries/0_stateless/02864_statistics_estimation.sql deleted file mode 100644 index 94fd7d12e0a..00000000000 --- a/tests/queries/0_stateless/02864_statistics_estimation.sql +++ /dev/null @@ -1,137 +0,0 @@ --- Tags: no-fasttest - -DROP TABLE IF EXISTS tab SYNC; -DROP TABLE IF EXISTS tab2 SYNC; - -SET allow_experimental_statistics = 1; -SET allow_statistics_optimize = 1; -SET allow_suspicious_low_cardinality_types=1; -SET allow_experimental_analyzer=1; -SET mutations_sync = 2; - -CREATE TABLE tab -( - a String, - b UInt64, - c Int64, - d DateTime, - pk String, -) Engine = MergeTree() ORDER BY pk -SETTINGS min_bytes_for_wide_part = 0; - -SHOW CREATE TABLE tab; - -INSERT INTO tab select toString(number % 10000), number % 1000, -(number % 100), cast(number, 'DateTime'), generateUUIDv4() FROM system.numbers LIMIT 10000; - - -SELECT 'Test statistics count_min:'; - -ALTER TABLE tab ADD STATISTICS a, b, c TYPE count_min; -ALTER TABLE tab MATERIALIZE STATISTICS a, b, c; -SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); -ALTER TABLE tab DROP STATISTICS a, b, c; - - -SELECT 'Test statistics minmax:'; - -ALTER TABLE tab ADD STATISTICS b, c, d TYPE minmax; -ALTER TABLE tab MATERIALIZE STATISTICS b, c, d; - -SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b > 0/*10000*/ and c < -1/*9990*/ and d > cast(9998, 'DateTime')/*1*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); - -ALTER TABLE tab DROP STATISTICS b, c, d; - - -SELECT 'Test statistics tdigest:'; - -ALTER TABLE tab ADD STATISTICS b, c, d TYPE tdigest; -ALTER TABLE tab MATERIALIZE STATISTICS b, c, d; -SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b > 0/*10000*/ and c < -1/*9990*/ and d > cast(9998, 'DateTime')/*1*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); -ALTER TABLE tab DROP STATISTICS b, c, d; - - -SELECT 'Test statistics uniq:'; - -ALTER TABLE tab ADD STATISTICS a, b, c, d TYPE uniq; -ALTER TABLE tab MATERIALIZE STATISTICS a, b, c, d; -SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = cast(1, 'DateTime')/*100*/ and c = 0/*1000*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); -ALTER TABLE tab DROP STATISTICS a, b, c, d; - - -SELECT 'Test statistics multi-types:'; - -ALTER TABLE tab ADD STATISTICS a TYPE count_min, uniq; -ALTER TABLE tab ADD STATISTICS b, c, d TYPE count_min, minmax, uniq, tdigest; -ALTER TABLE tab MATERIALIZE STATISTICS a, b, c, d; - -SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d = cast(1, 'DateTime')/*1*/ and c < -90/*900*/ and b > 900/*990*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); - -SELECT replaceRegexpAll(explain, '_CAST', 'CAST') FROM (SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String|_DateTime', '') as explain -FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE d > cast(1, 'DateTime')/*9999*/ and c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/) -WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%'); - -DROP TABLE IF EXISTS tab SYNC; - -SELECT 'Test statistics implicitly type conversion:'; - -CREATE TABLE tab2 -( - a String, - b UInt64, - c UInt8, - d DateTime, - e Boolean, - f Float64, - g Decimal32(1), - pk String, -) Engine = MergeTree() ORDER BY pk; - -ALTER TABLE tab2 ADD STATISTICS a TYPE count_min, uniq; -ALTER TABLE tab2 ADD STATISTICS b, c, d, e, f, g TYPE count_min, minmax, uniq, tdigest; - -INSERT INTO tab2 select toString(number), number, number, cast(number, 'DateTime'), number % 2, number, toDecimal32(number, 1), toString(number) FROM system.numbers LIMIT 100; - -SELECT count(*) FROM tab2 WHERE a = '0'; -SELECT count(*) FROM tab2 WHERE a = 0; -- { serverError NO_COMMON_TYPE } - -SELECT count(*) FROM tab2 WHERE b = 1; -SELECT count(*) FROM tab2 WHERE b = '1'; -SELECT count(*) FROM tab2 WHERE b = 1.1; -SELECT count(*) FROM tab2 WHERE b = '1.1'; -- { serverError TYPE_MISMATCH } - -SELECT count(*) FROM tab2 WHERE c = 1.1; -SELECT count(*) FROM tab2 WHERE c = 1000; -- out of range of UInt16 - -SELECT count(*) FROM tab2 WHERE d = '2024-08-06 09:58:09'; -SELECT count(*) FROM tab2 WHERE d = '2024-08-06 09:58:0'; -- { serverError CANNOT_PARSE_DATETIME } - -SELECT count(*) FROM tab2 WHERE e = true; -SELECT count(*) FROM tab2 WHERE e = 'true'; -- { serverError TYPE_MISMATCH } -SELECT count(*) FROM tab2 WHERE e = 1; -SELECT count(*) FROM tab2 WHERE e = 2; -SELECT count(*) FROM tab2 WHERE e = 1.1; -SELECT count(*) FROM tab2 WHERE e = '1'; - -SELECT count(*) FROM tab2 WHERE f = 1.1; -SELECT count(*) FROM tab2 WHERE f = '1.1'; -SELECT count(*) FROM tab2 WHERE f = 1; -SELECT count(*) FROM tab2 WHERE f = '1'; - -SELECT count(*) FROM tab2 WHERE g = toDecimal32(1.0, 1); -SELECT count(*) FROM tab2 WHERE g = toDecimal32(1.10, 1); -SELECT count(*) FROM tab2 WHERE g = toDecimal32(1.0, 2); -SELECT count(*) FROM tab2 WHERE g = 1.0; -SELECT count(*) FROM tab2 WHERE g = 1.0; -SELECT count(*) FROM tab2 WHERE g = '1.0'; - -DROP TABLE IF EXISTS tab2 SYNC; diff --git a/tests/queries/0_stateless/02864_statistics_predicates.reference b/tests/queries/0_stateless/02864_statistics_predicates.reference index ffbd7269e05..c2d74300fa9 100644 --- a/tests/queries/0_stateless/02864_statistics_predicates.reference +++ b/tests/queries/0_stateless/02864_statistics_predicates.reference @@ -3,10 +3,13 @@ u64 and = 10 10 10 +10 0 0 0 0 +0 +10 10 10 10 @@ -16,10 +19,13 @@ u64 and < 70 70 70 +70 80 80 80 80 +80 +70 70 70 70 @@ -29,6 +35,8 @@ f64 and = 10 10 10 +10 +0 0 0 0 @@ -37,6 +45,8 @@ f64 and = 10 10 10 +10 +0 0 0 0 @@ -46,6 +56,8 @@ f64 and < 70 70 70 +70 +80 80 80 80 @@ -54,6 +66,8 @@ f64 and < 70 70 70 +70 +80 80 80 80 @@ -63,6 +77,8 @@ dt and = 0 0 0 +0 +10 10 10 10 @@ -72,6 +88,8 @@ dt and < 10000 10000 10000 +10000 +70 70 70 70 @@ -89,6 +107,10 @@ b and = 5000 5000 5000 +5000 +5000 +5000 +0 0 0 0 @@ -96,3 +118,4 @@ b and = s and = 10 10 +10 diff --git a/tests/queries/0_stateless/02864_statistics_predicates.sql b/tests/queries/0_stateless/02864_statistics_predicates.sql index 779116cf19a..473a7bc95ad 100644 --- a/tests/queries/0_stateless/02864_statistics_predicates.sql +++ b/tests/queries/0_stateless/02864_statistics_predicates.sql @@ -12,46 +12,56 @@ CREATE TABLE tab ( u64 UInt64, u64_tdigest UInt64 STATISTICS(tdigest), + u64_minmax UInt64 STATISTICS(minmax), u64_count_min UInt64 STATISTICS(count_min), u64_uniq UInt64 STATISTICS(uniq), f64 Float64, f64_tdigest Float64 STATISTICS(tdigest), + f64_minmax Float64 STATISTICS(minmax), f64_count_min Float64 STATISTICS(count_min), f64_uniq Float64 STATISTICS(uniq), dt DateTime, dt_tdigest DateTime STATISTICS(tdigest), + dt_minmax DateTime STATISTICS(minmax), dt_count_min DateTime STATISTICS(count_min), dt_uniq DateTime STATISTICS(uniq), b Bool, b_tdigest Bool STATISTICS(tdigest), + b_minmax Bool STATISTICS(minmax), b_count_min Bool STATISTICS(count_min), b_uniq Bool STATISTICS(uniq), s String, -- s_tdigest String STATISTICS(tdigest), -- not supported by tdigest - s_count_min String STATISTICS(count_min) - -- s_uniq String STATISTICS(uniq), -- not supported by uniq + -- s_minmax String STATISTICS(minmax), -- not supported by minmax + s_count_min String STATISTICS(count_min), + s_uniq String STATISTICS(uniq) ) Engine = MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO tab -- SELECT number % 10000, number % 1000, -(number % 100) FROM system.numbers LIMIT 10000; -SELECT number % 1000, +SELECT number % 1000, -- u64 number % 1000, number % 1000, number % 1000, number % 1000, + number % 1000, -- f64 number % 1000, number % 1000, number % 1000, number % 1000, + number % 1000, -- dt number % 1000, number % 1000, number % 1000, + number % 1000, + number % 2, -- b number % 2, number % 2, number % 2, number % 2, toString(number % 1000), + toString(number % 1000), toString(number % 1000) FROM system.numbers LIMIT 10000; @@ -61,21 +71,25 @@ SELECT 'u64 and ='; SELECT count(*) FROM tab WHERE u64 = 7; SELECT count(*) FROM tab WHERE u64_tdigest = 7; +SELECT count(*) FROM tab WHERE u64_minmax = 7; SELECT count(*) FROM tab WHERE u64_count_min = 7; SELECT count(*) FROM tab WHERE u64_uniq = 7; SELECT count(*) FROM tab WHERE u64 = 7.7; SELECT count(*) FROM tab WHERE u64_tdigest = 7.7; +SELECT count(*) FROM tab WHERE u64_minmax = 7.7; SELECT count(*) FROM tab WHERE u64_count_min = 7.7; SELECT count(*) FROM tab WHERE u64_uniq = 7.7; SELECT count(*) FROM tab WHERE u64 = '7'; SELECT count(*) FROM tab WHERE u64_tdigest = '7'; +SELECT count(*) FROM tab WHERE u64_minmax = '7'; SELECT count(*) FROM tab WHERE u64_count_min = '7'; SELECT count(*) FROM tab WHERE u64_uniq = '7'; SELECT count(*) FROM tab WHERE u64 = '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_tdigest = '7.7'; -- { serverError TYPE_MISMATCH } +SELECT count(*) FROM tab WHERE u64_minmax = '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_count_min = '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_uniq = '7.7'; -- { serverError TYPE_MISMATCH } @@ -83,21 +97,25 @@ SELECT 'u64 and <'; SELECT count(*) FROM tab WHERE u64 < 7; SELECT count(*) FROM tab WHERE u64_tdigest < 7; +SELECT count(*) FROM tab WHERE u64_minmax < 7; SELECT count(*) FROM tab WHERE u64_count_min < 7; SELECT count(*) FROM tab WHERE u64_uniq < 7; SELECT count(*) FROM tab WHERE u64 < 7.7; SELECT count(*) FROM tab WHERE u64_tdigest < 7.7; +SELECT count(*) FROM tab WHERE u64_minmax < 7.7; SELECT count(*) FROM tab WHERE u64_count_min < 7.7; SELECT count(*) FROM tab WHERE u64_uniq < 7.7; SELECT count(*) FROM tab WHERE u64 < '7'; SELECT count(*) FROM tab WHERE u64_tdigest < '7'; +SELECT count(*) FROM tab WHERE u64_minmax < '7'; SELECT count(*) FROM tab WHERE u64_count_min < '7'; SELECT count(*) FROM tab WHERE u64_uniq < '7'; SELECT count(*) FROM tab WHERE u64 < '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_tdigest < '7.7'; -- { serverError TYPE_MISMATCH } +SELECT count(*) FROM tab WHERE u64_minmax < '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_count_min < '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_uniq < '7.7'; -- { serverError TYPE_MISMATCH } @@ -107,21 +125,25 @@ SELECT 'f64 and ='; SELECT count(*) FROM tab WHERE f64 = 7; SELECT count(*) FROM tab WHERE f64_tdigest = 7; +SELECT count(*) FROM tab WHERE f64_minmax = 7; SELECT count(*) FROM tab WHERE f64_count_min = 7; SELECT count(*) FROM tab WHERE f64_uniq = 7; SELECT count(*) FROM tab WHERE f64 = 7.7; SELECT count(*) FROM tab WHERE f64_tdigest = 7.7; +SELECT count(*) FROM tab WHERE f64_minmax = 7.7; SELECT count(*) FROM tab WHERE f64_count_min = 7.7; SELECT count(*) FROM tab WHERE f64_uniq = 7.7; SELECT count(*) FROM tab WHERE f64 = '7'; SELECT count(*) FROM tab WHERE f64_tdigest = '7'; +SELECT count(*) FROM tab WHERE f64_minmax = '7'; SELECT count(*) FROM tab WHERE f64_count_min = '7'; SELECT count(*) FROM tab WHERE f64_uniq = '7'; SELECT count(*) FROM tab WHERE f64 = '7.7'; SELECT count(*) FROM tab WHERE f64_tdigest = '7.7'; +SELECT count(*) FROM tab WHERE f64_minmax = '7.7'; SELECT count(*) FROM tab WHERE f64_count_min = '7.7'; SELECT count(*) FROM tab WHERE f64_uniq = '7.7'; @@ -129,21 +151,25 @@ SELECT 'f64 and <'; SELECT count(*) FROM tab WHERE f64 < 7; SELECT count(*) FROM tab WHERE f64_tdigest < 7; +SELECT count(*) FROM tab WHERE f64_minmax < 7; SELECT count(*) FROM tab WHERE f64_count_min < 7; SELECT count(*) FROM tab WHERE f64_uniq < 7; SELECT count(*) FROM tab WHERE f64 < 7.7; SELECT count(*) FROM tab WHERE f64_tdigest < 7.7; +SELECT count(*) FROM tab WHERE f64_minmax < 7.7; SELECT count(*) FROM tab WHERE f64_count_min < 7.7; SELECT count(*) FROM tab WHERE f64_uniq < 7.7; SELECT count(*) FROM tab WHERE f64 < '7'; SELECT count(*) FROM tab WHERE f64_tdigest < '7'; +SELECT count(*) FROM tab WHERE f64_minmax < '7'; SELECT count(*) FROM tab WHERE f64_count_min < '7'; SELECT count(*) FROM tab WHERE f64_uniq < '7'; SELECT count(*) FROM tab WHERE f64 < '7.7'; SELECT count(*) FROM tab WHERE f64_tdigest < '7.7'; +SELECT count(*) FROM tab WHERE f64_minmax < '7.7'; SELECT count(*) FROM tab WHERE f64_count_min < '7.7'; SELECT count(*) FROM tab WHERE f64_uniq < '7.7'; @@ -153,11 +179,13 @@ SELECT 'dt and ='; SELECT count(*) FROM tab WHERE dt = '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_tdigest = '2024-08-08 11:12:13'; +SELECT count(*) FROM tab WHERE dt_minmax = '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_count_min = '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_uniq = '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt = 7; SELECT count(*) FROM tab WHERE dt_tdigest = 7; +SELECT count(*) FROM tab WHERE dt_minmax = 7; SELECT count(*) FROM tab WHERE dt_count_min = 7; SELECT count(*) FROM tab WHERE dt_uniq = 7; @@ -165,11 +193,13 @@ SELECT 'dt and <'; SELECT count(*) FROM tab WHERE dt < '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_tdigest < '2024-08-08 11:12:13'; +SELECT count(*) FROM tab WHERE dt_minmax < '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_count_min < '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_uniq < '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt < 7; SELECT count(*) FROM tab WHERE dt_tdigest < 7; +SELECT count(*) FROM tab WHERE dt_minmax < 7; SELECT count(*) FROM tab WHERE dt_count_min < 7; SELECT count(*) FROM tab WHERE dt_uniq < 7; @@ -179,21 +209,25 @@ SELECT 'b and ='; SELECT count(*) FROM tab WHERE b = true; SELECT count(*) FROM tab WHERE b_tdigest = true; +SELECT count(*) FROM tab WHERE b_minmax = true; SELECT count(*) FROM tab WHERE b_count_min = true; SELECT count(*) FROM tab WHERE b_uniq = true; SELECT count(*) FROM tab WHERE b = 'true'; SELECT count(*) FROM tab WHERE b_tdigest = 'true'; +SELECT count(*) FROM tab WHERE b_minmax = 'true'; SELECT count(*) FROM tab WHERE b_count_min = 'true'; SELECT count(*) FROM tab WHERE b_uniq = 'true'; SELECT count(*) FROM tab WHERE b = 1; SELECT count(*) FROM tab WHERE b_tdigest = 1; +SELECT count(*) FROM tab WHERE b_minmax = 1; SELECT count(*) FROM tab WHERE b_count_min = 1; SELECT count(*) FROM tab WHERE b_uniq = 1; SELECT count(*) FROM tab WHERE b = 1.1; SELECT count(*) FROM tab WHERE b_tdigest = 1.1; +SELECT count(*) FROM tab WHERE b_minmax = 1.1; SELECT count(*) FROM tab WHERE b_count_min = 1.1; SELECT count(*) FROM tab WHERE b_uniq = 1.1; @@ -203,12 +237,14 @@ SELECT 's and ='; SELECT count(*) FROM tab WHERE s = 7; -- { serverError NO_COMMON_TYPE } -- SELECT count(*) FROM tab WHERE s_tdigest = 7; -- not supported +-- SELECT count(*) FROM tab WHERE s_minmax = 7; -- not supported SELECT count(*) FROM tab WHERE s_count_min = 7; -- { serverError NO_COMMON_TYPE } --- SELECT count(*) FROM tab WHERE s_uniq = 7; -- not supported +SELECT count(*) FROM tab WHERE s_uniq = 7; -- { serverError NO_COMMON_TYPE } SELECT count(*) FROM tab WHERE s = '7'; -- SELECT count(*) FROM tab WHERE s_tdigest = '7'; -- not supported +-- SELECT count(*) FROM tab WHERE s_minmax = '7'; -- not supported SELECT count(*) FROM tab WHERE s_count_min = '7'; --- SELECT count(*) FROM tab WHERE s_uniq = '7'; -- not supported +SELECT count(*) FROM tab WHERE s_uniq = '7'; DROP TABLE tab; From d71afba234f8260df19f66ada151e2f668c54aba Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Wed, 21 Aug 2024 18:58:34 +0800 Subject: [PATCH 121/289] Fix tests --- src/Storages/Statistics/StatisticsUniq.cpp | 2 +- tests/queries/0_stateless/02864_statistics_ddl.sql | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/Statistics/StatisticsUniq.cpp b/src/Storages/Statistics/StatisticsUniq.cpp index 21bc5d227ef..5e5b7a67b04 100644 --- a/src/Storages/Statistics/StatisticsUniq.cpp +++ b/src/Storages/Statistics/StatisticsUniq.cpp @@ -56,7 +56,7 @@ void uniqStatisticsValidator(const SingleStatisticsDescription & /*description*/ { DataTypePtr inner_data_type = removeNullable(data_type); inner_data_type = removeLowCardinalityAndNullable(inner_data_type); - if (!inner_data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type)) + if (!inner_data_type->isValueRepresentedByNumber() && !isStringOrFixedString(inner_data_type)) throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'uniq' do not support type {}", data_type->getName()); } diff --git a/tests/queries/0_stateless/02864_statistics_ddl.sql b/tests/queries/0_stateless/02864_statistics_ddl.sql index 0fc29d0c362..bcaaa9e7b61 100644 --- a/tests/queries/0_stateless/02864_statistics_ddl.sql +++ b/tests/queries/0_stateless/02864_statistics_ddl.sql @@ -47,7 +47,7 @@ CREATE TABLE tab (col Map(UInt64, UInt64) STATISTICS(tdigest)) Engine = MergeTre CREATE TABLE tab (col UUID STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } CREATE TABLE tab (col IPv6 STATISTICS(tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } --- uniq requires data_type.isValueRepresentedByInteger +-- uniq requires data_type.isValueRepresentedByInteger or (Fixed)String -- These types work: CREATE TABLE tab (col UInt8 STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; CREATE TABLE tab (col UInt256 STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; @@ -62,9 +62,9 @@ CREATE TABLE tab (col IPv4 STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple CREATE TABLE tab (col Nullable(UInt8) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; CREATE TABLE tab (col LowCardinality(UInt8) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; CREATE TABLE tab (col LowCardinality(Nullable(UInt8)) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col String STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col FixedString(1) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -- These types don't work: -CREATE TABLE tab (col String STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } -CREATE TABLE tab (col FixedString(1) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } CREATE TABLE tab (col Array(Float64) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } CREATE TABLE tab (col Tuple(Float64, Float64) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } CREATE TABLE tab (col Map(UInt64, UInt64) STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } From 83fa51f400fba43cd4bb961419f7dd572ec240a0 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Fri, 23 Aug 2024 21:42:54 +0200 Subject: [PATCH 122/289] PR fixes: update documentation on JSONCompactWithProgress format --- docs/en/interfaces/formats.md | 38 ++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 8795b71e0ac..f2ca803453e 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -930,7 +930,43 @@ Columns that are not present in the block will be filled with default values (yo In this format, ClickHouse outputs each row as a separated, newline-delimited JSON Object. -Each row is either a metadata object, data object, progress information or statistics object. +Each row is either a metadata object, data object, progress information or statistics object: + +1. **Metadata Object (`meta`)** + - Describes the structure of the data rows. + - Fields: `name` (column name), `type` (data type, e.g., `UInt32`, `String`, etc.). + - Example: `{"meta": [{"name":"id", "type":"UInt32"}, {"name":"name", "type":"String"}]}` + - Appears before any data objects. + +2. **Data Object (`data`)** + - Represents a row of query results. + - Fields: An array with values corresponding to the columns defined in the metadata. + - Example: `{"data":["1", "John Doe"]}` + - Appears after the metadata object, one per row. + +3. **Progress Information Object (`progress`)** + - Provides real-time progress feedback during query execution. + - Fields: `read_rows`, `read_bytes`, `written_rows`, `written_bytes`, `total_rows_to_read`, `result_rows`, `result_bytes`, `elapsed_ns`. + - Example: `{"progress":{"read_rows":"8","read_bytes":"168"}}` + - May appear intermittently. + +4. **Statistics Object (`statistics`)** + - Summarizes query execution statistics. + - Fields: `rows`, `rows_before_limit_at_least`, `elapsed`, `rows_read`, `bytes_read`. + - Example: `{"statistics": {"rows":2, "elapsed":0.001995, "rows_read":8}}` + - Appears at the end. + +5. **Exception Object (`exception`)** + - Represents an error that occurred during query execution. + - Fields: A single text field containing the error message. + - Example: `{"exception": "Code: 395. DB::Exception: Value passed to 'throwIf' function is non-zero..."}` + - Appears when an error is encountered. + +6. **Totals Object (`totals`)** + - Provides the totals for each numeric column in the result set. + - Fields: An array with total values corresponding to the columns defined in the metadata. + - Example: `{"totals": ["", "3"]}` + - Appears at the end of the data rows, if applicable. Example: From b772140514aa47ada3556f1d24712eae734a5645 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Fri, 23 Aug 2024 22:56:24 +0200 Subject: [PATCH 123/289] Fix PR comments: check exception in the test --- .../0_stateless/03174_json_compact_with_progress.reference | 2 ++ tests/queries/0_stateless/03174_json_compact_with_progress.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.reference b/tests/queries/0_stateless/03174_json_compact_with_progress.reference index b45f296e5c0..fe51e484bc1 100644 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.reference +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.reference @@ -11,3 +11,5 @@ {"data":["c", "1"]} {"totals": ["", "3"]} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":30}} +3 +Value passed to 'throwIf' function is non-zero: diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.sh b/tests/queries/0_stateless/03174_json_compact_with_progress.sh index 8f524cdff76..b15dc7cfdb2 100755 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.sh +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.sh @@ -16,4 +16,8 @@ $CLICKHOUSE_CLIENT -q "SELECT 2;" # Check Totals $CLICKHOUSE_CLIENT -q "SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | grep -v --text "progress" | sed -E 's/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' +$CLICKHOUSE_CLIENT -q "SELECT 3;" +# Check exceptions +${CLICKHOUSE_CURL} -sS "$CLICKHOUSE_URL" -d "SELECT throwIf(number = 15), 1::Int64 as a, '\"' from numbers(100) format JSONCompactWithProgress settings output_format_json_quote_64bit_integers=1, max_block_size=10" | grep "exception" | cut -c42-88 + $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_table;" From 06f0267ed142d76109b2d6d48d3c413bdaf0fc17 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 25 Aug 2024 11:27:43 +0200 Subject: [PATCH 124/289] Do not show statistics if exception happens --- ...JSONCompactWithProgressRowOutputFormat.cpp | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 0814e5f45d6..cb3398196f4 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -117,18 +117,16 @@ void JSONCompactWithProgressRowOutputFormat::writeProgress() void JSONCompactWithProgressRowOutputFormat::finalizeImpl() { - JSONUtils::writeCompactAdditionalInfo( - row_count, - statistics.rows_before_limit, - statistics.applied_limit, - statistics.watch, - statistics.progress, - settings.write_statistics && exception_message.empty(), - *ostr); - - if (!exception_message.empty()) - { - writeCString("\n", *ostr); + if (exception_message.empty()) { + JSONUtils::writeCompactAdditionalInfo( + row_count, + statistics.rows_before_limit, + statistics.applied_limit, + statistics.watch, + statistics.progress, + settings.write_statistics && exception_message.empty(), + *ostr); + } else { JSONUtils::writeCompactObjectStart(*ostr); JSONUtils::writeException(exception_message, *ostr, settings, 0); JSONUtils::writeCompactObjectEnd(*ostr); From 58aab9b8c789090834dd1db7bb00568db7b66cac Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 25 Aug 2024 11:29:07 +0200 Subject: [PATCH 125/289] Remove wrong bracket in the meta section --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 2 +- .../0_stateless/03174_json_compact_with_progress.reference | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index cb3398196f4..b5cbd6fb232 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -19,7 +19,7 @@ void JSONCompactWithProgressRowOutputFormat::writePrefix() JSONUtils::writeCompactObjectStart(*ostr); JSONUtils::writeCompactMetadata(names, types, settings, *ostr); JSONUtils::writeCompactObjectEnd(*ostr); - writeCString("}\n", *ostr); + writeCString("\n", *ostr); } void JSONCompactWithProgressRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.reference b/tests/queries/0_stateless/03174_json_compact_with_progress.reference index fe51e484bc1..cdbe7cfcb3e 100644 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.reference +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.reference @@ -1,11 +1,11 @@ 1 -{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}]}} +{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}]} {"data":[1, "a"]} {"data":[2, "b"]} {"data":[3, "c"]} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":33}} 2 -{"meta": [{"name":"name", "type":"String"}, {"name":"c", "type":"UInt64"}]}} +{"meta": [{"name":"name", "type":"String"}, {"name":"c", "type":"UInt64"}]} {"data":["a", "1"]} {"data":["b", "1"]} {"data":["c", "1"]} From af3f600b5cca7cadb304e0f69fc281b7015deca9 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 25 Aug 2024 11:35:35 +0200 Subject: [PATCH 126/289] Fix commas in the statistics section --- src/Formats/JSONUtils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Formats/JSONUtils.cpp b/src/Formats/JSONUtils.cpp index 0918658cf6f..6c2c6fc3700 100644 --- a/src/Formats/JSONUtils.cpp +++ b/src/Formats/JSONUtils.cpp @@ -563,17 +563,17 @@ namespace JSONUtils writeCompactObjectStart(out, 0, "statistics"); writeTitle("rows", out, 0, ""); writeIntText(rows, out); - writeFieldCompactDelimiter(out); if (applied_limit) { + writeFieldCompactDelimiter(out); writeTitle("rows_before_limit_at_least", out, 0, ""); writeIntText(rows_before_limit, out); - writeFieldCompactDelimiter(out); } if (write_statistics) { + writeFieldCompactDelimiter(out); writeTitle("elapsed", out, 0, ""); writeText(watch.elapsedSeconds(), out); writeFieldCompactDelimiter(out); From 0e4f2b17cc75a1f09994ad583df69fd067e582f8 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 25 Aug 2024 11:38:29 +0200 Subject: [PATCH 127/289] Remove unnecessary check --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index b5cbd6fb232..d1d7355724f 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -124,7 +124,7 @@ void JSONCompactWithProgressRowOutputFormat::finalizeImpl() statistics.applied_limit, statistics.watch, statistics.progress, - settings.write_statistics && exception_message.empty(), + settings.write_statistics, *ostr); } else { JSONUtils::writeCompactObjectStart(*ostr); From 312d32aaf972e01a88fceb2fea60fce424bfc49f Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 25 Aug 2024 12:14:18 +0200 Subject: [PATCH 128/289] Fix formatting --- .../Impl/JSONCompactWithProgressRowOutputFormat.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index d1d7355724f..e90864ecdf3 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -117,7 +117,8 @@ void JSONCompactWithProgressRowOutputFormat::writeProgress() void JSONCompactWithProgressRowOutputFormat::finalizeImpl() { - if (exception_message.empty()) { + if (exception_message.empty()) + { JSONUtils::writeCompactAdditionalInfo( row_count, statistics.rows_before_limit, @@ -126,7 +127,9 @@ void JSONCompactWithProgressRowOutputFormat::finalizeImpl() statistics.progress, settings.write_statistics, *ostr); - } else { + } + else + { JSONUtils::writeCompactObjectStart(*ostr); JSONUtils::writeException(exception_message, *ostr, settings, 0); JSONUtils::writeCompactObjectEnd(*ostr); From c1c4cabe9ffd5de032bc26fbb2ebd19b316e0336 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 27 Aug 2024 12:41:02 +0200 Subject: [PATCH 129/289] Update toReference*Num function documentation --- .../functions/date-time-functions.md | 306 +++++++++++++++++- 1 file changed, 298 insertions(+), 8 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 4f5e5a5d716..cb32d1a8d63 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1625,37 +1625,327 @@ If unit `WEEK` was specified, `toStartOfInterval` assumes that weeks start on Mo Converts a date with time to a certain fixed date, while preserving the time. +**Syntax** + +```sql +toTime(date[,timezone]) +``` + +**Arguments** + +- `date` — Date to convert to a time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). +- `timezone` (optional) — Timezone for the returned value. [String](../data-types/string.md). + +**Returned value** + +- DateTime with date equated to `1970-01-02` while preserving the time. [DateTime](../data-types/datetime.md). + +:::note +If the `date` input argument contained sub-second components, +they will be dropped in the returned `DateTime` value with second-accuracy. +::: + +**Example** + +Query: + +```sql +SELECT toTime(toDateTime64('1970-12-10 01:20:30.3000',3)) AS result, toTypeName(result); +``` + +Result: + +```response + +``` + ## toRelativeYearNum -Converts a date, or date with time, to the number of the year, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of years elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeYearNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of years from a fixed reference point in the past. [UInt16](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeYearNum(toDate('2002-12-08')) AS y1, + toRelativeYearNum(toDate('2010-10-26')) AS y2 +``` + +Result: + +```response +┌───y1─┬───y2─┐ +│ 2002 │ 2010 │ +└──────┴──────┘ +``` ## toRelativeQuarterNum -Converts a date, or date with time, to the number of the quarter, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of quarters elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeQuarterNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of quarters from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeQuarterNum(toDate('1993-11-25')) AS q1, + toRelativeQuarterNum(toDate('2005-01-05')) AS q2 +``` + +Result: + +```response +┌───q1─┬───q2─┐ +│ 7975 │ 8020 │ +└──────┴──────┘ +``` ## toRelativeMonthNum -Converts a date, or date with time, to the number of the month, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of months elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeMonthNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of months from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeMonthNum(toDate('2001-04-25')) AS m1, + toRelativeMonthNum(toDate('2009-07-08')) AS m2 +``` + +Result: + +```response +┌────m1─┬────m2─┐ +│ 24016 │ 24115 │ +└───────┴───────┘ +``` ## toRelativeWeekNum -Converts a date, or date with time, to the number of the week, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of weeks elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeWeekNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of weeks from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeWeekNum(toDate('2000-02-29')) AS w1, + toRelativeWeekNum(toDate('2001-01-12')) AS w2 +``` + +Result: + +```response +┌───w1─┬───w2─┐ +│ 1574 │ 1619 │ +└──────┴──────┘ +``` ## toRelativeDayNum -Converts a date, or date with time, to the number of the day, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of days elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeDayNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of days from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeDayNum(toDate('1993-10-05')) AS d1, + toRelativeDayNum(toDate('2000-09-20')) AS d2 +``` + +Result: + +```response +┌───d1─┬────d2─┐ +│ 8678 │ 11220 │ +└──────┴───────┘ +``` ## toRelativeHourNum -Converts a date, or date with time, to the number of the hour, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of hours elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeHourNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of hours from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeHourNum(toDateTime('1993-10-05 05:20:36')) AS h1, + toRelativeHourNum(toDateTime('2000-09-20 14:11:29')) AS h2 +``` + +Result: + +```response +┌─────h1─┬─────h2─┐ +│ 208276 │ 269292 │ +└────────┴────────┘ +``` ## toRelativeMinuteNum -Converts a date, or date with time, to the number of the minute, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of minutes elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeMinuteNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of minutes from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeMinuteNum(toDateTime('1993-10-05 05:20:36')) AS m1, + toRelativeMinuteNum(toDateTime('2000-09-20 14:11:29')) AS m2 +``` + +Result: + +```response +┌───────m1─┬───────m2─┐ +│ 12496580 │ 16157531 │ +└──────────┴──────────┘ +``` ## toRelativeSecondNum -Converts a date, or date with time, to the number of the second, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of the seconds elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeSecondNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of seconds from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeSecondNum(toDateTime('1993-10-05 05:20:36')) AS s1, + toRelativeSecondNum(toDateTime('2000-09-20 14:11:29')) AS s2 +``` + +Result: + +```response +┌────────s1─┬────────s2─┐ +│ 749794836 │ 969451889 │ +└───────────┴───────────┘ +``` ## toISOYear From 29664d12fc7eedb977147872bdebced05ecfd058 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 27 Aug 2024 12:44:28 +0200 Subject: [PATCH 130/289] update example of toTime --- docs/en/sql-reference/functions/date-time-functions.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index cb32d1a8d63..46d36dbe92f 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1656,7 +1656,9 @@ SELECT toTime(toDateTime64('1970-12-10 01:20:30.3000',3)) AS result, toTypeName( Result: ```response - +┌──────────────result─┬─toTypeName(result)─┐ +│ 1970-01-02 01:20:30 │ DateTime │ +└─────────────────────┴────────────────────┘ ``` ## toRelativeYearNum From 6febfc78b3418c27d2c5c12787da554ee3a13fb3 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 27 Aug 2024 14:52:46 +0200 Subject: [PATCH 131/289] updates docs for DateTime64OrZero/Null/Default --- .../functions/type-conversion-functions.md | 131 +++++++++++++++++- 1 file changed, 129 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index edd04580f27..625ed4462d7 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -3857,7 +3857,7 @@ Result: ## toDateTime64 -Converts the argument to the [DateTime64](../data-types/datetime64.md) data type. +Converts an input value to a value of type [DateTime64](../data-types/datetime64.md). **Syntax** @@ -3869,7 +3869,7 @@ toDateTime64(expr, scale, [timezone]) - `expr` — The value. [String](../data-types/string.md), [UInt32](../data-types/int-uint.md), [Float](../data-types/float.md) or [DateTime](../data-types/datetime.md). - `scale` - Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. -- `timezone` - Time zone of the specified datetime64 object. +- `timezone` (optional) - Time zone of the specified datetime64 object. **Returned value** @@ -3928,10 +3928,137 @@ SELECT toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul') AS value, toTypeN ## toDateTime64OrZero +Like [toDateTime64](#todatetime64), this function converts an input value to a value of type [DateTime64](../data-types/datetime64.md) but returns the min value of [DateTime64](../data-types/datetime64.md) if an invalid argument is received. + +**Syntax** + +``` sql +toDateTime64OrZero(expr, scale, [timezone]) +``` + +**Arguments** + +- `expr` — The value. [String](../data-types/string.md), [UInt32](../data-types/int-uint.md), [Float](../data-types/float.md) or [DateTime](../data-types/datetime.md). +- `scale` - Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. +- `timezone` (optional) - Time zone of the specified DateTime64 object. + +**Returned value** + +- A calendar date and time of day, with sub-second precision, otherwise the minimum value of `DateTime64`: `1970-01-01 01:00:00.000`. [DateTime64](../data-types/datetime64.md). + +**Example** + +Query: + +```sql +SELECT toDateTime64OrZero('2008-10-12 00:00:00 00:30:30', 3) AS invalid_arg +``` + +Result: + +```response +┌─────────────invalid_arg─┐ +│ 1970-01-01 01:00:00.000 │ +└─────────────────────────┘ +``` + +**See also** + +- [toDateTime64](#todatetime64). +- [toDateTime64OrNull](#todatetime64ornull). +- [toDateTime64OrDefault](#todatetime64ordefault). + ## toDateTime64OrNull +Like [toDateTime64](#todatetime64), this function converts an input value to a value of type [DateTime64](../data-types/datetime64.md) but returns `NULL` if an invalid argument is received. + +**Syntax** + +``` sql +toDateTime64OrNull(expr, scale, [timezone]) +``` + +**Arguments** + +- `expr` — The value. [String](../data-types/string.md), [UInt32](../data-types/int-uint.md), [Float](../data-types/float.md) or [DateTime](../data-types/datetime.md). +- `scale` - Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. +- `timezone` (optional) - Time zone of the specified DateTime64 object. + +**Returned value** + +- A calendar date and time of day, with sub-second precision, otherwise `NULL`. [DateTime64](../data-types/datetime64.md)/[NULL](../data-types/nullable.md). + +**Example** + +Query: + +```sql +SELECT + toDateTime64OrNull('1976-10-18 00:00:00.30', 3) AS valid_arg, + toDateTime64OrNull('1976-10-18 00:00:00 30', 3) AS invalid_arg +``` + +Result: + +```response +┌───────────────valid_arg─┬─invalid_arg─┐ +│ 1976-10-18 00:00:00.300 │ ᴺᵁᴸᴸ │ +└─────────────────────────┴─────────────┘ +``` + +**See also** + +- [toDateTime64](#todatetime64). +- [toDateTime64OrZero](#todatetime64orzero). +- [toDateTime64OrDefault](#todatetime64ordefault). + ## toDateTime64OrDefault +Like [toDateTime64](#todatetime64), this function converts an input value to a value of type [DateTime64](../data-types/datetime64.md), +but returns either the default value of [DateTime64](../data-types/datetime64.md) +or the provided default if an invalid argument is received. + +**Syntax** + +``` sql +toDateTime64OrNull(expr, scale, [timezone, default]) +``` + +**Arguments** + +- `expr` — The value. [String](../data-types/string.md), [UInt32](../data-types/int-uint.md), [Float](../data-types/float.md) or [DateTime](../data-types/datetime.md). +- `scale` - Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. +- `timezone` (optional) - Time zone of the specified DateTime64 object. +- `default` (optional) - Default value to return if an invalid argument is received. [DateTime64](../data-types/datetime64.md). + +**Returned value** + +- A calendar date and time of day, with sub-second precision, otherwise the minimum value of `DateTime64` or the `default` value if provided. [DateTime64](../data-types/datetime64.md). + +**Example** + +Query: + +```sql +SELECT + toDateTime64OrDefault('1976-10-18 00:00:00 30', 3) AS invalid_arg, + toDateTime64OrDefault('1976-10-18 00:00:00 30', 3, 'UTC', toDateTime64('2001-01-01 00:00:00.00',3)) AS invalid_arg_with_default +``` + +Result: + +```response +┌─────────────invalid_arg─┬─invalid_arg_with_default─┐ +│ 1970-01-01 01:00:00.000 │ 2000-12-31 23:00:00.000 │ +└─────────────────────────┴──────────────────────────┘ +``` + +**See also** + +- [toDateTime64](#todatetime64). +- [toDateTime64OrZero](#todatetime64orzero). +- [toDateTime64OrNull](#todatetime64ornull). + ## toDecimal32 Converts an input value to a value of type [`Decimal(9, S)`](../data-types/decimal.md) with scale of `S`. Throws an exception in case of an error. From d218b04fba8422049ec2a6f73bd646c55cd50c88 Mon Sep 17 00:00:00 2001 From: shiyer7474 Date: Wed, 28 Aug 2024 11:52:51 +0000 Subject: [PATCH 132/289] Fix test_role & test_keeper_s3_snapshot --- tests/integration/test_keeper_s3_snapshot/test.py | 2 +- tests/integration/test_role/test.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_keeper_s3_snapshot/test.py b/tests/integration/test_keeper_s3_snapshot/test.py index 84ffc964621..1e766cb974b 100644 --- a/tests/integration/test_keeper_s3_snapshot/test.py +++ b/tests/integration/test_keeper_s3_snapshot/test.py @@ -92,7 +92,7 @@ def test_s3_upload(started_cluster): # Keeper sends snapshots asynchornously, hence we need to retry. @retry(AssertionError, tries=10, delay=2) def _check_snapshots(): - assert set(get_saved_snapshots()) == set( + assert set(get_saved_snapshots()) >= set( [ "snapshot_50.bin.zstd", "snapshot_100.bin.zstd", diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py index 225cab975ff..b746af56083 100644 --- a/tests/integration/test_role/test.py +++ b/tests/integration/test_role/test.py @@ -629,5 +629,6 @@ def test_roles_cache(): check() instance.query("DROP USER " + ", ".join(users)) - instance.query("DROP ROLE " + ", ".join(roles)) + if roles: + instance.query("DROP ROLE " + ", ".join(roles)) instance.query("DROP TABLE tbl") From ad08db39e5309602736eefd1dbb535a223993c1f Mon Sep 17 00:00:00 2001 From: imddba Date: Wed, 28 Aug 2024 20:05:06 +0800 Subject: [PATCH 133/289] Add CKibana as third-party GUI --- docs/en/interfaces/third-party/gui.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index 8d9dce983bc..16d9b66e17f 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -233,6 +233,16 @@ Features: - Useful tools: Zookeeper data exploration, query EXPLAIN, kill queries, etc. - Visualization metric charts: queries and resource usage, number of merges/mutation, merge performance, query performance, etc. +### CKibana {#ckibana} + +[CKibana](https://github.com/TongchengOpenSource/ckibana) is a lightweight service that allows you to effortlessly search, explore, and visualize ClickHouse data using the native Kibana UI. + +Features: + +- Translates chart requests from the native Kibana UI into ClickHouse query syntax. +- Supports advanced features such as sampling and caching to enhance query performance. +- Minimizes the learning cost for users after migrating from ElasticSearch to ClickHouse. + ## Commercial {#commercial} ### DataGrip {#datagrip} From d88aa3952d34eeecbca363a1a66df111c5b3e587 Mon Sep 17 00:00:00 2001 From: flynn Date: Thu, 29 Aug 2024 10:56:26 +0000 Subject: [PATCH 134/289] Disable alter table add vector similarity index if setting does not enabled --- src/Storages/AlterCommands.cpp | 10 ++++++++++ src/Storages/AlterCommands.h | 2 ++ src/Storages/MergeTree/MergeTreeData.cpp | 5 +++++ ...rbid_add_vector_similarity_index.reference | 0 ...231_forbid_add_vector_similarity_index.sql | 19 +++++++++++++++++++ 5 files changed, 36 insertions(+) create mode 100644 tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.reference create mode 100644 tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index d92d8b59f6e..e0563f9f1c6 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1142,6 +1142,16 @@ bool AlterCommands::hasFullTextIndex(const StorageInMemoryMetadata & metadata) return false; } +bool AlterCommands::hasVectorSimilarityIndex(const StorageInMemoryMetadata & metadata) +{ + for (const auto & index : metadata.secondary_indices) + { + if (index.type == "vector_similarity") + return true; + } + return false; +} + void AlterCommands::apply(StorageInMemoryMetadata & metadata, ContextPtr context) const { if (!prepared) diff --git a/src/Storages/AlterCommands.h b/src/Storages/AlterCommands.h index a91bac10214..b2f0f9a6abd 100644 --- a/src/Storages/AlterCommands.h +++ b/src/Storages/AlterCommands.h @@ -237,6 +237,8 @@ public: /// Check if commands have any full-text index static bool hasFullTextIndex(const StorageInMemoryMetadata & metadata); + + static bool hasVectorSimilarityIndex(const StorageInMemoryMetadata & metadata); }; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 94f6d196b99..8b12330c1a4 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3230,6 +3230,11 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental full-text index feature is not enabled (turn on setting 'allow_experimental_full_text_index')"); + if (AlterCommands::hasVectorSimilarityIndex(new_metadata) && !settings.allow_experimental_vector_similarity_index) + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "Experimental vector_similarity index feature is not enabled (turn on setting 'allow_experimental_vector_similarity_index')"); + for (const auto & disk : getDisks()) if (!disk->supportsHardLinks() && !commands.isSettingsAlter() && !commands.isCommentAlter()) throw Exception( diff --git a/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.reference b/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql b/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql new file mode 100644 index 00000000000..a1e362e7bd1 --- /dev/null +++ b/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS test_embedding; + +CREATE TABLE test_embedding +( + id UInt32, + embedding Array(Float32), +) +ENGINE = MergeTree +ORDER BY tuple(); + +SET allow_experimental_vector_similarity_index = 0; + +alter table test_embedding add INDEX idx embedding TYPE vector_similarity('hnsw', 'cosineDistance'); -- { serverError SUPPORT_IS_DISABLED } + +SET allow_experimental_vector_similarity_index = 1; + +alter table test_embedding add INDEX idx embedding TYPE vector_similarity('hnsw', 'cosineDistance'); + +DROP TABLE test_embedding; From 0400dcb03eefdb6604d04f5ed3c70c179032f84d Mon Sep 17 00:00:00 2001 From: flynn Date: Thu, 29 Aug 2024 12:34:58 +0000 Subject: [PATCH 135/289] no-fastest --- .../0_stateless/03231_forbid_add_vector_similarity_index.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql b/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql index a1e362e7bd1..e91d7c71eac 100644 --- a/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql +++ b/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql @@ -1,3 +1,5 @@ +-- Tags: no-fasttest + DROP TABLE IF EXISTS test_embedding; CREATE TABLE test_embedding From 6ad8e5558a99b43e3452c057346b9c44e8e27517 Mon Sep 17 00:00:00 2001 From: flynn Date: Fri, 30 Aug 2024 07:25:25 +0000 Subject: [PATCH 136/289] Fix typo --- src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp index ae183d74782..58892d0dbf2 100644 --- a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp @@ -195,7 +195,7 @@ void MergeTreeIndexGranuleVectorSimilarity::serializeBinary(WriteBuffer & ostr) LOG_TRACE(logger, "Start writing vector similarity index"); if (empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to write empty minmax index {}", backQuote(index_name)); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to write empty vector similarity index {}", backQuote(index_name)); writeIntBinary(FILE_FORMAT_VERSION, ostr); From c5b92413cac091c4e28acb5160b89fcabc37a853 Mon Sep 17 00:00:00 2001 From: flynn Date: Fri, 30 Aug 2024 08:28:31 +0000 Subject: [PATCH 137/289] Fix vector similarity index does not work for cosineDistance --- src/Storages/MergeTree/VectorSimilarityCondition.cpp | 2 ++ src/Storages/MergeTree/VectorSimilarityCondition.h | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/VectorSimilarityCondition.cpp b/src/Storages/MergeTree/VectorSimilarityCondition.cpp index c8f33857640..251cdde65ab 100644 --- a/src/Storages/MergeTree/VectorSimilarityCondition.cpp +++ b/src/Storages/MergeTree/VectorSimilarityCondition.cpp @@ -44,6 +44,8 @@ VectorSimilarityCondition::Info::DistanceFunction stringToDistanceFunction(std:: { if (distance_function == "L2Distance") return VectorSimilarityCondition::Info::DistanceFunction::L2; + else if (distance_function == "cosineDistance") + return VectorSimilarityCondition::Info::DistanceFunction::Cosine; else return VectorSimilarityCondition::Info::DistanceFunction::Unknown; } diff --git a/src/Storages/MergeTree/VectorSimilarityCondition.h b/src/Storages/MergeTree/VectorSimilarityCondition.h index 2380f8f46b0..e2946222f49 100644 --- a/src/Storages/MergeTree/VectorSimilarityCondition.h +++ b/src/Storages/MergeTree/VectorSimilarityCondition.h @@ -57,7 +57,8 @@ public: enum class DistanceFunction : uint8_t { Unknown, - L2 + L2, + Cosine }; std::vector reference_vector; From af7adfe4b2f8e9b3b7354314a91d19a19b7955f9 Mon Sep 17 00:00:00 2001 From: flynn Date: Fri, 30 Aug 2024 09:03:23 +0000 Subject: [PATCH 138/289] Remove unused code --- src/Storages/MergeTree/VectorSimilarityCondition.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/VectorSimilarityCondition.h b/src/Storages/MergeTree/VectorSimilarityCondition.h index e2946222f49..83ae1e19bfb 100644 --- a/src/Storages/MergeTree/VectorSimilarityCondition.h +++ b/src/Storages/MergeTree/VectorSimilarityCondition.h @@ -143,6 +143,7 @@ private: void traverseOrderByAST(const ASTPtr & node, RPN & rpn); /// Returns true and stores ANNExpr if the query has valid WHERE section + /// TODO NOT implemented, WHERE does not supported. static bool matchRPNWhere(RPN & rpn, Info & info); /// Returns true and stores ANNExpr if the query has valid ORDERBY section @@ -151,9 +152,6 @@ private: /// Returns true and stores Length if we have valid LIMIT clause in query static bool matchRPNLimit(RPNElement & rpn, UInt64 & limit); - /// Matches dist function, reference vector, column name - static bool matchMainParts(RPN::iterator & iter, const RPN::iterator & end, Info & info); - /// Gets float or int from AST node static float getFloatOrIntLiteralOrPanic(const RPN::iterator& iter); From 81d0a04ecbcb9ad95a875c94467ab588bc5eeb7c Mon Sep 17 00:00:00 2001 From: shiyer7474 Date: Sun, 1 Sep 2024 13:53:56 +0000 Subject: [PATCH 139/289] Added restart of node1 for running multiple iterations of test --- tests/integration/test_keeper_s3_snapshot/test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_keeper_s3_snapshot/test.py b/tests/integration/test_keeper_s3_snapshot/test.py index 1e766cb974b..5e2545ae0a7 100644 --- a/tests/integration/test_keeper_s3_snapshot/test.py +++ b/tests/integration/test_keeper_s3_snapshot/test.py @@ -2,6 +2,7 @@ import pytest from helpers.cluster import ClickHouseCluster from time import sleep from retry import retry +import helpers.keeper_utils as keeper_utils from kazoo.client import KazooClient @@ -125,3 +126,5 @@ def test_s3_upload(started_cluster): ) destroy_zk_client(node2_zk) + node1.start_clickhouse() # for next iteration + keeper_utils.wait_until_connected(cluster, node1) From 4f16797cd127a4dff7169908890b0010d82027ae Mon Sep 17 00:00:00 2001 From: shiyer7474 Date: Mon, 2 Sep 2024 00:35:59 +0000 Subject: [PATCH 140/289] Formatting --- tests/integration/test_keeper_s3_snapshot/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_keeper_s3_snapshot/test.py b/tests/integration/test_keeper_s3_snapshot/test.py index 5e2545ae0a7..f47a26a77ad 100644 --- a/tests/integration/test_keeper_s3_snapshot/test.py +++ b/tests/integration/test_keeper_s3_snapshot/test.py @@ -126,5 +126,5 @@ def test_s3_upload(started_cluster): ) destroy_zk_client(node2_zk) - node1.start_clickhouse() # for next iteration + node1.start_clickhouse() # for next iteration keeper_utils.wait_until_connected(cluster, node1) From a3522072d4b38817559541353f3b70625f92b918 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 2 Sep 2024 13:08:34 +0000 Subject: [PATCH 141/289] Respect format settings in Values format during conversion from expression to the destination type --- src/Interpreters/convertFieldToType.cpp | 30 +++++++++---------- src/Interpreters/convertFieldToType.h | 7 +++-- .../Formats/Impl/ValuesBlockInputFormat.cpp | 2 +- ...at_settings_in_fields_conversion.reference | 1 + ...t_format_settings_in_fields_conversion.sql | 7 +++++ 5 files changed, 28 insertions(+), 19 deletions(-) create mode 100644 tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.reference create mode 100644 tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.sql diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 7e1b4e2fb0e..737353095b8 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -164,7 +164,7 @@ Field convertDecimalType(const Field & from, const To & type) } -Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const IDataType * from_type_hint) +Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const IDataType * from_type_hint, const FormatSettings & format_settings) { if (from_type_hint && from_type_hint->equals(type)) { @@ -359,7 +359,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID Array res(src_arr_size); for (size_t i = 0; i < src_arr_size; ++i) { - res[i] = convertFieldToType(src_arr[i], element_type); + res[i] = convertFieldToType(src_arr[i], element_type, nullptr, format_settings); if (res[i].isNull() && !canContainNull(element_type)) { // See the comment for Tuples below. @@ -387,7 +387,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID for (size_t i = 0; i < dst_tuple_size; ++i) { const auto & element_type = *(type_tuple->getElements()[i]); - res[i] = convertFieldToType(src_tuple[i], element_type); + res[i] = convertFieldToType(src_tuple[i], element_type, nullptr, format_settings); if (res[i].isNull() && !canContainNull(element_type)) { /* @@ -435,12 +435,12 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID Tuple updated_entry(2); - updated_entry[0] = convertFieldToType(key, key_type); + updated_entry[0] = convertFieldToType(key, key_type, nullptr, format_settings); if (updated_entry[0].isNull() && !canContainNull(key_type)) have_unconvertible_element = true; - updated_entry[1] = convertFieldToType(value, value_type); + updated_entry[1] = convertFieldToType(value, value_type, nullptr, format_settings); if (updated_entry[1].isNull() && !canContainNull(value_type)) have_unconvertible_element = true; @@ -551,7 +551,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID ReadBufferFromString in_buffer(src.safeGet()); try { - type_to_parse->getDefaultSerialization()->deserializeWholeText(*col, in_buffer, FormatSettings{}); + type_to_parse->getDefaultSerialization()->deserializeWholeText(*col, in_buffer, format_settings); } catch (Exception & e) { @@ -563,7 +563,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID } Field parsed = (*col)[0]; - return convertFieldToType(parsed, type, from_type_hint); + return convertFieldToType(parsed, type, from_type_hint, format_settings); } throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch in IN or VALUES section. Expected: {}. Got: {}", @@ -573,7 +573,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID } -Field convertFieldToType(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint) +Field convertFieldToType(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint, const FormatSettings & format_settings) { if (from_value.isNull()) return from_value; @@ -582,7 +582,7 @@ Field convertFieldToType(const Field & from_value, const IDataType & to_type, co return from_value; if (const auto * low_cardinality_type = typeid_cast(&to_type)) - return convertFieldToType(from_value, *low_cardinality_type->getDictionaryType(), from_type_hint); + return convertFieldToType(from_value, *low_cardinality_type->getDictionaryType(), from_type_hint, format_settings); else if (const auto * nullable_type = typeid_cast(&to_type)) { const IDataType & nested_type = *nullable_type->getNestedType(); @@ -593,20 +593,20 @@ Field convertFieldToType(const Field & from_value, const IDataType & to_type, co if (from_type_hint && from_type_hint->equals(nested_type)) return from_value; - return convertFieldToTypeImpl(from_value, nested_type, from_type_hint); + return convertFieldToTypeImpl(from_value, nested_type, from_type_hint, format_settings); } else - return convertFieldToTypeImpl(from_value, to_type, from_type_hint); + return convertFieldToTypeImpl(from_value, to_type, from_type_hint, format_settings); } -Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint) +Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint, const FormatSettings & format_settings) { bool is_null = from_value.isNull(); if (is_null && !canContainNull(to_type)) throw Exception(ErrorCodes::TYPE_MISMATCH, "Cannot convert NULL to {}", to_type.getName()); - Field converted = convertFieldToType(from_value, to_type, from_type_hint); + Field converted = convertFieldToType(from_value, to_type, from_type_hint, format_settings); if (!is_null && converted.isNull()) throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, @@ -626,9 +626,9 @@ static bool decimalEqualsFloat(Field field, Float64 float_value) return decimal_to_float == float_value; } -std::optional convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type) +std::optional convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type, const FormatSettings & format_settings) { - Field result_value = convertFieldToType(from_value, to_type, &from_type); + Field result_value = convertFieldToType(from_value, to_type, &from_type, format_settings); if (Field::isDecimal(from_value.getType()) && Field::isDecimal(result_value.getType())) { diff --git a/src/Interpreters/convertFieldToType.h b/src/Interpreters/convertFieldToType.h index 4aa09f8619e..c3c6271a157 100644 --- a/src/Interpreters/convertFieldToType.h +++ b/src/Interpreters/convertFieldToType.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -15,13 +16,13 @@ class IDataType; * Checks for the compatibility of types, checks values fall in the range of valid values of the type, makes type conversion. * If the value does not fall into the range - returns Null. */ -Field convertFieldToType(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint = nullptr); +Field convertFieldToType(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint = nullptr, const FormatSettings & format_settings = {}); /// Does the same, but throws ARGUMENT_OUT_OF_BOUND if value does not fall into the range. -Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint = nullptr); +Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint = nullptr, const FormatSettings & format_settings = {}); /// Applies stricter rules than convertFieldToType, doesn't allow loss of precision converting to Decimal. /// Returns `Field` if the conversion was successful and the result is equal to the original value, otherwise returns nullopt. -std::optional convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type); +std::optional convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type, const FormatSettings & format_settings = {}); } diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index 10d0e051665..16b88d0b8dc 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -542,7 +542,7 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx if (format_settings.null_as_default) tryToReplaceNullFieldsInComplexTypesWithDefaultValues(expression_value, type); - Field value = convertFieldToType(expression_value, type, value_raw.second.get()); + Field value = convertFieldToType(expression_value, type, value_raw.second.get(), format_settings); /// Check that we are indeed allowed to insert a NULL. if (value.isNull() && !type.isNullable() && !type.isLowCardinalityNullable()) diff --git a/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.reference b/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.reference new file mode 100644 index 00000000000..8dbf92d6590 --- /dev/null +++ b/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.reference @@ -0,0 +1 @@ +{'Hello':'2020-01-01 00:00:00'} diff --git a/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.sql b/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.sql new file mode 100644 index 00000000000..484a16bb22f --- /dev/null +++ b/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.sql @@ -0,0 +1,7 @@ +drop table if exists test; +create table test (map Map(String, DateTime)) engine=Memory; +set date_time_input_format='best_effort'; +insert into test values (map('Hello', '01/01/2020')); +select * from test; +drop table test; + From 45556278c917a0053e902ae9ebcee60f98062fe5 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 3 Sep 2024 11:06:28 +0000 Subject: [PATCH 142/289] Fix possible timeouts in `sh` tests with tsan, att. 2 --- src/Client/ClientBase.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index e34e263beb5..edcb98be90f 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1896,6 +1896,25 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin /// Temporarily apply query settings to context. std::optional old_settings; SCOPE_EXIT_SAFE({ + try + { + /// We need to park ParallelFormating threads, + /// because they can use settings from global context + /// and it can lead to data race with `setSettings` + if (output_format) + { + output_format->finalize(); + output_format.reset(); + } + } + catch (...) + { + if (!have_error) + { + client_exception = std::make_unique(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode()); + have_error = true; + } + } if (old_settings) client_context->setSettings(*old_settings); }); From ecfe9d97821658bd43c7034f3cb2dc85825c30bf Mon Sep 17 00:00:00 2001 From: Michael Stetsyuk Date: Fri, 30 Aug 2024 14:34:24 +0000 Subject: [PATCH 143/289] make replicated database creation operations idempotent --- src/Databases/DatabaseReplicated.cpp | 103 ++++++++++++++---- src/Interpreters/InterpreterCreateQuery.cpp | 21 ++-- .../test_replicated_database/test.py | 35 ++++++ 3 files changed, 127 insertions(+), 32 deletions(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 8e3378bcc12..26a834eea4c 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -441,7 +441,8 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessL bool is_create_query = mode == LoadingStrictnessLevel::CREATE; String replica_host_id; - if (current_zookeeper->tryGet(replica_path, replica_host_id)) + bool replica_exists_in_zk = current_zookeeper->tryGet(replica_path, replica_host_id); + if (replica_exists_in_zk) { if (replica_host_id == DROPPED_MARK && !is_create_query) { @@ -454,7 +455,7 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessL String host_id = getHostID(getContext(), db_uuid, cluster_auth_info.cluster_secure_connection); String host_id_default = getHostID(getContext(), db_uuid, false); - if (is_create_query || (replica_host_id != host_id && replica_host_id != host_id_default)) + if (replica_host_id != host_id && replica_host_id != host_id_default) { throw Exception( ErrorCodes::REPLICA_ALREADY_EXISTS, @@ -485,12 +486,13 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessL createEmptyLogEntry(current_zookeeper); } } - else if (is_create_query) + + if (is_create_query) { - /// Create new replica. Throws if replica with the same name already exists + /// Create replica nodes in ZooKeeper. If newly initialized nodes already exist, reuse them. createReplicaNodesInZooKeeper(current_zookeeper); } - else + else if (!replica_exists_in_zk) { /// It's not CREATE query, but replica does not exist. Probably it was dropped. /// Do not create anything, continue as readonly. @@ -606,37 +608,92 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt "already contains some data and it does not look like Replicated database path.", zookeeper_path); /// Write host name to replica_path, it will protect from multiple replicas with the same name - auto host_id = getHostID(getContext(), db_uuid, cluster_auth_info.cluster_secure_connection); + const auto host_id = getHostID(getContext(), db_uuid, cluster_auth_info.cluster_secure_connection); + + const std::vector check_paths = { + replica_path, + replica_path + "/replica_group", + replica_path + "/digest", + + /// Needed to mark all the queries + /// in the range (max log ptr at replica ZooKeeper nodes creation, max log ptr after replica recovery] as successful. + /// Previously, this method was not idempotent and max_log_ptr_at_creation could be stored in memory. + /// we need to store max_log_ptr_at_creation in ZooKeeper to make this method idempotent during replica creation. + replica_path + "/max_log_ptr_at_creation", + }; + bool nodes_exist = true; + auto check_responses = current_zookeeper->tryGet(check_paths); + for (size_t i = 0; i < check_responses.size(); ++i) + { + const auto response = check_responses[i]; + + if (response.error == Coordination::Error::ZNONODE) + { + nodes_exist = false; + break; + } else if (response.error != Coordination::Error::ZOK) + { + throw zkutil::KeeperException::fromPath(response.error, check_paths[i]); + } + } + + if (nodes_exist) + { + const std::vector expected_data = { + host_id, + replica_group_name, + "0", + }; + for (size_t i = 0; i != expected_data.size(); ++i) + { + if (check_responses[i].data != expected_data[i]) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Replica node {} in ZooKeeper already exists and contains unexpected value: {}", + quoteString(check_paths[i]), quoteString(check_responses[i].data)); + } + } + + LOG_DEBUG(log, "Newly initialized replica nodes found in ZooKeeper, reusing them"); + max_log_ptr_at_creation = parse(check_responses[check_responses.size() - 1].data); + createEmptyLogEntry(current_zookeeper); + return; + } for (int attempts = 10; attempts > 0; --attempts) { Coordination::Stat stat; - String max_log_ptr_str = current_zookeeper->get(zookeeper_path + "/max_log_ptr", &stat); + const String max_log_ptr_str = current_zookeeper->get(zookeeper_path + "/max_log_ptr", &stat); - Coordination::Requests ops; - ops.emplace_back(zkutil::makeCreateRequest(replica_path, host_id, zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/log_ptr", "0", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/digest", "0", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/replica_group", replica_group_name, zkutil::CreateMode::Persistent)); - /// In addition to creating the replica nodes, we record the max_log_ptr at the instant where - /// we declared ourself as an existing replica. We'll need this during recoverLostReplica to - /// notify other nodes that issued new queries while this node was recovering. - ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/max_log_ptr", stat.version)); + const Coordination::Requests ops = { + zkutil::makeCreateRequest(replica_path, host_id, zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest(replica_path + "/log_ptr", "0", zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest(replica_path + "/digest", "0", zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest(replica_path + "/replica_group", replica_group_name, zkutil::CreateMode::Persistent), + + /// In addition to creating the replica nodes, we record the max_log_ptr at the instant where + /// we declared ourself as an existing replica. We'll need this during recoverLostReplica to + /// notify other nodes that issued new queries while this node was recovering. + zkutil::makeCheckRequest(zookeeper_path + "/max_log_ptr", stat.version), + zkutil::makeCreateRequest(replica_path + "/max_log_ptr_at_creation", max_log_ptr_str, zkutil::CreateMode::Persistent), + }; + + Coordination::Responses ops_responses; + const auto code = current_zookeeper->tryMulti(ops, ops_responses); - Coordination::Responses responses; - const auto code = current_zookeeper->tryMulti(ops, responses); if (code == Coordination::Error::ZOK) { max_log_ptr_at_creation = parse(max_log_ptr_str); - break; + createEmptyLogEntry(current_zookeeper); + return; } - else if (code == Coordination::Error::ZNODEEXISTS || attempts == 1) + + if (attempts == 1) { - /// If its our last attempt, or if the replica already exists, fail immediately. - zkutil::KeeperMultiException::check(code, ops, responses); + zkutil::KeeperMultiException::check(code, ops, ops_responses); } } - createEmptyLogEntry(current_zookeeper); } void DatabaseReplicated::beforeLoadingMetadata(ContextMutablePtr context_, LoadingStrictnessLevel mode) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 80cb0510b35..a1ed9f40685 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -228,8 +228,8 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) metadata_path = metadata_path / "store" / DatabaseCatalog::getPathForUUID(create.uuid); - if (!create.attach && fs::exists(metadata_path)) - throw Exception(ErrorCodes::DATABASE_ALREADY_EXISTS, "Metadata directory {} already exists", metadata_path.string()); + if (!create.attach && fs::exists(metadata_path) && !fs::is_empty(metadata_path)) + throw Exception(ErrorCodes::DATABASE_ALREADY_EXISTS, "Metadata directory {} already exists and is not empty", metadata_path.string()); } else if (create.storage->engine->name == "MaterializeMySQL" || create.storage->engine->name == "MaterializedMySQL") @@ -329,6 +329,9 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) writeChar('\n', statement_buf); String statement = statement_buf.str(); + /// Needed to make database creation retriable if it fails after the file is created + fs::remove(metadata_file_tmp_path); + /// Exclusive flag guarantees, that database is not created right now in another thread. WriteBufferFromFile out(metadata_file_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL); writeString(statement, out); @@ -350,13 +353,6 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) DatabaseCatalog::instance().attachDatabase(database_name, database); added = true; - if (need_write_metadata) - { - /// Prevents from overwriting metadata of detached database - renameNoReplace(metadata_file_tmp_path, metadata_file_path); - renamed = true; - } - if (!load_database_without_tables) { /// We use global context here, because storages lifetime is bigger than query context lifetime @@ -368,6 +364,13 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) /// Only then prioritize, schedule and wait all the startup tasks waitLoad(currentPoolOr(TablesLoaderForegroundPoolId), startup_tasks); } + + if (need_write_metadata) + { + /// Prevents from overwriting metadata of detached database + renameNoReplace(metadata_file_tmp_path, metadata_file_path); + renamed = true; + } } catch (...) { diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 60a6e099b22..58e99d4c408 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -1,5 +1,8 @@ +import concurrent.futures import os +import random import shutil +import string import time import re import pytest @@ -1549,3 +1552,35 @@ def test_all_groups_cluster(started_cluster): assert "bad_settings_node\ndummy_node\n" == bad_settings_node.query( "select host_name from system.clusters where name='all_groups.db_cluster' order by host_name" ) + + +def test_database_creation_idempotency(started_cluster): + def randomize_database_name(database_name, random_suffix_length=20): + letters = string.ascii_letters + string.digits + return f"{database_name}{''.join(random.choice(letters) for _ in range(random_suffix_length))}" + + databases = [randomize_database_name("rdb") for _ in range(100)] + + def create_database(name): + main_node.query( + f""" + CREATE DATABASE {name} + ENGINE = Replicated('/test/test_database_creation_idempotency/{name}', 'shard', 'replica') + """ + ) + + with concurrent.futures.ThreadPoolExecutor() as executor: + for name in databases: + executor.submit(create_database, name) + + main_node.restart_clickhouse(kill=True) + + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [executor.submit(create_database, name) for name in databases] + concurrent.futures.wait(futures) + + assert int( + main_node.query( + f"SELECT count() FROM system.databases WHERE name IN {databases}" + ).strip() + ) == len(databases) From 4fad12ecb075dd1ba4faee65b7d2155085fa2e5f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 3 Sep 2024 19:53:28 +0200 Subject: [PATCH 144/289] rewrite the main logic --- src/Functions/DateTimeTransforms.h | 35 +-- src/Functions/toStartOfInterval.cpp | 199 ++++++------------ ...to_start_of_interval_with_origin.reference | 4 +- 3 files changed, 79 insertions(+), 159 deletions(-) diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 50907e3e87e..9824608f067 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -22,7 +22,6 @@ namespace DB { -static Int64 Int64_max_value = std::numeric_limits::max(); static constexpr auto millisecond_multiplier = 1'000; static constexpr auto microsecond_multiplier = 1'000'000; static constexpr auto nanosecond_multiplier = 1'000'000'000; @@ -701,12 +700,8 @@ struct ToStartOfInterval if (origin == 0) return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t / scale_multiplier), weeks); else - { - if (weeks < Int64_max_value / 7) // Check if multiplication doesn't overflow Int64 value - return ToStartOfInterval::execute(t, weeks * 7, time_zone, scale_multiplier, origin); - else - throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 7 is out of bounds for type Int64", weeks); - } + return ToStartOfInterval::execute(t, weeks * 7, time_zone, scale_multiplier, origin); + } }; @@ -727,18 +722,20 @@ struct ToStartOfInterval } static Int64 execute(Int64 t, Int64 months, const DateLUTImpl & time_zone, Int64 scale_multiplier, Int64 origin = 0) { + const Int64 scaled_time = t / scale_multiplier; if (origin == 0) - return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t / scale_multiplier), months); + return time_zone.toStartOfMonthInterval(time_zone.toDayNum(scaled_time), months); else { - Int64 days = time_zone.toDayOfMonth(t / scale_multiplier + origin) - time_zone.toDayOfMonth(origin); - Int64 months_to_add = time_zone.toMonth(t / scale_multiplier + origin) - time_zone.toMonth(origin); - Int64 years = time_zone.toYear(t / scale_multiplier + origin) - time_zone.toYear(origin); + const Int64 scaled_origin = origin / scale_multiplier; + const Int64 days = time_zone.toDayOfMonth(scaled_time + scaled_origin) - time_zone.toDayOfMonth(scaled_origin); + Int64 months_to_add = time_zone.toMonth(scaled_time + scaled_origin) - time_zone.toMonth(scaled_origin); + const Int64 years = time_zone.toYear(scaled_time + scaled_origin) - time_zone.toYear(scaled_origin); months_to_add = days < 0 ? months_to_add - 1 : months_to_add; months_to_add += years * 12; Int64 month_multiplier = (months_to_add / months) * months; - return time_zone.addMonths(time_zone.toDate(origin), month_multiplier) - time_zone.toDate(origin); + return (time_zone.addMonths(time_zone.toDate(scaled_origin), month_multiplier) - time_zone.toDate(scaled_origin)); } } }; @@ -763,12 +760,7 @@ struct ToStartOfInterval if (origin == 0) return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t / scale_multiplier), quarters); else - { - if (quarters < Int64_max_value / 3) // Check if multiplication doesn't overflow Int64 value - return ToStartOfInterval::execute(t, quarters * 3, time_zone, scale_multiplier, origin); - else - throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 3 is out of bounds for type Int64", quarters); - } + return ToStartOfInterval::execute(t, quarters * 3, time_zone, scale_multiplier, origin); } }; @@ -792,12 +784,7 @@ struct ToStartOfInterval if (origin == 0) return time_zone.toStartOfYearInterval(time_zone.toDayNum(t / scale_multiplier), years); else - { - if (years < Int64_max_value / 12) // Check if multiplication doesn't overflow Int64 value - return ToStartOfInterval::execute(t, years * 12, time_zone, scale_multiplier, origin); - else - throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} * 12 is out of bounds for type Int64", years); - } + return ToStartOfInterval::execute(t, years * 12, time_zone, scale_multiplier, origin);z } }; diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index cc5ffd56976..b749d92eeb6 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -1,6 +1,3 @@ -#include -#include -#include #include #include #include @@ -13,6 +10,8 @@ #include #include #include +#include +#include namespace DB @@ -176,7 +175,7 @@ public: else { throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Number of arguments for function {} doesn't match: passed {}, should be 2, 3 or 4", + "Number of arguments for function {} doesn't match: passed {}, must be 2, 3 or 4", getName(), arguments.size()); } @@ -193,13 +192,17 @@ public: { UInt32 scale = 0; if (isDateTime64(arguments[0].type) && overload == Overload::Origin) + { scale = assert_cast(*arguments[0].type.get()).getScale(); + if (assert_cast(*arguments[2].type.get()).getScale() != scale) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same scale", getName()); + } if (interval_type->getKind() == IntervalKind::Kind::Nanosecond) - scale = (9 > scale) ? 9 : scale; + scale = 9; else if (interval_type->getKind() == IntervalKind::Kind::Microsecond) - scale = (6 > scale) ? 6 : scale; + scale = 6; else if (interval_type->getKind() == IntervalKind::Kind::Millisecond) - scale = (3 > scale) ? 3 : scale; + scale = 3; const size_t time_zone_arg_num = (overload == Overload::Default) ? 2 : 3; return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, time_zone_arg_num, 0, false)); @@ -218,7 +221,7 @@ public: if (overload == Overload::Origin) origin_column = arguments[2]; - const size_t time_zone_arg_num = (overload == Overload::Origin) ? 3 : 2; + const size_t time_zone_arg_num = (overload == Overload::Default) ? 2 : 3; const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, time_zone_arg_num, 0); ColumnPtr result_column; @@ -272,25 +275,22 @@ private: if (!interval_type) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for 2nd argument of function {}, must be a time interval", getName()); - if (isDate(time_data_type) || isDateTime(time_data_type)) + switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) { - switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) - { - case IntervalKind::Kind::Nanosecond: - case IntervalKind::Kind::Microsecond: - case IntervalKind::Kind::Millisecond: - if (isDate(time_data_type) || isDateTime(time_data_type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal interval kind for argument data type {}", isDate(time_data_type) ? "Date" : "DateTime"); - break; - case IntervalKind::Kind::Second: - case IntervalKind::Kind::Minute: - case IntervalKind::Kind::Hour: - if (isDate(time_data_type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal interval kind for argument data type Date"); - break; - default: - break; - } + case IntervalKind::Kind::Nanosecond: + case IntervalKind::Kind::Microsecond: + case IntervalKind::Kind::Millisecond: + if (isDate(time_data_type) || isDateTime(time_data_type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal interval kind for argument data type {}", isDate(time_data_type) ? "Date" : "DateTime"); + break; + case IntervalKind::Kind::Second: + case IntervalKind::Kind::Minute: + case IntervalKind::Kind::Hour: + if (isDate(time_data_type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal interval kind for argument data type Date"); + break; + default: + break; } const auto * interval_column_const_int64 = checkAndGetColumnConst(interval_column.column.get()); @@ -330,27 +330,10 @@ private: std::unreachable(); } - template - static Int64 scaleFromInterval() - { - switch (unit) - { - case IntervalKind::Kind::Millisecond: - return 1'000; - case IntervalKind::Kind::Microsecond: - return 1'000'000; - case IntervalKind::Kind::Nanosecond: - return 1'000'000'000; - default: - return 1; - } - } - template ColumnPtr execute(const TimeDataType &, const ColumnType & time_column_type, Int64 num_units, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, UInt16 scale) const { using ResultColumnType = typename ResultDataType::ColumnType; - using ResultFieldType = typename ResultDataType::FieldType; const auto & time_data = time_column_type.getData(); size_t size = time_data.size(); @@ -360,114 +343,64 @@ private: auto & result_data = col_to->getData(); result_data.resize(size); - const Int64 scale_endtime = DecimalUtils::scaleMultiplier(scale); - const Int64 scale_interval = scaleFromInterval(); + Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale); - /// In case if we have a difference between time arguments and Interval, we need to calculate the difference between them - /// to get the right precision for the result. - const Int64 scale_diff = (scale_interval > scale_endtime) ? (scale_interval / scale_endtime) : (scale_endtime / scale_interval); + if (origin_column.column) // Overload: Origin + { + const bool is_small_interval = (unit == IntervalKind::Kind::Nanosecond || unit == IntervalKind::Kind::Microsecond || unit == IntervalKind::Kind::Millisecond); + const bool is_result_date = isDate(result_type); - if (origin_column.column == nullptr) - { - if (scale_endtime > scale_interval && scale_interval != 1) - { - for (size_t i = 0; i != size; ++i) - { - /// If we have a time argument that has bigger scale than the interval can contain and interval is not default, we need - /// to return a value with bigger precision and thus we should multiply result on the scale difference. - result_data[i] = 0; - result_data[i] += static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_endtime)); - } - } - else - { - for (size_t i = 0; i != size; ++i) - result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_endtime)); - } - } - else - { - UInt64 origin = origin_column.column->get64(0); + Int64 result_scale = scale_multiplier; Int64 origin_scale = 1; - if (isDateTime64(origin_column.type.get())) - origin_scale = assert_cast(*origin_column.type.get()).getScale(); + if (isDateTime64(result_type)) /// We have origin scale only in case if arguments are DateTime64. + origin_scale = assert_cast(*origin_column.type).getScaleMultiplier(); + else if (!is_small_interval) /// In case of large interval and arguments are not DateTime64, we should not have scale in result. + result_scale = 1; + + if (is_small_interval) + result_scale = assert_cast(*result_type).getScaleMultiplier(); + + /// In case if we have a difference between time arguments and Interval, we need to calculate the difference between them + /// to get the right precision for the result. In case of large intervals, we should not have scale difference. + Int64 scale_diff = is_small_interval ? std::max(result_scale / origin_scale, origin_scale / result_scale) : 1; + + static constexpr Int64 SECONDS_PER_DAY = 86'400; + + UInt64 origin = origin_column.column->get64(0); for (size_t i = 0; i != size; ++i) { - UInt64 end_time = time_data[i]; - - if (origin > static_cast(end_time) && origin_scale == scale) + UInt64 time_arg = time_data[i]; + if (origin > static_cast(time_arg)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The origin must be before the end date / date with time"); - else if (origin_scale > scale) - origin /= static_cast(std::pow(10, origin_scale - scale)); /// If arguments have different scales, we make - else if (origin_scale < scale) /// origin argument to have the same scale as the first argument. - origin *= static_cast(std::pow(10, scale - origin_scale)); - /// The trick to calculate the interval starting from an offset is to - /// 1. subtract the offset, - /// 2. perform the calculation, and - /// 3. add the offset to the result. - - static constexpr size_t SECONDS_PER_DAY = 86'400; - result_data[i] = 0; - - if (isDate(origin_column.type.get())) /// We need to perform calculations on dateTime (dateTime64) values only. + if (is_result_date) /// All internal calculations of ToStartOfInterval<...> expect arguments to be seconds or milli-, micro-, nanoseconds. { - end_time *= SECONDS_PER_DAY; + time_arg *= SECONDS_PER_DAY; origin *= SECONDS_PER_DAY; } - Int64 delta = (end_time - origin) * (isDateTime64(origin_column.type.get()) ? 1 : scale_endtime); /// No need to multiply on scale endtime if we have dateTime64 argument. - Int64 offset = 0; + Int64 offset = ToStartOfInterval::execute(time_arg - origin, num_units, time_zone, result_scale, origin); + /// In case if arguments are DateTime64 with large interval, we should apply scale on it. + offset *= (!is_small_interval) ? result_scale : 1; + + if (is_result_date) /// Convert back to date after calculations. { - auto origin_data = isDateTime64(result_type) ? origin / scale_endtime : origin; - offset = static_cast(ToStartOfInterval::execute(delta, num_units, time_zone, scale_endtime, origin_data)); + offset /= SECONDS_PER_DAY; + origin /= SECONDS_PER_DAY; } - - if (isDate(result_type)) /// The result should be a date and the calculations were as datetime. - result_data[i] += (origin + offset) / SECONDS_PER_DAY; - else if (unit == IntervalKind::Kind::Week || unit == IntervalKind::Kind::Month || unit == IntervalKind::Kind::Quarter || unit == IntervalKind::Kind::Year) - { - if (isDateTime64(result_type)) /// We need to have the right scale for offset, origin already has the right scale. - offset *= scale_endtime; - - result_data[i] += origin + offset; - } - else - { - /// ToStartOfInterval::execute() returns seconds. - - if (scale_interval == 1) - { - if (isDateTime64(result_type)) /// We need to have the right scale for offset, origin already has the correct scale. - offset *= scale_endtime; - - /// Interval has default scale, i.e. Year - Second. - - if (scale_endtime % 1000 != 0 && scale_endtime >= 1000) - /// The arguments are DateTime64 with precision like 4,5,7,8. Here offset has correct precision and origin doesn't. - result_data[i] += (origin + offset / scale_endtime) * scale_endtime; - else - /// Precision of DateTime64 is 1, 2, 3, 6, 9, e.g. has correct precision in offset and origin. - result_data[i] += (origin + offset); - } - else - { - /// Interval has some specific scale (3,6,9), i.e. Millisecond - Nanosecond. - - if (scale_interval < scale_endtime) - /// If we have a time argument that has bigger scale than the interval can contain, we need - /// to return a value with bigger precision and thus we should multiply result on the scale difference. - result_data[i] += origin + offset * scale_diff; - else - /// The other case: interval has bigger scale than the interval or they have the same scale, so offset has the right precision and origin doesn't. - result_data[i] += (origin + offset / scale_diff) * scale_diff; - } - } + result_data[i] = 0; + result_data[i] += (result_scale < origin_scale) ? (origin + offset) / scale_diff : (origin + offset) * scale_diff; } } + else // Overload: Default + { + for (size_t i = 0; i != size; ++i) + result_data[i] = static_cast(ToStartOfInterval::execute(time_data[i], num_units, time_zone, scale_multiplier)); + } + return result_col; } }; diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference index 552323be1a5..5e72df17027 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference @@ -23,8 +23,8 @@ Time and origin as DateTime64(9) 2023-10-09 10:10:07.123456789 2023-10-09 10:11:11.123456789 2023-10-09 10:11:12.123456789 -2023-10-09 10:11:12.987456789 -2023-10-09 10:11:12.987654789 +2023-10-09 10:11:12.987 +2023-10-09 10:11:12.987654 2023-10-09 10:11:12.987654321 Time and origin as DateTime64(3) 2023-02-01 09:08:07.123 From 0b1a0999e338325c2e5cd9ec25e9a5ecf3472d16 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 4 Sep 2024 08:29:55 +0000 Subject: [PATCH 145/289] Some fixups --- src/Interpreters/InterpreterCreateQuery.cpp | 4 +-- src/Storages/AlterCommands.h | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 5 ++- ...tor_search_experimental_setting.reference} | 0 ...354_vector_search_experimental_setting.sql | 32 +++++++++++++++++++ ...231_forbid_add_vector_similarity_index.sql | 21 ------------ 6 files changed, 37 insertions(+), 26 deletions(-) rename tests/queries/0_stateless/{03231_forbid_add_vector_similarity_index.reference => 02354_vector_search_experimental_setting.reference} (100%) create mode 100644 tests/queries/0_stateless/02354_vector_search_experimental_setting.sql delete mode 100644 tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index e9f40bdbaf5..c265154c9dd 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -781,14 +781,14 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti const auto & settings = getContext()->getSettingsRef(); if (index_desc.type == FULL_TEXT_INDEX_NAME && !settings.allow_experimental_full_text_index) - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental full-text index feature is not enabled (the setting 'allow_experimental_full_text_index')"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental full-text index feature is disabled. Turn on setting 'allow_experimental_full_text_index'"); /// ---- /// Temporary check during a transition period. Please remove at the end of 2024. if (index_desc.type == INVERTED_INDEX_NAME && !settings.allow_experimental_inverted_index) throw Exception(ErrorCodes::ILLEGAL_INDEX, "Please use index type 'full_text' instead of 'inverted'"); /// ---- if (index_desc.type == "vector_similarity" && !settings.allow_experimental_vector_similarity_index) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Vector similarity index is disabled. Turn on allow_experimental_vector_similarity_index"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental vector similarity index is disabled. Turn on setting 'allow_experimental_vector_similarity_index'"); properties.indices.push_back(index_desc); } diff --git a/src/Storages/AlterCommands.h b/src/Storages/AlterCommands.h index b2f0f9a6abd..c4c792e7dec 100644 --- a/src/Storages/AlterCommands.h +++ b/src/Storages/AlterCommands.h @@ -238,6 +238,7 @@ public: /// Check if commands have any full-text index static bool hasFullTextIndex(const StorageInMemoryMetadata & metadata); + /// Check if commands have any vector similarity index static bool hasVectorSimilarityIndex(const StorageInMemoryMetadata & metadata); }; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 8b12330c1a4..ac39831b3df 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3231,9 +3231,8 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context "Experimental full-text index feature is not enabled (turn on setting 'allow_experimental_full_text_index')"); if (AlterCommands::hasVectorSimilarityIndex(new_metadata) && !settings.allow_experimental_vector_similarity_index) - throw Exception( - ErrorCodes::SUPPORT_IS_DISABLED, - "Experimental vector_similarity index feature is not enabled (turn on setting 'allow_experimental_vector_similarity_index')"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "Experimental vector similarity index is disabled (turn on setting 'allow_experimental_vector_similarity_index')"); for (const auto & disk : getDisks()) if (!disk->supportsHardLinks() && !commands.isSettingsAlter() && !commands.isCommentAlter()) diff --git a/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.reference b/tests/queries/0_stateless/02354_vector_search_experimental_setting.reference similarity index 100% rename from tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.reference rename to tests/queries/0_stateless/02354_vector_search_experimental_setting.reference diff --git a/tests/queries/0_stateless/02354_vector_search_experimental_setting.sql b/tests/queries/0_stateless/02354_vector_search_experimental_setting.sql new file mode 100644 index 00000000000..cce838f8e02 --- /dev/null +++ b/tests/queries/0_stateless/02354_vector_search_experimental_setting.sql @@ -0,0 +1,32 @@ +-- Tags: no-fasttest, no-ordinary-database + +-- Tests that CREATE TABLE and ADD INDEX respect setting 'allow_experimental_vector_similarity_index'. + +DROP TABLE IF EXISTS tab; + +-- Test CREATE TABLE + +SET allow_experimental_vector_similarity_index = 0; +CREATE TABLE tab (id UInt32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance')) ENGINE = MergeTree ORDER BY tuple(); -- { serverError SUPPORT_IS_DISABLED } + +SET allow_experimental_vector_similarity_index = 1; +CREATE TABLE tab (id UInt32, vec Array(Float32), INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance')) ENGINE = MergeTree ORDER BY tuple(); +DROP TABLE tab; + +-- Test ADD INDEX + +CREATE TABLE tab (id UInt32, vec Array(Float32)) ENGINE = MergeTree ORDER BY tuple(); + +SET allow_experimental_vector_similarity_index = 0; +ALTER TABLE tab ADD INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance'); -- { serverError SUPPORT_IS_DISABLED } + +SET allow_experimental_vector_similarity_index = 1; +ALTER TABLE tab ADD INDEX idx vec TYPE vector_similarity('hnsw', 'L2Distance'); + +-- Other index DDL must work regardless of the setting +SET allow_experimental_vector_similarity_index = 0; +ALTER TABLE tab MATERIALIZE INDEX idx; +-- ALTER TABLE tab CLEAR INDEX idx; -- <-- Should work but doesn't w/o enabled setting. Unexpected but not terrible. +ALTER TABLE tab DROP INDEX idx; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql b/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql deleted file mode 100644 index e91d7c71eac..00000000000 --- a/tests/queries/0_stateless/03231_forbid_add_vector_similarity_index.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Tags: no-fasttest - -DROP TABLE IF EXISTS test_embedding; - -CREATE TABLE test_embedding -( - id UInt32, - embedding Array(Float32), -) -ENGINE = MergeTree -ORDER BY tuple(); - -SET allow_experimental_vector_similarity_index = 0; - -alter table test_embedding add INDEX idx embedding TYPE vector_similarity('hnsw', 'cosineDistance'); -- { serverError SUPPORT_IS_DISABLED } - -SET allow_experimental_vector_similarity_index = 1; - -alter table test_embedding add INDEX idx embedding TYPE vector_similarity('hnsw', 'cosineDistance'); - -DROP TABLE test_embedding; From 2620325b303b9318fdd347372f66faa131ca9401 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 4 Sep 2024 08:59:52 +0000 Subject: [PATCH 146/289] Minor fixups + add a tests --- .../MergeTree/VectorSimilarityCondition.cpp | 4 ++-- .../MergeTree/VectorSimilarityCondition.h | 6 +----- .../02354_vector_search_queries.reference | 15 +++++++++++++++ .../0_stateless/02354_vector_search_queries.sql | 7 +++++++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/Storages/MergeTree/VectorSimilarityCondition.cpp b/src/Storages/MergeTree/VectorSimilarityCondition.cpp index 251cdde65ab..641b0037e7b 100644 --- a/src/Storages/MergeTree/VectorSimilarityCondition.cpp +++ b/src/Storages/MergeTree/VectorSimilarityCondition.cpp @@ -40,7 +40,7 @@ void extractReferenceVectorFromLiteral(std::vector & reference_vector, } } -VectorSimilarityCondition::Info::DistanceFunction stringToDistanceFunction(std::string_view distance_function) +VectorSimilarityCondition::Info::DistanceFunction stringToDistanceFunction(const String & distance_function) { if (distance_function == "L2Distance") return VectorSimilarityCondition::Info::DistanceFunction::L2; @@ -59,7 +59,7 @@ VectorSimilarityCondition::VectorSimilarityCondition(const SelectQueryInfo & que , index_is_useful(checkQueryStructure(query_info)) {} -bool VectorSimilarityCondition::alwaysUnknownOrTrue(String distance_function) const +bool VectorSimilarityCondition::alwaysUnknownOrTrue(const String & distance_function) const { if (!index_is_useful) return true; /// query isn't supported diff --git a/src/Storages/MergeTree/VectorSimilarityCondition.h b/src/Storages/MergeTree/VectorSimilarityCondition.h index 83ae1e19bfb..2e9e06a31d0 100644 --- a/src/Storages/MergeTree/VectorSimilarityCondition.h +++ b/src/Storages/MergeTree/VectorSimilarityCondition.h @@ -69,7 +69,7 @@ public: }; /// Returns false if query can be speeded up by an ANN index, true otherwise. - bool alwaysUnknownOrTrue(String distance_function) const; + bool alwaysUnknownOrTrue(const String & distance_function) const; std::vector getReferenceVector() const; size_t getDimensions() const; @@ -142,10 +142,6 @@ private: /// Traverses the AST of ORDERBY section void traverseOrderByAST(const ASTPtr & node, RPN & rpn); - /// Returns true and stores ANNExpr if the query has valid WHERE section - /// TODO NOT implemented, WHERE does not supported. - static bool matchRPNWhere(RPN & rpn, Info & info); - /// Returns true and stores ANNExpr if the query has valid ORDERBY section static bool matchRPNOrderBy(RPN & rpn, Info & info); diff --git a/tests/queries/0_stateless/02354_vector_search_queries.reference b/tests/queries/0_stateless/02354_vector_search_queries.reference index faff306ef60..e42f91d05dc 100644 --- a/tests/queries/0_stateless/02354_vector_search_queries.reference +++ b/tests/queries/0_stateless/02354_vector_search_queries.reference @@ -41,6 +41,21 @@ Special cases 6 [1,9.3] 0.005731362878640178 1 [2,3.2] 0.15200169244542905 7 [5.5,4.7] 0.3503476876550442 +Expression (Projection) + Limit (preliminary LIMIT (without OFFSET)) + Sorting (Sorting for ORDER BY) + Expression (Before ORDER BY) + ReadFromMergeTree (default.tab) + Indexes: + PrimaryKey + Condition: true + Parts: 1/1 + Granules: 4/4 + Skip + Name: idx + Description: vector_similarity GRANULARITY 2 + Parts: 1/1 + Granules: 2/4 -- Setting "max_limit_for_ann_queries" Expression (Projection) Limit (preliminary LIMIT (without OFFSET)) diff --git a/tests/queries/0_stateless/02354_vector_search_queries.sql b/tests/queries/0_stateless/02354_vector_search_queries.sql index 17939992165..8769e5c56bb 100644 --- a/tests/queries/0_stateless/02354_vector_search_queries.sql +++ b/tests/queries/0_stateless/02354_vector_search_queries.sql @@ -63,6 +63,13 @@ FROM tab ORDER BY cosineDistance(vec, reference_vec) LIMIT 3; +EXPLAIN indexes = 1 +WITH [0.0, 2.0] AS reference_vec +SELECT id, vec, cosineDistance(vec, reference_vec) +FROM tab +ORDER BY cosineDistance(vec, reference_vec) +LIMIT 3; + SELECT '-- Setting "max_limit_for_ann_queries"'; EXPLAIN indexes=1 WITH [0.0, 2.0] as reference_vec From efdef8c186eb81ed90ef0efdaa70323c4f42e98f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:18:55 +0200 Subject: [PATCH 147/289] fix build --- src/Functions/DateTimeTransforms.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 043213dcb64..d6d533f16ed 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -786,7 +786,7 @@ struct ToStartOfInterval if (origin == 0) return time_zone.toStartOfYearInterval(time_zone.toDayNum(t / scale_multiplier), years); else - return ToStartOfInterval::execute(t, years * 12, time_zone, scale_multiplier, origin);z + return ToStartOfInterval::execute(t, years * 12, time_zone, scale_multiplier, origin); } }; From e28066645b16a1e6b2c7136bc2b2ca0a9d772f72 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 4 Sep 2024 09:31:06 +0000 Subject: [PATCH 148/289] Sort --- contrib/icu-cmake/CMakeLists.txt | 354 +++++++++++++++---------------- 1 file changed, 177 insertions(+), 177 deletions(-) diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt index adeaa7dcf33..178af2ade6c 100644 --- a/contrib/icu-cmake/CMakeLists.txt +++ b/contrib/icu-cmake/CMakeLists.txt @@ -15,162 +15,63 @@ set(ICUDATA_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/icudata/") # These lists of sources were generated from build log of the original ICU build system (configure + make). set(ICUUC_SOURCES -"${ICU_SOURCE_DIR}/common/errorcode.cpp" -"${ICU_SOURCE_DIR}/common/putil.cpp" -"${ICU_SOURCE_DIR}/common/umath.cpp" -"${ICU_SOURCE_DIR}/common/utypes.cpp" -"${ICU_SOURCE_DIR}/common/uinvchar.cpp" -"${ICU_SOURCE_DIR}/common/umutex.cpp" -"${ICU_SOURCE_DIR}/common/ucln_cmn.cpp" -"${ICU_SOURCE_DIR}/common/uinit.cpp" -"${ICU_SOURCE_DIR}/common/uobject.cpp" -"${ICU_SOURCE_DIR}/common/cmemory.cpp" -"${ICU_SOURCE_DIR}/common/charstr.cpp" -"${ICU_SOURCE_DIR}/common/cstr.cpp" -"${ICU_SOURCE_DIR}/common/udata.cpp" -"${ICU_SOURCE_DIR}/common/ucmndata.cpp" -"${ICU_SOURCE_DIR}/common/udatamem.cpp" -"${ICU_SOURCE_DIR}/common/umapfile.cpp" -"${ICU_SOURCE_DIR}/common/udataswp.cpp" -"${ICU_SOURCE_DIR}/common/utrie_swap.cpp" -"${ICU_SOURCE_DIR}/common/ucol_swp.cpp" -"${ICU_SOURCE_DIR}/common/utrace.cpp" -"${ICU_SOURCE_DIR}/common/uhash.cpp" -"${ICU_SOURCE_DIR}/common/uhash_us.cpp" -"${ICU_SOURCE_DIR}/common/uenum.cpp" -"${ICU_SOURCE_DIR}/common/ustrenum.cpp" -"${ICU_SOURCE_DIR}/common/uvector.cpp" -"${ICU_SOURCE_DIR}/common/ustack.cpp" -"${ICU_SOURCE_DIR}/common/uvectr32.cpp" -"${ICU_SOURCE_DIR}/common/uvectr64.cpp" -"${ICU_SOURCE_DIR}/common/ucnv.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_bld.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_cnv.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_io.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_cb.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_err.cpp" -"${ICU_SOURCE_DIR}/common/ucnvlat1.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_u7.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_u8.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_u16.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_u32.cpp" -"${ICU_SOURCE_DIR}/common/ucnvscsu.cpp" -"${ICU_SOURCE_DIR}/common/ucnvbocu.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_ext.cpp" -"${ICU_SOURCE_DIR}/common/ucnvmbcs.cpp" -"${ICU_SOURCE_DIR}/common/ucnv2022.cpp" -"${ICU_SOURCE_DIR}/common/ucnvhz.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_lmb.cpp" -"${ICU_SOURCE_DIR}/common/ucnvisci.cpp" -"${ICU_SOURCE_DIR}/common/ucnvdisp.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_set.cpp" -"${ICU_SOURCE_DIR}/common/ucnv_ct.cpp" -"${ICU_SOURCE_DIR}/common/resource.cpp" -"${ICU_SOURCE_DIR}/common/uresbund.cpp" -"${ICU_SOURCE_DIR}/common/ures_cnv.cpp" -"${ICU_SOURCE_DIR}/common/uresdata.cpp" -"${ICU_SOURCE_DIR}/common/resbund.cpp" -"${ICU_SOURCE_DIR}/common/resbund_cnv.cpp" -"${ICU_SOURCE_DIR}/common/ucurr.cpp" -"${ICU_SOURCE_DIR}/common/localebuilder.cpp" -"${ICU_SOURCE_DIR}/common/localeprioritylist.cpp" -"${ICU_SOURCE_DIR}/common/messagepattern.cpp" -"${ICU_SOURCE_DIR}/common/ucat.cpp" -"${ICU_SOURCE_DIR}/common/locmap.cpp" -"${ICU_SOURCE_DIR}/common/uloc.cpp" -"${ICU_SOURCE_DIR}/common/locid.cpp" -"${ICU_SOURCE_DIR}/common/locutil.cpp" -"${ICU_SOURCE_DIR}/common/locavailable.cpp" -"${ICU_SOURCE_DIR}/common/locdispnames.cpp" -"${ICU_SOURCE_DIR}/common/locdspnm.cpp" -"${ICU_SOURCE_DIR}/common/loclikely.cpp" -"${ICU_SOURCE_DIR}/common/locresdata.cpp" -"${ICU_SOURCE_DIR}/common/lsr.cpp" -"${ICU_SOURCE_DIR}/common/loclikelysubtags.cpp" -"${ICU_SOURCE_DIR}/common/locdistance.cpp" -"${ICU_SOURCE_DIR}/common/localematcher.cpp" -"${ICU_SOURCE_DIR}/common/bytestream.cpp" -"${ICU_SOURCE_DIR}/common/stringpiece.cpp" -"${ICU_SOURCE_DIR}/common/bytesinkutil.cpp" -"${ICU_SOURCE_DIR}/common/stringtriebuilder.cpp" -"${ICU_SOURCE_DIR}/common/bytestriebuilder.cpp" -"${ICU_SOURCE_DIR}/common/bytestrie.cpp" -"${ICU_SOURCE_DIR}/common/bytestrieiterator.cpp" -"${ICU_SOURCE_DIR}/common/ucharstrie.cpp" -"${ICU_SOURCE_DIR}/common/ucharstriebuilder.cpp" -"${ICU_SOURCE_DIR}/common/ucharstrieiterator.cpp" -"${ICU_SOURCE_DIR}/common/dictionarydata.cpp" -"${ICU_SOURCE_DIR}/common/edits.cpp" "${ICU_SOURCE_DIR}/common/appendable.cpp" -"${ICU_SOURCE_DIR}/common/ustr_cnv.cpp" -"${ICU_SOURCE_DIR}/common/unistr_cnv.cpp" -"${ICU_SOURCE_DIR}/common/unistr.cpp" -"${ICU_SOURCE_DIR}/common/unistr_case.cpp" -"${ICU_SOURCE_DIR}/common/unistr_props.cpp" -"${ICU_SOURCE_DIR}/common/utf_impl.cpp" -"${ICU_SOURCE_DIR}/common/ustring.cpp" -"${ICU_SOURCE_DIR}/common/ustrcase.cpp" -"${ICU_SOURCE_DIR}/common/ucasemap.cpp" -"${ICU_SOURCE_DIR}/common/ucasemap_titlecase_brkiter.cpp" -"${ICU_SOURCE_DIR}/common/cstring.cpp" -"${ICU_SOURCE_DIR}/common/ustrfmt.cpp" -"${ICU_SOURCE_DIR}/common/ustrtrns.cpp" -"${ICU_SOURCE_DIR}/common/ustr_wcs.cpp" -"${ICU_SOURCE_DIR}/common/utext.cpp" -"${ICU_SOURCE_DIR}/common/unistr_case_locale.cpp" -"${ICU_SOURCE_DIR}/common/ustrcase_locale.cpp" -"${ICU_SOURCE_DIR}/common/unistr_titlecase_brkiter.cpp" -"${ICU_SOURCE_DIR}/common/ustr_titlecase_brkiter.cpp" -"${ICU_SOURCE_DIR}/common/normalizer2impl.cpp" -"${ICU_SOURCE_DIR}/common/normalizer2.cpp" -"${ICU_SOURCE_DIR}/common/filterednormalizer2.cpp" -"${ICU_SOURCE_DIR}/common/normlzr.cpp" -"${ICU_SOURCE_DIR}/common/unorm.cpp" -"${ICU_SOURCE_DIR}/common/unormcmp.cpp" -"${ICU_SOURCE_DIR}/common/loadednormalizer2impl.cpp" -"${ICU_SOURCE_DIR}/common/chariter.cpp" -"${ICU_SOURCE_DIR}/common/schriter.cpp" -"${ICU_SOURCE_DIR}/common/uchriter.cpp" -"${ICU_SOURCE_DIR}/common/uiter.cpp" -"${ICU_SOURCE_DIR}/common/patternprops.cpp" -"${ICU_SOURCE_DIR}/common/uchar.cpp" -"${ICU_SOURCE_DIR}/common/uprops.cpp" -"${ICU_SOURCE_DIR}/common/ucase.cpp" -"${ICU_SOURCE_DIR}/common/propname.cpp" -"${ICU_SOURCE_DIR}/common/ubidi_props.cpp" -"${ICU_SOURCE_DIR}/common/characterproperties.cpp" -"${ICU_SOURCE_DIR}/common/ubidi.cpp" -"${ICU_SOURCE_DIR}/common/ubidiwrt.cpp" -"${ICU_SOURCE_DIR}/common/ubidiln.cpp" -"${ICU_SOURCE_DIR}/common/ushape.cpp" -"${ICU_SOURCE_DIR}/common/uscript.cpp" -"${ICU_SOURCE_DIR}/common/uscript_props.cpp" -"${ICU_SOURCE_DIR}/common/usc_impl.cpp" -"${ICU_SOURCE_DIR}/common/unames.cpp" -"${ICU_SOURCE_DIR}/common/utrie.cpp" -"${ICU_SOURCE_DIR}/common/utrie2.cpp" -"${ICU_SOURCE_DIR}/common/utrie2_builder.cpp" -"${ICU_SOURCE_DIR}/common/ucptrie.cpp" -"${ICU_SOURCE_DIR}/common/umutablecptrie.cpp" "${ICU_SOURCE_DIR}/common/bmpset.cpp" -"${ICU_SOURCE_DIR}/common/unisetspan.cpp" -"${ICU_SOURCE_DIR}/common/uset_props.cpp" -"${ICU_SOURCE_DIR}/common/uniset_props.cpp" -"${ICU_SOURCE_DIR}/common/uniset_closure.cpp" -"${ICU_SOURCE_DIR}/common/uset.cpp" -"${ICU_SOURCE_DIR}/common/uniset.cpp" -"${ICU_SOURCE_DIR}/common/usetiter.cpp" -"${ICU_SOURCE_DIR}/common/ruleiter.cpp" -"${ICU_SOURCE_DIR}/common/caniter.cpp" -"${ICU_SOURCE_DIR}/common/unifilt.cpp" -"${ICU_SOURCE_DIR}/common/unifunct.cpp" -"${ICU_SOURCE_DIR}/common/uarrsort.cpp" -"${ICU_SOURCE_DIR}/common/brkiter.cpp" -"${ICU_SOURCE_DIR}/common/ubrk.cpp" "${ICU_SOURCE_DIR}/common/brkeng.cpp" +"${ICU_SOURCE_DIR}/common/brkiter.cpp" +"${ICU_SOURCE_DIR}/common/bytesinkutil.cpp" +"${ICU_SOURCE_DIR}/common/bytestream.cpp" +"${ICU_SOURCE_DIR}/common/bytestrie.cpp" +"${ICU_SOURCE_DIR}/common/bytestriebuilder.cpp" +"${ICU_SOURCE_DIR}/common/bytestrieiterator.cpp" +"${ICU_SOURCE_DIR}/common/caniter.cpp" +"${ICU_SOURCE_DIR}/common/characterproperties.cpp" +"${ICU_SOURCE_DIR}/common/chariter.cpp" +"${ICU_SOURCE_DIR}/common/charstr.cpp" +"${ICU_SOURCE_DIR}/common/cmemory.cpp" +"${ICU_SOURCE_DIR}/common/cstr.cpp" +"${ICU_SOURCE_DIR}/common/cstring.cpp" +"${ICU_SOURCE_DIR}/common/cwchar.cpp" "${ICU_SOURCE_DIR}/common/dictbe.cpp" +"${ICU_SOURCE_DIR}/common/dictionarydata.cpp" +"${ICU_SOURCE_DIR}/common/dtintrv.cpp" +"${ICU_SOURCE_DIR}/common/edits.cpp" +"${ICU_SOURCE_DIR}/common/emojiprops.cpp" +"${ICU_SOURCE_DIR}/common/errorcode.cpp" "${ICU_SOURCE_DIR}/common/filteredbrk.cpp" +"${ICU_SOURCE_DIR}/common/filterednormalizer2.cpp" +"${ICU_SOURCE_DIR}/common/icudataver.cpp" +"${ICU_SOURCE_DIR}/common/icuplug.cpp" +"${ICU_SOURCE_DIR}/common/loadednormalizer2impl.cpp" +"${ICU_SOURCE_DIR}/common/localebuilder.cpp" +"${ICU_SOURCE_DIR}/common/localematcher.cpp" +"${ICU_SOURCE_DIR}/common/localeprioritylist.cpp" +"${ICU_SOURCE_DIR}/common/locavailable.cpp" +"${ICU_SOURCE_DIR}/common/locbased.cpp" +"${ICU_SOURCE_DIR}/common/locdispnames.cpp" +"${ICU_SOURCE_DIR}/common/locdistance.cpp" +"${ICU_SOURCE_DIR}/common/locdspnm.cpp" +"${ICU_SOURCE_DIR}/common/locid.cpp" +"${ICU_SOURCE_DIR}/common/loclikely.cpp" +"${ICU_SOURCE_DIR}/common/loclikelysubtags.cpp" +"${ICU_SOURCE_DIR}/common/locmap.cpp" +"${ICU_SOURCE_DIR}/common/locresdata.cpp" +"${ICU_SOURCE_DIR}/common/locutil.cpp" +"${ICU_SOURCE_DIR}/common/lsr.cpp" +"${ICU_SOURCE_DIR}/common/lstmbe.cpp") +"${ICU_SOURCE_DIR}/common/messagepattern.cpp" +"${ICU_SOURCE_DIR}/common/normalizer2.cpp" +"${ICU_SOURCE_DIR}/common/normalizer2impl.cpp" +"${ICU_SOURCE_DIR}/common/normlzr.cpp" +"${ICU_SOURCE_DIR}/common/parsepos.cpp" +"${ICU_SOURCE_DIR}/common/patternprops.cpp" +"${ICU_SOURCE_DIR}/common/pluralmap.cpp" +"${ICU_SOURCE_DIR}/common/propname.cpp" +"${ICU_SOURCE_DIR}/common/propsvec.cpp" +"${ICU_SOURCE_DIR}/common/punycode.cpp" +"${ICU_SOURCE_DIR}/common/putil.cpp" "${ICU_SOURCE_DIR}/common/rbbi.cpp" +"${ICU_SOURCE_DIR}/common/rbbi_cache.cpp" "${ICU_SOURCE_DIR}/common/rbbidata.cpp" "${ICU_SOURCE_DIR}/common/rbbinode.cpp" "${ICU_SOURCE_DIR}/common/rbbirb.cpp" @@ -178,41 +79,140 @@ set(ICUUC_SOURCES "${ICU_SOURCE_DIR}/common/rbbisetb.cpp" "${ICU_SOURCE_DIR}/common/rbbistbl.cpp" "${ICU_SOURCE_DIR}/common/rbbitblb.cpp" -"${ICU_SOURCE_DIR}/common/rbbi_cache.cpp" +"${ICU_SOURCE_DIR}/common/resbund.cpp" +"${ICU_SOURCE_DIR}/common/resbund_cnv.cpp" +"${ICU_SOURCE_DIR}/common/resource.cpp" +"${ICU_SOURCE_DIR}/common/restrace.cpp" +"${ICU_SOURCE_DIR}/common/ruleiter.cpp" +"${ICU_SOURCE_DIR}/common/schriter.cpp" "${ICU_SOURCE_DIR}/common/serv.cpp" -"${ICU_SOURCE_DIR}/common/servnotf.cpp" -"${ICU_SOURCE_DIR}/common/servls.cpp" "${ICU_SOURCE_DIR}/common/servlk.cpp" "${ICU_SOURCE_DIR}/common/servlkf.cpp" +"${ICU_SOURCE_DIR}/common/servls.cpp" +"${ICU_SOURCE_DIR}/common/servnotf.cpp" "${ICU_SOURCE_DIR}/common/servrbf.cpp" "${ICU_SOURCE_DIR}/common/servslkf.cpp" -"${ICU_SOURCE_DIR}/common/uidna.cpp" -"${ICU_SOURCE_DIR}/common/usprep.cpp" -"${ICU_SOURCE_DIR}/common/uts46.cpp" -"${ICU_SOURCE_DIR}/common/punycode.cpp" -"${ICU_SOURCE_DIR}/common/util.cpp" -"${ICU_SOURCE_DIR}/common/util_props.cpp" -"${ICU_SOURCE_DIR}/common/parsepos.cpp" -"${ICU_SOURCE_DIR}/common/locbased.cpp" -"${ICU_SOURCE_DIR}/common/cwchar.cpp" -"${ICU_SOURCE_DIR}/common/wintz.cpp" -"${ICU_SOURCE_DIR}/common/dtintrv.cpp" -"${ICU_SOURCE_DIR}/common/ucnvsel.cpp" -"${ICU_SOURCE_DIR}/common/propsvec.cpp" -"${ICU_SOURCE_DIR}/common/ulist.cpp" -"${ICU_SOURCE_DIR}/common/uloc_tag.cpp" -"${ICU_SOURCE_DIR}/common/icudataver.cpp" -"${ICU_SOURCE_DIR}/common/icuplug.cpp" "${ICU_SOURCE_DIR}/common/sharedobject.cpp" "${ICU_SOURCE_DIR}/common/simpleformatter.cpp" -"${ICU_SOURCE_DIR}/common/unifiedcache.cpp" -"${ICU_SOURCE_DIR}/common/uloc_keytype.cpp" -"${ICU_SOURCE_DIR}/common/ubiditransform.cpp" -"${ICU_SOURCE_DIR}/common/pluralmap.cpp" "${ICU_SOURCE_DIR}/common/static_unicode_sets.cpp" -"${ICU_SOURCE_DIR}/common/restrace.cpp" -"${ICU_SOURCE_DIR}/common/emojiprops.cpp" -"${ICU_SOURCE_DIR}/common/lstmbe.cpp") +"${ICU_SOURCE_DIR}/common/stringpiece.cpp" +"${ICU_SOURCE_DIR}/common/stringtriebuilder.cpp" +"${ICU_SOURCE_DIR}/common/uarrsort.cpp" +"${ICU_SOURCE_DIR}/common/ubidi.cpp" +"${ICU_SOURCE_DIR}/common/ubidi_props.cpp" +"${ICU_SOURCE_DIR}/common/ubidiln.cpp" +"${ICU_SOURCE_DIR}/common/ubiditransform.cpp" +"${ICU_SOURCE_DIR}/common/ubidiwrt.cpp" +"${ICU_SOURCE_DIR}/common/ubrk.cpp" +"${ICU_SOURCE_DIR}/common/ucase.cpp" +"${ICU_SOURCE_DIR}/common/ucasemap.cpp" +"${ICU_SOURCE_DIR}/common/ucasemap_titlecase_brkiter.cpp" +"${ICU_SOURCE_DIR}/common/ucat.cpp" +"${ICU_SOURCE_DIR}/common/uchar.cpp" +"${ICU_SOURCE_DIR}/common/ucharstrie.cpp" +"${ICU_SOURCE_DIR}/common/ucharstriebuilder.cpp" +"${ICU_SOURCE_DIR}/common/ucharstrieiterator.cpp" +"${ICU_SOURCE_DIR}/common/uchriter.cpp" +"${ICU_SOURCE_DIR}/common/ucln_cmn.cpp" +"${ICU_SOURCE_DIR}/common/ucmndata.cpp" +"${ICU_SOURCE_DIR}/common/ucnv.cpp" +"${ICU_SOURCE_DIR}/common/ucnv2022.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_bld.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_cb.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_cnv.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_ct.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_err.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_ext.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_io.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_lmb.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_set.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_u16.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_u32.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_u7.cpp" +"${ICU_SOURCE_DIR}/common/ucnv_u8.cpp" +"${ICU_SOURCE_DIR}/common/ucnvbocu.cpp" +"${ICU_SOURCE_DIR}/common/ucnvdisp.cpp" +"${ICU_SOURCE_DIR}/common/ucnvhz.cpp" +"${ICU_SOURCE_DIR}/common/ucnvisci.cpp" +"${ICU_SOURCE_DIR}/common/ucnvlat1.cpp" +"${ICU_SOURCE_DIR}/common/ucnvmbcs.cpp" +"${ICU_SOURCE_DIR}/common/ucnvscsu.cpp" +"${ICU_SOURCE_DIR}/common/ucnvsel.cpp" +"${ICU_SOURCE_DIR}/common/ucol_swp.cpp" +"${ICU_SOURCE_DIR}/common/ucptrie.cpp" +"${ICU_SOURCE_DIR}/common/ucurr.cpp" +"${ICU_SOURCE_DIR}/common/udata.cpp" +"${ICU_SOURCE_DIR}/common/udatamem.cpp" +"${ICU_SOURCE_DIR}/common/udataswp.cpp" +"${ICU_SOURCE_DIR}/common/uenum.cpp" +"${ICU_SOURCE_DIR}/common/uhash.cpp" +"${ICU_SOURCE_DIR}/common/uhash_us.cpp" +"${ICU_SOURCE_DIR}/common/uidna.cpp" +"${ICU_SOURCE_DIR}/common/uinit.cpp" +"${ICU_SOURCE_DIR}/common/uinvchar.cpp" +"${ICU_SOURCE_DIR}/common/uiter.cpp" +"${ICU_SOURCE_DIR}/common/ulist.cpp" +"${ICU_SOURCE_DIR}/common/uloc.cpp" +"${ICU_SOURCE_DIR}/common/uloc_keytype.cpp" +"${ICU_SOURCE_DIR}/common/uloc_tag.cpp" +"${ICU_SOURCE_DIR}/common/umapfile.cpp" +"${ICU_SOURCE_DIR}/common/umath.cpp" +"${ICU_SOURCE_DIR}/common/umutablecptrie.cpp" +"${ICU_SOURCE_DIR}/common/umutex.cpp" +"${ICU_SOURCE_DIR}/common/unames.cpp" +"${ICU_SOURCE_DIR}/common/unifiedcache.cpp" +"${ICU_SOURCE_DIR}/common/unifilt.cpp" +"${ICU_SOURCE_DIR}/common/unifunct.cpp" +"${ICU_SOURCE_DIR}/common/uniset.cpp" +"${ICU_SOURCE_DIR}/common/uniset_closure.cpp" +"${ICU_SOURCE_DIR}/common/uniset_props.cpp" +"${ICU_SOURCE_DIR}/common/unisetspan.cpp" +"${ICU_SOURCE_DIR}/common/unistr.cpp" +"${ICU_SOURCE_DIR}/common/unistr_case.cpp" +"${ICU_SOURCE_DIR}/common/unistr_case_locale.cpp" +"${ICU_SOURCE_DIR}/common/unistr_cnv.cpp" +"${ICU_SOURCE_DIR}/common/unistr_props.cpp" +"${ICU_SOURCE_DIR}/common/unistr_titlecase_brkiter.cpp" +"${ICU_SOURCE_DIR}/common/unorm.cpp" +"${ICU_SOURCE_DIR}/common/unormcmp.cpp" +"${ICU_SOURCE_DIR}/common/uobject.cpp" +"${ICU_SOURCE_DIR}/common/uprops.cpp" +"${ICU_SOURCE_DIR}/common/ures_cnv.cpp" +"${ICU_SOURCE_DIR}/common/uresbund.cpp" +"${ICU_SOURCE_DIR}/common/uresdata.cpp" +"${ICU_SOURCE_DIR}/common/usc_impl.cpp" +"${ICU_SOURCE_DIR}/common/uscript.cpp" +"${ICU_SOURCE_DIR}/common/uscript_props.cpp" +"${ICU_SOURCE_DIR}/common/uset.cpp" +"${ICU_SOURCE_DIR}/common/uset_props.cpp" +"${ICU_SOURCE_DIR}/common/usetiter.cpp" +"${ICU_SOURCE_DIR}/common/ushape.cpp" +"${ICU_SOURCE_DIR}/common/usprep.cpp" +"${ICU_SOURCE_DIR}/common/ustack.cpp" +"${ICU_SOURCE_DIR}/common/ustr_cnv.cpp" +"${ICU_SOURCE_DIR}/common/ustr_titlecase_brkiter.cpp" +"${ICU_SOURCE_DIR}/common/ustr_wcs.cpp" +"${ICU_SOURCE_DIR}/common/ustrcase.cpp" +"${ICU_SOURCE_DIR}/common/ustrcase_locale.cpp" +"${ICU_SOURCE_DIR}/common/ustrenum.cpp" +"${ICU_SOURCE_DIR}/common/ustrfmt.cpp" +"${ICU_SOURCE_DIR}/common/ustring.cpp" +"${ICU_SOURCE_DIR}/common/ustrtrns.cpp" +"${ICU_SOURCE_DIR}/common/utext.cpp" +"${ICU_SOURCE_DIR}/common/utf_impl.cpp" +"${ICU_SOURCE_DIR}/common/util.cpp" +"${ICU_SOURCE_DIR}/common/util_props.cpp" +"${ICU_SOURCE_DIR}/common/utrace.cpp" +"${ICU_SOURCE_DIR}/common/utrie.cpp" +"${ICU_SOURCE_DIR}/common/utrie2.cpp" +"${ICU_SOURCE_DIR}/common/utrie2_builder.cpp" +"${ICU_SOURCE_DIR}/common/utrie_swap.cpp" +"${ICU_SOURCE_DIR}/common/uts46.cpp" +"${ICU_SOURCE_DIR}/common/utypes.cpp" +"${ICU_SOURCE_DIR}/common/uvector.cpp" +"${ICU_SOURCE_DIR}/common/uvectr32.cpp" +"${ICU_SOURCE_DIR}/common/uvectr64.cpp" +"${ICU_SOURCE_DIR}/common/wintz.cpp" set(ICUI18N_SOURCES "${ICU_SOURCE_DIR}/i18n/ucln_in.cpp" From e047c7c4194bc32d2b0308a941ebd85f306f2489 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 4 Sep 2024 09:31:46 +0000 Subject: [PATCH 149/289] Update sources --- contrib/icu-cmake/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt index 178af2ade6c..ac849332429 100644 --- a/contrib/icu-cmake/CMakeLists.txt +++ b/contrib/icu-cmake/CMakeLists.txt @@ -58,8 +58,9 @@ set(ICUUC_SOURCES "${ICU_SOURCE_DIR}/common/locresdata.cpp" "${ICU_SOURCE_DIR}/common/locutil.cpp" "${ICU_SOURCE_DIR}/common/lsr.cpp" -"${ICU_SOURCE_DIR}/common/lstmbe.cpp") +"${ICU_SOURCE_DIR}/common/lstmbe.cpp" "${ICU_SOURCE_DIR}/common/messagepattern.cpp" +"${ICU_SOURCE_DIR}/common/mlbe.cpp" "${ICU_SOURCE_DIR}/common/normalizer2.cpp" "${ICU_SOURCE_DIR}/common/normalizer2impl.cpp" "${ICU_SOURCE_DIR}/common/normlzr.cpp" @@ -155,6 +156,8 @@ set(ICUUC_SOURCES "${ICU_SOURCE_DIR}/common/uloc.cpp" "${ICU_SOURCE_DIR}/common/uloc_keytype.cpp" "${ICU_SOURCE_DIR}/common/uloc_tag.cpp" +"${ICU_SOURCE_DIR}/common/ulocale.cpp" +"${ICU_SOURCE_DIR}/common/ulocbuilder.cpp" "${ICU_SOURCE_DIR}/common/umapfile.cpp" "${ICU_SOURCE_DIR}/common/umath.cpp" "${ICU_SOURCE_DIR}/common/umutablecptrie.cpp" @@ -212,7 +215,7 @@ set(ICUUC_SOURCES "${ICU_SOURCE_DIR}/common/uvector.cpp" "${ICU_SOURCE_DIR}/common/uvectr32.cpp" "${ICU_SOURCE_DIR}/common/uvectr64.cpp" -"${ICU_SOURCE_DIR}/common/wintz.cpp" +"${ICU_SOURCE_DIR}/common/wintz.cpp") set(ICUI18N_SOURCES "${ICU_SOURCE_DIR}/i18n/ucln_in.cpp" From 1c6e96071943bfa0f02412e23920625827fa6e35 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 4 Sep 2024 09:35:06 +0000 Subject: [PATCH 150/289] Sort --- contrib/icu-cmake/CMakeLists.txt | 370 +++++++++++++++---------------- 1 file changed, 185 insertions(+), 185 deletions(-) diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt index ac849332429..9d6e9ac0a62 100644 --- a/contrib/icu-cmake/CMakeLists.txt +++ b/contrib/icu-cmake/CMakeLists.txt @@ -218,129 +218,42 @@ set(ICUUC_SOURCES "${ICU_SOURCE_DIR}/common/wintz.cpp") set(ICUI18N_SOURCES -"${ICU_SOURCE_DIR}/i18n/ucln_in.cpp" -"${ICU_SOURCE_DIR}/i18n/fmtable.cpp" -"${ICU_SOURCE_DIR}/i18n/format.cpp" -"${ICU_SOURCE_DIR}/i18n/msgfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/umsg.cpp" -"${ICU_SOURCE_DIR}/i18n/numfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/unum.cpp" -"${ICU_SOURCE_DIR}/i18n/decimfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/dcfmtsym.cpp" -"${ICU_SOURCE_DIR}/i18n/fmtable_cnv.cpp" -"${ICU_SOURCE_DIR}/i18n/choicfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/datefmt.cpp" -"${ICU_SOURCE_DIR}/i18n/smpdtfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/reldtfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/dtfmtsym.cpp" -"${ICU_SOURCE_DIR}/i18n/udat.cpp" -"${ICU_SOURCE_DIR}/i18n/dtptngen.cpp" -"${ICU_SOURCE_DIR}/i18n/udatpg.cpp" -"${ICU_SOURCE_DIR}/i18n/nfrs.cpp" -"${ICU_SOURCE_DIR}/i18n/nfrule.cpp" -"${ICU_SOURCE_DIR}/i18n/nfsubs.cpp" -"${ICU_SOURCE_DIR}/i18n/rbnf.cpp" -"${ICU_SOURCE_DIR}/i18n/numsys.cpp" -"${ICU_SOURCE_DIR}/i18n/unumsys.cpp" -"${ICU_SOURCE_DIR}/i18n/ucsdet.cpp" -"${ICU_SOURCE_DIR}/i18n/ucal.cpp" -"${ICU_SOURCE_DIR}/i18n/calendar.cpp" -"${ICU_SOURCE_DIR}/i18n/gregocal.cpp" -"${ICU_SOURCE_DIR}/i18n/timezone.cpp" -"${ICU_SOURCE_DIR}/i18n/simpletz.cpp" -"${ICU_SOURCE_DIR}/i18n/olsontz.cpp" +"${ICU_SOURCE_DIR}/i18n/alphaindex.cpp" +"${ICU_SOURCE_DIR}/i18n/anytrans.cpp" "${ICU_SOURCE_DIR}/i18n/astro.cpp" -"${ICU_SOURCE_DIR}/i18n/taiwncal.cpp" +"${ICU_SOURCE_DIR}/i18n/basictz.cpp" +"${ICU_SOURCE_DIR}/i18n/bocsu.cpp" +"${ICU_SOURCE_DIR}/i18n/brktrans.cpp" "${ICU_SOURCE_DIR}/i18n/buddhcal.cpp" -"${ICU_SOURCE_DIR}/i18n/persncal.cpp" -"${ICU_SOURCE_DIR}/i18n/islamcal.cpp" -"${ICU_SOURCE_DIR}/i18n/japancal.cpp" -"${ICU_SOURCE_DIR}/i18n/gregoimp.cpp" -"${ICU_SOURCE_DIR}/i18n/hebrwcal.cpp" -"${ICU_SOURCE_DIR}/i18n/indiancal.cpp" -"${ICU_SOURCE_DIR}/i18n/chnsecal.cpp" +"${ICU_SOURCE_DIR}/i18n/calendar.cpp" +"${ICU_SOURCE_DIR}/i18n/casetrn.cpp" "${ICU_SOURCE_DIR}/i18n/cecal.cpp" -"${ICU_SOURCE_DIR}/i18n/coptccal.cpp" -"${ICU_SOURCE_DIR}/i18n/dangical.cpp" -"${ICU_SOURCE_DIR}/i18n/ethpccal.cpp" +"${ICU_SOURCE_DIR}/i18n/chnsecal.cpp" +"${ICU_SOURCE_DIR}/i18n/choicfmt.cpp" "${ICU_SOURCE_DIR}/i18n/coleitr.cpp" "${ICU_SOURCE_DIR}/i18n/coll.cpp" -"${ICU_SOURCE_DIR}/i18n/sortkey.cpp" -"${ICU_SOURCE_DIR}/i18n/bocsu.cpp" -"${ICU_SOURCE_DIR}/i18n/ucoleitr.cpp" -"${ICU_SOURCE_DIR}/i18n/ucol.cpp" -"${ICU_SOURCE_DIR}/i18n/ucol_res.cpp" -"${ICU_SOURCE_DIR}/i18n/ucol_sit.cpp" "${ICU_SOURCE_DIR}/i18n/collation.cpp" -"${ICU_SOURCE_DIR}/i18n/collationsettings.cpp" +"${ICU_SOURCE_DIR}/i18n/collationbuilder.cpp" +"${ICU_SOURCE_DIR}/i18n/collationcompare.cpp" "${ICU_SOURCE_DIR}/i18n/collationdata.cpp" -"${ICU_SOURCE_DIR}/i18n/collationtailoring.cpp" +"${ICU_SOURCE_DIR}/i18n/collationdatabuilder.cpp" "${ICU_SOURCE_DIR}/i18n/collationdatareader.cpp" "${ICU_SOURCE_DIR}/i18n/collationdatawriter.cpp" +"${ICU_SOURCE_DIR}/i18n/collationfastlatin.cpp" +"${ICU_SOURCE_DIR}/i18n/collationfastlatinbuilder.cpp" "${ICU_SOURCE_DIR}/i18n/collationfcd.cpp" "${ICU_SOURCE_DIR}/i18n/collationiterator.cpp" -"${ICU_SOURCE_DIR}/i18n/utf16collationiterator.cpp" -"${ICU_SOURCE_DIR}/i18n/utf8collationiterator.cpp" -"${ICU_SOURCE_DIR}/i18n/uitercollationiterator.cpp" -"${ICU_SOURCE_DIR}/i18n/collationsets.cpp" -"${ICU_SOURCE_DIR}/i18n/collationcompare.cpp" -"${ICU_SOURCE_DIR}/i18n/collationfastlatin.cpp" "${ICU_SOURCE_DIR}/i18n/collationkeys.cpp" -"${ICU_SOURCE_DIR}/i18n/rulebasedcollator.cpp" "${ICU_SOURCE_DIR}/i18n/collationroot.cpp" "${ICU_SOURCE_DIR}/i18n/collationrootelements.cpp" -"${ICU_SOURCE_DIR}/i18n/collationdatabuilder.cpp" -"${ICU_SOURCE_DIR}/i18n/collationweights.cpp" "${ICU_SOURCE_DIR}/i18n/collationruleparser.cpp" -"${ICU_SOURCE_DIR}/i18n/collationbuilder.cpp" -"${ICU_SOURCE_DIR}/i18n/collationfastlatinbuilder.cpp" -"${ICU_SOURCE_DIR}/i18n/listformatter.cpp" -"${ICU_SOURCE_DIR}/i18n/ulistformatter.cpp" -"${ICU_SOURCE_DIR}/i18n/strmatch.cpp" -"${ICU_SOURCE_DIR}/i18n/usearch.cpp" -"${ICU_SOURCE_DIR}/i18n/search.cpp" -"${ICU_SOURCE_DIR}/i18n/stsearch.cpp" -"${ICU_SOURCE_DIR}/i18n/translit.cpp" -"${ICU_SOURCE_DIR}/i18n/utrans.cpp" -"${ICU_SOURCE_DIR}/i18n/esctrn.cpp" -"${ICU_SOURCE_DIR}/i18n/unesctrn.cpp" -"${ICU_SOURCE_DIR}/i18n/funcrepl.cpp" -"${ICU_SOURCE_DIR}/i18n/strrepl.cpp" -"${ICU_SOURCE_DIR}/i18n/tridpars.cpp" +"${ICU_SOURCE_DIR}/i18n/collationsets.cpp" +"${ICU_SOURCE_DIR}/i18n/collationsettings.cpp" +"${ICU_SOURCE_DIR}/i18n/collationtailoring.cpp" +"${ICU_SOURCE_DIR}/i18n/collationweights.cpp" +"${ICU_SOURCE_DIR}/i18n/compactdecimalformat.cpp" +"${ICU_SOURCE_DIR}/i18n/coptccal.cpp" "${ICU_SOURCE_DIR}/i18n/cpdtrans.cpp" -"${ICU_SOURCE_DIR}/i18n/rbt.cpp" -"${ICU_SOURCE_DIR}/i18n/rbt_data.cpp" -"${ICU_SOURCE_DIR}/i18n/rbt_pars.cpp" -"${ICU_SOURCE_DIR}/i18n/rbt_rule.cpp" -"${ICU_SOURCE_DIR}/i18n/rbt_set.cpp" -"${ICU_SOURCE_DIR}/i18n/nultrans.cpp" -"${ICU_SOURCE_DIR}/i18n/remtrans.cpp" -"${ICU_SOURCE_DIR}/i18n/casetrn.cpp" -"${ICU_SOURCE_DIR}/i18n/titletrn.cpp" -"${ICU_SOURCE_DIR}/i18n/tolowtrn.cpp" -"${ICU_SOURCE_DIR}/i18n/toupptrn.cpp" -"${ICU_SOURCE_DIR}/i18n/anytrans.cpp" -"${ICU_SOURCE_DIR}/i18n/name2uni.cpp" -"${ICU_SOURCE_DIR}/i18n/uni2name.cpp" -"${ICU_SOURCE_DIR}/i18n/nortrans.cpp" -"${ICU_SOURCE_DIR}/i18n/quant.cpp" -"${ICU_SOURCE_DIR}/i18n/transreg.cpp" -"${ICU_SOURCE_DIR}/i18n/brktrans.cpp" -"${ICU_SOURCE_DIR}/i18n/regexcmp.cpp" -"${ICU_SOURCE_DIR}/i18n/rematch.cpp" -"${ICU_SOURCE_DIR}/i18n/repattrn.cpp" -"${ICU_SOURCE_DIR}/i18n/regexst.cpp" -"${ICU_SOURCE_DIR}/i18n/regextxt.cpp" -"${ICU_SOURCE_DIR}/i18n/regeximp.cpp" -"${ICU_SOURCE_DIR}/i18n/uregex.cpp" -"${ICU_SOURCE_DIR}/i18n/uregexc.cpp" -"${ICU_SOURCE_DIR}/i18n/ulocdata.cpp" -"${ICU_SOURCE_DIR}/i18n/measfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/currfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/curramt.cpp" -"${ICU_SOURCE_DIR}/i18n/currunit.cpp" -"${ICU_SOURCE_DIR}/i18n/measure.cpp" -"${ICU_SOURCE_DIR}/i18n/utmscale.cpp" "${ICU_SOURCE_DIR}/i18n/csdetect.cpp" "${ICU_SOURCE_DIR}/i18n/csmatch.cpp" "${ICU_SOURCE_DIR}/i18n/csr2022.cpp" @@ -349,60 +262,67 @@ set(ICUI18N_SOURCES "${ICU_SOURCE_DIR}/i18n/csrsbcs.cpp" "${ICU_SOURCE_DIR}/i18n/csrucode.cpp" "${ICU_SOURCE_DIR}/i18n/csrutf8.cpp" -"${ICU_SOURCE_DIR}/i18n/inputext.cpp" -"${ICU_SOURCE_DIR}/i18n/wintzimpl.cpp" -"${ICU_SOURCE_DIR}/i18n/windtfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/winnmfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/basictz.cpp" -"${ICU_SOURCE_DIR}/i18n/dtrule.cpp" -"${ICU_SOURCE_DIR}/i18n/rbtz.cpp" -"${ICU_SOURCE_DIR}/i18n/tzrule.cpp" -"${ICU_SOURCE_DIR}/i18n/tztrans.cpp" -"${ICU_SOURCE_DIR}/i18n/vtzone.cpp" -"${ICU_SOURCE_DIR}/i18n/zonemeta.cpp" -"${ICU_SOURCE_DIR}/i18n/standardplural.cpp" -"${ICU_SOURCE_DIR}/i18n/upluralrules.cpp" -"${ICU_SOURCE_DIR}/i18n/plurrule.cpp" -"${ICU_SOURCE_DIR}/i18n/plurfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/selfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/curramt.cpp" +"${ICU_SOURCE_DIR}/i18n/currfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/currpinf.cpp" +"${ICU_SOURCE_DIR}/i18n/currunit.cpp" +"${ICU_SOURCE_DIR}/i18n/dangical.cpp" +"${ICU_SOURCE_DIR}/i18n/datefmt.cpp" +"${ICU_SOURCE_DIR}/i18n/dayperiodrules.cpp" +"${ICU_SOURCE_DIR}/i18n/dcfmtsym.cpp" +"${ICU_SOURCE_DIR}/i18n/decContext.cpp" +"${ICU_SOURCE_DIR}/i18n/decNumber.cpp" +"${ICU_SOURCE_DIR}/i18n/decimfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/double-conversion-bignum-dtoa.cpp" +"${ICU_SOURCE_DIR}/i18n/double-conversion-bignum.cpp" +"${ICU_SOURCE_DIR}/i18n/double-conversion-cached-powers.cpp" +"${ICU_SOURCE_DIR}/i18n/double-conversion-double-to-string.cpp" +"${ICU_SOURCE_DIR}/i18n/double-conversion-fast-dtoa.cpp" +"${ICU_SOURCE_DIR}/i18n/double-conversion-string-to-double.cpp" +"${ICU_SOURCE_DIR}/i18n/double-conversion-strtod.cpp" +"${ICU_SOURCE_DIR}/i18n/dtfmtsym.cpp" "${ICU_SOURCE_DIR}/i18n/dtitvfmt.cpp" "${ICU_SOURCE_DIR}/i18n/dtitvinf.cpp" -"${ICU_SOURCE_DIR}/i18n/udateintervalformat.cpp" -"${ICU_SOURCE_DIR}/i18n/tmunit.cpp" -"${ICU_SOURCE_DIR}/i18n/tmutamt.cpp" -"${ICU_SOURCE_DIR}/i18n/tmutfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/currpinf.cpp" -"${ICU_SOURCE_DIR}/i18n/uspoof.cpp" -"${ICU_SOURCE_DIR}/i18n/uspoof_impl.cpp" -"${ICU_SOURCE_DIR}/i18n/uspoof_build.cpp" -"${ICU_SOURCE_DIR}/i18n/uspoof_conf.cpp" -"${ICU_SOURCE_DIR}/i18n/smpdtfst.cpp" -"${ICU_SOURCE_DIR}/i18n/ztrans.cpp" -"${ICU_SOURCE_DIR}/i18n/zrule.cpp" -"${ICU_SOURCE_DIR}/i18n/vzone.cpp" +"${ICU_SOURCE_DIR}/i18n/dtptngen.cpp" +"${ICU_SOURCE_DIR}/i18n/dtrule.cpp" +"${ICU_SOURCE_DIR}/i18n/erarules.cpp" +"${ICU_SOURCE_DIR}/i18n/esctrn.cpp" +"${ICU_SOURCE_DIR}/i18n/ethpccal.cpp" +"${ICU_SOURCE_DIR}/i18n/fmtable.cpp" +"${ICU_SOURCE_DIR}/i18n/fmtable_cnv.cpp" +"${ICU_SOURCE_DIR}/i18n/format.cpp" +"${ICU_SOURCE_DIR}/i18n/formatted_string_builder.cpp" +"${ICU_SOURCE_DIR}/i18n/formattedval_iterimpl.cpp" +"${ICU_SOURCE_DIR}/i18n/formattedval_sbimpl.cpp" +"${ICU_SOURCE_DIR}/i18n/formattedvalue.cpp" "${ICU_SOURCE_DIR}/i18n/fphdlimp.cpp" "${ICU_SOURCE_DIR}/i18n/fpositer.cpp" -"${ICU_SOURCE_DIR}/i18n/ufieldpositer.cpp" -"${ICU_SOURCE_DIR}/i18n/decNumber.cpp" -"${ICU_SOURCE_DIR}/i18n/decContext.cpp" -"${ICU_SOURCE_DIR}/i18n/alphaindex.cpp" -"${ICU_SOURCE_DIR}/i18n/tznames.cpp" -"${ICU_SOURCE_DIR}/i18n/tznames_impl.cpp" -"${ICU_SOURCE_DIR}/i18n/tzgnames.cpp" -"${ICU_SOURCE_DIR}/i18n/tzfmt.cpp" -"${ICU_SOURCE_DIR}/i18n/compactdecimalformat.cpp" +"${ICU_SOURCE_DIR}/i18n/funcrepl.cpp" "${ICU_SOURCE_DIR}/i18n/gender.cpp" -"${ICU_SOURCE_DIR}/i18n/region.cpp" -"${ICU_SOURCE_DIR}/i18n/scriptset.cpp" -"${ICU_SOURCE_DIR}/i18n/uregion.cpp" -"${ICU_SOURCE_DIR}/i18n/reldatefmt.cpp" -"${ICU_SOURCE_DIR}/i18n/quantityformatter.cpp" +"${ICU_SOURCE_DIR}/i18n/gregocal.cpp" +"${ICU_SOURCE_DIR}/i18n/gregoimp.cpp" +"${ICU_SOURCE_DIR}/i18n/hebrwcal.cpp" +"${ICU_SOURCE_DIR}/i18n/indiancal.cpp" +"${ICU_SOURCE_DIR}/i18n/inputext.cpp" +"${ICU_SOURCE_DIR}/i18n/islamcal.cpp" +"${ICU_SOURCE_DIR}/i18n/japancal.cpp" +"${ICU_SOURCE_DIR}/i18n/listformatter.cpp" +"${ICU_SOURCE_DIR}/i18n/measfmt.cpp" "${ICU_SOURCE_DIR}/i18n/measunit.cpp" -"${ICU_SOURCE_DIR}/i18n/sharedbreakiterator.cpp" -"${ICU_SOURCE_DIR}/i18n/scientificnumberformatter.cpp" -"${ICU_SOURCE_DIR}/i18n/dayperiodrules.cpp" +"${ICU_SOURCE_DIR}/i18n/measunit_extra.cpp" +"${ICU_SOURCE_DIR}/i18n/measure.cpp" +"${ICU_SOURCE_DIR}/i18n/msgfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/name2uni.cpp" +"${ICU_SOURCE_DIR}/i18n/nfrs.cpp" +"${ICU_SOURCE_DIR}/i18n/nfrule.cpp" +"${ICU_SOURCE_DIR}/i18n/nfsubs.cpp" +"${ICU_SOURCE_DIR}/i18n/nortrans.cpp" +"${ICU_SOURCE_DIR}/i18n/nultrans.cpp" "${ICU_SOURCE_DIR}/i18n/number_affixutils.cpp" +"${ICU_SOURCE_DIR}/i18n/number_asformat.cpp" +"${ICU_SOURCE_DIR}/i18n/number_capi.cpp" "${ICU_SOURCE_DIR}/i18n/number_compact.cpp" +"${ICU_SOURCE_DIR}/i18n/number_currencysymbols.cpp" "${ICU_SOURCE_DIR}/i18n/number_decimalquantity.cpp" "${ICU_SOURCE_DIR}/i18n/number_decimfmtprops.cpp" "${ICU_SOURCE_DIR}/i18n/number_fluent.cpp" @@ -410,7 +330,9 @@ set(ICUI18N_SOURCES "${ICU_SOURCE_DIR}/i18n/number_grouping.cpp" "${ICU_SOURCE_DIR}/i18n/number_integerwidth.cpp" "${ICU_SOURCE_DIR}/i18n/number_longnames.cpp" +"${ICU_SOURCE_DIR}/i18n/number_mapper.cpp" "${ICU_SOURCE_DIR}/i18n/number_modifiers.cpp" +"${ICU_SOURCE_DIR}/i18n/number_multiplier.cpp" "${ICU_SOURCE_DIR}/i18n/number_notation.cpp" "${ICU_SOURCE_DIR}/i18n/number_output.cpp" "${ICU_SOURCE_DIR}/i18n/number_padding.cpp" @@ -418,46 +340,124 @@ set(ICUI18N_SOURCES "${ICU_SOURCE_DIR}/i18n/number_patternstring.cpp" "${ICU_SOURCE_DIR}/i18n/number_rounding.cpp" "${ICU_SOURCE_DIR}/i18n/number_scientific.cpp" -"${ICU_SOURCE_DIR}/i18n/number_utils.cpp" -"${ICU_SOURCE_DIR}/i18n/number_asformat.cpp" -"${ICU_SOURCE_DIR}/i18n/number_mapper.cpp" -"${ICU_SOURCE_DIR}/i18n/number_multiplier.cpp" -"${ICU_SOURCE_DIR}/i18n/number_currencysymbols.cpp" "${ICU_SOURCE_DIR}/i18n/number_skeletons.cpp" -"${ICU_SOURCE_DIR}/i18n/number_capi.cpp" -"${ICU_SOURCE_DIR}/i18n/double-conversion-string-to-double.cpp" -"${ICU_SOURCE_DIR}/i18n/double-conversion-double-to-string.cpp" -"${ICU_SOURCE_DIR}/i18n/double-conversion-bignum-dtoa.cpp" -"${ICU_SOURCE_DIR}/i18n/double-conversion-bignum.cpp" -"${ICU_SOURCE_DIR}/i18n/double-conversion-cached-powers.cpp" -"${ICU_SOURCE_DIR}/i18n/double-conversion-fast-dtoa.cpp" -"${ICU_SOURCE_DIR}/i18n/double-conversion-strtod.cpp" -"${ICU_SOURCE_DIR}/i18n/string_segment.cpp" -"${ICU_SOURCE_DIR}/i18n/numparse_parsednumber.cpp" -"${ICU_SOURCE_DIR}/i18n/numparse_impl.cpp" -"${ICU_SOURCE_DIR}/i18n/numparse_symbols.cpp" -"${ICU_SOURCE_DIR}/i18n/numparse_decimal.cpp" -"${ICU_SOURCE_DIR}/i18n/numparse_scientific.cpp" -"${ICU_SOURCE_DIR}/i18n/numparse_currency.cpp" -"${ICU_SOURCE_DIR}/i18n/numparse_affixes.cpp" -"${ICU_SOURCE_DIR}/i18n/numparse_compositions.cpp" -"${ICU_SOURCE_DIR}/i18n/numparse_validators.cpp" -"${ICU_SOURCE_DIR}/i18n/numrange_fluent.cpp" -"${ICU_SOURCE_DIR}/i18n/numrange_impl.cpp" -"${ICU_SOURCE_DIR}/i18n/erarules.cpp" -"${ICU_SOURCE_DIR}/i18n/formattedvalue.cpp" -"${ICU_SOURCE_DIR}/i18n/formattedval_iterimpl.cpp" -"${ICU_SOURCE_DIR}/i18n/formattedval_sbimpl.cpp" -"${ICU_SOURCE_DIR}/i18n/formatted_string_builder.cpp" -"${ICU_SOURCE_DIR}/i18n/measunit_extra.cpp" "${ICU_SOURCE_DIR}/i18n/number_symbolswrapper.cpp" "${ICU_SOURCE_DIR}/i18n/number_usageprefs.cpp" +"${ICU_SOURCE_DIR}/i18n/number_utils.cpp" +"${ICU_SOURCE_DIR}/i18n/numfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/numparse_affixes.cpp" +"${ICU_SOURCE_DIR}/i18n/numparse_compositions.cpp" +"${ICU_SOURCE_DIR}/i18n/numparse_currency.cpp" +"${ICU_SOURCE_DIR}/i18n/numparse_decimal.cpp" +"${ICU_SOURCE_DIR}/i18n/numparse_impl.cpp" +"${ICU_SOURCE_DIR}/i18n/numparse_parsednumber.cpp" +"${ICU_SOURCE_DIR}/i18n/numparse_scientific.cpp" +"${ICU_SOURCE_DIR}/i18n/numparse_symbols.cpp" +"${ICU_SOURCE_DIR}/i18n/numparse_validators.cpp" "${ICU_SOURCE_DIR}/i18n/numrange_capi.cpp" +"${ICU_SOURCE_DIR}/i18n/numrange_fluent.cpp" +"${ICU_SOURCE_DIR}/i18n/numrange_impl.cpp" +"${ICU_SOURCE_DIR}/i18n/numsys.cpp" +"${ICU_SOURCE_DIR}/i18n/olsontz.cpp" +"${ICU_SOURCE_DIR}/i18n/persncal.cpp" "${ICU_SOURCE_DIR}/i18n/pluralranges.cpp" +"${ICU_SOURCE_DIR}/i18n/plurfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/plurrule.cpp" +"${ICU_SOURCE_DIR}/i18n/quant.cpp" +"${ICU_SOURCE_DIR}/i18n/quantityformatter.cpp" +"${ICU_SOURCE_DIR}/i18n/rbnf.cpp" +"${ICU_SOURCE_DIR}/i18n/rbt.cpp" +"${ICU_SOURCE_DIR}/i18n/rbt_data.cpp" +"${ICU_SOURCE_DIR}/i18n/rbt_pars.cpp" +"${ICU_SOURCE_DIR}/i18n/rbt_rule.cpp" +"${ICU_SOURCE_DIR}/i18n/rbt_set.cpp" +"${ICU_SOURCE_DIR}/i18n/rbtz.cpp" +"${ICU_SOURCE_DIR}/i18n/regexcmp.cpp" +"${ICU_SOURCE_DIR}/i18n/regeximp.cpp" +"${ICU_SOURCE_DIR}/i18n/regexst.cpp" +"${ICU_SOURCE_DIR}/i18n/regextxt.cpp" +"${ICU_SOURCE_DIR}/i18n/region.cpp" +"${ICU_SOURCE_DIR}/i18n/reldatefmt.cpp" +"${ICU_SOURCE_DIR}/i18n/reldtfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/rematch.cpp" +"${ICU_SOURCE_DIR}/i18n/remtrans.cpp" +"${ICU_SOURCE_DIR}/i18n/repattrn.cpp" +"${ICU_SOURCE_DIR}/i18n/rulebasedcollator.cpp" +"${ICU_SOURCE_DIR}/i18n/scientificnumberformatter.cpp" +"${ICU_SOURCE_DIR}/i18n/scriptset.cpp" +"${ICU_SOURCE_DIR}/i18n/search.cpp" +"${ICU_SOURCE_DIR}/i18n/selfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/sharedbreakiterator.cpp" +"${ICU_SOURCE_DIR}/i18n/simpletz.cpp" +"${ICU_SOURCE_DIR}/i18n/smpdtfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/smpdtfst.cpp" +"${ICU_SOURCE_DIR}/i18n/sortkey.cpp" +"${ICU_SOURCE_DIR}/i18n/standardplural.cpp" +"${ICU_SOURCE_DIR}/i18n/string_segment.cpp" +"${ICU_SOURCE_DIR}/i18n/strmatch.cpp" +"${ICU_SOURCE_DIR}/i18n/strrepl.cpp" +"${ICU_SOURCE_DIR}/i18n/stsearch.cpp" +"${ICU_SOURCE_DIR}/i18n/taiwncal.cpp" +"${ICU_SOURCE_DIR}/i18n/timezone.cpp" +"${ICU_SOURCE_DIR}/i18n/titletrn.cpp" +"${ICU_SOURCE_DIR}/i18n/tmunit.cpp" +"${ICU_SOURCE_DIR}/i18n/tmutamt.cpp" +"${ICU_SOURCE_DIR}/i18n/tmutfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/tolowtrn.cpp" +"${ICU_SOURCE_DIR}/i18n/toupptrn.cpp" +"${ICU_SOURCE_DIR}/i18n/translit.cpp" +"${ICU_SOURCE_DIR}/i18n/transreg.cpp" +"${ICU_SOURCE_DIR}/i18n/tridpars.cpp" +"${ICU_SOURCE_DIR}/i18n/tzfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/tzgnames.cpp" +"${ICU_SOURCE_DIR}/i18n/tznames.cpp" +"${ICU_SOURCE_DIR}/i18n/tznames_impl.cpp" +"${ICU_SOURCE_DIR}/i18n/tzrule.cpp" +"${ICU_SOURCE_DIR}/i18n/tztrans.cpp" +"${ICU_SOURCE_DIR}/i18n/ucal.cpp" +"${ICU_SOURCE_DIR}/i18n/ucln_in.cpp" +"${ICU_SOURCE_DIR}/i18n/ucol.cpp" +"${ICU_SOURCE_DIR}/i18n/ucol_res.cpp" +"${ICU_SOURCE_DIR}/i18n/ucol_sit.cpp" +"${ICU_SOURCE_DIR}/i18n/ucoleitr.cpp" +"${ICU_SOURCE_DIR}/i18n/ucsdet.cpp" +"${ICU_SOURCE_DIR}/i18n/udat.cpp" +"${ICU_SOURCE_DIR}/i18n/udateintervalformat.cpp" +"${ICU_SOURCE_DIR}/i18n/udatpg.cpp" +"${ICU_SOURCE_DIR}/i18n/ufieldpositer.cpp" +"${ICU_SOURCE_DIR}/i18n/uitercollationiterator.cpp" +"${ICU_SOURCE_DIR}/i18n/ulistformatter.cpp" +"${ICU_SOURCE_DIR}/i18n/ulocdata.cpp" +"${ICU_SOURCE_DIR}/i18n/umsg.cpp" +"${ICU_SOURCE_DIR}/i18n/unesctrn.cpp" +"${ICU_SOURCE_DIR}/i18n/uni2name.cpp" "${ICU_SOURCE_DIR}/i18n/units_complexconverter.cpp" "${ICU_SOURCE_DIR}/i18n/units_converter.cpp" "${ICU_SOURCE_DIR}/i18n/units_data.cpp" -"${ICU_SOURCE_DIR}/i18n/units_router.cpp") +"${ICU_SOURCE_DIR}/i18n/units_router.cpp" +"${ICU_SOURCE_DIR}/i18n/unum.cpp" +"${ICU_SOURCE_DIR}/i18n/unumsys.cpp" +"${ICU_SOURCE_DIR}/i18n/upluralrules.cpp" +"${ICU_SOURCE_DIR}/i18n/uregex.cpp" +"${ICU_SOURCE_DIR}/i18n/uregexc.cpp" +"${ICU_SOURCE_DIR}/i18n/uregion.cpp" +"${ICU_SOURCE_DIR}/i18n/usearch.cpp" +"${ICU_SOURCE_DIR}/i18n/uspoof.cpp" +"${ICU_SOURCE_DIR}/i18n/uspoof_build.cpp" +"${ICU_SOURCE_DIR}/i18n/uspoof_conf.cpp" +"${ICU_SOURCE_DIR}/i18n/uspoof_impl.cpp" +"${ICU_SOURCE_DIR}/i18n/utf16collationiterator.cpp" +"${ICU_SOURCE_DIR}/i18n/utf8collationiterator.cpp" +"${ICU_SOURCE_DIR}/i18n/utmscale.cpp" +"${ICU_SOURCE_DIR}/i18n/utrans.cpp" +"${ICU_SOURCE_DIR}/i18n/vtzone.cpp" +"${ICU_SOURCE_DIR}/i18n/vzone.cpp" +"${ICU_SOURCE_DIR}/i18n/windtfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/winnmfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/wintzimpl.cpp" +"${ICU_SOURCE_DIR}/i18n/zonemeta.cpp" +"${ICU_SOURCE_DIR}/i18n/zrule.cpp" +"${ICU_SOURCE_DIR}/i18n/ztrans.cpp") file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/empty.cpp" CONTENT " ") enable_language(ASM) From 6f03c51368e950ca5bc734616fd9a55a65504345 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 4 Sep 2024 09:40:45 +0000 Subject: [PATCH 151/289] Update --- contrib/icu-cmake/CMakeLists.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt index 9d6e9ac0a62..afaa189701d 100644 --- a/contrib/icu-cmake/CMakeLists.txt +++ b/contrib/icu-cmake/CMakeLists.txt @@ -273,6 +273,7 @@ set(ICUI18N_SOURCES "${ICU_SOURCE_DIR}/i18n/decContext.cpp" "${ICU_SOURCE_DIR}/i18n/decNumber.cpp" "${ICU_SOURCE_DIR}/i18n/decimfmt.cpp" +"${ICU_SOURCE_DIR}/i18n/displayoptions.cpp" "${ICU_SOURCE_DIR}/i18n/double-conversion-bignum-dtoa.cpp" "${ICU_SOURCE_DIR}/i18n/double-conversion-bignum.cpp" "${ICU_SOURCE_DIR}/i18n/double-conversion-cached-powers.cpp" @@ -305,12 +306,24 @@ set(ICUI18N_SOURCES "${ICU_SOURCE_DIR}/i18n/indiancal.cpp" "${ICU_SOURCE_DIR}/i18n/inputext.cpp" "${ICU_SOURCE_DIR}/i18n/islamcal.cpp" +"${ICU_SOURCE_DIR}/i18n/iso8601cal.cpp" "${ICU_SOURCE_DIR}/i18n/japancal.cpp" "${ICU_SOURCE_DIR}/i18n/listformatter.cpp" "${ICU_SOURCE_DIR}/i18n/measfmt.cpp" "${ICU_SOURCE_DIR}/i18n/measunit.cpp" "${ICU_SOURCE_DIR}/i18n/measunit_extra.cpp" "${ICU_SOURCE_DIR}/i18n/measure.cpp" +"${ICU_SOURCE_DIR}/i18n/messageformat2.cpp" +"${ICU_SOURCE_DIR}/i18n/messageformat2_arguments.cpp" +"${ICU_SOURCE_DIR}/i18n/messageformat2_checker.cpp" +"${ICU_SOURCE_DIR}/i18n/messageformat2_data_model.cpp" +"${ICU_SOURCE_DIR}/i18n/messageformat2_errors.cpp" +"${ICU_SOURCE_DIR}/i18n/messageformat2_evaluation.cpp" +"${ICU_SOURCE_DIR}/i18n/messageformat2_formattable.cpp" +"${ICU_SOURCE_DIR}/i18n/messageformat2_formatter.cpp" +"${ICU_SOURCE_DIR}/i18n/messageformat2_function_registry.cpp" +"${ICU_SOURCE_DIR}/i18n/messageformat2_parser.cpp" +"${ICU_SOURCE_DIR}/i18n/messageformat2_serializer.cpp" "${ICU_SOURCE_DIR}/i18n/msgfmt.cpp" "${ICU_SOURCE_DIR}/i18n/name2uni.cpp" "${ICU_SOURCE_DIR}/i18n/nfrs.cpp" @@ -340,6 +353,7 @@ set(ICUI18N_SOURCES "${ICU_SOURCE_DIR}/i18n/number_patternstring.cpp" "${ICU_SOURCE_DIR}/i18n/number_rounding.cpp" "${ICU_SOURCE_DIR}/i18n/number_scientific.cpp" +"${ICU_SOURCE_DIR}/i18n/number_simple.cpp" "${ICU_SOURCE_DIR}/i18n/number_skeletons.cpp" "${ICU_SOURCE_DIR}/i18n/number_symbolswrapper.cpp" "${ICU_SOURCE_DIR}/i18n/number_usageprefs.cpp" From 54bdd392ee5dc8ca1541a52f1d61b442f3c164aa Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:50:09 +0200 Subject: [PATCH 152/289] fix build --- src/Functions/toStartOfInterval.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 35e9a93acef..3a0df85dcf0 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -215,7 +215,7 @@ public: std::unreachable(); } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /* input_rows_count */) const override { const auto & time_column = arguments[0]; const auto & interval_column = arguments[1]; @@ -252,6 +252,12 @@ private: if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } + else if (isDate32(time_column_type)) + { + const auto * time_column_vec = checkAndGetColumn(&time_column_col); + if (time_column_vec) + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); + } else if (isDateTime(time_column_type)) { const auto * time_column_vec = checkAndGetColumn(&time_column_col); @@ -266,12 +272,6 @@ private: if (time_column_vec) return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone, scale); } - else if (isDate32(time_column_type)) - { - const auto * time_column_vec = checkAndGetColumn(&time_column_col); - if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, result_type, time_zone, input_rows_count); - } throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for 1st argument of function {}, expected a Date, Date32, DateTime or DateTime64", getName()); } @@ -345,11 +345,12 @@ private: using ResultColumnType = typename ResultDataType::ColumnType; const auto & time_data = time_column_type.getData(); + size_t size = time_data.size(); auto result_col = result_type->createColumn(); auto * col_to = assert_cast(result_col.get()); auto & result_data = col_to->getData(); - result_data.resize(input_rows_count); + result_data.resize(size); Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale); From e16c85d94fd2f0143f6e31d456ccc07abfa8c816 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:32:57 +0200 Subject: [PATCH 153/289] fix suggestions --- src/Functions/toStartOfInterval.cpp | 12 ++++++------ ...02916_to_start_of_interval_with_origin.reference | 6 ++++++ .../02916_to_start_of_interval_with_origin.sql | 13 +++++++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 3a0df85dcf0..3301974da7f 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -18,11 +18,11 @@ namespace DB { namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ARGUMENT_OUT_OF_BOUND; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -275,7 +275,7 @@ private: throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for 1st argument of function {}, expected a Date, Date32, DateTime or DateTime64", getName()); } - template + template ColumnPtr dispatchForIntervalColumn( const TimeDataType & time_data_type, const TimeColumnType & time_column, const ColumnWithTypeAndName & interval_column, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, UInt16 scale = 1) const @@ -339,8 +339,8 @@ private: std::unreachable(); } - template - ColumnPtr execute(const TimeDataType &, const ColumnType & time_column_type, Int64 num_units, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, UInt16 scale) const + template + ColumnPtr execute(const TimeDataType &, const TimeColumnType & time_column_type, Int64 num_units, const ColumnWithTypeAndName & origin_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone, UInt16 scale) const { using ResultColumnType = typename ResultDataType::ColumnType; diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference index 5e72df17027..f0afdf03963 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.reference @@ -5,6 +5,12 @@ Time and origin as Date 2023-10-08 2023-10-08 2023-10-09 +Time and origin as Date32 +2023-02-01 +2023-08-01 +2023-10-08 +2023-10-08 +2023-10-09 Time and origin as DateTime 2023-02-01 09:08:07 2023-08-01 09:08:07 diff --git a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql index 4f8a96b093d..b03ccae31d9 100644 --- a/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql +++ b/tests/queries/0_stateless/02916_to_start_of_interval_with_origin.sql @@ -36,6 +36,19 @@ SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalMillisecond(1), toDate( SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalMicrosecond(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT toStartOfInterval(toDate('2023-10-09'), toIntervalNanosecond(1), toDate('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT 'Time and origin as Date32'; +SELECT toStartOfInterval(toDate32('2023-10-09'), toIntervalYear(1), toDate32('2022-02-01')); +SELECT toStartOfInterval(toDate32('2023-10-09'), toIntervalQuarter(1), toDate32('2022-02-01')); +SELECT toStartOfInterval(toDate32('2023-10-09'), toIntervalMonth(1), toDate32('2023-09-08')); +SELECT toStartOfInterval(toDate32('2023-10-09'), toIntervalWeek(1), toDate32('2023-10-01')); +SELECT toStartOfInterval(toDate32('2023-10-09'), toIntervalDay(1), toDate32('2023-10-08')); +SELECT toStartOfInterval(toDate32('2023-10-09'), toIntervalHour(1), toDate32('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate32('2023-10-09'), toIntervalMinute(1), toDate32('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate32('2023-10-09'), toIntervalSecond(1), toDate32('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate32('2023-10-09'), toIntervalMillisecond(1), toDate32('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate32('2023-10-09'), toIntervalMicrosecond(1), toDate32('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toStartOfInterval(toDate32('2023-10-09'), toIntervalNanosecond(1), toDate32('2023-10-09')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + SELECT 'Time and origin as DateTime'; SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalYear(1), toDateTime('2022-02-01 09:08:07')); SELECT toStartOfInterval(toDateTime('2023-10-09 10:11:12'), toIntervalQuarter(1), toDateTime('2022-02-01 09:08:07')); From 945c93032207a7bf7e5b153fb759342c8e172308 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:53:19 +0200 Subject: [PATCH 154/289] fix build --- src/Functions/toStartOfInterval.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 3301974da7f..dc629160aff 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -313,27 +313,27 @@ private: switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) { case IntervalKind::Kind::Nanosecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Microsecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Millisecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Second: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Minute: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Hour: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Day: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Week: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Month: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Quarter: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Year: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); } std::unreachable(); From f497fde2ca8a4fba7e9fb94c1ac11d5db5e11c87 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:08:12 +0200 Subject: [PATCH 155/289] fix templates --- src/Functions/toStartOfInterval.cpp | 30 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index dc629160aff..b2b0c2dbfed 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -250,19 +250,19 @@ private: const auto * time_column_vec = checkAndGetColumn(&time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDate32(time_column_type)) { const auto * time_column_vec = checkAndGetColumn(&time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDateTime(time_column_type)) { const auto * time_column_vec = checkAndGetColumn(&time_column_col); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone); } else if (isDateTime64(time_column_type)) { @@ -270,7 +270,7 @@ private: auto scale = assert_cast(time_column_type).getScale(); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone, scale); + return dispatchForIntervalColumn(assert_cast(time_column_type), *time_column_vec, interval_column, origin_column, result_type, time_zone, scale); } throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for 1st argument of function {}, expected a Date, Date32, DateTime or DateTime64", getName()); } @@ -313,27 +313,27 @@ private: switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case) { case IntervalKind::Kind::Nanosecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Microsecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Millisecond: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Second: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Minute: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Hour: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Day: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Week: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Month: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Quarter: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); case IntervalKind::Kind::Year: - return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); + return execute(time_data_type, time_column, num_units, origin_column, result_type, time_zone, scale); } std::unreachable(); From 51464be3e1b53b839fd142ee6e7eed7de64f6c2e Mon Sep 17 00:00:00 2001 From: shiyer7474 Date: Wed, 4 Sep 2024 13:26:09 +0000 Subject: [PATCH 156/289] Set use_cluster=false --- .../test_keeper_s3_snapshot/configs/keeper_config1.xml | 1 + .../test_keeper_s3_snapshot/configs/keeper_config2.xml | 1 + .../test_keeper_s3_snapshot/configs/keeper_config3.xml | 1 + 3 files changed, 3 insertions(+) diff --git a/tests/integration/test_keeper_s3_snapshot/configs/keeper_config1.xml b/tests/integration/test_keeper_s3_snapshot/configs/keeper_config1.xml index 8459ea3e068..6af17946eec 100644 --- a/tests/integration/test_keeper_s3_snapshot/configs/keeper_config1.xml +++ b/tests/integration/test_keeper_s3_snapshot/configs/keeper_config1.xml @@ -5,6 +5,7 @@ minio minio123 + false 9181 1 /var/lib/clickhouse/coordination/log diff --git a/tests/integration/test_keeper_s3_snapshot/configs/keeper_config2.xml b/tests/integration/test_keeper_s3_snapshot/configs/keeper_config2.xml index dfe73628f66..25f2b0de812 100644 --- a/tests/integration/test_keeper_s3_snapshot/configs/keeper_config2.xml +++ b/tests/integration/test_keeper_s3_snapshot/configs/keeper_config2.xml @@ -5,6 +5,7 @@ minio minio123 + false 9181 2 /var/lib/clickhouse/coordination/log diff --git a/tests/integration/test_keeper_s3_snapshot/configs/keeper_config3.xml b/tests/integration/test_keeper_s3_snapshot/configs/keeper_config3.xml index 948d9527718..e274b5184f1 100644 --- a/tests/integration/test_keeper_s3_snapshot/configs/keeper_config3.xml +++ b/tests/integration/test_keeper_s3_snapshot/configs/keeper_config3.xml @@ -5,6 +5,7 @@ minio minio123 + false 9181 3 /var/lib/clickhouse/coordination/log From d38551a1788d2ebc1b7970cc912f894f87f3db1c Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 4 Sep 2024 13:29:45 +0000 Subject: [PATCH 157/289] resetOutput --- src/Client/ClientBase.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index edcb98be90f..445500a5f0f 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1901,11 +1901,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin /// We need to park ParallelFormating threads, /// because they can use settings from global context /// and it can lead to data race with `setSettings` - if (output_format) - { - output_format->finalize(); - output_format.reset(); - } + resetOutput(); } catch (...) { From 6acb5ab5a18a453d45231d51f14cc2e51d11b207 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:52:23 +0200 Subject: [PATCH 158/289] add Date32 support --- src/Functions/toStartOfInterval.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index b2b0c2dbfed..77c72e5b5f2 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include @@ -118,16 +117,18 @@ public: "A timezone argument of function {} with interval type {} is allowed only when the 1st argument has the type DateTime or DateTime64", getName(), interval_type->getKind().toString()); } - else if (isDate(type_arg3) || isDateTime(type_arg3) || isDateTime64(type_arg3)) + else if (isDateOrDate32OrDateTimeOrDateTime64(type_arg3)) { overload = Overload::Origin; const DataTypePtr & type_arg1 = arguments[0].type; - if (isDateTime64(type_arg1) && isDateTime64(type_arg3)) - result_type = ResultType::DateTime64; + if (isDate(type_arg1) && isDate(type_arg3)) + result_type = ResultType::Date; + else if (isDate32(type_arg1) && isDate32(type_arg3)) + result_type = ResultType::Date32; else if (isDateTime(type_arg1) && isDateTime(type_arg3)) result_type = ResultType::DateTime; - else if (isDate(type_arg1) && isDate(type_arg3)) - result_type = ResultType::Date; + else if (isDateTime64(type_arg1) && isDateTime64(type_arg3)) + result_type = ResultType::DateTime64; else throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Datetime argument and origin argument for function {} must have the same type", getName()); } @@ -230,6 +231,8 @@ public: ColumnPtr result_column; if (isDate(result_type)) result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); + else if (isDate32(result_type)) + result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); else if (isDateTime(result_type)) result_column = dispatchForTimeColumn(time_column, interval_column, origin_column, result_type, time_zone); else if (isDateTime64(result_type)) @@ -357,7 +360,7 @@ private: if (origin_column.column) // Overload: Origin { const bool is_small_interval = (unit == IntervalKind::Kind::Nanosecond || unit == IntervalKind::Kind::Microsecond || unit == IntervalKind::Kind::Millisecond); - const bool is_result_date = isDate(result_type); + const bool is_result_date = isDateOrDate32(result_type); Int64 result_scale = scale_multiplier; Int64 origin_scale = 1; From 018a62777be68cbabce86e3ee9ee74692f6caf3f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:56:07 +0200 Subject: [PATCH 159/289] small fixes for Date32 --- src/Functions/toStartOfInterval.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 77c72e5b5f2..6573fef6634 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -292,13 +292,13 @@ private: case IntervalKind::Kind::Nanosecond: case IntervalKind::Kind::Microsecond: case IntervalKind::Kind::Millisecond: - if (isDate(time_data_type) || isDateTime(time_data_type)) + if (isDateOrDate32(time_data_type) || isDateTime(time_data_type)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal interval kind for argument data type {}", isDate(time_data_type) ? "Date" : "DateTime"); break; case IntervalKind::Kind::Second: case IntervalKind::Kind::Minute: case IntervalKind::Kind::Hour: - if (isDate(time_data_type)) + if (isDateOrDate32(time_data_type)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal interval kind for argument data type Date"); break; default: From 5709dc585f35a62251f91fe7a069670ba68f17f3 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 4 Sep 2024 15:09:21 +0000 Subject: [PATCH 160/289] Fix: Not-ready Set with parallel replicas --- src/Planner/Planner.cpp | 1 + .../QueryPlan/Optimizations/Optimizations.h | 2 +- .../QueryPlanOptimizationSettings.h | 2 ++ .../QueryPlan/Optimizations/optimizeTree.cpp | 2 +- src/Processors/QueryPlan/QueryPlan.cpp | 3 ++- .../03232_pr_not_ready_set.reference | 0 .../0_stateless/03232_pr_not_ready_set.sql | 18 ++++++++++++++++++ 7 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/03232_pr_not_ready_set.reference create mode 100644 tests/queries/0_stateless/03232_pr_not_ready_set.sql diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 69a652a74a0..7ed16f17087 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -198,6 +198,7 @@ FiltersForTableExpressionMap collectFiltersForAnalysis(const QueryTreeNodePtr & auto & result_query_plan = planner.getQueryPlan(); auto optimization_settings = QueryPlanOptimizationSettings::fromContext(query_context); + optimization_settings.build_sets = false; // no need to build sets to collect filters result_query_plan.optimize(optimization_settings); FiltersForTableExpressionMap res; diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h index c48bdf1552a..43f07ced696 100644 --- a/src/Processors/QueryPlan/Optimizations/Optimizations.h +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -16,7 +16,7 @@ void optimizeTreeFirstPass(const QueryPlanOptimizationSettings & settings, Query void optimizeTreeSecondPass(const QueryPlanOptimizationSettings & optimization_settings, QueryPlan::Node & root, QueryPlan::Nodes & nodes); /// Third pass is used to apply filters such as key conditions and skip indexes to the storages that support them. /// After that it add CreateSetsStep for the subqueries that has not be used in the filters. -void optimizeTreeThirdPass(QueryPlan & plan, QueryPlan::Node & root, QueryPlan::Nodes & nodes); +void addStepsToBuildSets(QueryPlan & plan, QueryPlan::Node & root, QueryPlan::Nodes & nodes); /// Optimization (first pass) is a function applied to QueryPlan::Node. /// It can read and update subtree of specified node. diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index 539ff2eafbb..a2b22495800 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -75,6 +75,8 @@ struct QueryPlanOptimizationSettings String force_projection_name; bool optimize_use_implicit_projections = false; + bool build_sets = true; + static QueryPlanOptimizationSettings fromSettings(const Settings & from); static QueryPlanOptimizationSettings fromContext(ContextPtr from); }; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp index 25895788e2e..f8504d84d12 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -216,7 +216,7 @@ void optimizeTreeSecondPass(const QueryPlanOptimizationSettings & optimization_s optimization_settings.force_projection_name); } -void optimizeTreeThirdPass(QueryPlan & plan, QueryPlan::Node & root, QueryPlan::Nodes & nodes) +void addStepsToBuildSets(QueryPlan & plan, QueryPlan::Node & root, QueryPlan::Nodes & nodes) { Stack stack; stack.push_back({.node = &root}); diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index b78f7a29cde..9a39df26241 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -504,7 +504,8 @@ void QueryPlan::optimize(const QueryPlanOptimizationSettings & optimization_sett QueryPlanOptimizations::optimizeTreeFirstPass(optimization_settings, *root, nodes); QueryPlanOptimizations::optimizeTreeSecondPass(optimization_settings, *root, nodes); - QueryPlanOptimizations::optimizeTreeThirdPass(*this, *root, nodes); + if (optimization_settings.build_sets) + QueryPlanOptimizations::addStepsToBuildSets(*this, *root, nodes); updateDataStreams(*root); } diff --git a/tests/queries/0_stateless/03232_pr_not_ready_set.reference b/tests/queries/0_stateless/03232_pr_not_ready_set.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03232_pr_not_ready_set.sql b/tests/queries/0_stateless/03232_pr_not_ready_set.sql new file mode 100644 index 00000000000..1a724085903 --- /dev/null +++ b/tests/queries/0_stateless/03232_pr_not_ready_set.sql @@ -0,0 +1,18 @@ +SELECT + is_initial_query, + count() AS c, + replaceRegexpAll(query, '_data_(\\d+)_(\\d+)', '_data_') AS query +FROM system.query_log +WHERE (event_date >= yesterday()) AND (type = 'QueryFinish') AND (ignore(54, 0, ignore('QueryFinish', 11, toLowCardinality(toLowCardinality(11)), 11, 11, 11), 'QueryFinish', materialize(11), toUInt128(11)) IN ( + SELECT query_id + FROM system.query_log + WHERE (current_database = currentDatabase()) AND (event_date >= yesterday()) AND (type = 'QueryFinish') AND (query LIKE '-- Parallel inner query alone%') +)) +GROUP BY + is_initial_query, + query +ORDER BY + is_initial_query ASC, + c ASC, + query ASC +SETTINGS allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost', parallel_replicas_for_non_replicated_merge_tree=1, parallel_replicas_min_number_of_rows_per_replica=10; From 35ab639d1a602724204286f4b43460770f058272 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 3 Sep 2024 23:18:52 +0200 Subject: [PATCH 161/289] fix --- src/Core/SettingsChangesHistory.cpp | 263 +----------------- src/Databases/DatabaseReplicated.cpp | 3 +- src/Interpreters/InterpreterCreateQuery.cpp | 5 +- .../configs/settings.xml | 1 + .../configs/settings2.xml | 1 + 5 files changed, 10 insertions(+), 263 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 2e89cb85682..8e92f848bdd 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -57,265 +57,6 @@ String ClickHouseVersion::toString() const /// Note: please check if the key already exists to prevent duplicate entries. static std::initializer_list> settings_changes_history_initializer = { - {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."}, - {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"}, - {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"}, - {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"}, - {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"}, - {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"}, - {"optimize_functions_to_subcolumns", false, true, "Enable optimization by default"}, - {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."}, - {"input_format_json_ignore_key_case", false, false, "Ignore json key case while read json field from string."}, - {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."}, - {"collect_hash_table_stats_during_joins", false, true, "New setting."}, - {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."}, - {"input_format_orc_read_use_writer_time_zone", false, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT."}, - {"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."}, - {"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"}, - {"database_replicated_allow_replicated_engine_arguments", 1, 0, "Don't allow explicit arguments by default"}, - {"database_replicated_allow_explicit_uuid", 0, 0, "Added a new setting to disallow explicitly specifying table UUID"}, - {"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"}, - {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"}, - {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"}, - {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"}, - {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."}, - {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."}, - {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."} - }}, - {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"}, - {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"}, - {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."}, - {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"}, - {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"}, - {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"}, - {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"}, - {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"}, - {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"}, - {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"}, - {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"}, - {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"}, - {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"}, - {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."}, - {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."}, - {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"}, - {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."}, - {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."}, - {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."}, - {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."}, - {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."}, - {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"}, - {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"}, - {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"}, - {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."}, - {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."}, - {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."}, - }}, - {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"}, - {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."}, - {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"}, - {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."}, - {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."}, - {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."}, - {"http_max_chunk_size", 0, 0, "Internal limitation"}, - {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."}, - {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"}, - {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"}, - {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"}, - {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."}, - }}, - {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"}, - {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"}, - {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"}, - {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"}, - {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"}, - {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"}, - {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."}, - {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"}, - {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"}, - {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"}, - {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."}, - {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"}, - {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"}, - }}, - {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"}, - {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"}, - {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"}, - {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"}, - {"page_cache_inject_eviction", false, false, "Added userspace page cache"}, - {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"}, - {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"}, - {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."}, - {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."}, - {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"}, - {"log_processors_profiles", false, true, "Enable by default"}, - {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."}, - {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"}, - {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"}, - {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"}, - {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"}, - {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"}, - {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"}, - {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"}, - {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"}, - {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"}, - {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."}, - {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."}, - {"allow_get_client_http_header", false, false, "Introduced a new function."}, - {"output_format_pretty_row_numbers", false, true, "It is better for usability."}, - {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."}, - {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, - {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, - {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, - {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."}, - {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."}, - {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."}, - {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."}, - {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."}, - {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."}, - {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."}, - {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."}, - {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."}, - {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."}, - {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."}, - }}, - {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, - {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, - {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"}, - {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"}, - {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"}, - {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"}, - {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."}, - {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"}, - {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"}, - {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"}, - {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"}, - {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"}, - {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"}, - {"format_template_row_format", "", "", "Template row format string can be set directly in query"}, - {"format_template_resultset_format", "", "", "Template result set format string can be set in query"}, - {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, - {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}, - {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."}, - {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"}, - {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."}, - {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."}, - {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"}, - {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."}, - {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"}, - {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"}, - {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"}, - {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, - {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, - }}, - {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, - {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, - {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"}, - {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"}, - {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"}, - {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, - {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"}, - {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"}, - {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"}, - {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"}, - {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"}, - {"enable_vertical_final", false, true, "Use vertical final by default"}, - {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, - {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"}, - {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"}, - {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"}, - {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"}, - {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"}, - {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"}, - {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"}, - {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"}, - {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, - {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}}, - {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."}, - {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"}, - {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"}, - {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}}, - {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, - {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"}, - {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"}, - {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"}, - {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"}, - {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"}, - {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"}, - {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}}, - {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}}, - {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}}, - {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."}, - {"http_receive_timeout", 180, 30, "See http_send_timeout."}}}, - {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."}, - {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."}, - {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"}, - {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}}, - {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"}, - {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."}, - {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"}, - {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"}, - {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"}, - {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"}, - {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, - {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"}, - {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}, - {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"}, - {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"}, - {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"}, - {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"}, - {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}}, - {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, - {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, - {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, - {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"}, - {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}}, - {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}, - {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}, - {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"}, - {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"}, - {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"}, - {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}}, - {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"}, - {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"}, - {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}}, - {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}}, - {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}}, - {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"}, - {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"}, - {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}}, - {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"}, - {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}}, - {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"}, - {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}}, - {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}}, - {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}}, - {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}}, - {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"}, - {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}}, - {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}}, - {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}}, - {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"}, - {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"}, - {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}}, - {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}}, - {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"}, - {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"}, - {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"}, - {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}}, - {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}}, - {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing UID of the table in its CREATE query for Engine=Atomic"}}}, - {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"}, - {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}}, - {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}}, - {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}}, - {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}}, - {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}}, - {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}}, - {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}}, {"24.12", { } @@ -335,7 +76,9 @@ static std::initializer_listgetSettingsRef().database_replicated_allow_replicated_engine_arguments == 0) + /// We will replace it with default arguments if the setting is 2 + if (query_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments != 2) throw Exception(ErrorCodes::INCORRECT_QUERY, "Explicit zookeeper_path and replica_name are specified in ReplicatedMergeTree arguments. " "If you really want to specify it explicitly, then you should use some macros " diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index d34ba4c267b..7efaac029c7 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1239,7 +1239,7 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data bool from_path = create.attach_from_path.has_value(); bool is_on_cluster = getContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; - if (database->getEngineName() == "Replicated" && create.uuid != UUIDHelpers::Nil && !is_replicated_database_internal && !create.attach) + if (database->getEngineName() == "Replicated" && create.uuid != UUIDHelpers::Nil && !is_replicated_database_internal && !is_on_cluster && !create.attach) { if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 0) { @@ -1253,7 +1253,8 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data else if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 2) { UUID old_uuid = create.uuid; - create.generateRandomUUID(/*always_generate_new_uuid*/ true); + create.uuid = UUIDHelpers::Nil; + create.generateRandomUUIDs(); LOG_WARNING(&Poco::Logger::get("InterpreterCreateQuery"), "Replaced a user-provided UUID ({}) with a random one ({}) " "to make sure it's unique", old_uuid, create.uuid); } diff --git a/tests/integration/test_replicated_database/configs/settings.xml b/tests/integration/test_replicated_database/configs/settings.xml index 4b1122b1b5f..41799c5bed2 100644 --- a/tests/integration/test_replicated_database/configs/settings.xml +++ b/tests/integration/test_replicated_database/configs/settings.xml @@ -6,6 +6,7 @@ 0 0 3 + 3 diff --git a/tests/integration/test_replicated_database/configs/settings2.xml b/tests/integration/test_replicated_database/configs/settings2.xml index 9a3038871e1..fb7f0c8d4d3 100644 --- a/tests/integration/test_replicated_database/configs/settings2.xml +++ b/tests/integration/test_replicated_database/configs/settings2.xml @@ -6,6 +6,7 @@ 0 0 3 + 3 0 From 4b322ee3c5e7a40f9bc3bc4fec8100f33dabf750 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 4 Sep 2024 17:12:17 +0000 Subject: [PATCH 162/289] Reduce memory usage of inserts to JSON by using adaptive write buffer size --- src/Compression/CompressedWriteBuffer.cpp | 23 +++++++++++++++++-- src/Compression/CompressedWriteBuffer.h | 11 ++++++++- src/Core/Defines.h | 3 +++ .../Serializations/ISerialization.cpp | 14 +++++++++++ src/DataTypes/Serializations/ISerialization.h | 3 +++ src/Disks/DiskLocal.cpp | 12 +++++++++- .../IO/WriteBufferFromAzureBlobStorage.cpp | 19 ++++++++++++++- .../IO/WriteBufferFromAzureBlobStorage.h | 1 + .../ObjectStorages/S3/S3ObjectStorage.cpp | 2 +- src/IO/WriteBufferFromFile.cpp | 12 ++++++---- src/IO/WriteBufferFromFile.h | 8 +++++-- src/IO/WriteBufferFromFileDescriptor.cpp | 16 +++++++++++-- src/IO/WriteBufferFromFileDescriptor.h | 10 +++++++- src/IO/WriteBufferFromS3.cpp | 12 +++++++--- src/IO/WriteSettings.h | 3 +++ .../MergeTreeDataPartWriterOnDisk.cpp | 6 ++--- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 6 ++++- .../MergeTree/MergeTreeIOSettings.cpp | 2 ++ src/Storages/MergeTree/MergeTreeIOSettings.h | 2 ++ src/Storages/MergeTree/MergeTreeSettings.h | 2 ++ 20 files changed, 145 insertions(+), 22 deletions(-) diff --git a/src/Compression/CompressedWriteBuffer.cpp b/src/Compression/CompressedWriteBuffer.cpp index 83c9fbc9573..c3acfcb7da6 100644 --- a/src/Compression/CompressedWriteBuffer.cpp +++ b/src/Compression/CompressedWriteBuffer.cpp @@ -55,10 +55,29 @@ void CompressedWriteBuffer::nextImpl() out.write(compressed_buffer.data(), compressed_size); } + + /// Increase buffer size for next data if adaptive buffer size is used and nextImpl was called because of end of buffer. + if (!available() && use_adaptive_buffer_size && memory.size() < adaptive_buffer_max_size) + { + memory.resize(std::min(memory.size() * 2, adaptive_buffer_max_size)); + BufferBase::set(memory.data(), memory.size(), 0); + } } -CompressedWriteBuffer::CompressedWriteBuffer(WriteBuffer & out_, CompressionCodecPtr codec_, size_t buf_size) - : BufferWithOwnMemory(buf_size), out(out_), codec(std::move(codec_)) +void CompressedWriteBuffer::finalizeImpl() +{ + /// Don't try to resize buffer in nextImpl. + use_adaptive_buffer_size = false; + next(); +} + +CompressedWriteBuffer::CompressedWriteBuffer( + WriteBuffer & out_, CompressionCodecPtr codec_, size_t buf_size, bool use_adaptive_buffer_size_, size_t adaptive_buffer_initial_size) + : BufferWithOwnMemory(use_adaptive_buffer_size_ ? adaptive_buffer_initial_size : buf_size) + , out(out_) + , codec(std::move(codec_)) + , use_adaptive_buffer_size(use_adaptive_buffer_size_) + , adaptive_buffer_max_size(buf_size) { } diff --git a/src/Compression/CompressedWriteBuffer.h b/src/Compression/CompressedWriteBuffer.h index 6ae1fbee9cc..a3aae6b0c61 100644 --- a/src/Compression/CompressedWriteBuffer.h +++ b/src/Compression/CompressedWriteBuffer.h @@ -19,7 +19,9 @@ public: explicit CompressedWriteBuffer( WriteBuffer & out_, CompressionCodecPtr codec_ = CompressionCodecFactory::instance().getDefaultCodec(), - size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); + size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, + bool use_adaptive_buffer_size_ = false, + size_t adaptive_buffer_initial_size = DBMS_DEFAULT_INITIAL_ADAPTIVE_BUFFER_SIZE); ~CompressedWriteBuffer() override; @@ -45,10 +47,17 @@ public: private: void nextImpl() override; + void finalizeImpl() override; WriteBuffer & out; CompressionCodecPtr codec; + /// If true, the size of internal buffer will be exponentially increased up to + /// adaptive_buffer_max_size after each nextImpl call. It can be used to avoid + /// large buffer allocation when actual size of writen data is small. + bool use_adaptive_buffer_size; + size_t adaptive_buffer_max_size; + PODArray compressed_buffer; }; diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 6df335a9c8f..629ec58a936 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -20,6 +20,9 @@ static constexpr auto DBMS_DEFAULT_POLL_INTERVAL = 10; /// The size of the I/O buffer by default. static constexpr auto DBMS_DEFAULT_BUFFER_SIZE = 1048576ULL; +/// The initial size of adaptive I/O buffer by default. +static constexpr auto DBMS_DEFAULT_INITIAL_ADAPTIVE_BUFFER_SIZE = 16384ULL; + static constexpr auto PADDING_FOR_SIMD = 64; /** Which blocks by default read the data (by number of rows). diff --git a/src/DataTypes/Serializations/ISerialization.cpp b/src/DataTypes/Serializations/ISerialization.cpp index 338edc3a144..81efa921c9e 100644 --- a/src/DataTypes/Serializations/ISerialization.cpp +++ b/src/DataTypes/Serializations/ISerialization.cpp @@ -420,6 +420,20 @@ bool ISerialization::isEphemeralSubcolumn(const DB::ISerialization::SubstreamPat return path[last_elem].type == Substream::VariantElementNullMap; } +bool ISerialization::isDynamicSubcolumn(const DB::ISerialization::SubstreamPath & path, size_t prefix_len) +{ + if (prefix_len == 0 || prefix_len > path.size()) + return false; + + for (size_t i = 0; i != prefix_len; ++i) + { + if (path[i].type == SubstreamType::DynamicData || path[i].type == SubstreamType::ObjectData) + return true; + } + + return false; +} + ISerialization::SubstreamData ISerialization::createFromPath(const SubstreamPath & path, size_t prefix_len) { assert(prefix_len <= path.size()); diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h index 33575a07177..32f418e9132 100644 --- a/src/DataTypes/Serializations/ISerialization.h +++ b/src/DataTypes/Serializations/ISerialization.h @@ -457,6 +457,9 @@ public: /// for writing/reading data. For example, it's a null-map subcolumn of Variant type (it's always constructed from discriminators);. static bool isEphemeralSubcolumn(const SubstreamPath & path, size_t prefix_len); + /// Returns true if stream with specified path corresponds to dynamic subcolumn. + static bool isDynamicSubcolumn(const SubstreamPath & path, size_t prefix_len); + protected: template State * checkAndGetState(const StatePtr & state) const; diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index d1f0a928b1d..03b8140ce5c 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -23,6 +23,8 @@ #include #include #include +#include +#include #include @@ -339,7 +341,15 @@ DiskLocal::writeFile(const String & path, size_t buf_size, WriteMode mode, const { int flags = (mode == WriteMode::Append) ? (O_APPEND | O_CREAT | O_WRONLY) : -1; return std::make_unique( - fs::path(disk_path) / path, buf_size, flags, settings.local_throttler); + fs::path(disk_path) / path, + buf_size, + flags, + settings.local_throttler, + 0666, + nullptr, + 0, + settings.use_adaptive_write_buffer, + settings.adaptive_write_buffer_initial_size); } std::vector DiskLocal::getBlobPath(const String & path) const diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index 60fa2997c50..dcd625f7ee0 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -251,12 +251,29 @@ void WriteBufferFromAzureBlobStorage::allocateBuffer() auto size = buffer_allocation_policy->getBufferSize(); if (buffer_allocation_policy->getBufferNumber() == 1) - size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), size); + { + allocateFirstBuffer(); + return; + } memory = Memory(size); WriteBuffer::set(memory.data(), memory.size()); } +void WriteBufferFromAzureBlobStorage::allocateFirstBuffer() +{ + /// First buffer was already allocated in BufferWithOwnMemory constructor with buffer size provided in constructor. + /// It will be reallocated in subsequent nextImpl calls up to the desired buffer size from buffer_allocation_policy. + /// But it may happen that buffer size provided in constructor is larger then desired buffer size from buffer_allocation_policy. + /// Resize memory in this case to the desired size. + const auto max_first_buffer = buffer_allocation_policy->getBufferSize(); + if (memory.size() > max_first_buffer) + { + memory.resize(max_first_buffer); + WriteBuffer::set(memory.data(), memory.size()); + } +} + void WriteBufferFromAzureBlobStorage::detachBuffer() { size_t data_size = size_t(position() - memory.data()); diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h index 3ee497c4e44..c2d65928cfa 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h @@ -53,6 +53,7 @@ private: void detachBuffer(); void reallocateFirstBuffer(); void allocateBuffer(); + void allocateFirstBuffer(); void hidePartialData(); void setFakeBufferWhenPreFinalized(); diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 8de80971238..f26a3a8bd9d 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -282,7 +282,7 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN client.get(), uri.bucket, object.remote_path, - buf_size, + write_settings.use_adaptive_write_buffer ? write_settings.adaptive_write_buffer_initial_size : buf_size, request_settings, std::move(blob_storage_log), attributes, diff --git a/src/IO/WriteBufferFromFile.cpp b/src/IO/WriteBufferFromFile.cpp index f1825ce1e22..d68203029c1 100644 --- a/src/IO/WriteBufferFromFile.cpp +++ b/src/IO/WriteBufferFromFile.cpp @@ -32,8 +32,10 @@ WriteBufferFromFile::WriteBufferFromFile( ThrottlerPtr throttler_, mode_t mode, char * existing_memory, - size_t alignment) - : WriteBufferFromFileDescriptor(-1, buf_size, existing_memory, throttler_, alignment, file_name_) + size_t alignment, + bool use_adaptive_buffer_size_, + size_t adaptive_buffer_initial_size) + : WriteBufferFromFileDescriptor(-1, buf_size, existing_memory, throttler_, alignment, file_name_, use_adaptive_buffer_size_, adaptive_buffer_initial_size) { ProfileEvents::increment(ProfileEvents::FileOpen); @@ -66,8 +68,10 @@ WriteBufferFromFile::WriteBufferFromFile( size_t buf_size, ThrottlerPtr throttler_, char * existing_memory, - size_t alignment) - : WriteBufferFromFileDescriptor(fd_, buf_size, existing_memory, throttler_, alignment, original_file_name) + size_t alignment, + bool use_adaptive_buffer_size_, + size_t adaptive_buffer_initial_size) + : WriteBufferFromFileDescriptor(fd_, buf_size, existing_memory, throttler_, alignment, original_file_name, use_adaptive_buffer_size_, adaptive_buffer_initial_size) { fd_ = -1; } diff --git a/src/IO/WriteBufferFromFile.h b/src/IO/WriteBufferFromFile.h index 57847d893af..c0fa7f0b233 100644 --- a/src/IO/WriteBufferFromFile.h +++ b/src/IO/WriteBufferFromFile.h @@ -36,7 +36,9 @@ public: ThrottlerPtr throttler_ = {}, mode_t mode = 0666, char * existing_memory = nullptr, - size_t alignment = 0); + size_t alignment = 0, + bool use_adaptive_buffer_size_ = false, + size_t adaptive_buffer_initial_size = DBMS_DEFAULT_INITIAL_ADAPTIVE_BUFFER_SIZE); /// Use pre-opened file descriptor. explicit WriteBufferFromFile( @@ -45,7 +47,9 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, ThrottlerPtr throttler_ = {}, char * existing_memory = nullptr, - size_t alignment = 0); + size_t alignment = 0, + bool use_adaptive_buffer_size_ = false, + size_t adaptive_buffer_initial_size = DBMS_DEFAULT_INITIAL_ADAPTIVE_BUFFER_SIZE); ~WriteBufferFromFile() override; diff --git a/src/IO/WriteBufferFromFileDescriptor.cpp b/src/IO/WriteBufferFromFileDescriptor.cpp index f1207edc55b..b60a792e11c 100644 --- a/src/IO/WriteBufferFromFileDescriptor.cpp +++ b/src/IO/WriteBufferFromFileDescriptor.cpp @@ -83,6 +83,13 @@ void WriteBufferFromFileDescriptor::nextImpl() ProfileEvents::increment(ProfileEvents::DiskWriteElapsedMicroseconds, watch.elapsedMicroseconds()); ProfileEvents::increment(ProfileEvents::WriteBufferFromFileDescriptorWriteBytes, bytes_written); + + /// Increase buffer size for next data if adaptive buffer size is used and nextImpl was called because of end of buffer. + if (!available() && use_adaptive_buffer_size && memory.size() < adaptive_max_buffer_size) + { + memory.resize(std::min(memory.size() * 2, adaptive_max_buffer_size)); + BufferBase::set(memory.data(), memory.size(), 0); + } } /// NOTE: This class can be used as a very low-level building block, for example @@ -94,11 +101,15 @@ WriteBufferFromFileDescriptor::WriteBufferFromFileDescriptor( char * existing_memory, ThrottlerPtr throttler_, size_t alignment, - std::string file_name_) - : WriteBufferFromFileBase(buf_size, existing_memory, alignment) + std::string file_name_, + bool use_adaptive_buffer_size_, + size_t adaptive_buffer_initial_size) + : WriteBufferFromFileBase(use_adaptive_buffer_size_ ? adaptive_buffer_initial_size : buf_size, existing_memory, alignment) , fd(fd_) , throttler(throttler_) , file_name(std::move(file_name_)) + , use_adaptive_buffer_size(use_adaptive_buffer_size_) + , adaptive_max_buffer_size(buf_size) { } @@ -124,6 +135,7 @@ void WriteBufferFromFileDescriptor::finalizeImpl() return; } + use_adaptive_buffer_size = false; next(); } diff --git a/src/IO/WriteBufferFromFileDescriptor.h b/src/IO/WriteBufferFromFileDescriptor.h index cb73b1e1d08..1008c7cd8d9 100644 --- a/src/IO/WriteBufferFromFileDescriptor.h +++ b/src/IO/WriteBufferFromFileDescriptor.h @@ -18,7 +18,9 @@ public: char * existing_memory = nullptr, ThrottlerPtr throttler_ = {}, size_t alignment = 0, - std::string file_name_ = ""); + std::string file_name_ = "", + bool use_adaptive_buffer_size_ = false, + size_t adaptive_buffer_initial_size = DBMS_DEFAULT_INITIAL_ADAPTIVE_BUFFER_SIZE); /** Could be used before initialization if needed 'fd' was not passed to constructor. * It's not possible to change 'fd' during work. @@ -56,6 +58,12 @@ protected: /// If file has name contains filename, otherwise contains string "(fd=...)" std::string file_name; + /// If true, the size of internal buffer will be exponentially increased up to + /// adaptive_buffer_max_size after each nextImpl call. It can be used to avoid + /// large buffer allocation when actual size of writen data is small. + bool use_adaptive_buffer_size; + size_t adaptive_max_buffer_size; + void finalizeImpl() override; }; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index e702b4d35ad..d88e393e4e1 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -364,10 +364,16 @@ void WriteBufferFromS3::allocateBuffer() void WriteBufferFromS3::allocateFirstBuffer() { + /// First buffer was already allocated in BufferWithOwnMemory constructor with provided in constructor buffer size. + /// It will be reallocated in subsequent nextImpl calls up to the desired buffer size from buffer_allocation_policy. + /// But it may happen that buffer size provided in constructor is larger then desired buffer size from buffer_allocation_policy. + /// Resize memory in this case to the desired size. const auto max_first_buffer = buffer_allocation_policy->getBufferSize(); - const auto size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), max_first_buffer); - memory = Memory(size); - WriteBuffer::set(memory.data(), memory.size()); + if (memory.size() > max_first_buffer) + { + memory.resize(max_first_buffer); + WriteBuffer::set(memory.data(), memory.size()); + } } void WriteBufferFromS3::setFakeBufferWhenPreFinalized() diff --git a/src/IO/WriteSettings.h b/src/IO/WriteSettings.h index 84bb25439b5..ce78a3fd26b 100644 --- a/src/IO/WriteSettings.h +++ b/src/IO/WriteSettings.h @@ -25,6 +25,9 @@ struct WriteSettings bool s3_allow_parallel_part_upload = true; bool azure_allow_parallel_part_upload = true; + bool use_adaptive_write_buffer = false; + size_t adaptive_write_buffer_initial_size = 16 * 1024; + bool operator==(const WriteSettings & other) const = default; }; diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index b0e70e94b73..9bfc87135d9 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -85,11 +85,11 @@ MergeTreeDataPartWriterOnDisk::Stream::Stream( marks_file_extension{marks_file_extension_}, plain_file(data_part_storage->writeFile(data_path_ + data_file_extension, max_compress_block_size_, query_write_settings)), plain_hashing(*plain_file), - compressor(plain_hashing, compression_codec_, max_compress_block_size_), + compressor(plain_hashing, compression_codec_, max_compress_block_size_, query_write_settings.use_adaptive_write_buffer, query_write_settings.adaptive_write_buffer_initial_size), compressed_hashing(compressor), marks_file(data_part_storage->writeFile(marks_path_ + marks_file_extension, 4096, query_write_settings)), marks_hashing(*marks_file), - marks_compressor(marks_hashing, marks_compression_codec_, marks_compress_block_size_), + marks_compressor(marks_hashing, marks_compression_codec_, marks_compress_block_size_, query_write_settings.use_adaptive_write_buffer, query_write_settings.adaptive_write_buffer_initial_size), marks_compressed_hashing(marks_compressor), compress_marks(MarkType(marks_file_extension).compressed) { @@ -108,7 +108,7 @@ MergeTreeDataPartWriterOnDisk::Stream::Stream( data_file_extension{data_file_extension_}, plain_file(data_part_storage->writeFile(data_path_ + data_file_extension, max_compress_block_size_, query_write_settings)), plain_hashing(*plain_file), - compressor(plain_hashing, compression_codec_, max_compress_block_size_), + compressor(plain_hashing, compression_codec_, max_compress_block_size_, query_write_settings.use_adaptive_write_buffer, query_write_settings.adaptive_write_buffer_initial_size), compressed_hashing(compressor), compress_marks(false) { diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 8b6735e0fe2..f050accd7a1 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -177,6 +177,10 @@ void MergeTreeDataPartWriterWide::addStreams( if (!max_compress_block_size) max_compress_block_size = settings.max_compress_block_size; + WriteSettings query_write_settings = settings.query_write_settings; + query_write_settings.use_adaptive_write_buffer = settings.use_adaptive_write_buffer_for_dynamic_subcolumns && ISerialization::isDynamicSubcolumn(substream_path, substream_path.size()); + query_write_settings.adaptive_write_buffer_initial_size = settings.adaptive_write_buffer_initial_size; + column_streams[stream_name] = std::make_unique>( stream_name, data_part_storage, @@ -186,7 +190,7 @@ void MergeTreeDataPartWriterWide::addStreams( max_compress_block_size, marks_compression_codec, settings.marks_compress_block_size, - settings.query_write_settings); + query_write_settings); full_name_to_stream_name.emplace(full_stream_name, stream_name); stream_name_to_full_name.emplace(stream_name, full_stream_name); diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.cpp b/src/Storages/MergeTree/MergeTreeIOSettings.cpp index 24cb25afe47..19365a90a14 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeIOSettings.cpp @@ -30,6 +30,8 @@ MergeTreeWriterSettings::MergeTreeWriterSettings( , low_cardinality_max_dictionary_size(global_settings.low_cardinality_max_dictionary_size) , low_cardinality_use_single_dictionary_for_part(global_settings.low_cardinality_use_single_dictionary_for_part != 0) , use_compact_variant_discriminators_serialization(storage_settings->use_compact_variant_discriminators_serialization) + , use_adaptive_write_buffer_for_dynamic_subcolumns(storage_settings->use_adaptive_write_buffer_for_dynamic_subcolumns) + , adaptive_write_buffer_initial_size(storage_settings->adaptive_write_buffer_initial_size) { } diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index 47b174b2e29..fcc72815d8f 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -80,6 +80,8 @@ struct MergeTreeWriterSettings size_t low_cardinality_max_dictionary_size; bool low_cardinality_use_single_dictionary_for_part; bool use_compact_variant_discriminators_serialization; + bool use_adaptive_write_buffer_for_dynamic_subcolumns; + size_t adaptive_write_buffer_initial_size; }; } diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 0769b60dc6b..dcb18155114 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -99,6 +99,8 @@ struct Settings; M(Bool, add_implicit_sign_column_constraint_for_collapsing_engine, false, "If true, add implicit constraint for sign column for CollapsingMergeTree engine.", 0) \ M(Milliseconds, sleep_before_commit_local_part_in_replicated_table_ms, 0, "For testing. Do not change it.", 0) \ M(Bool, optimize_row_order, false, "Allow reshuffling of rows during part inserts and merges to improve the compressibility of the new part", 0) \ + M(Bool, use_adaptive_write_buffer_for_dynamic_subcolumns, true, "Allow to use adaptive writer buffers during writing dynamic subcolumns to reduce memory usage", 0) \ + M(UInt64, adaptive_write_buffer_initial_size, 16 * 1024, "Initial size of an adaptive write buffer", 0) \ \ /* Part removal settings. */ \ M(UInt64, simultaneous_parts_removal_limit, 0, "Maximum number of parts to remove during one CleanupThread iteration (0 means unlimited).", 0) \ From 68a8b5a3a1e70469ff47f81669a5b25d1d40fb96 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 4 Sep 2024 18:22:20 +0000 Subject: [PATCH 163/289] Better --- src/Disks/DiskLocal.cpp | 2 -- .../ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 03b8140ce5c..12a5b615234 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -23,8 +23,6 @@ #include #include #include -#include -#include #include diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index f85b5f45b37..fa48825e1a6 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -289,7 +289,7 @@ std::unique_ptr AzureObjectStorage::writeObject( /// NO return std::make_unique( client.get(), object.remote_path, - buf_size, + write_settings.use_adaptive_write_buffer ? write_settings.adaptive_write_buffer_initial_size : buf_size, patchSettings(write_settings), settings.get(), std::move(scheduler)); From 0dad8b088a0452831c89b88b470b8df963fd741a Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 4 Sep 2024 18:35:10 +0000 Subject: [PATCH 164/289] Fix typo --- src/Compression/CompressedWriteBuffer.h | 2 +- src/IO/WriteBufferFromFileDescriptor.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Compression/CompressedWriteBuffer.h b/src/Compression/CompressedWriteBuffer.h index a3aae6b0c61..41596703bfe 100644 --- a/src/Compression/CompressedWriteBuffer.h +++ b/src/Compression/CompressedWriteBuffer.h @@ -54,7 +54,7 @@ private: /// If true, the size of internal buffer will be exponentially increased up to /// adaptive_buffer_max_size after each nextImpl call. It can be used to avoid - /// large buffer allocation when actual size of writen data is small. + /// large buffer allocation when actual size of written data is small. bool use_adaptive_buffer_size; size_t adaptive_buffer_max_size; diff --git a/src/IO/WriteBufferFromFileDescriptor.h b/src/IO/WriteBufferFromFileDescriptor.h index 1008c7cd8d9..e893ecd80fb 100644 --- a/src/IO/WriteBufferFromFileDescriptor.h +++ b/src/IO/WriteBufferFromFileDescriptor.h @@ -60,7 +60,7 @@ protected: /// If true, the size of internal buffer will be exponentially increased up to /// adaptive_buffer_max_size after each nextImpl call. It can be used to avoid - /// large buffer allocation when actual size of writen data is small. + /// large buffer allocation when actual size of written data is small. bool use_adaptive_buffer_size; size_t adaptive_max_buffer_size; From b7b88737ad996b43e27c22e697b9a947fff2a3c0 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 4 Sep 2024 18:37:33 +0000 Subject: [PATCH 165/289] Treat dynamic/object structure streams as dynamic --- src/DataTypes/Serializations/ISerialization.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/DataTypes/Serializations/ISerialization.cpp b/src/DataTypes/Serializations/ISerialization.cpp index 81efa921c9e..dcf637c7d2b 100644 --- a/src/DataTypes/Serializations/ISerialization.cpp +++ b/src/DataTypes/Serializations/ISerialization.cpp @@ -427,7 +427,8 @@ bool ISerialization::isDynamicSubcolumn(const DB::ISerialization::SubstreamPath for (size_t i = 0; i != prefix_len; ++i) { - if (path[i].type == SubstreamType::DynamicData || path[i].type == SubstreamType::ObjectData) + if (path[i].type == SubstreamType::DynamicData || path[i].type == SubstreamType::DynamicStructure + || path[i].type == SubstreamType::ObjectData || path[i].type == SubstreamType::ObjectStructure) return true; } From 6eef6a7759c8cb1c3da7c113f18572f248a30210 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 4 Sep 2024 21:43:36 +0200 Subject: [PATCH 166/289] fix --- .../configs/settings.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_replicated_database_alter_modify_order_by/configs/settings.xml b/tests/integration/test_replicated_database_alter_modify_order_by/configs/settings.xml index 16caee9ba20..a5e45ead44e 100644 --- a/tests/integration/test_replicated_database_alter_modify_order_by/configs/settings.xml +++ b/tests/integration/test_replicated_database_alter_modify_order_by/configs/settings.xml @@ -1,6 +1,8 @@ + 3 + 3 From b4ef10ad1c075b5c16cafb911f67d4623cd40ceb Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 5 Sep 2024 08:52:22 +0000 Subject: [PATCH 167/289] Make better --- .../IO/WriteBufferFromAzureBlobStorage.cpp | 30 +++++-------------- .../IO/WriteBufferFromAzureBlobStorage.h | 1 - src/IO/WriteBufferFromS3.cpp | 21 ++----------- src/IO/WriteBufferFromS3.h | 1 - 4 files changed, 10 insertions(+), 43 deletions(-) diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index dcd625f7ee0..cbcfe0bdb97 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -59,7 +59,7 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( const WriteSettings & write_settings_, std::shared_ptr settings_, ThreadPoolCallbackRunnerUnsafe schedule_) - : WriteBufferFromFileBase(buf_size_, nullptr, 0) + : WriteBufferFromFileBase(std::min(buf_size_, static_cast(DBMS_DEFAULT_BUFFER_SIZE)), nullptr, 0) , log(getLogger("WriteBufferFromAzureBlobStorage")) , buffer_allocation_policy(createBufferAllocationPolicy(*settings_)) , max_single_part_upload_size(settings_->max_single_part_upload_size) @@ -248,30 +248,14 @@ void WriteBufferFromAzureBlobStorage::allocateBuffer() buffer_allocation_policy->nextBuffer(); chassert(0 == hidden_size); - auto size = buffer_allocation_policy->getBufferSize(); - - if (buffer_allocation_policy->getBufferNumber() == 1) - { - allocateFirstBuffer(); - return; - } - - memory = Memory(size); - WriteBuffer::set(memory.data(), memory.size()); -} - -void WriteBufferFromAzureBlobStorage::allocateFirstBuffer() -{ /// First buffer was already allocated in BufferWithOwnMemory constructor with buffer size provided in constructor. /// It will be reallocated in subsequent nextImpl calls up to the desired buffer size from buffer_allocation_policy. - /// But it may happen that buffer size provided in constructor is larger then desired buffer size from buffer_allocation_policy. - /// Resize memory in this case to the desired size. - const auto max_first_buffer = buffer_allocation_policy->getBufferSize(); - if (memory.size() > max_first_buffer) - { - memory.resize(max_first_buffer); - WriteBuffer::set(memory.data(), memory.size()); - } + if (buffer_allocation_policy->getBufferNumber() == 1) + return; + + auto size = buffer_allocation_policy->getBufferSize(); + memory = Memory(size); + WriteBuffer::set(memory.data(), memory.size()); } void WriteBufferFromAzureBlobStorage::detachBuffer() diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h index c2d65928cfa..3ee497c4e44 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.h @@ -53,7 +53,6 @@ private: void detachBuffer(); void reallocateFirstBuffer(); void allocateBuffer(); - void allocateFirstBuffer(); void hidePartialData(); void setFakeBufferWhenPreFinalized(); diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index d88e393e4e1..424708cf2c8 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -96,7 +96,7 @@ WriteBufferFromS3::WriteBufferFromS3( std::optional> object_metadata_, ThreadPoolCallbackRunnerUnsafe schedule_, const WriteSettings & write_settings_) - : WriteBufferFromFileBase(buf_size_, nullptr, 0) + : WriteBufferFromFileBase(std::min(buf_size_, static_cast(DBMS_DEFAULT_BUFFER_SIZE)), nullptr, 0) , bucket(bucket_) , key(key_) , request_settings(request_settings_) @@ -352,30 +352,15 @@ void WriteBufferFromS3::allocateBuffer() buffer_allocation_policy->nextBuffer(); chassert(0 == hidden_size); + /// First buffer was already allocated in BufferWithOwnMemory constructor with provided in constructor buffer size. + /// It will be reallocated in subsequent nextImpl calls up to the desired buffer size from buffer_allocation_policy. if (buffer_allocation_policy->getBufferNumber() == 1) - { - allocateFirstBuffer(); return; - } memory = Memory(buffer_allocation_policy->getBufferSize()); WriteBuffer::set(memory.data(), memory.size()); } -void WriteBufferFromS3::allocateFirstBuffer() -{ - /// First buffer was already allocated in BufferWithOwnMemory constructor with provided in constructor buffer size. - /// It will be reallocated in subsequent nextImpl calls up to the desired buffer size from buffer_allocation_policy. - /// But it may happen that buffer size provided in constructor is larger then desired buffer size from buffer_allocation_policy. - /// Resize memory in this case to the desired size. - const auto max_first_buffer = buffer_allocation_policy->getBufferSize(); - if (memory.size() > max_first_buffer) - { - memory.resize(max_first_buffer); - WriteBuffer::set(memory.data(), memory.size()); - } -} - void WriteBufferFromS3::setFakeBufferWhenPreFinalized() { WriteBuffer::set(fake_buffer_when_prefinalized, sizeof(fake_buffer_when_prefinalized)); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index b026da607c5..604f036fcb8 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -64,7 +64,6 @@ private: void reallocateFirstBuffer(); void detachBuffer(); void allocateBuffer(); - void allocateFirstBuffer(); void setFakeBufferWhenPreFinalized(); S3::UploadPartRequest getUploadRequest(size_t part_number, PartData & data); From ef177d2c9fff4e3594f0b11e3e72de3cb5d09055 Mon Sep 17 00:00:00 2001 From: shiyer7474 Date: Thu, 5 Sep 2024 11:37:57 +0000 Subject: [PATCH 168/289] Proper cleanup & restart for subsequent iterations --- .../test_keeper_s3_snapshot/test.py | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_keeper_s3_snapshot/test.py b/tests/integration/test_keeper_s3_snapshot/test.py index f47a26a77ad..bcb9c7cbd13 100644 --- a/tests/integration/test_keeper_s3_snapshot/test.py +++ b/tests/integration/test_keeper_s3_snapshot/test.py @@ -2,7 +2,9 @@ import pytest from helpers.cluster import ClickHouseCluster from time import sleep from retry import retry +from multiprocessing.dummy import Pool import helpers.keeper_utils as keeper_utils +from minio.deleteobjects import DeleteObject from kazoo.client import KazooClient @@ -76,7 +78,18 @@ def wait_node(node): raise Exception("Can't wait node", node.name, "to become ready") +def delete_keeper_snapshots_logs(nodex): + nodex.exec_in_container( + [ + "bash", + "-c", + "rm -rf /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots" + ] + ) + + def test_s3_upload(started_cluster): + node1_zk = get_fake_zk(node1.name) # we defined in configs snapshot_distance as 50 @@ -90,10 +103,17 @@ def test_s3_upload(started_cluster): for obj in list(cluster.minio_client.list_objects("snapshots")) ] + def delete_s3_snapshots(): + snapshots = cluster.minio_client.list_objects("snapshots") + for s in snapshots: + cluster.minio_client.remove_object( + "snapshots", + s.object_name) + # Keeper sends snapshots asynchornously, hence we need to retry. @retry(AssertionError, tries=10, delay=2) def _check_snapshots(): - assert set(get_saved_snapshots()) >= set( + assert set(get_saved_snapshots()) == set( [ "snapshot_50.bin.zstd", "snapshot_100.bin.zstd", @@ -126,5 +146,25 @@ def test_s3_upload(started_cluster): ) destroy_zk_client(node2_zk) - node1.start_clickhouse() # for next iteration + node2.stop_clickhouse() + delete_keeper_snapshots_logs(node2) + node3.stop_clickhouse() + delete_keeper_snapshots_logs(node3) + delete_keeper_snapshots_logs(node1) + p = Pool(3) + waiters = [] + def start_clickhouse(node): + node.start_clickhouse() + + waiters.append(p.apply_async(start_clickhouse, args=(node1,))) + waiters.append(p.apply_async(start_clickhouse, args=(node2,))) + waiters.append(p.apply_async(start_clickhouse, args=(node3,))) + + delete_s3_snapshots() # for next iteration + + for waiter in waiters: + waiter.wait() + keeper_utils.wait_until_connected(cluster, node1) + keeper_utils.wait_until_connected(cluster, node2) + keeper_utils.wait_until_connected(cluster, node3) From a991a58766bb7b8480d9dff908b025186828bdd3 Mon Sep 17 00:00:00 2001 From: Michael Stetsyuk Date: Wed, 4 Sep 2024 12:11:58 +0000 Subject: [PATCH 169/289] rm test_database_creation_idempotency --- .../test_replicated_database/test.py | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 58e99d4c408..60a6e099b22 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -1,8 +1,5 @@ -import concurrent.futures import os -import random import shutil -import string import time import re import pytest @@ -1552,35 +1549,3 @@ def test_all_groups_cluster(started_cluster): assert "bad_settings_node\ndummy_node\n" == bad_settings_node.query( "select host_name from system.clusters where name='all_groups.db_cluster' order by host_name" ) - - -def test_database_creation_idempotency(started_cluster): - def randomize_database_name(database_name, random_suffix_length=20): - letters = string.ascii_letters + string.digits - return f"{database_name}{''.join(random.choice(letters) for _ in range(random_suffix_length))}" - - databases = [randomize_database_name("rdb") for _ in range(100)] - - def create_database(name): - main_node.query( - f""" - CREATE DATABASE {name} - ENGINE = Replicated('/test/test_database_creation_idempotency/{name}', 'shard', 'replica') - """ - ) - - with concurrent.futures.ThreadPoolExecutor() as executor: - for name in databases: - executor.submit(create_database, name) - - main_node.restart_clickhouse(kill=True) - - with concurrent.futures.ThreadPoolExecutor() as executor: - futures = [executor.submit(create_database, name) for name in databases] - concurrent.futures.wait(futures) - - assert int( - main_node.query( - f"SELECT count() FROM system.databases WHERE name IN {databases}" - ).strip() - ) == len(databases) From 57f80473cc74a826bd11f96f9741f762ce639f82 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Thu, 5 Sep 2024 13:42:09 +0100 Subject: [PATCH 170/289] Docs: Add info on lightweight_deletes_sync --- docs/en/sql-reference/statements/delete.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/delete.md b/docs/en/sql-reference/statements/delete.md index 8ddb5840f2e..ded0d7dacac 100644 --- a/docs/en/sql-reference/statements/delete.md +++ b/docs/en/sql-reference/statements/delete.md @@ -24,9 +24,15 @@ DELETE FROM hits WHERE Title LIKE '%hello%'; ## Lightweight `DELETE` does not delete data immediately -Lightweight `DELETE` is implemented as a [mutation](/en/sql-reference/statements/alter#mutations), which is executed asynchronously in the background by default. The statement is going to return almost immediately, but the data can still be visible to queries until the mutation is finished. +Lightweight `DELETE` is implemented as a [mutation](/en/sql-reference/statements/alter#mutations) that marks rows as deleted but does not immediately physically delete them. -The mutation marks rows as deleted, and at that point, they will no longer show up in query results. It does not physically delete the data, this will happen during the next merge. As a result, it is possible that for an unspecified period, data is not actually deleted from storage and is only marked as deleted. +By default, `DELETE` statements wait until marking the rows as deleted is completed before returning. This can take a long time if the amount of data is large. Alternatively, you can run it asynchronously in the background using the setting [`lightweight_deletes_sync`](/en/operations/settings/settings#lightweight_deletes_sync). If disabled, the `DELETE` statement is going to return immediately, but the data can still be visible to queries until the background mutation is finished. + +:::note +Before version 24.4, lightweight deletes were asynchronous by default. +::: + +The mutation does not physically delete the rows that have been marked as deleted, this will only happen during the next merge. As a result, it is possible that for an unspecified period, data is not actually deleted from storage and is only marked as deleted. If you need to guarantee that your data is deleted from storage in a predictable time, consider using the table setting [`min_age_to_force_merge_seconds`](https://clickhouse.com/docs/en/operations/settings/merge-tree-settings#min_age_to_force_merge_seconds). Or you can use the [ALTER TABLE ... DELETE](/en/sql-reference/statements/alter/delete) command. Note that deleting data using `ALTER TABLE ... DELETE` may consume significant resources as it recreates all affected parts. From df3c5c97914a62b3ab8dcdfd9f9980bc62094c47 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 5 Sep 2024 15:51:03 +0200 Subject: [PATCH 171/289] Update 00609_mv_index_in_in.sql --- tests/queries/0_stateless/00609_mv_index_in_in.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00609_mv_index_in_in.sql b/tests/queries/0_stateless/00609_mv_index_in_in.sql index 89fc85651ad..848938780c2 100644 --- a/tests/queries/0_stateless/00609_mv_index_in_in.sql +++ b/tests/queries/0_stateless/00609_mv_index_in_in.sql @@ -1,4 +1,4 @@ --- Tags: no-ordinary-database +-- Tags: no-ordinary-database, no-parallel DROP TABLE IF EXISTS test_00609; DROP TABLE IF EXISTS test_mv_00609; From 4d87e349a04173cff7dd0814a6425a1865cf74a8 Mon Sep 17 00:00:00 2001 From: shiyer7474 Date: Thu, 5 Sep 2024 13:57:46 +0000 Subject: [PATCH 172/289] Python formatting --- tests/integration/test_keeper_s3_snapshot/test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_keeper_s3_snapshot/test.py b/tests/integration/test_keeper_s3_snapshot/test.py index bcb9c7cbd13..b6c25305aef 100644 --- a/tests/integration/test_keeper_s3_snapshot/test.py +++ b/tests/integration/test_keeper_s3_snapshot/test.py @@ -83,7 +83,7 @@ def delete_keeper_snapshots_logs(nodex): [ "bash", "-c", - "rm -rf /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots" + "rm -rf /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots", ] ) @@ -106,9 +106,7 @@ def test_s3_upload(started_cluster): def delete_s3_snapshots(): snapshots = cluster.minio_client.list_objects("snapshots") for s in snapshots: - cluster.minio_client.remove_object( - "snapshots", - s.object_name) + cluster.minio_client.remove_object("snapshots", s.object_name) # Keeper sends snapshots asynchornously, hence we need to retry. @retry(AssertionError, tries=10, delay=2) @@ -153,6 +151,7 @@ def test_s3_upload(started_cluster): delete_keeper_snapshots_logs(node1) p = Pool(3) waiters = [] + def start_clickhouse(node): node.start_clickhouse() From 18a6b970ebd512568f69203c169c69928e89e15b Mon Sep 17 00:00:00 2001 From: Pablo Marcos Date: Thu, 5 Sep 2024 13:58:21 +0000 Subject: [PATCH 173/289] Improve logical error trace This will provide meaningful information whenever the issue happens again. --- src/Client/ConnectionPoolWithFailover.h | 2 +- src/Common/PoolWithFailoverBase.h | 8 ++++++++ src/Storages/Distributed/DistributedAsyncInsertBatch.cpp | 9 ++------- .../Distributed/DistributedAsyncInsertDirectoryQueue.cpp | 4 +--- src/Storages/Distributed/DistributedSink.cpp | 4 +--- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index a2dc188eb7d..308644ebbdb 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -42,7 +42,7 @@ public: size_t max_error_cap = DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT); using Entry = IConnectionPool::Entry; - using PoolWithFailoverBase::isTryResultInvalid; + using PoolWithFailoverBase::checkTryResultIsValid; /** Allocates connection to work. */ Entry get(const ConnectionTimeouts & timeouts) override; diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index c44ab7df53a..53a746c316e 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -122,6 +122,14 @@ public: return result.entry.isNull() || !result.is_usable || (skip_read_only_replicas && result.is_readonly); } + void checkTryResultIsValid(const TryResult & result, bool skip_read_only_replicas) const + { + if (isTryResultInvalid(result, skip_read_only_replicas)) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, + "Got an invalid connection result: entry.isNull {}, is_usable {}, is_up_to_date {}, delay {}, is_readonly {}, skip_read_only_replicas {}", + result.entry.isNull(), result.is_usable, result.is_up_to_date, result.delay, result.is_readonly, skip_read_only_replicas); + } + size_t getPoolSize() const { return nested_pools.size(); } protected: diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp index 2cf69b9f6b7..2db2bdf3981 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp @@ -28,7 +28,6 @@ namespace ErrorCodes extern const int TOO_MANY_PARTITIONS; extern const int DISTRIBUTED_TOO_MANY_PENDING_BYTES; extern const int ARGUMENT_OUT_OF_BOUND; - extern const int LOGICAL_ERROR; } /// Can the batch be split and send files from batch one-by-one instead? @@ -244,9 +243,7 @@ void DistributedAsyncInsertBatch::sendBatch(const SettingsChanges & settings_cha auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName()); auto result = results.front(); - if (parent.pool->isTryResultInvalid(result, insert_settings.distributed_insert_skip_read_only_replicas)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Got an invalid connection result"); - + parent.pool->checkTryResultIsValid(result, insert_settings.distributed_insert_skip_read_only_replicas); connection = std::move(result.entry); compression_expected = connection->getCompression() == Protocol::Compression::Enable; @@ -306,9 +303,7 @@ void DistributedAsyncInsertBatch::sendSeparateFiles(const SettingsChanges & sett auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName()); auto result = results.front(); - if (parent.pool->isTryResultInvalid(result, insert_settings.distributed_insert_skip_read_only_replicas)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Got an invalid connection result"); - + parent.pool->checkTryResultIsValid(result, insert_settings.distributed_insert_skip_read_only_replicas); auto connection = std::move(result.entry); bool compression_expected = connection->getCompression() == Protocol::Compression::Enable; diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp index 7616b384860..2400de4c07c 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -416,9 +416,7 @@ void DistributedAsyncInsertDirectoryQueue::processFile(std::string & file_path, auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName()); auto result = results.front(); - if (pool->isTryResultInvalid(result, insert_settings.distributed_insert_skip_read_only_replicas)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Got an invalid connection result"); - + pool->checkTryResultIsValid(result, insert_settings.distributed_insert_skip_read_only_replicas); auto connection = std::move(result.entry); LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)", diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index e3e73e42096..daccbb78f2c 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -378,9 +378,7 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si /// (anyway fallback_to_stale_replicas_for_distributed_queries=true by default) auto results = shard_info.pool->getManyCheckedForInsert(timeouts, settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName()); auto result = results.front(); - if (shard_info.pool->isTryResultInvalid(result, settings.distributed_insert_skip_read_only_replicas)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Got an invalid connection result"); - + shard_info.pool->checkTryResultIsValid(result, settings.distributed_insert_skip_read_only_replicas); job.connection_entry = std::move(result.entry); } else From 6621365526c4fc5072c254522a22088192f785b9 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 5 Sep 2024 16:40:44 +0200 Subject: [PATCH 174/289] Update 01287_max_execution_speed.sql --- tests/queries/0_stateless/01287_max_execution_speed.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01287_max_execution_speed.sql b/tests/queries/0_stateless/01287_max_execution_speed.sql index 35bc4e02d38..0d132999481 100644 --- a/tests/queries/0_stateless/01287_max_execution_speed.sql +++ b/tests/queries/0_stateless/01287_max_execution_speed.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-debug, no-tsan, no-msan, no-asan SET min_execution_speed = 100000000000, timeout_before_checking_execution_speed = 0; SELECT count() FROM system.numbers; -- { serverError TOO_SLOW } From 0996ed5246b2b48aab375ccc870bc529f969cdec Mon Sep 17 00:00:00 2001 From: Pablo Marcos Date: Thu, 5 Sep 2024 14:53:14 +0000 Subject: [PATCH 175/289] Create a deep copy of Settings to ensure they don't change --- src/Storages/Distributed/DistributedSink.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index daccbb78f2c..39f75fd7d56 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -347,7 +347,7 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si } const Block & shard_block = (num_shards > 1) ? job.current_shard_block : current_block; - const Settings & settings = context->getSettingsRef(); + const Settings settings = context->getSettingsCopy(); size_t rows = shard_block.rows(); From 5e0673d207b409b8a93d56975e1d05ddb378f6be Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Thu, 5 Sep 2024 16:07:33 +0100 Subject: [PATCH 176/289] Remove version note --- docs/en/sql-reference/statements/delete.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/en/sql-reference/statements/delete.md b/docs/en/sql-reference/statements/delete.md index ded0d7dacac..5733efbf8f2 100644 --- a/docs/en/sql-reference/statements/delete.md +++ b/docs/en/sql-reference/statements/delete.md @@ -28,10 +28,6 @@ Lightweight `DELETE` is implemented as a [mutation](/en/sql-reference/statements By default, `DELETE` statements wait until marking the rows as deleted is completed before returning. This can take a long time if the amount of data is large. Alternatively, you can run it asynchronously in the background using the setting [`lightweight_deletes_sync`](/en/operations/settings/settings#lightweight_deletes_sync). If disabled, the `DELETE` statement is going to return immediately, but the data can still be visible to queries until the background mutation is finished. -:::note -Before version 24.4, lightweight deletes were asynchronous by default. -::: - The mutation does not physically delete the rows that have been marked as deleted, this will only happen during the next merge. As a result, it is possible that for an unspecified period, data is not actually deleted from storage and is only marked as deleted. If you need to guarantee that your data is deleted from storage in a predictable time, consider using the table setting [`min_age_to_force_merge_seconds`](https://clickhouse.com/docs/en/operations/settings/merge-tree-settings#min_age_to_force_merge_seconds). Or you can use the [ALTER TABLE ... DELETE](/en/sql-reference/statements/alter/delete) command. Note that deleting data using `ALTER TABLE ... DELETE` may consume significant resources as it recreates all affected parts. From 364ad4315b1b5dff72a21f42e8b1a7a912373dbd Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 5 Sep 2024 18:20:46 +0200 Subject: [PATCH 177/289] Add checks agains segfault in DeltaLakeMetadata --- .../DataLakes/DeltaLakeMetadata.cpp | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp index 667a925d11e..f04e868ee5a 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadata.cpp @@ -207,18 +207,28 @@ struct DeltaLakeMetadataImpl Poco::Dynamic::Var json = parser.parse(json_str); Poco::JSON::Object::Ptr object = json.extract(); + if (!object) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to parse metadata file"); + +#ifdef ABORT_ON_LOGICAL_ERROR std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM object->stringify(oss); LOG_TEST(log, "Metadata: {}", oss.str()); +#endif if (object->has("metaData")) { const auto metadata_object = object->get("metaData").extract(); + if (!metadata_object) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to extract `metaData` field"); + const auto schema_object = metadata_object->getValue("schemaString"); Poco::JSON::Parser p; Poco::Dynamic::Var fields_json = parser.parse(schema_object); const Poco::JSON::Object::Ptr & fields_object = fields_json.extract(); + if (!fields_object) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to extract `fields` field"); auto current_schema = parseMetadata(fields_object); if (file_schema.empty()) @@ -237,6 +247,9 @@ struct DeltaLakeMetadataImpl if (object->has("add")) { auto add_object = object->get("add").extract(); + if (!add_object) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to extract `add` field"); + auto path = add_object->getValue("path"); result.insert(fs::path(configuration->getPath()) / path); @@ -247,6 +260,9 @@ struct DeltaLakeMetadataImpl if (add_object->has("partitionValues")) { auto partition_values = add_object->get("partitionValues").extract(); + if (!partition_values) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to extract `partitionValues` field"); + if (partition_values->size()) { auto & current_partition_columns = file_partition_columns[filename]; @@ -274,7 +290,11 @@ struct DeltaLakeMetadataImpl } else if (object->has("remove")) { - auto path = object->get("remove").extract()->getValue("path"); + auto remove_object = object->get("remove").extract(); + if (!remove_object) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to extract `remove` field"); + + auto path = remove_object->getValue("path"); result.erase(fs::path(configuration->getPath()) / path); } } From c6f0e29ecad8385954fc26f5578c6c666ff6e07d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 5 Sep 2024 18:36:13 +0200 Subject: [PATCH 178/289] Update StackTrace.cpp --- src/Common/StackTrace.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index bd01b639913..dc5434677ee 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -70,7 +70,7 @@ std::string SigsegvErrorString(const siginfo_t & info, [[maybe_unused]] const uc #if defined(__x86_64__) && !defined(OS_FREEBSD) && !defined(OS_DARWIN) && !defined(__arm__) && !defined(__powerpc__) (context.uc_mcontext.gregs[REG_ERR] & 0x02) ? "write" : "read"; #else - ""; + ""; #endif std::string_view message; From cedddf6fa48c8f84088cb3f3c98ec89fe9d7a849 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 26 Aug 2024 19:10:49 +0200 Subject: [PATCH 179/289] PoC --- .../integration/helper_container/Dockerfile | 4 +- tests/integration/helpers/network.py | 135 ++++++++++++++++-- 2 files changed, 129 insertions(+), 10 deletions(-) diff --git a/docker/test/integration/helper_container/Dockerfile b/docker/test/integration/helper_container/Dockerfile index 49a3d3cd84b..1084d087e53 100644 --- a/docker/test/integration/helper_container/Dockerfile +++ b/docker/test/integration/helper_container/Dockerfile @@ -3,6 +3,8 @@ FROM alpine:3.18 RUN apk add --no-cache -U iproute2 \ - && for bin in iptables iptables-restore iptables-save; \ + && for bin in \ + iptables iptables-restore iptables-save \ + ip6tables ip6tables-restore ip6tables-save; \ do ln -sf xtables-nft-multi "/sbin/$bin"; \ done diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index e6e79dc7947..5219ac22f71 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -3,6 +3,7 @@ import subprocess import time import logging import docker +import ipaddress class PartitionManager: @@ -26,25 +27,76 @@ class PartitionManager: self._check_instance(instance) self._add_rule( - {"source": instance.ip_address, "destination_port": 2181, "action": action} + { + "source": instance.ipv4_address, + "destination_port": 2181, + "action": action, + } ) self._add_rule( - {"destination": instance.ip_address, "source_port": 2181, "action": action} + { + "destination": instance.ipv4_address, + "source_port": 2181, + "action": action, + } ) + if instance.ipv6_address: + self._add_rule( + { + "source": instance.ipv6_address, + "destination_port": 2181, + "action": action, + } + ) + self._add_rule( + { + "destination": instance.ipv6_address, + "source_port": 2181, + "action": action, + } + ) + def dump_rules(self): - return _NetworkManager.get().dump_rules() + v4 = _NetworkManager.get().dump_rules() + v6 = _NetworkManager.get().dump_v6_rules() + + return v4 + v6 def restore_instance_zk_connections(self, instance, action="DROP"): self._check_instance(instance) self._delete_rule( - {"source": instance.ip_address, "destination_port": 2181, "action": action} + { + "source": instance.ipv4_address, + "destination_port": 2181, + "action": action, + } ) self._delete_rule( - {"destination": instance.ip_address, "source_port": 2181, "action": action} + { + "destination": instance.ipv4_address, + "source_port": 2181, + "action": action, + } ) + if instance.ipv6_address: + self._delete_rule( + { + "source": instance.ipv6_address, + "destination_port": 2181, + "action": action, + } + ) + self._delete_rule( + { + "destination": instance.ipv6_address, + "source_port": 2181, + "action": action, + } + ) + def partition_instances(self, left, right, port=None, action="DROP"): self._check_instance(left) self._check_instance(right) @@ -59,16 +111,36 @@ class PartitionManager: rule["destination_port"] = port return rule + def create_rule_v6(src, dst): + rule = { + "source": src.ipv6_address, + "destination": dst.ipv6_address, + "action": action, + } + if port is not None: + rule["destination_port"] = port + return rule + self._add_rule(create_rule(left, right)) self._add_rule(create_rule(right, left)) + if left.ipv6_address and right.ipv6_address: + self._add_rule(create_rule_v6(left, right)) + self._add_rule(create_rule_v6(right, left)) + def add_network_delay(self, instance, delay_ms): self._add_tc_netem_delay(instance, delay_ms) def heal_all(self): while self._iptables_rules: rule = self._iptables_rules.pop() - _NetworkManager.get().delete_iptables_rule(**rule) + + if self._is_ipv6_rule(rule): + _NetworkManager.get().delete_ip6tables_rule(**rule) + else: + _NetworkManager.get().delete_iptables_rule(**rule) + # _NetworkManager.get().delete_iptables_rule(**rule) + # _NetworkManager.get().delete_ip6tables_rule(**rule) while self._netem_delayed_instances: instance = self._netem_delayed_instances.pop() @@ -90,12 +162,29 @@ class PartitionManager: if instance.ip_address is None: raise Exception("Instance + " + instance.name + " is not launched!") + @staticmethod + def _is_ipv6_rule(rule): + is_ipv6 = False + + if "source" in rule: + is_ipv6 = ipaddress.ip_address(rule["source"]).version == 6 + if "destination" in rule: + is_ipv6 = ipaddress.ip_address(rule["source"]).version == 6 + + return is_ipv6 + def _add_rule(self, rule): - _NetworkManager.get().add_iptables_rule(**rule) + if self._is_ipv6_rule(rule): + _NetworkManager.get().add_ip6tables_rule(**rule) + else: + _NetworkManager.get().add_iptables_rule(**rule) self._iptables_rules.append(rule) def _delete_rule(self, rule): - _NetworkManager.get().delete_iptables_rule(**rule) + if self._is_ipv6_rule(rule): + _NetworkManager.get().delete_ip6tables_rule(**rule) + else: + _NetworkManager.get().delete_iptables_rule(**rule) self._iptables_rules.remove(rule) def _add_tc_netem_delay(self, instance, delay_ms): @@ -155,15 +244,29 @@ class _NetworkManager: cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) + def add_ip6tables_rule(self, **kwargs): + cmd = ["ip6tables-legacy", "--wait", "-I", "DOCKER-USER", "1"] + cmd.extend(self._iptables_cmd_suffix(**kwargs)) + self._exec_run(cmd, privileged=True) + def delete_iptables_rule(self, **kwargs): cmd = ["iptables", "--wait", "-D", "DOCKER-USER"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) + def delete_ip6tables_rule(self, **kwargs): + cmd = ["ip6tables-legacy", "--wait", "-D", "DOCKER-USER"] + cmd.extend(self._iptables_cmd_suffix(**kwargs)) + self._exec_run(cmd, privileged=True) + def dump_rules(self): cmd = ["iptables", "-L", "DOCKER-USER"] return self._exec_run(cmd, privileged=True) + def dump_v6_rules(self): + cmd = ["ip6tables-legacy", "-L", "DOCKER-USER"] + return self._exec_run(cmd, privileged=True) + @staticmethod def clean_all_user_iptables_rules(): for i in range(1000): @@ -178,6 +281,20 @@ class _NetworkManager: + " iterations, last error: " + str(res.stderr) ) + break + + for i in range(1000): + iptables_iter = i + # when rules will be empty, it will return error + res = subprocess.run("ip6tables-legacy --wait -D DOCKER-USER 1", shell=True) + + if res.returncode != 0: + logging.info( + "All ip6tables rules cleared, " + + str(iptables_iter) + + " iterations, last error: " + + str(res.stderr) + ) return @staticmethod @@ -244,7 +361,7 @@ class _NetworkManager: def _ensure_container(self): if self._container is None or self._container_expire_time <= time.time(): image_name = "clickhouse/integration-helper:" + os.getenv( - "DOCKER_HELPER_TAG", "latest" + "DOCKER_HELPER_TAG", "" ) for i in range(5): if self._container is not None: From fe42299928dddd9d2e705305524566ded756f6ec Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 26 Aug 2024 19:11:12 +0200 Subject: [PATCH 180/289] Typo --- tests/integration/helpers/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 5219ac22f71..a92843a313b 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -361,7 +361,7 @@ class _NetworkManager: def _ensure_container(self): if self._container is None or self._container_expire_time <= time.time(): image_name = "clickhouse/integration-helper:" + os.getenv( - "DOCKER_HELPER_TAG", "" + "DOCKER_HELPER_TAG", "latest" ) for i in range(5): if self._container is not None: From d49d413c8d0ae7995bf4188ff8ce87ac85587c25 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 26 Aug 2024 19:11:48 +0200 Subject: [PATCH 181/289] No legacy --- tests/integration/helpers/network.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index a92843a313b..e62034a5104 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -245,7 +245,7 @@ class _NetworkManager: self._exec_run(cmd, privileged=True) def add_ip6tables_rule(self, **kwargs): - cmd = ["ip6tables-legacy", "--wait", "-I", "DOCKER-USER", "1"] + cmd = ["ip6tables", "--wait", "-I", "DOCKER-USER", "1"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) @@ -255,7 +255,7 @@ class _NetworkManager: self._exec_run(cmd, privileged=True) def delete_ip6tables_rule(self, **kwargs): - cmd = ["ip6tables-legacy", "--wait", "-D", "DOCKER-USER"] + cmd = ["ip6tables", "--wait", "-D", "DOCKER-USER"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) @@ -264,7 +264,7 @@ class _NetworkManager: return self._exec_run(cmd, privileged=True) def dump_v6_rules(self): - cmd = ["ip6tables-legacy", "-L", "DOCKER-USER"] + cmd = ["ip6tables", "-L", "DOCKER-USER"] return self._exec_run(cmd, privileged=True) @staticmethod @@ -286,7 +286,7 @@ class _NetworkManager: for i in range(1000): iptables_iter = i # when rules will be empty, it will return error - res = subprocess.run("ip6tables-legacy --wait -D DOCKER-USER 1", shell=True) + res = subprocess.run("ip6tables --wait -D DOCKER-USER 1", shell=True) if res.returncode != 0: logging.info( From 5ee5c8224ea715b9a78a9ee79f79419ae36db6f2 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 26 Aug 2024 22:16:05 +0200 Subject: [PATCH 182/289] Fix --- tests/integration/helpers/network.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index e62034a5104..c35ab65eef5 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -164,14 +164,10 @@ class PartitionManager: @staticmethod def _is_ipv6_rule(rule): - is_ipv6 = False - if "source" in rule: - is_ipv6 = ipaddress.ip_address(rule["source"]).version == 6 + return ipaddress.ip_address(rule["source"]).version == 6 if "destination" in rule: - is_ipv6 = ipaddress.ip_address(rule["source"]).version == 6 - - return is_ipv6 + return ipaddress.ip_address(rule["destination"]).version == 6 def _add_rule(self, rule): if self._is_ipv6_rule(rule): From 9be79614a3fd3d97ed1be76dddcd4182e75d3f8b Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Wed, 28 Aug 2024 15:20:56 +0200 Subject: [PATCH 183/289] Fix --- tests/integration/helpers/network.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index c35ab65eef5..d3de4660acb 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -164,11 +164,13 @@ class PartitionManager: @staticmethod def _is_ipv6_rule(rule): - if "source" in rule: + if rule.get("source"): return ipaddress.ip_address(rule["source"]).version == 6 - if "destination" in rule: + if rule.get("destination"): return ipaddress.ip_address(rule["destination"]).version == 6 + return False + def _add_rule(self, rule): if self._is_ipv6_rule(rule): _NetworkManager.get().add_ip6tables_rule(**rule) From 335ca75174329ecabb9d8de59c1cfd4f799842ba Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Sep 2024 00:16:40 +0200 Subject: [PATCH 184/289] Lint --- tests/integration/helpers/network.py | 41 +++++++++------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index d3de4660acb..4b02f99876f 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -139,8 +139,6 @@ class PartitionManager: _NetworkManager.get().delete_ip6tables_rule(**rule) else: _NetworkManager.get().delete_iptables_rule(**rule) - # _NetworkManager.get().delete_iptables_rule(**rule) - # _NetworkManager.get().delete_ip6tables_rule(**rule) while self._netem_delayed_instances: instance = self._netem_delayed_instances.pop() @@ -267,33 +265,20 @@ class _NetworkManager: @staticmethod def clean_all_user_iptables_rules(): - for i in range(1000): - iptables_iter = i - # when rules will be empty, it will return error - res = subprocess.run("iptables --wait -D DOCKER-USER 1", shell=True) + for iptables in ("iptables", "ip6tables"): + for i in range(1000): + iptables_iter = i + # when rules will be empty, it will return error + res = subprocess.run(f"{iptables}--wait -D DOCKER-USER 1", shell=True) - if res.returncode != 0: - logging.info( - "All iptables rules cleared, " - + str(iptables_iter) - + " iterations, last error: " - + str(res.stderr) - ) - break - - for i in range(1000): - iptables_iter = i - # when rules will be empty, it will return error - res = subprocess.run("ip6tables --wait -D DOCKER-USER 1", shell=True) - - if res.returncode != 0: - logging.info( - "All ip6tables rules cleared, " - + str(iptables_iter) - + " iterations, last error: " - + str(res.stderr) - ) - return + if res.returncode != 0: + logging.info( + f"All {iptables} rules cleared, " + + str(iptables_iter) + + " iterations, last error: " + + str(res.stderr) + ) + break @staticmethod def _iptables_cmd_suffix( From d46065360b13c9dfbab11f642d54702610cd550d Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Sep 2024 05:24:03 +0200 Subject: [PATCH 185/289] Fix --- tests/integration/helpers/network.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 4b02f99876f..8d96add9501 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -28,14 +28,14 @@ class PartitionManager: self._add_rule( { - "source": instance.ipv4_address, + "source": instance.ip_address, "destination_port": 2181, "action": action, } ) self._add_rule( { - "destination": instance.ipv4_address, + "destination": instance.ip_address, "source_port": 2181, "action": action, } @@ -68,14 +68,14 @@ class PartitionManager: self._delete_rule( { - "source": instance.ipv4_address, + "source": instance.ip_address, "destination_port": 2181, "action": action, } ) self._delete_rule( { - "destination": instance.ipv4_address, + "destination": instance.ip_address, "source_port": 2181, "action": action, } From 58b84e9e3d75846428b6839aa03207a4ce8f5cbd Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Sep 2024 17:29:44 +0200 Subject: [PATCH 186/289] Update network.py --- tests/integration/helpers/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 8d96add9501..c9e2df0b2a0 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -269,7 +269,7 @@ class _NetworkManager: for i in range(1000): iptables_iter = i # when rules will be empty, it will return error - res = subprocess.run(f"{iptables}--wait -D DOCKER-USER 1", shell=True) + res = subprocess.run(f"{iptables} --wait -D DOCKER-USER 1", shell=True) if res.returncode != 0: logging.info( From 1f5eb2cd2c1de8639d017f769d04975d21cc590a Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Sep 2024 18:36:32 +0200 Subject: [PATCH 187/289] Set IPv6 instance to the node object --- tests/integration/helpers/cluster.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 53f4f1e1f26..821bb887435 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -2112,6 +2112,7 @@ class ClickHouseCluster: self.base_cmd + ["up", "--force-recreate", "--no-deps", "-d", node.name] ) node.ip_address = self.get_instance_ip(node.name) + node.ipv6_address = self.get_instance_global_ipv6(node.name) node.client = Client(node.ip_address, command=self.client_bin_path) logging.info("Restart node with ip change") @@ -3182,6 +3183,7 @@ class ClickHouseCluster: for instance in self.instances.values(): instance.docker_client = self.docker_client instance.ip_address = self.get_instance_ip(instance.name) + instance.ipv6_address = self.get_instance_global_ipv6(instance.name) logging.debug( f"Waiting for ClickHouse start in {instance.name}, ip: {instance.ip_address}..." From 26add45c70cef4dd2c0fc18891c3db9bf90bb06e Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Sep 2024 18:49:04 +0200 Subject: [PATCH 188/289] Poke CI From da3a7d069173b7508584a63190a5e24a9452e5bd Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Thu, 5 Sep 2024 06:23:20 +0200 Subject: [PATCH 189/289] Setup missing ip6tables chain --- tests/integration/helpers/network.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index c9e2df0b2a0..39f413cdf3b 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -235,12 +235,29 @@ class _NetworkManager: cls._instance = cls(**kwargs) return cls._instance + def setup_ip6tables_docker_user_chain(self): + _rules = subprocess.check_output( + f"ip6tables-save", shell=True + ) + if "DOCKER-USER" in _rules.decode("utf-8"): + return + + setup_cmds = [ + ["ip6tables", "--wait", "-N", "DOCKER-USER"], + ["ip6tables", "--wait", "-I", "FORWARD", "-j", "DOCKER-USER"], + ["ip6tables", "--wait", "-A", "DOCKER-USER", "-j", "RETURN"], + ] + for cmd in setup_cmds: + self._exec_run(cmd, privileged=True) + def add_iptables_rule(self, **kwargs): cmd = ["iptables", "--wait", "-I", "DOCKER-USER", "1"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) def add_ip6tables_rule(self, **kwargs): + self.setup_ip6tables_docker_user_chain() + cmd = ["ip6tables", "--wait", "-I", "DOCKER-USER", "1"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) From 798f4b4c3b4667b6cd09760449758b6b7e503e56 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Sep 2024 04:35:04 +0000 Subject: [PATCH 190/289] Automatic style fix --- tests/integration/helpers/network.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 39f413cdf3b..065836396f3 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -236,9 +236,7 @@ class _NetworkManager: return cls._instance def setup_ip6tables_docker_user_chain(self): - _rules = subprocess.check_output( - f"ip6tables-save", shell=True - ) + _rules = subprocess.check_output(f"ip6tables-save", shell=True) if "DOCKER-USER" in _rules.decode("utf-8"): return From a463b2b44c9162a9ee7cb3d830a3ad6a3b5a7ff6 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Thu, 5 Sep 2024 15:59:21 +0200 Subject: [PATCH 191/289] Poke CI From f22876302e2c284aa29f0e5e2733860fe4f9c253 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 5 Sep 2024 23:37:29 +0000 Subject: [PATCH 192/289] Mark Dynamic type as not safe primary key type to avoid issues with Fields --- src/Processors/QueryPlan/PartsSplitter.cpp | 13 ++----- .../queries/0_stateless/00000_test.reference | 37 +++++++++++++++++++ tests/queries/0_stateless/00000_test.sql | 12 ++++++ ...231_dynamic_not_safe_primary_key.reference | 0 .../03231_dynamic_not_safe_primary_key.sql | 10 +++++ 5 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/00000_test.reference create mode 100644 tests/queries/0_stateless/00000_test.sql create mode 100644 tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.reference create mode 100644 tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.sql diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp index 63c10a11913..935af914827 100644 --- a/src/Processors/QueryPlan/PartsSplitter.cpp +++ b/src/Processors/QueryPlan/PartsSplitter.cpp @@ -50,6 +50,9 @@ bool isSafePrimaryDataKeyType(const IDataType & data_type) case TypeIndex::Float64: case TypeIndex::Nullable: case TypeIndex::ObjectDeprecated: + case TypeIndex::Object: + case TypeIndex::Variant: + case TypeIndex::Dynamic: return false; case TypeIndex::Array: { @@ -76,16 +79,6 @@ bool isSafePrimaryDataKeyType(const IDataType & data_type) const auto & data_type_map = static_cast(data_type); return isSafePrimaryDataKeyType(*data_type_map.getKeyType()) && isSafePrimaryDataKeyType(*data_type_map.getValueType()); } - case TypeIndex::Variant: - { - const auto & data_type_variant = static_cast(data_type); - const auto & data_type_variant_elements = data_type_variant.getVariants(); - for (const auto & data_type_variant_element : data_type_variant_elements) - if (!isSafePrimaryDataKeyType(*data_type_variant_element)) - return false; - - return false; - } default: { break; diff --git a/tests/queries/0_stateless/00000_test.reference b/tests/queries/0_stateless/00000_test.reference new file mode 100644 index 00000000000..5374971d501 --- /dev/null +++ b/tests/queries/0_stateless/00000_test.reference @@ -0,0 +1,37 @@ +a +a +a +b +xx +xx +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +--- + 4 + xx + xx +a 2020-01-01 00:00:00.000 +a 2020-01-01 10:10:10.000 +a 2020-01-01 10:10:10.110 +a 1 +a 1 +a 1 +a a +a a +a b +a \N +a \N +a \N +a \N +a \N +z a diff --git a/tests/queries/0_stateless/00000_test.sql b/tests/queries/0_stateless/00000_test.sql new file mode 100644 index 00000000000..d380f6f69de --- /dev/null +++ b/tests/queries/0_stateless/00000_test.sql @@ -0,0 +1,12 @@ +set max_threads = 1, read_in_order_two_level_merge_threshold = 1; + +CREATE OR REPLACE TABLE test( + key String, + val Map(String, String) +) engine = MergeTree +order by key settings min_bytes_for_wide_part = 0; +insert into test VALUES ('', {'x':'xx'}); +insert into test VALUES ('', {}); +SELECT key, arrayJoin(mapValues(val)) as v FROM test ORDER BY key, v; +DROP TABLE test; + diff --git a/tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.reference b/tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.sql b/tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.sql new file mode 100644 index 00000000000..f207581f482 --- /dev/null +++ b/tests/queries/0_stateless/03231_dynamic_not_safe_primary_key.sql @@ -0,0 +1,10 @@ +SET allow_experimental_dynamic_type = 1; +DROP TABLE IF EXISTS t0; +DROP TABLE IF EXISTS t1; +CREATE TABLE t0 (c0 Int) ENGINE = AggregatingMergeTree() ORDER BY (c0); +CREATE TABLE t1 (c0 Array(Dynamic), c1 Int) ENGINE = MergeTree() ORDER BY (c0); +INSERT INTO t1 (c0, c1) VALUES ([18446717433683171873], 13623876564923702671), ([-4], 6111684076076982207); +SELECT 1 FROM t0 FINAL JOIN t1 ON TRUE; +DROP TABLE t0; +DROP TABLE t1; + From ff86b1872f7b0e11367d67dbdaee46a93fa00684 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 5 Sep 2024 23:38:31 +0000 Subject: [PATCH 193/289] Remove extra files --- .../queries/0_stateless/00000_test.reference | 37 ------------------- tests/queries/0_stateless/00000_test.sql | 12 ------ 2 files changed, 49 deletions(-) delete mode 100644 tests/queries/0_stateless/00000_test.reference delete mode 100644 tests/queries/0_stateless/00000_test.sql diff --git a/tests/queries/0_stateless/00000_test.reference b/tests/queries/0_stateless/00000_test.reference deleted file mode 100644 index 5374971d501..00000000000 --- a/tests/queries/0_stateless/00000_test.reference +++ /dev/null @@ -1,37 +0,0 @@ -a -a -a -b -xx -xx -\N -\N -\N -\N -\N -\N -\N -\N -\N -\N -\N -\N ---- - 4 - xx - xx -a 2020-01-01 00:00:00.000 -a 2020-01-01 10:10:10.000 -a 2020-01-01 10:10:10.110 -a 1 -a 1 -a 1 -a a -a a -a b -a \N -a \N -a \N -a \N -a \N -z a diff --git a/tests/queries/0_stateless/00000_test.sql b/tests/queries/0_stateless/00000_test.sql deleted file mode 100644 index d380f6f69de..00000000000 --- a/tests/queries/0_stateless/00000_test.sql +++ /dev/null @@ -1,12 +0,0 @@ -set max_threads = 1, read_in_order_two_level_merge_threshold = 1; - -CREATE OR REPLACE TABLE test( - key String, - val Map(String, String) -) engine = MergeTree -order by key settings min_bytes_for_wide_part = 0; -insert into test VALUES ('', {'x':'xx'}); -insert into test VALUES ('', {}); -SELECT key, arrayJoin(mapValues(val)) as v FROM test ORDER BY key, v; -DROP TABLE test; - From 2ace0c55eb7e51919aa26422e73c1c6d16722430 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 6 Sep 2024 08:35:39 +0000 Subject: [PATCH 194/289] Increase timeout for intergration test tsan job --- tests/ci/ci_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index cd499aee581..30b2ce5f361 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -410,7 +410,9 @@ class CI: num_batches=6, ), JobNames.INTEGRATION_TEST_TSAN: CommonJobConfigs.INTEGRATION_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_TSAN], num_batches=6 + required_builds=[BuildNames.PACKAGE_TSAN], + num_batches=6, + timeout = 9000 # the job timed out with default value (7200) ), JobNames.INTEGRATION_TEST_ARM: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], From b3b8abb4133f8bc80aec84b7647e5cafdc8a7bc3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Sep 2024 09:13:37 +0000 Subject: [PATCH 195/289] Automatic style fix --- tests/ci/ci_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 30b2ce5f361..436fc5b6bb2 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -412,7 +412,7 @@ class CI: JobNames.INTEGRATION_TEST_TSAN: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_TSAN], num_batches=6, - timeout = 9000 # the job timed out with default value (7200) + timeout=9000, # the job timed out with default value (7200) ), JobNames.INTEGRATION_TEST_ARM: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], From e2a7a1f66292366be35f2e1d61d01937befb180d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 6 Sep 2024 12:08:14 +0200 Subject: [PATCH 196/289] Update StackTrace.cpp --- src/Common/StackTrace.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index dc5434677ee..3dce34803b2 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -67,10 +67,18 @@ std::string SigsegvErrorString(const siginfo_t & info, [[maybe_unused]] const uc = info.si_addr == nullptr ? "NULL pointer"s : (shouldShowAddress(info.si_addr) ? fmt::format("{}", info.si_addr) : ""s); const std::string_view access = -#if defined(__x86_64__) && !defined(OS_FREEBSD) && !defined(OS_DARWIN) && !defined(__arm__) && !defined(__powerpc__) - (context.uc_mcontext.gregs[REG_ERR] & 0x02) ? "write" : "read"; -#else +#if defined(__arm__) ""; +#elif defined(__powerpc__) + ""; +#elif defined(OS_DARWIN) + ""; +#elif defined(OS_FREEBSD) + ""; +#elif !defined(__x86_64__) + ""; +#else + (context.uc_mcontext.gregs[REG_ERR] & 0x02) ? "write" : "read"; #endif std::string_view message; From 3974e9060a4a8da4bd1f9e4c1f96f2a25bc36518 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Fri, 6 Sep 2024 18:50:02 +0800 Subject: [PATCH 197/289] Fix docs and some fixups --- .../mergetree-family/mergetree.md | 33 +++++++++---------- src/Storages/Statistics/Statistics.cpp | 2 +- src/Storages/Statistics/StatisticsMinMax.cpp | 3 ++ 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index d63ab54ed1f..74f1295ef96 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -989,44 +989,43 @@ ALTER TABLE tab DROP STATISTICS a; These lightweight statistics aggregate information about distribution of values in columns. Statistics are stored in every part and updated when every insert comes. They can be used for prewhere optimization only if we enable `set allow_statistics_optimize = 1`. -#### Available Types of Column Statistics {#available-types-of-column-statistics} +### Available Types of Column Statistics {#available-types-of-column-statistics} - `MinMax` The minimum and maximum column value which allows to estimate the selectivity of range filters on numeric columns. - Supported data types: (U)Int*, Float*, Decimal(*), Boolean and Date*. - - `TDigest` [TDigest](https://github.com/tdunning/t-digest) sketches which allow to compute approximate percentiles (e.g. the 90th percentile) for numeric columns. - Supported data types: (U)Int*, Float*, Decimal(*), Boolean and Date*. - - `Uniq` [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) sketches which provide an estimation how many distinct values a column contains. - Supported data types: (U)Int*, Float*, Decimal*, Boolean, Date* and (Fixed)String. - - `count_min` [Count-min](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch) sketches which provide an approximate count of the frequency of each value in a column. - Supported data types: (U)Int*, Float*, Decimal*, Boolean, Date* and (Fixed)String. -Note that all statistics types support `LowCardinality` and `Nullable` modifiers to data types. +### Supported Data Types {#supported-data-types} -#### Supported operations of Column Statistics {#supported-operations-of-column-statistics} +| | (U)Int* | Float* | Decimal(*) | Date* | Boolean | Enum* | (Fixed)String | +|-----------|---------|--------|------------|-------|---------|-------|------------------| +| count_min | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| MinMax | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✗ | +| TDigest | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✗ | +| Uniq | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | -| | Equals | Range | -|-----------|---------|-------| -| count_min | ✔ | ✗ | -| MinMax | ✗ | ✔ | -| TDigest | ✗ | ✔ | -| Uniq | ✔ | ✗ | -Please note that operation `Range` represents >, >=, < or <=. +### Supported Operations {#supported-operations} + +| | Equality filters (==) | Range filters (>, >=, <, <=) | +|-----------|-----------------------|------------------------------| +| count_min | ✔ | ✗ | +| MinMax | ✗ | ✔ | +| TDigest | ✗ | ✔ | +| Uniq | ✔ | ✗ | ## Column-level Settings {#column-level-settings} diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp index 5e22c5d121a..795963bd55d 100644 --- a/src/Storages/Statistics/Statistics.cpp +++ b/src/Storages/Statistics/Statistics.cpp @@ -127,7 +127,7 @@ Float64 ColumnStatistics::estimateEqual(const Field & val) const if (stats.contains(StatisticsType::Uniq)) { UInt64 cardinality = stats.at(StatisticsType::Uniq)->estimateCardinality(); - if (cardinality == 0) + if (cardinality == 0 || rows == 0) return 0; return 1.0 / cardinality * rows; /// assume uniform distribution } diff --git a/src/Storages/Statistics/StatisticsMinMax.cpp b/src/Storages/Statistics/StatisticsMinMax.cpp index 6ae05cb0a6b..27072d225de 100644 --- a/src/Storages/Statistics/StatisticsMinMax.cpp +++ b/src/Storages/Statistics/StatisticsMinMax.cpp @@ -51,6 +51,9 @@ void StatisticsMinMax::deserialize(ReadBuffer & buf) Float64 StatisticsMinMax::estimateLess(const Field & val) const { + if (row_count == 0) + return 0; + auto val_as_float = StatisticsUtils::tryConvertToFloat64(val, data_type); if (!val_as_float.has_value()) return 0; From fde9f23829bfd6d0c69af1fa90938b0c69be9afb Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 6 Sep 2024 13:48:20 +0200 Subject: [PATCH 198/289] Disable long tests with sanitizers --- tests/integration/test_backup_restore_new/test.py | 7 +++++++ .../test_backward_compatibility/test_functions.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 4806625f3f0..56e007dcf5d 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -343,6 +343,13 @@ def test_increment_backup_without_changes(): def test_incremental_backup_overflow(): + if ( + instance.is_built_with_thread_sanitizer() + or instance.is_built_with_memory_sanitizer() + or instance.is_built_with_address_sanitizer() + ): + pytest.skip("The test is slow in builds with sanitizer") + backup_name = new_backup_name() incremental_backup_name = new_backup_name() diff --git a/tests/integration/test_backward_compatibility/test_functions.py b/tests/integration/test_backward_compatibility/test_functions.py index 202a741bfb5..acf4bd28c9b 100644 --- a/tests/integration/test_backward_compatibility/test_functions.py +++ b/tests/integration/test_backward_compatibility/test_functions.py @@ -154,6 +154,13 @@ def test_aggregate_states(start_cluster): def test_string_functions(start_cluster): + if ( + upstream.is_built_with_thread_sanitizer() + or upstream.is_built_with_memory_sanitizer() + or upstream.is_built_with_address_sanitizer() + ): + pytest.skip("The test is slow in builds with sanitizer") + functions = backward.query( """ SELECT if(NOT empty(alias_to), alias_to, name) From 238c9b5789cf9eb03dc688211d0e5f9633b46b9c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 6 Sep 2024 13:51:06 +0200 Subject: [PATCH 199/289] Update 01153_attach_mv_uuid.sql --- tests/queries/0_stateless/01153_attach_mv_uuid.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01153_attach_mv_uuid.sql b/tests/queries/0_stateless/01153_attach_mv_uuid.sql index 6b167253d8f..0ef16449096 100644 --- a/tests/queries/0_stateless/01153_attach_mv_uuid.sql +++ b/tests/queries/0_stateless/01153_attach_mv_uuid.sql @@ -15,6 +15,7 @@ SELECT * FROM mv ORDER BY n; DROP TABLE mv SYNC; SET database_replicated_allow_explicit_uuid=3; + SET show_table_uuid_in_table_create_query_if_not_nil=1; CREATE TABLE ".inner_id.e15f3ab5-6cae-4df3-b879-f40deafd82c2" (n Int32, n2 Int64) ENGINE = MergeTree PARTITION BY n % 10 ORDER BY n; ATTACH MATERIALIZED VIEW mv UUID 'e15f3ab5-6cae-4df3-b879-f40deafd82c2' (n Int32, n2 Int64) ENGINE = MergeTree PARTITION BY n % 10 ORDER BY n AS SELECT n, n * n AS n2 FROM src; From 66bd32bd189a55fd7007d4c1fefad4a725fb23cd Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 12:02:29 +0000 Subject: [PATCH 200/289] Bump curl to v8.8.0 --- contrib/curl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/curl b/contrib/curl index de7b3e89218..fd567d4f068 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit de7b3e89218467159a7af72d58cea8425946e97d +Subproject commit fd567d4f06857f4fc8e2f64ea727b1318f76ad33 From 1b6fa8ef53a209b1bdffe988180f49a5373f14f4 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 12:03:08 +0000 Subject: [PATCH 201/289] Bump curl to v8.9.0 --- contrib/curl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/curl b/contrib/curl index fd567d4f068..5040f7e94cd 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit fd567d4f06857f4fc8e2f64ea727b1318f76ad33 +Subproject commit 5040f7e94cd01decbe7ba8fdacbf489182d503dc From b3f4f3459bf1907eb8c6b92f639206c2df9d2175 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 12:03:53 +0000 Subject: [PATCH 202/289] Bump curl to v8.9.1 --- contrib/curl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/curl b/contrib/curl index 5040f7e94cd..83bedbd730d 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit 5040f7e94cd01decbe7ba8fdacbf489182d503dc +Subproject commit 83bedbd730d62b83744cc26fa0433d3f6e2e4cd6 From 06507190d414b2537d3759128fdf40332d0fe1b8 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 6 Sep 2024 14:29:26 +0200 Subject: [PATCH 203/289] Speed up some Kafka tests with multiprocessing --- tests/integration/test_storage_kafka/test.py | 196 +++++++++++-------- 1 file changed, 116 insertions(+), 80 deletions(-) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index bef90e1b9d3..6802a0b7785 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -10,6 +10,8 @@ import string import ast import math +from multiprocessing.dummy import Pool + import avro.schema import avro.io import avro.datafile @@ -4831,27 +4833,13 @@ def test_max_rows_per_message(kafka_cluster, create_query_generator): def test_row_based_formats(kafka_cluster, create_query_generator): admin_client = get_admin_client(kafka_cluster) - for format_name in [ - "TSV", - "TSVWithNamesAndTypes", - "TSKV", - "CSV", - "CSVWithNamesAndTypes", - "CustomSeparatedWithNamesAndTypes", - "Values", - "JSON", - "JSONEachRow", - "JSONCompactEachRow", - "JSONCompactEachRowWithNamesAndTypes", - "JSONObjectEachRow", - "Avro", - "RowBinary", - "RowBinaryWithNamesAndTypes", - "MsgPack", - ]: + p = Pool(10) + + def run_for_format_name(format_name): logging.debug("Checking {format_name}") topic_name = format_name + get_topic_postfix(create_query_generator) + view_name = f"kafka_view_{format_name}" table_name = f"kafka_{format_name}" with kafka_topic(admin_client, topic_name): @@ -4870,12 +4858,12 @@ def test_row_based_formats(kafka_cluster, create_query_generator): instance.query( f""" - DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.{view_name}; DROP TABLE IF EXISTS test.{table_name}; {create_query}; - CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY (key, value) AS + CREATE MATERIALIZED VIEW test.{view_name} ENGINE=MergeTree ORDER BY (key, value) AS SELECT key, value FROM test.{table_name}; INSERT INTO test.{table_name} SELECT number * 10 as key, number * 100 as value FROM numbers({num_rows}); @@ -4886,18 +4874,44 @@ def test_row_based_formats(kafka_cluster, create_query_generator): kafka_cluster, topic_name, message_count, need_decode=False ) - assert len(messages) == message_count + assert ( + len(messages) == message_count + ), f"Invalid message count for {format_name}" instance.query_with_retry( - "SELECT count() FROM test.view", + f"SELECT count() FROM test.{view_name}", check_callback=lambda res: int(res) == num_rows, ) - result = instance.query("SELECT * FROM test.view") + result = instance.query(f"SELECT * FROM test.{view_name}") expected = "" for i in range(num_rows): expected += str(i * 10) + "\t" + str(i * 100) + "\n" - assert result == expected + assert result == expected, f"Invalid result for {format_name}" + + results = [] + for format_name in [ + "TSV", + "TSVWithNamesAndTypes", + "TSKV", + "CSV", + "CSVWithNamesAndTypes", + "CustomSeparatedWithNamesAndTypes", + "Values", + "JSON", + "JSONEachRow", + "JSONCompactEachRow", + "JSONCompactEachRowWithNamesAndTypes", + "JSONObjectEachRow", + "Avro", + "RowBinary", + "RowBinaryWithNamesAndTypes", + "MsgPack", + ]: + results.append(p.apply_async(run_for_format_name, args=(format_name,))) + + for result in results: + result.get() @pytest.mark.parametrize( @@ -4955,16 +4969,12 @@ def test_block_based_formats_2(kafka_cluster, create_query_generator): num_rows = 100 message_count = 9 - for format_name in [ - "JSONColumns", - "Native", - "Arrow", - "Parquet", - "ORC", - "JSONCompactColumns", - ]: + p = Pool(10) + + def run_for_format_name(format_name): topic_name = format_name + get_topic_postfix(create_query_generator) table_name = f"kafka_{format_name}" + view_name = f"kafka_view_{format_name}" logging.debug(f"Checking format {format_name}") with kafka_topic(admin_client, topic_name): create_query = create_query_generator( @@ -4977,12 +4987,12 @@ def test_block_based_formats_2(kafka_cluster, create_query_generator): instance.query( f""" - DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.{view_name}; DROP TABLE IF EXISTS test.{table_name}; {create_query}; - CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY (key, value) AS + CREATE MATERIALIZED VIEW test.{view_name} ENGINE=MergeTree ORDER BY (key, value) AS SELECT key, value FROM test.{table_name}; INSERT INTO test.{table_name} SELECT number * 10 as key, number * 100 as value FROM numbers({num_rows}) settings max_block_size=12, optimize_trivial_insert_select=0; @@ -4991,22 +5001,38 @@ def test_block_based_formats_2(kafka_cluster, create_query_generator): messages = kafka_consume_with_retry( kafka_cluster, topic_name, message_count, need_decode=False ) - assert len(messages) == message_count + assert ( + len(messages) == message_count + ), f"Invalid message count for {format_name}" rows = int( instance.query_with_retry( - "SELECT count() FROM test.view", + f"SELECT count() FROM test.{view_name}", check_callback=lambda res: int(res) == num_rows, ) ) - assert rows == num_rows + assert rows == num_rows, f"Invalid row count for {format_name}" - result = instance.query("SELECT * FROM test.view ORDER by key") + result = instance.query(f"SELECT * FROM test.{view_name} ORDER by key") expected = "" for i in range(num_rows): expected += str(i * 10) + "\t" + str(i * 100) + "\n" - assert result == expected + assert result == expected, f"Invalid result for {format_name}" + + results = [] + for format_name in [ + "JSONColumns", + "Native", + "Arrow", + "Parquet", + "ORC", + "JSONCompactColumns", + ]: + results.append(p.apply_async(run_for_format_name, args=(format_name,))) + + for result in results: + result.get() def test_system_kafka_consumers(kafka_cluster): @@ -5300,6 +5326,54 @@ def test_formats_errors(kafka_cluster): bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) ) + p = Pool(10) + + def run_for_format_name(format_name): + with kafka_topic(admin_client, format_name): + table_name = f"kafka_{format_name}" + view_name = f"kafka_view_{format_name}" + + instance.query( + f""" + DROP TABLE IF EXISTS test.{view_name}; + DROP TABLE IF EXISTS test.{table_name}; + + CREATE TABLE test.{table_name} (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = '{format_name}', + kafka_group_name = '{format_name}', + kafka_format = '{format_name}', + kafka_max_rows_per_message = 5, + format_template_row='template_row.format', + format_regexp='id: (.+?)', + input_format_with_names_use_header=0, + format_schema='key_value_message:Message'; + + CREATE MATERIALIZED VIEW test.{view_name} ENGINE=MergeTree ORDER BY (key, value) AS + SELECT key, value FROM test.{table_name}; + """ + ) + + kafka_produce( + kafka_cluster, + format_name, + ["Broken message\nBroken message\nBroken message\n"], + ) + + num_errors = int( + instance.query_with_retry( + f"SELECT length(exceptions.text) from system.kafka_consumers where database = 'test' and table = '{table_name}'", + check_callback=lambda res: int(res) > 0, + ) + ) + + assert num_errors > 0, f"No errors for {format_name}" + + instance.query(f"DROP TABLE test.{table_name}") + instance.query(f"DROP TABLE test.{view_name}") + + results = [] for format_name in [ "Template", "Regexp", @@ -5342,48 +5416,10 @@ def test_formats_errors(kafka_cluster): "HiveText", "MySQLDump", ]: - with kafka_topic(admin_client, format_name): - table_name = f"kafka_{format_name}" + results.append(p.apply_async(run_for_format_name, args=(format_name,))) - instance.query( - f""" - DROP TABLE IF EXISTS test.view; - DROP TABLE IF EXISTS test.{table_name}; - - CREATE TABLE test.{table_name} (key UInt64, value UInt64) - ENGINE = Kafka - SETTINGS kafka_broker_list = 'kafka1:19092', - kafka_topic_list = '{format_name}', - kafka_group_name = '{format_name}', - kafka_format = '{format_name}', - kafka_max_rows_per_message = 5, - format_template_row='template_row.format', - format_regexp='id: (.+?)', - input_format_with_names_use_header=0, - format_schema='key_value_message:Message'; - - CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY (key, value) AS - SELECT key, value FROM test.{table_name}; - """ - ) - - kafka_produce( - kafka_cluster, - format_name, - ["Broken message\nBroken message\nBroken message\n"], - ) - - num_errors = int( - instance.query_with_retry( - f"SELECT length(exceptions.text) from system.kafka_consumers where database = 'test' and table = '{table_name}'", - check_callback=lambda res: int(res) > 0, - ) - ) - - assert num_errors > 0 - - instance.query(f"DROP TABLE test.{table_name}") - instance.query("DROP TABLE test.view") + for result in results: + result.get() @pytest.mark.parametrize( From 84a5c88b50f197dc4a6f3bf9e06d245959011805 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 4 Sep 2024 12:45:13 +0000 Subject: [PATCH 204/289] Minor follow-up to #66933 --- .../functions/string-replace-functions.md | 41 ++++++++++++------- src/Functions/overlay.cpp | 16 ++++---- tests/queries/0_stateless/03205_overlay.sql | 4 +- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index 0cc6b0b27d5..283d41d8b73 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -20,10 +20,10 @@ overlay(s, replace, offset[, length]) **Parameters** -- `input`: A string type [String](../data-types/string.md). +- `s`: A string type [String](../data-types/string.md). - `replace`: A string type [String](../data-types/string.md). -- `offset`: An integer type [Int](../data-types/int-uint.md). If `offset` is negative, it is counted from the end of the `input` string. -- `length`: Optional. An integer type [Int](../data-types/int-uint.md). `length` specifies the length of the snippet within input to be replaced. If `length` is not specified, the number of bytes removed from `input` equals the length of `replace`; otherwise `length` bytes are removed. +- `offset`: An integer type [Int](../data-types/int-uint.md). If `offset` is negative, it is counted from the end of the string `s`. +- `length`: Optional. An integer type [Int](../data-types/int-uint.md). `length` specifies the length of the snippet within the input string `s` to be replaced. If `length` is not specified, the number of bytes removed from `s` equals the length of `replace`; otherwise `length` bytes are removed. **Returned value** @@ -32,22 +32,35 @@ overlay(s, replace, offset[, length]) **Example** ```sql -SELECT overlay('ClickHouse SQL', 'CORE', 12) AS res; +SELECT overlay('My father is from Mexico.', 'mother', 4) AS res; ``` Result: ```text -┌─res─────────────┐ -│ ClickHouse CORE │ -└─────────────────┘ +┌─res──────────────────────┐ +│ My mother is from Mexico.│ +└──────────────────────────┘ +``` + +```sql +SELECT overlay('My father is from Mexico.', 'dad', 4, 6) AS res; +``` + +Result: + +```text +┌─res───────────────────┐ +│ My dad is from Mexico.│ +└───────────────────────┘ ``` ## overlayUTF8 Replace part of the string `input` with another string `replace`, starting at the 1-based index `offset`. -Assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined. +Assumes that the string contains valid UTF-8 encoded text. +If this assumption is violated, no exception is thrown and the result is undefined. **Syntax** @@ -59,8 +72,8 @@ overlayUTF8(s, replace, offset[, length]) - `s`: A string type [String](../data-types/string.md). - `replace`: A string type [String](../data-types/string.md). -- `offset`: An integer type [Int](../data-types/int-uint.md). If `offset` is negative, it is counted from the end of the `input` string. -- `length`: Optional. An integer type [Int](../data-types/int-uint.md). `length` specifies the length of the snippet within input to be replaced. If `length` is not specified, the number of characters removed from `input` equals the length of `replace`; otherwise `length` characters are removed. +- `offset`: An integer type [Int](../data-types/int-uint.md). If `offset` is negative, it is counted from the end of the input string `s`. +- `length`: Optional. An integer type [Int](../data-types/int-uint.md). `length` specifies the length of the snippet within the input string `s` to be replaced. If `length` is not specified, the number of characters removed from `s` equals the length of `replace`; otherwise `length` characters are removed. **Returned value** @@ -69,15 +82,15 @@ overlayUTF8(s, replace, offset[, length]) **Example** ```sql -SELECT overlayUTF8('ClickHouse是一款OLAP数据库', '开源', 12, 2) AS res; +SELECT overlay('Mein Vater ist aus Österreich.', 'der Türkei', 20) AS res; ``` Result: ```text -┌─res────────────────────────┐ -│ ClickHouse是开源OLAP数据库 │ -└────────────────────────────┘ +┌─res───────────────────────────┐ +│ Mein Vater ist aus der Türkei.│ +└───────────────────────────────┘ ``` ## replaceOne diff --git a/src/Functions/overlay.cpp b/src/Functions/overlay.cpp index df8b825eabe..eddb7590cab 100644 --- a/src/Functions/overlay.cpp +++ b/src/Functions/overlay.cpp @@ -1,12 +1,12 @@ #include #include +#include +#include #include #include #include #include #include -#include -#include namespace DB { @@ -16,8 +16,8 @@ namespace /// If 'is_utf8' - measure offset and length in code points instead of bytes. /// Syntax: -/// - overlay(input, replace, offset[, length]) -/// - overlayUTF8(input, replace, offset[, length]) - measure offset and length in code points instead of bytes +/// - overlay(s, replace, offset[, length]) +/// - overlayUTF8(s, replace, offset[, length]) - measure offset and length in code points instead of bytes template class FunctionOverlay : public IFunction { @@ -34,7 +34,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { FunctionArgumentDescriptors mandatory_args{ - {"input", static_cast(&isString), nullptr, "String"}, + {"s", static_cast(&isString), nullptr, "String"}, {"replace", static_cast(&isString), nullptr, "String"}, {"offset", static_cast(&isNativeInteger), nullptr, "(U)Int8/16/32/64"}, }; @@ -100,7 +100,6 @@ public: res_data.reserve(col_input_string->getChars().size()); } - #define OVERLAY_EXECUTE_CASE(HAS_FOUR_ARGS, OFFSET_IS_CONST, LENGTH_IS_CONST) \ if (input_is_const && replace_is_const) \ constantConstant( \ @@ -186,7 +185,6 @@ public: return res_col; } - private: /// input offset is 1-based, maybe negative /// output result is 0-based valid offset, within [0, input_size] @@ -229,6 +227,7 @@ private: ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets) const { + /// Free us from handling negative length in the code below if (has_four_args && length_is_const && const_length < 0) { constantConstant( @@ -343,6 +342,7 @@ private: ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets) const { + /// Free us from handling negative length in the code below if (has_four_args && length_is_const && const_length < 0) { vectorConstant( @@ -461,6 +461,7 @@ private: ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets) const { + /// Free us from handling negative length in the code below if (has_four_args && length_is_const && const_length < 0) { constantVector( @@ -577,6 +578,7 @@ private: ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets) const { + /// Free us from handling negative length in the code below if (has_four_args && length_is_const && const_length < 0) { vectorVector( diff --git a/tests/queries/0_stateless/03205_overlay.sql b/tests/queries/0_stateless/03205_overlay.sql index 765b29f93ec..b692cc0c5ab 100644 --- a/tests/queries/0_stateless/03205_overlay.sql +++ b/tests/queries/0_stateless/03205_overlay.sql @@ -35,10 +35,10 @@ SELECT overlay('Spark SQL', materialize('ANSI '), materialize(7), materialize(0) SELECT overlay(materialize('Spark SQL'), materialize('ANSI '), materialize(7), materialize(0)), overlayUTF8(materialize('Spark SQL和CH'), materialize('ANSI '), materialize(7), materialize(0)); SELECT 'Test with special offset values'; -WITH number - 12 AS offset SELECT offset, overlay('Spark SQL', '__', offset), overlayUTF8('Spark SQL和CH', '之', offset) FROM numbers(26); +WITH number - 12 AS offset SELECT offset, overlay('Spark SQL', '__', offset), overlayUTF8('Spark SQL和CH', '之', offset) FROM numbers(26) ORDER BY number; SELECT 'Test with special length values'; -WITH number - 1 AS length SELECT length, overlay('Spark SQL', 'ANSI ', 7, length), overlayUTF8('Spark SQL和CH', 'ANSI ', 7, length) FROM numbers(8); +WITH number - 1 AS length SELECT length, overlay('Spark SQL', 'ANSI ', 7, length), overlayUTF8('Spark SQL和CH', 'ANSI ', 7, length) FROM numbers(8) ORDER BY number; SELECT 'Test with special input and replace values'; SELECT overlay('', '_', 6), overlayUTF8('', '_', 6); From dea714d1696e0ff0ceb277b37e7f36f5e3af03cc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Sep 2024 14:25:02 +0000 Subject: [PATCH 205/289] Update version_date.tsv and changelogs after v24.3.11.7-lts --- docs/changelogs/v24.3.11.7-lts.md | 17 +++++++++++++++++ utils/list-versions/version_date.tsv | 3 +++ 2 files changed, 20 insertions(+) create mode 100644 docs/changelogs/v24.3.11.7-lts.md diff --git a/docs/changelogs/v24.3.11.7-lts.md b/docs/changelogs/v24.3.11.7-lts.md new file mode 100644 index 00000000000..2d97dfbaad2 --- /dev/null +++ b/docs/changelogs/v24.3.11.7-lts.md @@ -0,0 +1,17 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.3.11.7-lts (28795d0a47e) FIXME as compared to v24.3.10.33-lts (37b6502ebf0) + +#### Bug Fix (user-visible misbehavior in an official stable release) +* Backported in [#67479](https://github.com/ClickHouse/ClickHouse/issues/67479): In rare cases ClickHouse could consider parts as broken because of some unexpected projections on disk. Now it's fixed. [#66898](https://github.com/ClickHouse/ClickHouse/pull/66898) ([alesapin](https://github.com/alesapin)). +* Backported in [#69243](https://github.com/ClickHouse/ClickHouse/issues/69243): `UNION` clause in subqueries wasn't handled correctly in queries with parallel replicas and lead to LOGICAL_ERROR `Duplicate announcement received for replica`. [#69146](https://github.com/ClickHouse/ClickHouse/pull/69146) ([Igor Nikonov](https://github.com/devcrafter)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backported in [#69221](https://github.com/ClickHouse/ClickHouse/issues/69221): Disable memory test with sanitizer. [#69193](https://github.com/ClickHouse/ClickHouse/pull/69193) ([alesapin](https://github.com/alesapin)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 4fa387731ce..d979577677d 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -3,12 +3,14 @@ v24.8.2.3-lts 2024-08-22 v24.8.1.2684-lts 2024-08-21 v24.7.5.37-stable 2024-09-03 v24.7.4.51-stable 2024-08-23 +v24.7.3.47-stable 2024-09-04 v24.7.3.42-stable 2024-08-08 v24.7.2.13-stable 2024-08-01 v24.7.1.2915-stable 2024-07-30 v24.6.5.30-stable 2024-09-03 v24.6.4.42-stable 2024-08-23 v24.6.3.95-stable 2024-08-06 +v24.6.3.38-stable 2024-09-04 v24.6.2.17-stable 2024-07-05 v24.6.1.4423-stable 2024-07-01 v24.5.7.31-stable 2024-09-03 @@ -22,6 +24,7 @@ v24.4.4.113-stable 2024-08-02 v24.4.3.25-stable 2024-06-14 v24.4.2.141-stable 2024-06-07 v24.4.1.2088-stable 2024-05-01 +v24.3.11.7-lts 2024-09-06 v24.3.10.33-lts 2024-09-03 v24.3.9.5-lts 2024-08-22 v24.3.8.13-lts 2024-08-20 From 86f11e221e2b2880d56f330c9a5aa0c2ab129727 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 6 Sep 2024 17:07:21 +0200 Subject: [PATCH 206/289] Speed up more tests --- tests/integration/test_storage_kafka/test.py | 45 ++++++++++++++++++-- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 6802a0b7785..440b7c526b9 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -1000,7 +1000,11 @@ def test_kafka_formats(kafka_cluster, create_query_generator): } topic_postfix = str(hash(create_query_generator)) - for format_name, format_opts in list(all_formats.items()): + + p = Pool(10) + results = [] + + def run_for_format(format_name, format_opts): logging.debug(f"Set up {format_name}") topic_name = f"format_tests_{format_name}-{topic_postfix}" data_sample = format_opts["data_sample"] @@ -1036,6 +1040,13 @@ def test_kafka_formats(kafka_cluster, create_query_generator): ), ) ) + + for format_name, format_opts in list(all_formats.items()): + results.append(p.apply_async(run_for_format, args=(format_name, format_opts))) + + for result in results: + result.get() + raw_expected = """\ 0 0 AM 0.5 1 {topic_name} 0 {offset_0} 1 0 AM 0.5 1 {topic_name} 0 {offset_1} @@ -1066,7 +1077,9 @@ def test_kafka_formats(kafka_cluster, create_query_generator): ) assert result_checker(res) - for format_name, format_opts in list(all_formats.items()): + results = [] + + def run_for_format2(format_name, format_opts): logging.debug(("Checking {}".format(format_name))) topic_name = f"format_tests_{format_name}-{topic_postfix}" # shift offsets by 1 if format supports empty value @@ -1090,6 +1103,12 @@ def test_kafka_formats(kafka_cluster, create_query_generator): ) kafka_delete_topic(get_admin_client(kafka_cluster), topic_name) + for format_name, format_opts in list(all_formats.items()): + results.append(p.apply_async(run_for_format2, args=(format_name, format_opts))) + + for result in results: + result.get() + # Since everything is async and shaky when receiving messages from Kafka, # we may want to try and check results multiple times in a loop. @@ -4239,7 +4258,11 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator topic_name_prefix = "format_tests_4_stream_" topic_name_postfix = get_topic_postfix(create_query_generator) - for format_name, format_opts in list(all_formats.items()): + + p = Pool(10) + results = [] + + def run_for_format(format_name, format_opts): logging.debug(f"Set up {format_name}") topic_name = f"{topic_name_prefix}{format_name}{topic_name_postfix}" data_sample = format_opts["data_sample"] @@ -4280,6 +4303,12 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator """ ) + for format_name, format_opts in list(all_formats.items()): + results.append(p.apply_async(run_for_format, args=(format_name, format_opts))) + + for result in results: + result.get() + raw_expected = """\ 0 0 AM 0.5 1 {topic_name} 0 {offset_0} 1 0 AM 0.5 1 {topic_name} 0 {offset_1} @@ -4310,7 +4339,9 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator ) assert result_checker(res) - for format_name, format_opts in list(all_formats.items()): + results = [] + + def run_for_format2(format_name, format_opts): logging.debug(f"Checking {format_name}") topic_name = f"{topic_name_prefix}{format_name}{topic_name_postfix}" # shift offsets by 1 if format supports empty value @@ -4350,6 +4381,12 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator ), "Proper error for format: {}".format(format_name) kafka_delete_topic(admin_client, topic_name) + for format_name, format_opts in list(all_formats.items()): + results.append(p.apply_async(run_for_format2, args=(format_name, format_opts))) + + for result in results: + result.get() + @pytest.mark.parametrize( "create_query_generator", From 9b87f2e6d40135c8f6fc9eb3dcd352e85a10ab21 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Sep 2024 15:10:13 +0000 Subject: [PATCH 207/289] Update version_date.tsv and changelogs after v24.5.8.10-stable --- docs/changelogs/v24.5.8.10-stable.md | 18 ++++++++++++++++++ utils/list-versions/version_date.tsv | 4 ++++ 2 files changed, 22 insertions(+) create mode 100644 docs/changelogs/v24.5.8.10-stable.md diff --git a/docs/changelogs/v24.5.8.10-stable.md b/docs/changelogs/v24.5.8.10-stable.md new file mode 100644 index 00000000000..c7a2baa4e4c --- /dev/null +++ b/docs/changelogs/v24.5.8.10-stable.md @@ -0,0 +1,18 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.5.8.10-stable (f11729638ea) FIXME as compared to v24.5.7.31-stable (6c185e9aec1) + +#### Bug Fix (user-visible misbehavior in an official stable release) +* Backported in [#69295](https://github.com/ClickHouse/ClickHouse/issues/69295): TODO. [#68744](https://github.com/ClickHouse/ClickHouse/pull/68744) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#69245](https://github.com/ClickHouse/ClickHouse/issues/69245): `UNION` clause in subqueries wasn't handled correctly in queries with parallel replicas and lead to LOGICAL_ERROR `Duplicate announcement received for replica`. [#69146](https://github.com/ClickHouse/ClickHouse/pull/69146) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix crash when using `s3` table function with GLOB paths and filters. [#69176](https://github.com/ClickHouse/ClickHouse/pull/69176) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backported in [#69223](https://github.com/ClickHouse/ClickHouse/issues/69223): Disable memory test with sanitizer. [#69193](https://github.com/ClickHouse/ClickHouse/pull/69193) ([alesapin](https://github.com/alesapin)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 4fa387731ce..09edea9af21 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -3,14 +3,17 @@ v24.8.2.3-lts 2024-08-22 v24.8.1.2684-lts 2024-08-21 v24.7.5.37-stable 2024-09-03 v24.7.4.51-stable 2024-08-23 +v24.7.3.47-stable 2024-09-04 v24.7.3.42-stable 2024-08-08 v24.7.2.13-stable 2024-08-01 v24.7.1.2915-stable 2024-07-30 v24.6.5.30-stable 2024-09-03 v24.6.4.42-stable 2024-08-23 v24.6.3.95-stable 2024-08-06 +v24.6.3.38-stable 2024-09-04 v24.6.2.17-stable 2024-07-05 v24.6.1.4423-stable 2024-07-01 +v24.5.8.10-stable 2024-09-06 v24.5.7.31-stable 2024-09-03 v24.5.6.45-stable 2024-08-23 v24.5.5.78-stable 2024-08-05 @@ -22,6 +25,7 @@ v24.4.4.113-stable 2024-08-02 v24.4.3.25-stable 2024-06-14 v24.4.2.141-stable 2024-06-07 v24.4.1.2088-stable 2024-05-01 +v24.3.11.7-lts 2024-09-06 v24.3.10.33-lts 2024-09-03 v24.3.9.5-lts 2024-08-22 v24.3.8.13-lts 2024-08-20 From b430e27a5090e05e261d6a7961353e44972afa91 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 15:33:53 +0000 Subject: [PATCH 208/289] Add CITATION.cff --- CITATION.cff | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000000..e4008c6940b --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,31 @@ +# This CITATION.cff file was generated with cffinit. + +cff-version: 1.2.0 +title: "ClickHouse" +message: "If you use this software, please cite it as below." +type: software +authors: + - family-names: "Milovidov" + given-names: "Alexey" +repository-code: 'https://github.com/ClickHouse/ClickHouse' +url: 'https://clickhouse.com' +license: Apache-2.0 +preferred-citation: + type: article + authors: + - family-names: "Schulze" + given-names: "Robert" + - family-names: "Schreiber" + given-names: "Tom" + - family-names: "Yatsishin" + given-names: "Ilya" + - family-names: "Dahimene" + given-names: "Ryadh" + - family-names: "Milovidov" + given-names: "Alexey" + journal: "Proceedings of the VLDB Endowment" + title: "ClickHouse - Lightning Fast Analytics for Everyone" + year: 2024 + volume: 17 + issue: 12 + doi: 10.14778/3685800.3685802 From 4665867400e70fbf002759cf9114bbff826fbe56 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Sep 2024 15:39:40 +0000 Subject: [PATCH 209/289] Update version_date.tsv and changelogs after v24.6.6.6-stable --- docs/changelogs/v24.6.6.6-stable.md | 16 ++++++++++++++++ utils/list-versions/version_date.tsv | 5 +++++ 2 files changed, 21 insertions(+) create mode 100644 docs/changelogs/v24.6.6.6-stable.md diff --git a/docs/changelogs/v24.6.6.6-stable.md b/docs/changelogs/v24.6.6.6-stable.md new file mode 100644 index 00000000000..8cf83e8fc18 --- /dev/null +++ b/docs/changelogs/v24.6.6.6-stable.md @@ -0,0 +1,16 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.6.6.6-stable (a4c4580e639) FIXME as compared to v24.6.5.30-stable (e6e196c92d6) + +#### Bug Fix (user-visible misbehavior in an official stable release) +* Backported in [#69197](https://github.com/ClickHouse/ClickHouse/issues/69197): TODO. [#68744](https://github.com/ClickHouse/ClickHouse/pull/68744) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backported in [#69225](https://github.com/ClickHouse/ClickHouse/issues/69225): Disable memory test with sanitizer. [#69193](https://github.com/ClickHouse/ClickHouse/pull/69193) ([alesapin](https://github.com/alesapin)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 4fa387731ce..f0ae5f641f0 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -3,14 +3,18 @@ v24.8.2.3-lts 2024-08-22 v24.8.1.2684-lts 2024-08-21 v24.7.5.37-stable 2024-09-03 v24.7.4.51-stable 2024-08-23 +v24.7.3.47-stable 2024-09-04 v24.7.3.42-stable 2024-08-08 v24.7.2.13-stable 2024-08-01 v24.7.1.2915-stable 2024-07-30 +v24.6.6.6-stable 2024-09-06 v24.6.5.30-stable 2024-09-03 v24.6.4.42-stable 2024-08-23 v24.6.3.95-stable 2024-08-06 +v24.6.3.38-stable 2024-09-04 v24.6.2.17-stable 2024-07-05 v24.6.1.4423-stable 2024-07-01 +v24.5.8.10-stable 2024-09-06 v24.5.7.31-stable 2024-09-03 v24.5.6.45-stable 2024-08-23 v24.5.5.78-stable 2024-08-05 @@ -22,6 +26,7 @@ v24.4.4.113-stable 2024-08-02 v24.4.3.25-stable 2024-06-14 v24.4.2.141-stable 2024-06-07 v24.4.1.2088-stable 2024-05-01 +v24.3.11.7-lts 2024-09-06 v24.3.10.33-lts 2024-09-03 v24.3.9.5-lts 2024-08-22 v24.3.8.13-lts 2024-08-20 From 06632b9bcd5f6cb0b18848cea806012e9ea6baf9 Mon Sep 17 00:00:00 2001 From: Michael Stetsyuk Date: Fri, 6 Sep 2024 15:58:46 +0000 Subject: [PATCH 210/289] fxs --- src/Databases/DatabaseReplicated.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 26a834eea4c..0a79fc45d0f 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -649,7 +649,7 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt if (check_responses[i].data != expected_data[i]) { throw Exception( - ErrorCodes::LOGICAL_ERROR, + ErrorCodes::REPLICA_ALREADY_EXISTS, "Replica node {} in ZooKeeper already exists and contains unexpected value: {}", quoteString(check_paths[i]), quoteString(check_responses[i].data)); } From 4c9713717ff83f4cf1676398a432209d783a8d62 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Sep 2024 16:10:11 +0000 Subject: [PATCH 211/289] Update version_date.tsv and changelogs after v24.7.6.8-stable --- docs/changelogs/v24.7.6.8-stable.md | 17 +++++++++++++++++ utils/list-versions/version_date.tsv | 6 ++++++ 2 files changed, 23 insertions(+) create mode 100644 docs/changelogs/v24.7.6.8-stable.md diff --git a/docs/changelogs/v24.7.6.8-stable.md b/docs/changelogs/v24.7.6.8-stable.md new file mode 100644 index 00000000000..13102a97b40 --- /dev/null +++ b/docs/changelogs/v24.7.6.8-stable.md @@ -0,0 +1,17 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.7.6.8-stable (7779883593a) FIXME as compared to v24.7.5.37-stable (f2533ca97be) + +#### Bug Fix (user-visible misbehavior in an official stable release) +* Backported in [#69198](https://github.com/ClickHouse/ClickHouse/issues/69198): TODO. [#68744](https://github.com/ClickHouse/ClickHouse/pull/68744) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#69249](https://github.com/ClickHouse/ClickHouse/issues/69249): `UNION` clause in subqueries wasn't handled correctly in queries with parallel replicas and lead to LOGICAL_ERROR `Duplicate announcement received for replica`. [#69146](https://github.com/ClickHouse/ClickHouse/pull/69146) ([Igor Nikonov](https://github.com/devcrafter)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backported in [#69227](https://github.com/ClickHouse/ClickHouse/issues/69227): Disable memory test with sanitizer. [#69193](https://github.com/ClickHouse/ClickHouse/pull/69193) ([alesapin](https://github.com/alesapin)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 4fa387731ce..f3876e1c51d 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,16 +1,21 @@ v24.8.3.59-lts 2024-09-03 v24.8.2.3-lts 2024-08-22 v24.8.1.2684-lts 2024-08-21 +v24.7.6.8-stable 2024-09-06 v24.7.5.37-stable 2024-09-03 v24.7.4.51-stable 2024-08-23 +v24.7.3.47-stable 2024-09-04 v24.7.3.42-stable 2024-08-08 v24.7.2.13-stable 2024-08-01 v24.7.1.2915-stable 2024-07-30 +v24.6.6.6-stable 2024-09-06 v24.6.5.30-stable 2024-09-03 v24.6.4.42-stable 2024-08-23 v24.6.3.95-stable 2024-08-06 +v24.6.3.38-stable 2024-09-04 v24.6.2.17-stable 2024-07-05 v24.6.1.4423-stable 2024-07-01 +v24.5.8.10-stable 2024-09-06 v24.5.7.31-stable 2024-09-03 v24.5.6.45-stable 2024-08-23 v24.5.5.78-stable 2024-08-05 @@ -22,6 +27,7 @@ v24.4.4.113-stable 2024-08-02 v24.4.3.25-stable 2024-06-14 v24.4.2.141-stable 2024-06-07 v24.4.1.2088-stable 2024-05-01 +v24.3.11.7-lts 2024-09-06 v24.3.10.33-lts 2024-09-03 v24.3.9.5-lts 2024-08-22 v24.3.8.13-lts 2024-08-20 From 9d07d002f109a45b1348297342dce7f575f31bfe Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 6 Sep 2024 16:20:30 +0000 Subject: [PATCH 212/289] update keeper client docs --- docs/en/operations/utilities/clickhouse-keeper-client.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/operations/utilities/clickhouse-keeper-client.md b/docs/en/operations/utilities/clickhouse-keeper-client.md index a66ecbc1372..fbfdd66d1a3 100644 --- a/docs/en/operations/utilities/clickhouse-keeper-client.md +++ b/docs/en/operations/utilities/clickhouse-keeper-client.md @@ -47,6 +47,8 @@ keeper foo bar - `ls '[path]'` -- Lists the nodes for the given path (default: cwd) - `cd '[path]'` -- Changes the working path (default `.`) +- `cp '' ''` -- Copies 'src' node to 'dest' path +- `mv '' ''` -- Moves 'src' node to the 'dest' path - `exists ''` -- Returns `1` if node exists, `0` otherwise - `set '' [version]` -- Updates the node's value. Only updates if version matches (default: -1) - `create '' [mode]` -- Creates new node with the set value From 8544718d329ff772af2e06b19d949a676fb84af0 Mon Sep 17 00:00:00 2001 From: Michael Stetsyuk Date: Fri, 6 Sep 2024 16:31:39 +0000 Subject: [PATCH 213/289] initialize max_log_ptr_at_creation on attach --- src/Databases/DatabaseReplicated.cpp | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 0a79fc45d0f..fd886d37f0e 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -485,6 +485,12 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessL current_zookeeper->set(replica_path + "/replica_group", replica_group_name, -1); createEmptyLogEntry(current_zookeeper); } + + /// Needed to mark all the queries + /// in the range (max log ptr at replica ZooKeeper nodes creation, max log ptr after replica recovery] as successful. + String max_log_ptr_at_creation_str; + if (current_zookeeper->tryGet(replica_path + "/max_log_ptr_at_creation", max_log_ptr_at_creation_str)) + max_log_ptr_at_creation = parse(max_log_ptr_at_creation_str); } if (is_create_query) @@ -614,12 +620,6 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt replica_path, replica_path + "/replica_group", replica_path + "/digest", - - /// Needed to mark all the queries - /// in the range (max log ptr at replica ZooKeeper nodes creation, max log ptr after replica recovery] as successful. - /// Previously, this method was not idempotent and max_log_ptr_at_creation could be stored in memory. - /// we need to store max_log_ptr_at_creation in ZooKeeper to make this method idempotent during replica creation. - replica_path + "/max_log_ptr_at_creation", }; bool nodes_exist = true; auto check_responses = current_zookeeper->tryGet(check_paths); @@ -656,7 +656,6 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt } LOG_DEBUG(log, "Newly initialized replica nodes found in ZooKeeper, reusing them"); - max_log_ptr_at_creation = parse(check_responses[check_responses.size() - 1].data); createEmptyLogEntry(current_zookeeper); return; } @@ -672,11 +671,10 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt zkutil::makeCreateRequest(replica_path + "/digest", "0", zkutil::CreateMode::Persistent), zkutil::makeCreateRequest(replica_path + "/replica_group", replica_group_name, zkutil::CreateMode::Persistent), - /// In addition to creating the replica nodes, we record the max_log_ptr at the instant where - /// we declared ourself as an existing replica. We'll need this during recoverLostReplica to - /// notify other nodes that issued new queries while this node was recovering. - zkutil::makeCheckRequest(zookeeper_path + "/max_log_ptr", stat.version), + /// Previously, this method was not idempotent and max_log_ptr_at_creation could be stored in memory. + /// we need to store max_log_ptr_at_creation in ZooKeeper to make this method idempotent during replica creation. zkutil::makeCreateRequest(replica_path + "/max_log_ptr_at_creation", max_log_ptr_str, zkutil::CreateMode::Persistent), + zkutil::makeCheckRequest(zookeeper_path + "/max_log_ptr", stat.version), }; Coordination::Responses ops_responses; From 11fae9f321104e70746b21d1c12d35fff719db5f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 16:37:42 +0000 Subject: [PATCH 214/289] Bump libarchive to v3.7.1 --- contrib/libarchive | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libarchive b/contrib/libarchive index ee457961713..0c21691b177 160000 --- a/contrib/libarchive +++ b/contrib/libarchive @@ -1 +1 @@ -Subproject commit ee45796171324519f0c0bfd012018dd099296336 +Subproject commit 0c21691b177fac5f4cceca2a1ff2ddfa5d60f51c From 8be745e8c223de9150552cc2a561bfb325fd0163 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Sep 2024 16:43:55 +0000 Subject: [PATCH 215/289] Update version_date.tsv and changelogs after v24.8.4.13-lts --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v24.8.4.13-lts.md | 22 ++++++++++++++++++++++ utils/list-versions/version_date.tsv | 7 +++++++ 5 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v24.8.4.13-lts.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 39b9fa7c395..19c5903461c 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.8.3.59" +ARG VERSION="24.8.4.13" ARG PACKAGES="clickhouse-keeper" ARG DIRECT_DOWNLOAD_URLS="" diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 60b038d0da9..e74dcf6d73d 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.8.3.59" +ARG VERSION="24.8.4.13" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" ARG DIRECT_DOWNLOAD_URLS="" diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 77c403bb046..456bed14a7e 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="24.8.3.59" +ARG VERSION="24.8.4.13" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" #docker-official-library:off diff --git a/docs/changelogs/v24.8.4.13-lts.md b/docs/changelogs/v24.8.4.13-lts.md new file mode 100644 index 00000000000..66385172cf8 --- /dev/null +++ b/docs/changelogs/v24.8.4.13-lts.md @@ -0,0 +1,22 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.8.4.13-lts (53195bc189b) FIXME as compared to v24.8.3.59-lts (e729b9fa40e) + +#### Improvement +* Backported in [#68699](https://github.com/ClickHouse/ClickHouse/issues/68699): Delete old code of named collections from dictionaries and substitute it to the new, which allows to use DDL created named collections in dictionaries. Closes [#60936](https://github.com/ClickHouse/ClickHouse/issues/60936), closes [#36890](https://github.com/ClickHouse/ClickHouse/issues/36890). [#68412](https://github.com/ClickHouse/ClickHouse/pull/68412) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Bug Fix (user-visible misbehavior in an official stable release) +* Backported in [#69231](https://github.com/ClickHouse/ClickHouse/issues/69231): Fix parsing error when null should be inserted as default in some cases during JSON type parsing. [#68955](https://github.com/ClickHouse/ClickHouse/pull/68955) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#69251](https://github.com/ClickHouse/ClickHouse/issues/69251): `UNION` clause in subqueries wasn't handled correctly in queries with parallel replicas and lead to LOGICAL_ERROR `Duplicate announcement received for replica`. [#69146](https://github.com/ClickHouse/ClickHouse/pull/69146) ([Igor Nikonov](https://github.com/devcrafter)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backported in [#69189](https://github.com/ClickHouse/ClickHouse/issues/69189): Don't create Object type if use_json_alias_for_old_object_type=1 but allow_experimental_object_type=0. [#69150](https://github.com/ClickHouse/ClickHouse/pull/69150) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#69229](https://github.com/ClickHouse/ClickHouse/issues/69229): Disable memory test with sanitizer. [#69193](https://github.com/ClickHouse/ClickHouse/pull/69193) ([alesapin](https://github.com/alesapin)). +* Backported in [#69219](https://github.com/ClickHouse/ClickHouse/issues/69219): Disable perf-like test with sanitizers. [#69194](https://github.com/ClickHouse/ClickHouse/pull/69194) ([alesapin](https://github.com/alesapin)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 4fa387731ce..a621f5a7ddf 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,16 +1,22 @@ +v24.8.4.13-lts 2024-09-06 v24.8.3.59-lts 2024-09-03 v24.8.2.3-lts 2024-08-22 v24.8.1.2684-lts 2024-08-21 +v24.7.6.8-stable 2024-09-06 v24.7.5.37-stable 2024-09-03 v24.7.4.51-stable 2024-08-23 +v24.7.3.47-stable 2024-09-04 v24.7.3.42-stable 2024-08-08 v24.7.2.13-stable 2024-08-01 v24.7.1.2915-stable 2024-07-30 +v24.6.6.6-stable 2024-09-06 v24.6.5.30-stable 2024-09-03 v24.6.4.42-stable 2024-08-23 v24.6.3.95-stable 2024-08-06 +v24.6.3.38-stable 2024-09-04 v24.6.2.17-stable 2024-07-05 v24.6.1.4423-stable 2024-07-01 +v24.5.8.10-stable 2024-09-06 v24.5.7.31-stable 2024-09-03 v24.5.6.45-stable 2024-08-23 v24.5.5.78-stable 2024-08-05 @@ -22,6 +28,7 @@ v24.4.4.113-stable 2024-08-02 v24.4.3.25-stable 2024-06-14 v24.4.2.141-stable 2024-06-07 v24.4.1.2088-stable 2024-05-01 +v24.3.11.7-lts 2024-09-06 v24.3.10.33-lts 2024-09-03 v24.3.9.5-lts 2024-08-22 v24.3.8.13-lts 2024-08-20 From 3431e2ec4ab7239d04b98212ea52171d8bb8e5ae Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 6 Sep 2024 16:52:31 +0000 Subject: [PATCH 216/289] add dest to aspell ignore --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index d10db5f0d3d..d4166e221a7 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -2583,6 +2583,7 @@ sqlinsert sqlite sqrt src +dest srcReplicas sshkey stackoverflow From 86842766e430ad080c58ca1eb4265dc2e6816f0e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 16:59:14 +0000 Subject: [PATCH 217/289] Bump OpenSSL to 3.2.2 --- contrib/openssl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/openssl b/contrib/openssl index 66deddc1e53..b7e86862e3a 160000 --- a/contrib/openssl +++ b/contrib/openssl @@ -1 +1 @@ -Subproject commit 66deddc1e53cda8706604a019777259372d1bd62 +Subproject commit b7e86862e3af97c33bd99442519cd8553d5a1c83 From fdf8ff0e66cee78694606696d03774533b5c29fc Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 17:25:14 +0000 Subject: [PATCH 218/289] Don't forget to call ucasemap_close for every ucasemap_open --- src/Functions/LowerUpperUTF8Impl.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Functions/LowerUpperUTF8Impl.h b/src/Functions/LowerUpperUTF8Impl.h index 7d60bd54d2f..f174bcd5f3e 100644 --- a/src/Functions/LowerUpperUTF8Impl.h +++ b/src/Functions/LowerUpperUTF8Impl.h @@ -6,6 +6,7 @@ # include # include +# include # include # include # include @@ -49,6 +50,11 @@ struct LowerUpperUTF8Impl if (U_FAILURE(error_code)) throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Error calling ucasemap_open: {}", u_errorName(error_code)); + SCOPE_EXIT( + { + ucasemap_close(case_map); + }); + size_t curr_offset = 0; for (size_t row_i = 0; row_i < input_rows_count; ++row_i) { From 4bd66f41879b8bc4a3f57bad790cc87e0cf2b5fb Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 17:31:20 +0000 Subject: [PATCH 219/289] Add test --- .../00170_lower_upper_utf8_memleak.reference | 1 + .../0_stateless/00170_lower_upper_utf8_memleak.sh | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/queries/0_stateless/00170_lower_upper_utf8_memleak.reference create mode 100755 tests/queries/0_stateless/00170_lower_upper_utf8_memleak.sh diff --git a/tests/queries/0_stateless/00170_lower_upper_utf8_memleak.reference b/tests/queries/0_stateless/00170_lower_upper_utf8_memleak.reference new file mode 100644 index 00000000000..7f6160bace3 --- /dev/null +++ b/tests/queries/0_stateless/00170_lower_upper_utf8_memleak.reference @@ -0,0 +1 @@ +españa diff --git a/tests/queries/0_stateless/00170_lower_upper_utf8_memleak.sh b/tests/queries/0_stateless/00170_lower_upper_utf8_memleak.sh new file mode 100755 index 00000000000..8436659c6a8 --- /dev/null +++ b/tests/queries/0_stateless/00170_lower_upper_utf8_memleak.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: upper/lowerUTF8 use ICU + +# Test for issue #69336 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL --query "SELECT lowerUTF8('ESPAÑA')" From 52024b7af3958ad25ffe9212e517854f18b4d582 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 17:51:03 +0000 Subject: [PATCH 220/289] Bump openssl to 3.2.3 --- contrib/openssl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/openssl b/contrib/openssl index b7e86862e3a..b3e62c440f3 160000 --- a/contrib/openssl +++ b/contrib/openssl @@ -1 +1 @@ -Subproject commit b7e86862e3af97c33bd99442519cd8553d5a1c83 +Subproject commit b3e62c440f390e12e77c80675f883af82ad3d5ed From 2bd36997f734c4e743d2ad814365d2d775ea27b8 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 12:11:10 +0000 Subject: [PATCH 221/289] Bump grpc to v1.59.5 CVE-2024-7246 --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index 1716359d2e2..7bc3abe952a 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit 1716359d2e28d304a250f9df0e6c0ccad03de8db +Subproject commit 7bc3abe952aba1dc7bce7f2f790dc781cb51a41e From fb3de3451cbf61ac69e8c5cdd30b47d029911158 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 18:04:01 +0000 Subject: [PATCH 222/289] Make a clean start with v1.44.2 --- contrib/libuv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libuv b/contrib/libuv index 4482964660c..04cc9419e49 160000 --- a/contrib/libuv +++ b/contrib/libuv @@ -1 +1 @@ -Subproject commit 4482964660c77eec1166cd7d14fb915e3dbd774a +Subproject commit 04cc9419e4976c2ea33108f206b50f5922c2bbfe From cc2467542b1c8d4fce916b84008b6f52b82701cc Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 18:26:50 +0000 Subject: [PATCH 223/289] Bump libuv to 1.45.0 --- contrib/libuv | 2 +- contrib/libuv-cmake/CMakeLists.txt | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/contrib/libuv b/contrib/libuv index 04cc9419e49..3dd84dd6aed 160000 --- a/contrib/libuv +++ b/contrib/libuv @@ -1 +1 @@ -Subproject commit 04cc9419e4976c2ea33108f206b50f5922c2bbfe +Subproject commit 3dd84dd6aed6d2f726d2720d12ec240ff240bc00 diff --git a/contrib/libuv-cmake/CMakeLists.txt b/contrib/libuv-cmake/CMakeLists.txt index 928fdcdd7e6..22df1e9a102 100644 --- a/contrib/libuv-cmake/CMakeLists.txt +++ b/contrib/libuv-cmake/CMakeLists.txt @@ -10,6 +10,7 @@ set(uv_sources src/random.c src/strscpy.c src/strtok.c + src/thread-common.c src/threadpool.c src/timer.c src/uv-common.c @@ -70,10 +71,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") list(APPEND uv_defines _GNU_SOURCE _POSIX_C_SOURCE=200112) list(APPEND uv_libraries rt) list(APPEND uv_sources - src/unix/epoll.c - src/unix/linux-core.c - src/unix/linux-inotify.c - src/unix/linux-syscalls.c + src/unix/linux.c src/unix/procfs-exepath.c src/unix/random-getrandom.c src/unix/random-sysctl-linux.c) From 22f1d9752e896a9583f8ffd0a0051c3af1f87981 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 18:48:40 +0000 Subject: [PATCH 224/289] Bump libuv to 1.46.0 --- contrib/libuv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libuv b/contrib/libuv index 3dd84dd6aed..0e4fcb11681 160000 --- a/contrib/libuv +++ b/contrib/libuv @@ -1 +1 @@ -Subproject commit 3dd84dd6aed6d2f726d2720d12ec240ff240bc00 +Subproject commit 0e4fcb11681ce480a5db15868021bb4424717d82 From ac6eb1c52092c0dcb0bda66dfda6d7a6b7337be8 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 18:50:59 +0000 Subject: [PATCH 225/289] Bump libuv to 1.47.0 --- contrib/libuv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libuv b/contrib/libuv index 0e4fcb11681..7a297688d08 160000 --- a/contrib/libuv +++ b/contrib/libuv @@ -1 +1 @@ -Subproject commit 0e4fcb11681ce480a5db15868021bb4424717d82 +Subproject commit 7a297688d08c487e1c5e7f63f6f5950396ed9b81 From f65c869d4c5c661decc70b027f2f4e864dc4b261 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 18:52:43 +0000 Subject: [PATCH 226/289] Bump libuv to v1.48.0 --- contrib/libuv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libuv b/contrib/libuv index 7a297688d08..714b58b9849 160000 --- a/contrib/libuv +++ b/contrib/libuv @@ -1 +1 @@ -Subproject commit 7a297688d08c487e1c5e7f63f6f5950396ed9b81 +Subproject commit 714b58b9849568211ade86b44dd91d37f8a2175e From ba038cba4732197c31d791d20c7e973ee4e11750 Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Tue, 27 Aug 2024 00:31:53 +0000 Subject: [PATCH 227/289] Fix 02378_part_log_profile_events flakiness --- .../02378_part_log_profile_events.sh | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 tests/queries/0_stateless/02378_part_log_profile_events.sh diff --git a/tests/queries/0_stateless/02378_part_log_profile_events.sh b/tests/queries/0_stateless/02378_part_log_profile_events.sh new file mode 100755 index 00000000000..8dd8b1eca91 --- /dev/null +++ b/tests/queries/0_stateless/02378_part_log_profile_events.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Tags: no-shared-merge-tree +# Tag no-shared-merge-tree: depend on events with local disk + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS test; + + CREATE TABLE test (key UInt64, val UInt64) engine = MergeTree Order by key PARTITION BY key >= 128; + SET max_block_size = 64, max_insert_block_size = 64, min_insert_block_size_rows = 64; + INSERT INTO test SELECT number AS key, sipHash64(number) AS val FROM numbers(512); +" + +${CLICKHOUSE_CLIENT} --query " + SYSTEM FLUSH LOGS; + SELECT + if(count(DISTINCT query_id) == 1, 'Ok', 'Error: ' || toString(count(DISTINCT query_id))), + if(count() == 512 / 64, 'Ok', 'Error: ' || toString(count())), -- 512 rows inserted, 64 rows per block + if(SUM(ProfileEvents['MergeTreeDataWriterRows']) == 512, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterRows']))), + if(SUM(ProfileEvents['MergeTreeDataWriterUncompressedBytes']) >= 1024, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterUncompressedBytes']))), + if(SUM(ProfileEvents['MergeTreeDataWriterCompressedBytes']) >= 1024, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterCompressedBytes']))), + if(SUM(ProfileEvents['MergeTreeDataWriterBlocks']) >= 8, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterBlocks']))) + FROM system.part_log + WHERE event_time > now() - INTERVAL 10 MINUTE + AND database == currentDatabase() AND table == 'test' + AND event_type == 'NewPart'; +" + +${CLICKHOUSE_CLIENT} --query "OPTIMIZE TABLE test FINAL;" + +${CLICKHOUSE_CLIENT} --query " + SYSTEM FLUSH LOGS; + SELECT + if(count() > 2, 'Ok', 'Error: ' || toString(count())), + if(SUM(ProfileEvents['MergedRows']) >= 512, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergedRows']))) + FROM system.part_log + WHERE event_time > now() - INTERVAL 10 MINUTE + AND database == currentDatabase() AND table == 'test' + AND event_type == 'MergeParts'; +" + +${CLICKHOUSE_CLIENT} --query " + ALTER TABLE test UPDATE val = 0 WHERE key % 2 == 0 SETTINGS mutations_sync = 2 +" + +# The mutation query may return before the entry is added to the system.part_log table. +# Retry SYSTEM FLUSH LOGS until all entries are fully flushed. +for _ in {1..10}; do + ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" + res=$(${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM system.part_log + WHERE event_time > now() - INTERVAL 10 MINUTE + AND database == currentDatabase() AND table == 'test' + AND event_type == 'MutatePart';" + ) + if [[ $res -eq 2 ]]; then + break + fi + + sleep 2.0 +done + +${CLICKHOUSE_CLIENT} --query " + SELECT + if(count() == 2, 'Ok', 'Error: ' || toString(count())), + if(SUM(ProfileEvents['MutatedRows']) == 512, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MutatedRows']))), + if(SUM(ProfileEvents['FileOpen']) > 1, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['FileOpen']))) + FROM system.part_log + WHERE event_time > now() - INTERVAL 10 MINUTE + AND database == currentDatabase() AND table == 'test' + AND event_type == 'MutatePart'; +" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE test" From 99667c42f3af47c2d2ee8bb9652d2f5ab6fa73aa Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Fri, 6 Sep 2024 20:59:48 +0000 Subject: [PATCH 228/289] sync with the private --- tests/queries/0_stateless/02378_part_log_profile_events.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02378_part_log_profile_events.sql b/tests/queries/0_stateless/02378_part_log_profile_events.sql index eec76d6f50e..706731fb644 100644 --- a/tests/queries/0_stateless/02378_part_log_profile_events.sql +++ b/tests/queries/0_stateless/02378_part_log_profile_events.sql @@ -1,3 +1,5 @@ +-- Tags: no-shared-merge-tree +-- no-shared-merge-tree: depend on events with local disk DROP TABLE IF EXISTS test; CREATE TABLE test (key UInt64, val UInt64) engine = MergeTree Order by key PARTITION BY key >= 128; From bf74b600e506d59df3c12c3f830fc2a04763b97c Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Fri, 6 Sep 2024 21:00:16 +0000 Subject: [PATCH 229/289] rm tests/queries/0_stateless/02378_part_log_profile_events.sql --- .../02378_part_log_profile_events.sql | 52 ------------------- 1 file changed, 52 deletions(-) delete mode 100644 tests/queries/0_stateless/02378_part_log_profile_events.sql diff --git a/tests/queries/0_stateless/02378_part_log_profile_events.sql b/tests/queries/0_stateless/02378_part_log_profile_events.sql deleted file mode 100644 index 706731fb644..00000000000 --- a/tests/queries/0_stateless/02378_part_log_profile_events.sql +++ /dev/null @@ -1,52 +0,0 @@ --- Tags: no-shared-merge-tree --- no-shared-merge-tree: depend on events with local disk -DROP TABLE IF EXISTS test; - -CREATE TABLE test (key UInt64, val UInt64) engine = MergeTree Order by key PARTITION BY key >= 128; - -SET max_block_size = 64, max_insert_block_size = 64, min_insert_block_size_rows = 64; - -INSERT INTO test SELECT number AS key, sipHash64(number) AS val FROM numbers(512); - -SYSTEM FLUSH LOGS; - -SELECT - if(count(DISTINCT query_id) == 1, 'Ok', 'Error: ' || toString(count(DISTINCT query_id))), - if(count() == 512 / 64, 'Ok', 'Error: ' || toString(count())), -- 512 rows inserted, 64 rows per block - if(SUM(ProfileEvents['MergeTreeDataWriterRows']) == 512, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterRows']))), - if(SUM(ProfileEvents['MergeTreeDataWriterUncompressedBytes']) >= 1024, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterUncompressedBytes']))), - if(SUM(ProfileEvents['MergeTreeDataWriterCompressedBytes']) >= 1024, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterCompressedBytes']))), - if(SUM(ProfileEvents['MergeTreeDataWriterBlocks']) >= 8, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterBlocks']))) -FROM system.part_log -WHERE event_time > now() - INTERVAL 10 MINUTE - AND database == currentDatabase() AND table == 'test' - AND event_type == 'NewPart' -; - -OPTIMIZE TABLE test FINAL; - -SYSTEM FLUSH LOGS; -SELECT - if(count() > 2, 'Ok', 'Error: ' || toString(count())), - if(SUM(ProfileEvents['MergedRows']) >= 512, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergedRows']))) -FROM system.part_log -WHERE event_time > now() - INTERVAL 10 MINUTE - AND database == currentDatabase() AND table == 'test' - AND event_type == 'MergeParts' -; - -ALTER TABLE test UPDATE val = 0 WHERE key % 2 == 0 SETTINGS mutations_sync = 2; - -SYSTEM FLUSH LOGS; - -SELECT - if(count() == 2, 'Ok', 'Error: ' || toString(count())), - if(SUM(ProfileEvents['MutatedRows']) == 512, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MutatedRows']))), - if(SUM(ProfileEvents['FileOpen']) > 1, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['FileOpen']))) -FROM system.part_log -WHERE event_time > now() - INTERVAL 10 MINUTE - AND database == currentDatabase() AND table == 'test' - AND event_type == 'MutatePart' -; - -DROP TABLE test; From 7f637f7acfb83888e4c3de73c04de018da340c30 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Fri, 30 Aug 2024 21:46:56 +0000 Subject: [PATCH 230/289] kdc_network_reject: test passed --- tests/integration/helpers/network.py | 3 +- .../test_storage_kerberized_kafka/test.py | 48 ++++++++++++------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 065836396f3..f24b5924e73 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -303,6 +303,7 @@ class _NetworkManager: destination_port=None, action=None, probability=None, + protocol=None, custom_args=None, ): ret = [] @@ -317,7 +318,7 @@ class _NetworkManager: str(probability), ] ) - ret.extend(["-p", "tcp"]) + ret.extend(["-p", "tcp" if protocol is None else protocol]) if source is not None: ret.extend(["-s", source]) if destination is not None: diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 24d10d7ff83..c5e91831fc2 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -8,6 +8,7 @@ import logging from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV from helpers.client import QueryRuntimeException +from helpers.network import PartitionManager import json import subprocess @@ -204,27 +205,40 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): ], ) - kafka_cluster.pause_container("kafka_kerberos") - time.sleep(45) # wait for ticket expiration + # temporary prevent CH - KDC communications + with PartitionManager() as pm: + for other_node in ["kafka_kerberos"]: + for node in kafka_cluster.instances.values(): + source = node.ip_address + destination = kafka_cluster.get_instance_ip(other_node) + logging.debug(f"partitioning source {source}, destination {destination}") + pm._add_rule( + { + "source": source, + "destination": destination, + "action": "REJECT", + "protocol": "all" + } + ) - instance.query( - """ - CREATE TABLE test.kafka_no_kdc (field String) - ENGINE = Kafka - SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', - kafka_topic_list = 'kafka_json_as_string_no_kdc', - kafka_group_name = 'kafka_json_as_string_no_kdc', - kafka_commit_on_select = 1, - kafka_format = 'JSONAsString', - kafka_flush_interval_ms=1000; - """ - ) + time.sleep(45) # wait for ticket expiration - result = instance.query("SELECT * FROM test.kafka_no_kdc;") + instance.query( + """ + CREATE TABLE test.kafka_no_kdc (field String) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', + kafka_topic_list = 'kafka_json_as_string_no_kdc', + kafka_group_name = 'kafka_json_as_string_no_kdc', + kafka_commit_on_select = 1, + kafka_format = 'JSONAsString', + kafka_flush_interval_ms=1000; + """ + ) + + result = instance.query("SELECT * FROM test.kafka_no_kdc;") expected = "" - kafka_cluster.unpause_container("kafka_kerberos") - assert TSV(result) == TSV(expected) assert instance.contains_in_log("StorageKafka (kafka_no_kdc): Nothing to commit") assert instance.contains_in_log("Ticket expired") From c264f1c4681b3a84d1578ce6ad02213a868c01de Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 2 Sep 2024 07:45:27 +0000 Subject: [PATCH 231/289] kdc_network_reject: no redundant loop --- .../test_storage_kerberized_kafka/test.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index c5e91831fc2..dd5ee673b9e 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -207,19 +207,19 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): # temporary prevent CH - KDC communications with PartitionManager() as pm: - for other_node in ["kafka_kerberos"]: - for node in kafka_cluster.instances.values(): - source = node.ip_address - destination = kafka_cluster.get_instance_ip(other_node) - logging.debug(f"partitioning source {source}, destination {destination}") - pm._add_rule( - { - "source": source, - "destination": destination, - "action": "REJECT", - "protocol": "all" - } - ) + other_node = "kafka_kerberos" + for node in kafka_cluster.instances.values(): + source = node.ip_address + destination = kafka_cluster.get_instance_ip(other_node) + logging.debug(f"partitioning source {source}, destination {destination}") + pm._add_rule( + { + "source": source, + "destination": destination, + "action": "REJECT", + "protocol": "all" + } + ) time.sleep(45) # wait for ticket expiration From 4212ad02e3f1798d27588aa7648fb482e332f9ff Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 3 Sep 2024 13:32:02 +0000 Subject: [PATCH 232/289] kdc_network_reject: make black formatter happy --- tests/integration/test_storage_kerberized_kafka/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index dd5ee673b9e..3ac3415b6f3 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -217,7 +217,7 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): "source": source, "destination": destination, "action": "REJECT", - "protocol": "all" + "protocol": "all", } ) From a9fcd61b92a0a0e126036d4d31a88d9ae776662c Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 4 Sep 2024 15:48:59 +0000 Subject: [PATCH 233/289] kdc_network_reject: fix test_kafka_config_from_sql_named_collection --- tests/integration/test_storage_kerberized_kafka/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 3ac3415b6f3..cb8693c95ad 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -259,6 +259,7 @@ def test_kafka_config_from_sql_named_collection(kafka_cluster): instance.query( """ + DROP NAMED COLLECTION IF EXISTS kafka_config CREATE NAMED COLLECTION kafka_config AS kafka.security_protocol = 'SASL_PLAINTEXT', kafka.sasl_mechanism = 'GSSAPI', From f5c6612a0bf4276a8a664634eacb6449f4985401 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 4 Sep 2024 21:15:38 +0000 Subject: [PATCH 234/289] kdc_network_reject: bugfix --- .../test_storage_kerberized_kafka/test.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index cb8693c95ad..4ffb5dab906 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -139,7 +139,7 @@ def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster) kafka_produce( kafka_cluster, - "kafka_json_as_string", + "kafka_json_as_string_after_expiration", [ '{"t": 123, "e": {"x": "woof"} }', "", @@ -153,9 +153,9 @@ def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster) CREATE TABLE test.kafka (field String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', - kafka_topic_list = 'kafka_json_as_string', + kafka_topic_list = 'kafka_json_as_string_after_expiration', kafka_commit_on_select = 1, - kafka_group_name = 'kafka_json_as_string', + kafka_group_name = 'kafka_json_as_string_after_expiration', kafka_format = 'JSONAsString', kafka_flush_interval_ms=1000; """ @@ -248,7 +248,7 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): def test_kafka_config_from_sql_named_collection(kafka_cluster): kafka_produce( kafka_cluster, - "kafka_json_as_string", + "kafka_json_as_string_named_collection", [ '{"t": 123, "e": {"x": "woof"} }', "", @@ -259,7 +259,7 @@ def test_kafka_config_from_sql_named_collection(kafka_cluster): instance.query( """ - DROP NAMED COLLECTION IF EXISTS kafka_config + DROP NAMED COLLECTION IF EXISTS kafka_config; CREATE NAMED COLLECTION kafka_config AS kafka.security_protocol = 'SASL_PLAINTEXT', kafka.sasl_mechanism = 'GSSAPI', @@ -270,9 +270,9 @@ def test_kafka_config_from_sql_named_collection(kafka_cluster): kafka.api_version_request = 'false', kafka_broker_list = 'kerberized_kafka1:19092', - kafka_topic_list = 'kafka_json_as_string', + kafka_topic_list = 'kafka_json_as_string_named_collection', kafka_commit_on_select = 1, - kafka_group_name = 'kafka_json_as_string', + kafka_group_name = 'kafka_json_as_string_named_collection', kafka_format = 'JSONAsString', kafka_flush_interval_ms=1000; """ From f16d4c9567ccf6fa5f206614925e860da4587071 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 5 Sep 2024 07:31:17 +0000 Subject: [PATCH 235/289] kdc_network_reject: bugfix (again) --- tests/integration/test_storage_kerberized_kafka/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 4ffb5dab906..a00914543c6 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -171,7 +171,7 @@ def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster) """ assert TSV(result) == TSV(expected) assert instance.contains_in_log( - "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" + "Parsing of message (topic: kafka_json_as_string_after_expiration, partition: 0, offset: 1) return no rows" ) @@ -294,7 +294,7 @@ def test_kafka_config_from_sql_named_collection(kafka_cluster): """ assert TSV(result) == TSV(expected) assert instance.contains_in_log( - "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" + "Parsing of message (topic: kafka_json_as_string_named_collection, partition: 0, offset: 1) return no rows" ) From 9667c19b6ea81bf5227202824879c3d9b34a33c9 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Sat, 7 Sep 2024 09:16:38 +0200 Subject: [PATCH 236/289] Revert "Speed up some Kafka tests with multiprocessing" --- tests/integration/test_storage_kafka/test.py | 289 +++++++------------ 1 file changed, 108 insertions(+), 181 deletions(-) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 440b7c526b9..bef90e1b9d3 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -10,8 +10,6 @@ import string import ast import math -from multiprocessing.dummy import Pool - import avro.schema import avro.io import avro.datafile @@ -1000,11 +998,7 @@ def test_kafka_formats(kafka_cluster, create_query_generator): } topic_postfix = str(hash(create_query_generator)) - - p = Pool(10) - results = [] - - def run_for_format(format_name, format_opts): + for format_name, format_opts in list(all_formats.items()): logging.debug(f"Set up {format_name}") topic_name = f"format_tests_{format_name}-{topic_postfix}" data_sample = format_opts["data_sample"] @@ -1040,13 +1034,6 @@ def test_kafka_formats(kafka_cluster, create_query_generator): ), ) ) - - for format_name, format_opts in list(all_formats.items()): - results.append(p.apply_async(run_for_format, args=(format_name, format_opts))) - - for result in results: - result.get() - raw_expected = """\ 0 0 AM 0.5 1 {topic_name} 0 {offset_0} 1 0 AM 0.5 1 {topic_name} 0 {offset_1} @@ -1077,9 +1064,7 @@ def test_kafka_formats(kafka_cluster, create_query_generator): ) assert result_checker(res) - results = [] - - def run_for_format2(format_name, format_opts): + for format_name, format_opts in list(all_formats.items()): logging.debug(("Checking {}".format(format_name))) topic_name = f"format_tests_{format_name}-{topic_postfix}" # shift offsets by 1 if format supports empty value @@ -1103,12 +1088,6 @@ def test_kafka_formats(kafka_cluster, create_query_generator): ) kafka_delete_topic(get_admin_client(kafka_cluster), topic_name) - for format_name, format_opts in list(all_formats.items()): - results.append(p.apply_async(run_for_format2, args=(format_name, format_opts))) - - for result in results: - result.get() - # Since everything is async and shaky when receiving messages from Kafka, # we may want to try and check results multiple times in a loop. @@ -4258,11 +4237,7 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator topic_name_prefix = "format_tests_4_stream_" topic_name_postfix = get_topic_postfix(create_query_generator) - - p = Pool(10) - results = [] - - def run_for_format(format_name, format_opts): + for format_name, format_opts in list(all_formats.items()): logging.debug(f"Set up {format_name}") topic_name = f"{topic_name_prefix}{format_name}{topic_name_postfix}" data_sample = format_opts["data_sample"] @@ -4303,12 +4278,6 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator """ ) - for format_name, format_opts in list(all_formats.items()): - results.append(p.apply_async(run_for_format, args=(format_name, format_opts))) - - for result in results: - result.get() - raw_expected = """\ 0 0 AM 0.5 1 {topic_name} 0 {offset_0} 1 0 AM 0.5 1 {topic_name} 0 {offset_1} @@ -4339,9 +4308,7 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator ) assert result_checker(res) - results = [] - - def run_for_format2(format_name, format_opts): + for format_name, format_opts in list(all_formats.items()): logging.debug(f"Checking {format_name}") topic_name = f"{topic_name_prefix}{format_name}{topic_name_postfix}" # shift offsets by 1 if format supports empty value @@ -4381,12 +4348,6 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator ), "Proper error for format: {}".format(format_name) kafka_delete_topic(admin_client, topic_name) - for format_name, format_opts in list(all_formats.items()): - results.append(p.apply_async(run_for_format2, args=(format_name, format_opts))) - - for result in results: - result.get() - @pytest.mark.parametrize( "create_query_generator", @@ -4870,63 +4831,6 @@ def test_max_rows_per_message(kafka_cluster, create_query_generator): def test_row_based_formats(kafka_cluster, create_query_generator): admin_client = get_admin_client(kafka_cluster) - p = Pool(10) - - def run_for_format_name(format_name): - logging.debug("Checking {format_name}") - - topic_name = format_name + get_topic_postfix(create_query_generator) - view_name = f"kafka_view_{format_name}" - table_name = f"kafka_{format_name}" - - with kafka_topic(admin_client, topic_name): - num_rows = 10 - max_rows_per_message = 5 - message_count = num_rows / max_rows_per_message - - create_query = create_query_generator( - table_name, - "key UInt64, value UInt64", - topic_list=topic_name, - consumer_group=topic_name, - format=format_name, - settings={"kafka_max_rows_per_message": max_rows_per_message}, - ) - - instance.query( - f""" - DROP TABLE IF EXISTS test.{view_name}; - DROP TABLE IF EXISTS test.{table_name}; - - {create_query}; - - CREATE MATERIALIZED VIEW test.{view_name} ENGINE=MergeTree ORDER BY (key, value) AS - SELECT key, value FROM test.{table_name}; - - INSERT INTO test.{table_name} SELECT number * 10 as key, number * 100 as value FROM numbers({num_rows}); - """ - ) - - messages = kafka_consume_with_retry( - kafka_cluster, topic_name, message_count, need_decode=False - ) - - assert ( - len(messages) == message_count - ), f"Invalid message count for {format_name}" - - instance.query_with_retry( - f"SELECT count() FROM test.{view_name}", - check_callback=lambda res: int(res) == num_rows, - ) - - result = instance.query(f"SELECT * FROM test.{view_name}") - expected = "" - for i in range(num_rows): - expected += str(i * 10) + "\t" + str(i * 100) + "\n" - assert result == expected, f"Invalid result for {format_name}" - - results = [] for format_name in [ "TSV", "TSVWithNamesAndTypes", @@ -4945,10 +4849,55 @@ def test_row_based_formats(kafka_cluster, create_query_generator): "RowBinaryWithNamesAndTypes", "MsgPack", ]: - results.append(p.apply_async(run_for_format_name, args=(format_name,))) + logging.debug("Checking {format_name}") - for result in results: - result.get() + topic_name = format_name + get_topic_postfix(create_query_generator) + table_name = f"kafka_{format_name}" + + with kafka_topic(admin_client, topic_name): + num_rows = 10 + max_rows_per_message = 5 + message_count = num_rows / max_rows_per_message + + create_query = create_query_generator( + table_name, + "key UInt64, value UInt64", + topic_list=topic_name, + consumer_group=topic_name, + format=format_name, + settings={"kafka_max_rows_per_message": max_rows_per_message}, + ) + + instance.query( + f""" + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.{table_name}; + + {create_query}; + + CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY (key, value) AS + SELECT key, value FROM test.{table_name}; + + INSERT INTO test.{table_name} SELECT number * 10 as key, number * 100 as value FROM numbers({num_rows}); + """ + ) + + messages = kafka_consume_with_retry( + kafka_cluster, topic_name, message_count, need_decode=False + ) + + assert len(messages) == message_count + + instance.query_with_retry( + "SELECT count() FROM test.view", + check_callback=lambda res: int(res) == num_rows, + ) + + result = instance.query("SELECT * FROM test.view") + expected = "" + for i in range(num_rows): + expected += str(i * 10) + "\t" + str(i * 100) + "\n" + assert result == expected @pytest.mark.parametrize( @@ -5006,12 +4955,16 @@ def test_block_based_formats_2(kafka_cluster, create_query_generator): num_rows = 100 message_count = 9 - p = Pool(10) - - def run_for_format_name(format_name): + for format_name in [ + "JSONColumns", + "Native", + "Arrow", + "Parquet", + "ORC", + "JSONCompactColumns", + ]: topic_name = format_name + get_topic_postfix(create_query_generator) table_name = f"kafka_{format_name}" - view_name = f"kafka_view_{format_name}" logging.debug(f"Checking format {format_name}") with kafka_topic(admin_client, topic_name): create_query = create_query_generator( @@ -5024,12 +4977,12 @@ def test_block_based_formats_2(kafka_cluster, create_query_generator): instance.query( f""" - DROP TABLE IF EXISTS test.{view_name}; + DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.{table_name}; {create_query}; - CREATE MATERIALIZED VIEW test.{view_name} ENGINE=MergeTree ORDER BY (key, value) AS + CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY (key, value) AS SELECT key, value FROM test.{table_name}; INSERT INTO test.{table_name} SELECT number * 10 as key, number * 100 as value FROM numbers({num_rows}) settings max_block_size=12, optimize_trivial_insert_select=0; @@ -5038,38 +4991,22 @@ def test_block_based_formats_2(kafka_cluster, create_query_generator): messages = kafka_consume_with_retry( kafka_cluster, topic_name, message_count, need_decode=False ) - assert ( - len(messages) == message_count - ), f"Invalid message count for {format_name}" + assert len(messages) == message_count rows = int( instance.query_with_retry( - f"SELECT count() FROM test.{view_name}", + "SELECT count() FROM test.view", check_callback=lambda res: int(res) == num_rows, ) ) - assert rows == num_rows, f"Invalid row count for {format_name}" + assert rows == num_rows - result = instance.query(f"SELECT * FROM test.{view_name} ORDER by key") + result = instance.query("SELECT * FROM test.view ORDER by key") expected = "" for i in range(num_rows): expected += str(i * 10) + "\t" + str(i * 100) + "\n" - assert result == expected, f"Invalid result for {format_name}" - - results = [] - for format_name in [ - "JSONColumns", - "Native", - "Arrow", - "Parquet", - "ORC", - "JSONCompactColumns", - ]: - results.append(p.apply_async(run_for_format_name, args=(format_name,))) - - for result in results: - result.get() + assert result == expected def test_system_kafka_consumers(kafka_cluster): @@ -5363,54 +5300,6 @@ def test_formats_errors(kafka_cluster): bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) ) - p = Pool(10) - - def run_for_format_name(format_name): - with kafka_topic(admin_client, format_name): - table_name = f"kafka_{format_name}" - view_name = f"kafka_view_{format_name}" - - instance.query( - f""" - DROP TABLE IF EXISTS test.{view_name}; - DROP TABLE IF EXISTS test.{table_name}; - - CREATE TABLE test.{table_name} (key UInt64, value UInt64) - ENGINE = Kafka - SETTINGS kafka_broker_list = 'kafka1:19092', - kafka_topic_list = '{format_name}', - kafka_group_name = '{format_name}', - kafka_format = '{format_name}', - kafka_max_rows_per_message = 5, - format_template_row='template_row.format', - format_regexp='id: (.+?)', - input_format_with_names_use_header=0, - format_schema='key_value_message:Message'; - - CREATE MATERIALIZED VIEW test.{view_name} ENGINE=MergeTree ORDER BY (key, value) AS - SELECT key, value FROM test.{table_name}; - """ - ) - - kafka_produce( - kafka_cluster, - format_name, - ["Broken message\nBroken message\nBroken message\n"], - ) - - num_errors = int( - instance.query_with_retry( - f"SELECT length(exceptions.text) from system.kafka_consumers where database = 'test' and table = '{table_name}'", - check_callback=lambda res: int(res) > 0, - ) - ) - - assert num_errors > 0, f"No errors for {format_name}" - - instance.query(f"DROP TABLE test.{table_name}") - instance.query(f"DROP TABLE test.{view_name}") - - results = [] for format_name in [ "Template", "Regexp", @@ -5453,10 +5342,48 @@ def test_formats_errors(kafka_cluster): "HiveText", "MySQLDump", ]: - results.append(p.apply_async(run_for_format_name, args=(format_name,))) + with kafka_topic(admin_client, format_name): + table_name = f"kafka_{format_name}" - for result in results: - result.get() + instance.query( + f""" + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.{table_name}; + + CREATE TABLE test.{table_name} (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = '{format_name}', + kafka_group_name = '{format_name}', + kafka_format = '{format_name}', + kafka_max_rows_per_message = 5, + format_template_row='template_row.format', + format_regexp='id: (.+?)', + input_format_with_names_use_header=0, + format_schema='key_value_message:Message'; + + CREATE MATERIALIZED VIEW test.view ENGINE=MergeTree ORDER BY (key, value) AS + SELECT key, value FROM test.{table_name}; + """ + ) + + kafka_produce( + kafka_cluster, + format_name, + ["Broken message\nBroken message\nBroken message\n"], + ) + + num_errors = int( + instance.query_with_retry( + f"SELECT length(exceptions.text) from system.kafka_consumers where database = 'test' and table = '{table_name}'", + check_callback=lambda res: int(res) > 0, + ) + ) + + assert num_errors > 0 + + instance.query(f"DROP TABLE test.{table_name}") + instance.query("DROP TABLE test.view") @pytest.mark.parametrize( From c335f4b1e409390b90579c6d9dbe80b05f594538 Mon Sep 17 00:00:00 2001 From: Tanya Bragin Date: Sat, 7 Sep 2024 07:39:31 -0700 Subject: [PATCH 237/289] Update README.md - Meetups Adjusting meetup listings per latest --- README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 652f5e0751d..5fa04fe29e7 100644 --- a/README.md +++ b/README.md @@ -42,21 +42,19 @@ Keep an eye out for upcoming meetups and events around the world. Somewhere else The following upcoming meetups are featuring creator of ClickHouse & CTO, Alexey Milovidov: -* [ClickHouse Guangzhou User Group Meetup](https://mp.weixin.qq.com/s/GSvo-7xUoVzCsuUvlLTpCw) - August 25 -* [San Francisco Meetup (Cloudflare)](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/302540575) - September 5 * [Raleigh Meetup (Deutsche Bank)](https://www.meetup.com/triangletechtalks/events/302723486/) - September 9 * [New York Meetup (Rokt)](https://www.meetup.com/clickhouse-new-york-user-group/events/302575342) - September 10 * [Chicago Meetup (Jump Capital)](https://lu.ma/43tvmrfw) - September 12 Other upcoming meetups -* [Seattle Meetup (Statsig)](https://www.meetup.com/clickhouse-seattle-user-group/events/302518075/) - August 27 -* [Melbourne Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/302732666/) - August 27 -* [Sydney Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/302862966/) - September 5 -* [Zurich Meetup](https://www.meetup.com/clickhouse-switzerland-meetup-group/events/302267429/) - September 5 + * [Toronto Meetup (Shopify)](https://www.meetup.com/clickhouse-toronto-user-group/events/301490855/) - September 10 * [Austin Meetup](https://www.meetup.com/clickhouse-austin-user-group/events/302558689/) - September 17 * [London Meetup](https://www.meetup.com/clickhouse-london-user-group/events/302977267) - September 17 +* [Bangalore Meetup](https://www.meetup.com/clickhouse-bangalore-user-group/events/303208274/) - September 18 * [Tel Aviv Meetup](https://www.meetup.com/clickhouse-meetup-israel/events/303095121) - September 22 +* [Jakarta Meetup](https://www.meetup.com/clickhouse-indonesia-user-group/events/303191359/) - October 1 +* [Singapore Meetup](https://www.meetup.com/clickhouse-singapore-meetup-group/events/303212064/) - October 3 * [Madrid Meetup](https://www.meetup.com/clickhouse-spain-user-group/events/303096564/) - October 22 * [Barcelona Meetup](https://www.meetup.com/clickhouse-spain-user-group/events/303096876/) - October 29 * [Oslo Meetup](https://www.meetup.com/open-source-real-time-data-warehouse-real-time-analytics/events/302938622) - October 31 @@ -64,7 +62,13 @@ Other upcoming meetups * [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - November 21 * [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/303096434) - November 26 - +Recently completed events +* [ClickHouse Guangzhou User Group Meetup](https://mp.weixin.qq.com/s/GSvo-7xUoVzCsuUvlLTpCw) - August 25 +* [Seattle Meetup (Statsig)](https://www.meetup.com/clickhouse-seattle-user-group/events/302518075/) - August 27 +* [Melbourne Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/302732666/) - August 27 +* [Sydney Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/302862966/) - September 5 +* [Zurich Meetup](https://www.meetup.com/clickhouse-switzerland-meetup-group/events/302267429/) - September 5 +* [San Francisco Meetup (Cloudflare)](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/302540575) - September 5 ## Recent Recordings * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments" From 5362b117bfbb17a79d31e3e40152c1cb0d56fb34 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 7 Sep 2024 16:20:20 +0000 Subject: [PATCH 238/289] Remove superfluous --multiquery/-n --- tests/docker_scripts/stateful_runner.sh | 2 +- tests/docker_scripts/stress_runner.sh | 2 +- tests/integration/helpers/client.py | 1 - tests/performance/scripts/compare.sh | 12 +-- ...006_client_test_hint_no_such_error_name.sh | 2 +- .../0_stateless/02030_rocksdb_race_long.sh | 6 +- .../02151_hash_table_sizes_stats_joins.sh | 2 +- .../02229_client_stop_multiquery_in_SIGINT.sh | 4 +- ...41_filesystem_cache_on_write_operations.sh | 8 +- .../0_stateless/02242_delete_user_race.sh | 2 +- .../0_stateless/02243_drop_user_grant_race.sh | 4 +- .../02266_protobuf_format_google_wrappers.sh | 16 ++-- .../02286_drop_filesystem_cache.sh | 6 +- .../02313_filesystem_cache_seeks.sh | 2 +- .../02340_parts_refcnt_mergetree.sh | 6 +- .../0_stateless/02344_describe_cache.sh | 4 +- tests/queries/0_stateless/02352_rwlock.sh | 4 +- .../0_stateless/02380_insert_mv_race.sh | 4 +- ...entelemetry_insert_on_distributed_table.sh | 8 +- .../02419_keeper_map_primary_key.sh | 2 +- .../02443_detach_attach_partition.sh | 2 +- .../0_stateless/02473_optimize_old_parts.sh | 8 +- .../0_stateless/02477_s3_request_throttler.sh | 2 +- ...02494_zero_copy_projection_cancel_fetch.sh | 4 +- .../02504_regexp_dictionary_ua_parser.sh | 8 +- .../02504_regexp_dictionary_yaml_source.sh | 18 ++-- .../02515_cleanup_async_insert_block_ids.sh | 6 +- .../02585_query_status_deadlock.sh | 2 +- ...de_client_info_initial_query_start_time.sh | 14 ++-- ...rofile_events_from_query_log_and_client.sh | 16 ++-- .../02676_optimize_old_parts_replicated.sh | 8 +- .../02686_postgres_protocol_decimal_256.sh | 2 +- .../0_stateless/02700_s3_part_INT_MAX.sh | 2 +- .../0_stateless/02725_start_stop_fetches.sh | 2 +- .../02731_zero_objects_in_metadata.sh | 6 +- tests/queries/0_stateless/02766_prql.sh | 2 +- .../0_stateless/02770_async_buffer_ignore.sh | 4 +- .../02789_filesystem_cache_alignment.sh | 18 ++-- ...89_reading_from_s3_with_connection_pool.sh | 8 +- .../0_stateless/02801_backup_native_copy.sh | 10 +-- .../0_stateless/02803_backup_tmp_files.sh | 2 +- ...2808_custom_disk_with_user_defined_name.sh | 8 +- .../02808_filesystem_cache_drop_query.sh | 22 ++--- ...ackup_use_same_password_for_base_backup.sh | 2 +- ...use_same_s3_credentials_for_base_backup.sh | 2 +- .../02844_max_backup_bandwidth_s3.sh | 6 +- .../0_stateless/02867_storage_set_tsan.sh | 4 +- .../02900_union_schema_inference_mode.sh | 10 +-- .../02908_many_requests_to_system_replicas.sh | 2 +- .../02922_deduplication_with_zero_copy.sh | 10 +-- .../02932_refreshable_materialized_views_1.sh | 72 ++++++++-------- .../02932_refreshable_materialized_views_2.sh | 82 +++++++++---------- .../0_stateless/02941_variant_type_1.sh | 10 +-- .../0_stateless/02941_variant_type_2.sh | 4 +- .../0_stateless/02941_variant_type_3.sh | 4 +- .../0_stateless/02941_variant_type_4.sh | 2 +- ...ynamically_change_filesystem_cache_size.sh | 10 +-- .../0_stateless/02960_polygon_bound_bug.sh | 2 +- .../02961_storage_config_volume_priority.sh | 4 +- ...02963_remote_read_small_buffer_size_bug.sh | 6 +- ...allel_replicas_join_algo_and_analyzer_1.sh | 2 +- ...allel_replicas_join_algo_and_analyzer_2.sh | 2 +- ...allel_replicas_join_algo_and_analyzer_3.sh | 2 +- .../02969_auto_format_detection.sh | 6 +- .../02980_s3_plain_DROP_TABLE_MergeTree.sh | 4 +- ...s3_plain_DROP_TABLE_ReplicatedMergeTree.sh | 6 +- ...ion_insert_several_blocks_nonreplicated.sh | 2 +- ...cation_insert_several_blocks_replicated.sh | 2 +- ..._generates_several_blocks_nonreplicated.sh | 2 +- ..._mv_generates_several_blocks_replicated.sh | 2 +- ...several_mv_into_one_table_nonreplicated.sh | 2 +- ...on_several_mv_into_one_table_replicated.sh | 2 +- .../0_stateless/03008_s3_plain_rewritable.sh | 8 +- ...2_dynamically_resize_filesystem_cache_2.sh | 6 +- .../03039_dynamic_aggregating_merge_tree.sh | 2 +- .../03039_dynamic_collapsing_merge_tree.sh | 2 +- .../03039_dynamic_replacing_merge_tree.sh | 2 +- .../03039_dynamic_summing_merge_tree.sh | 2 +- ...dynamic_versioned_collapsing_merge_tree.sh | 2 +- .../0_stateless/03151_unload_index_race.sh | 6 +- .../03172_error_log_table_not_empty.sh | 8 +- .../03203_hive_style_partitioning.sh | 18 ++-- ...kup_and_clear_old_temporary_directories.sh | 4 +- 83 files changed, 298 insertions(+), 299 deletions(-) diff --git a/tests/docker_scripts/stateful_runner.sh b/tests/docker_scripts/stateful_runner.sh index 86f6a299ad3..defc24781a1 100755 --- a/tests/docker_scripts/stateful_runner.sh +++ b/tests/docker_scripts/stateful_runner.sh @@ -105,7 +105,7 @@ setup_logs_replication clickhouse-client --query "SHOW DATABASES" clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql +clickhouse-client < /repo/tests/docker_scripts/create.sql clickhouse-client --query "SHOW TABLES FROM datasets" if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index 039c60c8e4e..af93c93f202 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -62,7 +62,7 @@ start_server setup_logs_replication clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql +clickhouse-client < /repo/tests/docker_scripts/create.sql clickhouse-client --query "SHOW TABLES FROM datasets" clickhouse-client --query "CREATE DATABASE IF NOT EXISTS test" diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index 8ba7b342020..97405860784 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -89,7 +89,6 @@ class Client: command = self.command[:] if stdin is None: - command += ["--multiquery"] stdin = sql else: command += ["--query", sql] diff --git a/tests/performance/scripts/compare.sh b/tests/performance/scripts/compare.sh index da7bbf77a28..c3566c51a16 100755 --- a/tests/performance/scripts/compare.sh +++ b/tests/performance/scripts/compare.sh @@ -427,7 +427,7 @@ do done # for each query run, prepare array of metrics from query log -clickhouse-local --multiquery --query " +clickhouse-local --query " create view query_runs as select * from file('analyze/query-runs.tsv', TSV, 'test text, query_index int, query_id text, version UInt8, time float'); @@ -582,7 +582,7 @@ numactl --cpunodebind=all --membind=all numactl --show # If the available memory falls below 2 * size, GNU parallel will suspend some of the running jobs. numactl --cpunodebind=all --membind=all parallel -v --joblog analyze/parallel-log.txt --memsuspend 15G --null < analyze/commands.txt 2>> analyze/errors.log -clickhouse-local --multiquery --query " +clickhouse-local --query " -- Join the metric names back to the metric statistics we've calculated, and make -- a denormalized table of them -- statistics for all metrics for all queries. -- The WITH, ARRAY JOIN and CROSS JOIN do not like each other: @@ -680,7 +680,7 @@ rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.ts cat analyze/errors.log >> report/errors.log ||: cat profile-errors.log >> report/errors.log ||: -clickhouse-local --multiquery --query " +clickhouse-local --query " create view query_display_names as select * from file('analyze/query-display-names.tsv', TSV, 'test text, query_index int, query_display_name text') @@ -981,7 +981,7 @@ create table all_query_metrics_tsv engine File(TSV, 'report/all-query-metrics.ts for version in {right,left} do rm -rf data - clickhouse-local --multiquery --query " + clickhouse-local --query " create view query_profiles as with 0 as left, 1 as right select * from file('analyze/query-profiles.tsv', TSV, @@ -1151,7 +1151,7 @@ function report_metrics rm -rf metrics ||: mkdir metrics -clickhouse-local --multiquery --query " +clickhouse-local --query " create view right_async_metric_log as select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes) ; @@ -1211,7 +1211,7 @@ function upload_results # Prepare info for the CI checks table. rm -f ci-checks.tsv - clickhouse-local --multiquery --query " + clickhouse-local --query " create view queries as select * from file('report/queries.tsv', TSVWithNamesAndTypes); create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv') diff --git a/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh b/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh index 972ff3ba73f..7bf8100a53d 100755 --- a/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh +++ b/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh @@ -5,4 +5,4 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -n -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*' +$CLICKHOUSE_CLIENT -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*' diff --git a/tests/queries/0_stateless/02030_rocksdb_race_long.sh b/tests/queries/0_stateless/02030_rocksdb_race_long.sh index da31861991c..7d329f78d98 100755 --- a/tests/queries/0_stateless/02030_rocksdb_race_long.sh +++ b/tests/queries/0_stateless/02030_rocksdb_race_long.sh @@ -12,14 +12,14 @@ echo " DROP TABLE IF EXISTS rocksdb_race; CREATE TABLE rocksdb_race (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key); INSERT INTO rocksdb_race SELECT '1_' || toString(number), number FROM numbers(100000); -" | $CLICKHOUSE_CLIENT -n +" | $CLICKHOUSE_CLIENT function read_stat_thread() { while true; do echo " SELECT * FROM system.rocksdb FORMAT Null; - " | $CLICKHOUSE_CLIENT -n + " | $CLICKHOUSE_CLIENT done } @@ -29,7 +29,7 @@ function truncate_thread() sleep 3s; echo " TRUNCATE TABLE rocksdb_race; - " | $CLICKHOUSE_CLIENT -n + " | $CLICKHOUSE_CLIENT done } diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh b/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh index 007dae6e427..a9b5c461c5a 100755 --- a/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh +++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh @@ -12,7 +12,7 @@ opts=( --join_algorithm='parallel_hash' ) -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " CREATE TABLE t1(a UInt32, b UInt32) ENGINE=MergeTree ORDER BY (); INSERT INTO t1 SELECT number, number FROM numbers_mt(1e6); diff --git a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh index e5d00bc1a1c..b23164f8354 100755 --- a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh +++ b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh @@ -5,12 +5,12 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -timeout -s INT 3s $CLICKHOUSE_CLIENT --max_block_size 1 -nm -q " +timeout -s INT 3s $CLICKHOUSE_CLIENT --max_block_size 1 -m -q " SELECT sleep(1) FROM numbers(100) FORMAT Null; SELECT 'FAIL'; " -timeout -s INT 3s $CLICKHOUSE_LOCAL --max_block_size 1 -nm -q " +timeout -s INT 3s $CLICKHOUSE_LOCAL --max_block_size 1 -m -q " SELECT sleep(1) FROM numbers(100) FORMAT Null; SELECT 'FAIL'; " diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh index f8e7b7e7e72..8a44b65e6d1 100755 --- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh +++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh @@ -16,7 +16,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --query "SYSTEM DROP FILESYSTEM CACHE" - $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -37,7 +37,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)" - $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -70,7 +70,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100, 200)" - $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -109,7 +109,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --query "SYSTEM FLUSH LOGS" - $CLICKHOUSE_CLIENT -n --query "SELECT + $CLICKHOUSE_CLIENT --query "SELECT query, ProfileEvents['RemoteFSReadBytes'] > 0 as remote_fs_read FROM system.query_log diff --git a/tests/queries/0_stateless/02242_delete_user_race.sh b/tests/queries/0_stateless/02242_delete_user_race.sh index 2af54276469..3cb79d69955 100755 --- a/tests/queries/0_stateless/02242_delete_user_race.sh +++ b/tests/queries/0_stateless/02242_delete_user_race.sh @@ -15,7 +15,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP ROLE IF EXISTS test_role_02242; CREATE ROLE test_role_02242; " diff --git a/tests/queries/0_stateless/02243_drop_user_grant_race.sh b/tests/queries/0_stateless/02243_drop_user_grant_race.sh index 4dce8e8124c..1972ecade06 100755 --- a/tests/queries/0_stateless/02243_drop_user_grant_race.sh +++ b/tests/queries/0_stateless/02243_drop_user_grant_race.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP ROLE IF EXISTS test_role_02244; CREATE ROLE test_role_02244; DROP USER IF EXISTS kek_02243; @@ -37,4 +37,4 @@ $CLICKHOUSE_CLIENT --user kek_02243 -q "SELECT * FROM test" 2>&1| grep -Fa "Exce $CLICKHOUSE_CLIENT -q "DROP ROLE IF EXISTS test_role_02243" $CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS test_user_02243" -$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS kek_02243" \ No newline at end of file +$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS kek_02243" diff --git a/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh b/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh index ae2a2351c6b..eb1109aaac1 100755 --- a/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh +++ b/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh @@ -44,7 +44,7 @@ protobuf_info() { fi } -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE IF EXISTS $MAIN_TABLE; DROP TABLE IF EXISTS $ROUNDTRIP_TABLE; DROP TABLE IF EXISTS $COMPATIBILITY_TABLE; @@ -78,14 +78,14 @@ echo $SET_OUTPUT echo echo "Insert $INITIAL_INSERT_VALUES into table (Nullable(String), Int32):" -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " INSERT INTO $MAIN_TABLE VALUES $INITIAL_INSERT_VALUES; SELECT * FROM $MAIN_TABLE; " echo echo "Protobuf representation of the second row:" -$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $MAIN_TABLE WHERE ref = 2 LIMIT 1 $(protobuf_info output ProtobufSingle Message)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $MAIN_TABLE WHERE ref = 2 LIMIT 1 $(protobuf_info output ProtobufSingle Message)" > "$BINARY_FILE_PATH" hexdump -C $BINARY_FILE_PATH echo @@ -101,12 +101,12 @@ hexdump -C $MESSAGE_FILE_PATH echo echo "Insert proto message into table (Nullable(String), Int32):" -$CLICKHOUSE_CLIENT -n --query "$SET_INPUT INSERT INTO $ROUNDTRIP_TABLE $(protobuf_info input Protobuf Message)" < "$MESSAGE_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_INPUT INSERT INTO $ROUNDTRIP_TABLE $(protobuf_info input Protobuf Message)" < "$MESSAGE_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM $ROUNDTRIP_TABLE" echo echo "Proto output of the table using Google wrapper:" -$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $ROUNDTRIP_TABLE $(protobuf_info output Protobuf Message)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $ROUNDTRIP_TABLE $(protobuf_info output Protobuf Message)" > "$BINARY_FILE_PATH" hexdump -C $BINARY_FILE_PATH echo @@ -124,14 +124,14 @@ echo echo "Insert $MULTI_WRAPPER_VALUES and reinsert using Google wrappers into:" echo "Table (Nullable(Int32), Nullable(Int32), Int32):" $CLICKHOUSE_CLIENT --query "INSERT INTO $MULTI_TABLE VALUES $MULTI_WRAPPER_VALUES" -$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $MULTI_TABLE $(protobuf_info output Protobuf MessageMultiWrapper)" > "$BINARY_FILE_PATH" -$CLICKHOUSE_CLIENT -n --query "$SET_INPUT INSERT INTO $MULTI_TABLE $(protobuf_info input Protobuf MessageMultiWrapper)" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $MULTI_TABLE $(protobuf_info output Protobuf MessageMultiWrapper)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_INPUT INSERT INTO $MULTI_TABLE $(protobuf_info input Protobuf MessageMultiWrapper)" < "$BINARY_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM $MULTI_TABLE" rm "$BINARY_FILE_PATH" rm "$MESSAGE_FILE_PATH" -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE $MAIN_TABLE; DROP TABLE $ROUNDTRIP_TABLE; DROP TABLE $COMPATIBILITY_TABLE; diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh index dab777fcc31..672bf4d068b 100755 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh @@ -11,7 +11,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do echo "Using storage policy: $STORAGE_POLICY" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02286" - $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_02286 (key UInt32, value String) + $CLICKHOUSE_CLIENT --query "CREATE TABLE test_02286 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='$STORAGE_POLICY', min_bytes_for_wide_part = 10485760" @@ -38,7 +38,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --query "SELECT * FROM test_02286 FORMAT Null" $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache" - $CLICKHOUSE_CLIENT -n --query "SELECT count() + $CLICKHOUSE_CLIENT --query "SELECT count() FROM ( SELECT arrayJoin(cache_paths) AS cache_path, @@ -54,7 +54,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache" $CLICKHOUSE_CLIENT --query "SELECT cache_path FROM system.filesystem_cache" - $CLICKHOUSE_CLIENT -n --query "SELECT cache_path, local_path + $CLICKHOUSE_CLIENT --query "SELECT cache_path, local_path FROM ( SELECT arrayJoin(cache_paths) AS cache_path, diff --git a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh index b7adde6fcbb..64fae06f220 100755 --- a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh +++ b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh @@ -23,7 +23,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; d ORDER BY tuple() SETTINGS storage_policy = '$STORAGE_POLICY'" > /dev/null - $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=0 -n --query "INSERT INTO test_02313 + $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=0 --query "INSERT INTO test_02313 SELECT * FROM generateRandom('id Int32, val String') LIMIT 100000" diff --git a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh index bd018018789..f252b9304cd 100755 --- a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh +++ b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh @@ -9,7 +9,7 @@ function check_refcnt_for_table() { local table=$1 && shift - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " system stop merges $table; -- cleanup thread may hold the parts lock system stop cleanup $table; @@ -66,14 +66,14 @@ function check_refcnt_for_table() # NOTE: index_granularity=1 to cancel ASAP -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " drop table if exists data_02340; create table data_02340 (key Int, part Int) engine=MergeTree() partition by part order by key settings index_granularity=1; " || exit 1 check_refcnt_for_table data_02340 $CLICKHOUSE_CLIENT -q "drop table data_02340 sync" -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " drop table if exists data_02340_rep sync; create table data_02340_rep (key Int, part Int) engine=ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') partition by part order by key settings index_granularity=1; " || exit 1 diff --git a/tests/queries/0_stateless/02344_describe_cache.sh b/tests/queries/0_stateless/02344_describe_cache.sh index c5373b4d7e3..4cc4c415f3f 100755 --- a/tests/queries/0_stateless/02344_describe_cache.sh +++ b/tests/queries/0_stateless/02344_describe_cache.sh @@ -7,14 +7,14 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="02344_describe_cache_test" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = '$disk_name', disk = 's3_disk', load_metadata_asynchronously = 0); """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ diff --git a/tests/queries/0_stateless/02352_rwlock.sh b/tests/queries/0_stateless/02352_rwlock.sh index b4a77e0b08a..7f02b3ee935 100755 --- a/tests/queries/0_stateless/02352_rwlock.sh +++ b/tests/queries/0_stateless/02352_rwlock.sh @@ -24,7 +24,7 @@ function wait_query_by_id_started() # wait for query to be started while [ "$($CLICKHOUSE_CLIENT "$@" -q "select count() from system.processes where query_id = '$query_id'")" -ne 1 ]; do if [ "$( - $CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 -nm -q " + $CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 -m -q " system flush logs; select count() from system.query_log @@ -52,7 +52,7 @@ $CLICKHOUSE_CLIENT -q "CREATE DATABASE ${CLICKHOUSE_DATABASE}_ordinary Engine=Or # debug build on CI, so if this will happen, then DROP query will be # finished instantly, and to avoid flakiness we will retry in this case while :; do - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}_ordinary.data_02352; CREATE TABLE ${CLICKHOUSE_DATABASE}_ordinary.data_02352 (key Int) Engine=Null(); " diff --git a/tests/queries/0_stateless/02380_insert_mv_race.sh b/tests/queries/0_stateless/02380_insert_mv_race.sh index 725c7eacce6..a5f05365180 100755 --- a/tests/queries/0_stateless/02380_insert_mv_race.sh +++ b/tests/queries/0_stateless/02380_insert_mv_race.sh @@ -9,13 +9,13 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q "ATTACH TABLE mv" |& { +$CLICKHOUSE_CLIENT -m -q "ATTACH TABLE mv" |& { # CANNOT_GET_CREATE_TABLE_QUERY -- ATTACH TABLE IF EXISTS # TABLE_ALREADY_EXISTS -- ATTACH TABLE IF NOT EXISTS grep -F -m1 Exception | grep -v -e CANNOT_GET_CREATE_TABLE_QUERY -e TABLE_ALREADY_EXISTS } -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS null; CREATE TABLE null (key Int) ENGINE = Null; DROP TABLE IF EXISTS mv; diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh index c692c37f27f..9e8b854375b 100755 --- a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh +++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh @@ -27,7 +27,7 @@ function insert() function check_span() { -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " SYSTEM FLUSH LOGS; SELECT operation_name, @@ -50,7 +50,7 @@ ${CLICKHOUSE_CLIENT} -nq " # $2 - value of distributed_foreground_insert function check_span_kind() { -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " SYSTEM FLUSH LOGS; SELECT count() @@ -65,7 +65,7 @@ ${CLICKHOUSE_CLIENT} -nq " # # Prepare tables for tests # -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.dist_opentelemetry; DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.local_opentelemetry; @@ -122,7 +122,7 @@ check_span_kind $trace_id 'CLIENT' # # Cleanup # -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " DROP TABLE ${CLICKHOUSE_DATABASE}.dist_opentelemetry; DROP TABLE ${CLICKHOUSE_DATABASE}.local_opentelemetry; " diff --git a/tests/queries/0_stateless/02419_keeper_map_primary_key.sh b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh index c43c5bb6408..6a23f666f36 100755 --- a/tests/queries/0_stateless/02419_keeper_map_primary_key.sh +++ b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh @@ -9,7 +9,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS 02419_test SYNC;" test_primary_key() { - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " CREATE TABLE 02419_test (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 3) PRIMARY KEY($1); INSERT INTO 02419_test VALUES (1, 1.1), (2, 2.2); SELECT value FROM 02419_test WHERE key = 1; diff --git a/tests/queries/0_stateless/02443_detach_attach_partition.sh b/tests/queries/0_stateless/02443_detach_attach_partition.sh index 6a47b7d8d61..4a5377a952a 100755 --- a/tests/queries/0_stateless/02443_detach_attach_partition.sh +++ b/tests/queries/0_stateless/02443_detach_attach_partition.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/replication.lib -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS alter_table0; DROP TABLE IF EXISTS alter_table1; diff --git a/tests/queries/0_stateless/02473_optimize_old_parts.sh b/tests/queries/0_stateless/02473_optimize_old_parts.sh index b563bc31b39..3a4e6145f12 100755 --- a/tests/queries/0_stateless/02473_optimize_old_parts.sh +++ b/tests/queries/0_stateless/02473_optimize_old_parts.sh @@ -21,7 +21,7 @@ wait_for_number_of_parts() { echo "$res" } -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE IF EXISTS test_without_merge; DROP TABLE IF EXISTS test_with_merge; @@ -34,7 +34,7 @@ INSERT INTO test_without_merge SELECT 3;" wait_for_number_of_parts 'test_without_merge' 1 10 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_without_merge; SELECT 'With merge any part range'; @@ -47,7 +47,7 @@ INSERT INTO test_with_merge SELECT 3;" wait_for_number_of_parts 'test_with_merge' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_with_merge; SELECT 'With merge partition only'; @@ -60,7 +60,7 @@ INSERT INTO test_with_merge SELECT 3;" wait_for_number_of_parts 'test_with_merge' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_with_merge' AND active; diff --git a/tests/queries/0_stateless/02477_s3_request_throttler.sh b/tests/queries/0_stateless/02477_s3_request_throttler.sh index c74cb598d42..5e6185e7304 100755 --- a/tests/queries/0_stateless/02477_s3_request_throttler.sh +++ b/tests/queries/0_stateless/02477_s3_request_throttler.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " -- Limit S3 PUT request per second rate SET s3_max_put_rps = 2; SET s3_max_put_burst = 1; diff --git a/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh b/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh index b72c3eb56c7..122684abe9c 100755 --- a/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh +++ b/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE IF EXISTS wikistat1 SYNC; DROP TABLE IF EXISTS wikistat2 SYNC; " @@ -60,7 +60,7 @@ wait $CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat1 WHERE NOT ignore(*)" $CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat2 WHERE NOT ignore(*)" -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE IF EXISTS wikistat1 SYNC; DROP TABLE IF EXISTS wikistat2 SYNC; " diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh index e389cf410e8..77c8049bd58 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh @@ -11,7 +11,7 @@ cp $CURDIR/data_ua_parser/os.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ cp $CURDIR/data_ua_parser/browser.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ cp $CURDIR/data_ua_parser/device.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" drop dictionary if exists regexp_os; drop dictionary if exists regexp_browser; drop dictionary if exists regexp_device; @@ -61,10 +61,10 @@ create table user_agents Engine = Log(); " -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" insert into user_agents select ua from input('ua String') FORMAT LineAsString" < $CURDIR/data_ua_parser/useragents.txt -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" select ua, device, concat(tupleElement(browser, 1), ' ', tupleElement(browser, 2), '.', tupleElement(browser, 3)) as browser , concat(tupleElement(os, 1), ' ', tupleElement(os, 2), '.', tupleElement(os, 3), '.', tupleElement(os, 4)) as os @@ -74,7 +74,7 @@ from ( dictGet('regexp_device', 'device_replacement', ua) device from user_agents) order by ua; " -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" drop dictionary if exists regexp_os; drop dictionary if exists regexp_browser; drop dictionary if exists regexp_device; diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh index 68a87a14320..dec03bcaaa0 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh @@ -27,7 +27,7 @@ cat > "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < /dev/null & diff --git a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh b/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh index 3b0d2309784..9f743581e1a 100755 --- a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh +++ b/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh @@ -13,7 +13,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function get_query_id() { random_str 10; } -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists buf; drop table if exists dist; drop table if exists data; @@ -31,7 +31,7 @@ query_id="$(get_query_id)" # test, since we care about the difference between NOW() and there should # not be any significant difference. $CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist" -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; " @@ -42,25 +42,25 @@ query_id="$(get_query_id)" # this query (and all subsequent) should reuse the previous connection (at least most of the time) $CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist" -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; " echo "INSERT" query_id="$(get_query_id)" -$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -nm -q " +$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -m -q " insert into dist_dist values (1),(2); select * from data; " sleep 1 -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist_dist" +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "system flush distributed dist_dist" sleep 1 -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist" +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "system flush distributed dist" echo "CHECK" -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " select * from data order by key; system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; diff --git a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh index ff534a6a2e6..5015521ca4d 100755 --- a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh +++ b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh echo "INSERT TO S3" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " INSERT INTO TABLE FUNCTION s3('http://localhost:11111/test/profile_events.csv', 'test', 'testtest', 'CSV', 'number UInt64') SELECT number FROM numbers(1000000) SETTINGS s3_max_single_part_upload_size = 10, s3_truncate_on_insert = 1; " 2>&1 | $CLICKHOUSE_LOCAL -q " WITH '(\\w+): (\\d+)' AS pattern, @@ -30,7 +30,7 @@ SELECT * FROM ( " echo "CHECK WITH query_log" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " SYSTEM FLUSH LOGS; SELECT type, 'S3CreateMultipartUpload', ProfileEvents['S3CreateMultipartUpload'], @@ -45,7 +45,7 @@ ORDER BY query_start_time DESC; " echo "CREATE" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS times; CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t SETTINGS @@ -56,29 +56,29 @@ CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t " echo "INSERT" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " INSERT INTO times SELECT now() + INTERVAL 1 day SETTINGS optimize_on_insert = 0; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "READ" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " SELECT '1', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "INSERT and READ INSERT" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " INSERT INTO times SELECT now() + INTERVAL 2 day SETTINGS optimize_on_insert = 0; SELECT '2', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1; INSERT INTO times SELECT now() + INTERVAL 3 day SETTINGS optimize_on_insert = 0; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "DROP" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " DROP TABLE times; " echo "CHECK with query_log" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " SYSTEM FLUSH LOGS; SELECT type, query, diff --git a/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh b/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh index c1f28f9f079..84be5959344 100755 --- a/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh +++ b/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh @@ -21,7 +21,7 @@ wait_for_number_of_parts() { echo "$res" } -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE IF EXISTS test_without_merge; DROP TABLE IF EXISTS test_replicated; @@ -34,7 +34,7 @@ INSERT INTO test_without_merge SELECT 3;" wait_for_number_of_parts 'test_without_merge' 1 10 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_without_merge; SELECT 'With merge replicated any part range'; @@ -47,7 +47,7 @@ INSERT INTO test_replicated SELECT 3;" wait_for_number_of_parts 'test_replicated' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_replicated; SELECT 'With merge replicated partition only'; @@ -60,7 +60,7 @@ INSERT INTO test_replicated SELECT 3;" wait_for_number_of_parts 'test_replicated' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_replicated' AND active; diff --git a/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh b/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh index 2a94f940327..a540a0981b3 100755 --- a/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh +++ b/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh @@ -9,6 +9,6 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) echo " DROP USER IF EXISTS postgresql_user; CREATE USER postgresql_user HOST IP '127.0.0.1' IDENTIFIED WITH no_password; -" | $CLICKHOUSE_CLIENT -n +" | $CLICKHOUSE_CLIENT psql --host localhost --port ${CLICKHOUSE_PORT_POSTGRESQL} ${CLICKHOUSE_DATABASE} --user postgresql_user -c "SELECT 1.23::Decimal256(70) AS test;" diff --git a/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh b/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh index cfb38c60615..dcd83f9fec3 100755 --- a/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh +++ b/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh @@ -12,7 +12,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # too slow with this. # # Unfortunately, the test has to buffer it in memory. -$CLICKHOUSE_CLIENT --max_memory_usage 16G -nm -q " +$CLICKHOUSE_CLIENT --max_memory_usage 16G -m -q " INSERT INTO FUNCTION s3('http://localhost:11111/test/$CLICKHOUSE_DATABASE/test_INT_MAX.tsv', '', '', 'TSV') SELECT repeat('a', 1024) FROM numbers((pow(2, 30) * 2) / 1024) SETTINGS s3_max_single_part_upload_size = '5Gi'; diff --git a/tests/queries/0_stateless/02725_start_stop_fetches.sh b/tests/queries/0_stateless/02725_start_stop_fetches.sh index c9922455d94..604d0774b83 100755 --- a/tests/queries/0_stateless/02725_start_stop_fetches.sh +++ b/tests/queries/0_stateless/02725_start_stop_fetches.sh @@ -10,7 +10,7 @@ set -e NUM_REPLICAS=5 for i in $(seq 1 $NUM_REPLICAS); do - $CLICKHOUSE_CLIENT -n -q " + $CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS r$i SYNC; CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x SETTINGS replicated_deduplication_window = 1, allow_remote_fs_zero_copy_replication = 1; " diff --git a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh index 78659b70129..6e40495157b 100755 --- a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh +++ b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) for DISK in s3_disk s3_cache do - ${CLICKHOUSE_CLIENT} -n --query " + ${CLICKHOUSE_CLIENT} --query " DROP TABLE IF EXISTS test; CREATE TABLE test (id Int32, empty Array(Int32)) ENGINE=MergeTree ORDER BY id @@ -17,13 +17,13 @@ do SELECT * FROM test; " - ${CLICKHOUSE_CLIENT} -n --query " + ${CLICKHOUSE_CLIENT} --query " BACKUP TABLE test TO Disk('backups', 'test_s3_backup'); DROP TABLE test; RESTORE TABLE test FROM Disk('backups', 'test_s3_backup'); " &>/dev/null - ${CLICKHOUSE_CLIENT} -n --query " + ${CLICKHOUSE_CLIENT} --query " SELECT * FROM test; SELECT empty FROM test; " diff --git a/tests/queries/0_stateless/02766_prql.sh b/tests/queries/0_stateless/02766_prql.sh index 85b1167027c..d1e4dca070c 100755 --- a/tests/queries/0_stateless/02766_prql.sh +++ b/tests/queries/0_stateless/02766_prql.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -q " CREATE TEMPORARY TABLE IF NOT EXISTS aboba ( user_id UInt32, diff --git a/tests/queries/0_stateless/02770_async_buffer_ignore.sh b/tests/queries/0_stateless/02770_async_buffer_ignore.sh index 37f002767d6..af9f18b232f 100755 --- a/tests/queries/0_stateless/02770_async_buffer_ignore.sh +++ b/tests/queries/0_stateless/02770_async_buffer_ignore.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " DROP TABLE IF EXISTS test_s3; CREATE TABLE test_s3 (a UInt64, b UInt64) @@ -17,7 +17,7 @@ INSERT INTO test_s3 SELECT number, number FROM numbers(1000000); query="SELECT sum(b) FROM test_s3 WHERE a >= 100000 AND a <= 102000" query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " SELECT ProfileEvents['S3ReadRequestsCount'], ProfileEvents['ReadBufferFromS3Bytes'], diff --git a/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh b/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh index c69c635f6ed..53d2832c589 100755 --- a/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh +++ b/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() @@ -22,7 +22,7 @@ INSERT INTO test SELECT number, randomString(100) FROM numbers(1000000); " QUERY_ID=$RANDOM -$CLICKHOUSE_CLIENT --query_id "$QUERY_ID" -nm -q " +$CLICKHOUSE_CLIENT --query_id "$QUERY_ID" -m -q " SET enable_filesystem_cache_log = 1; SYSTEM DROP FILESYSTEM CACHE; SELECT * FROM test WHERE NOT ignore() LIMIT 1 FORMAT Null; @@ -49,14 +49,14 @@ WHERE query_id = '$QUERY_ID' " # File segments cannot be less that 20Mi, # except for last file segment in a file or if file size is less. -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size AND file_segment_size < 20 * 1024 * 1024; " -all=$($CLICKHOUSE_CLIENT -nm -q " +all=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size; ") @@ -68,7 +68,7 @@ else echo "FAIL" fi -count=$($CLICKHOUSE_CLIENT -nm -q " +count=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size @@ -87,21 +87,21 @@ FROM (SELECT * FROM ($query)) AS cache_log INNER JOIN system.filesystem_cache AS cache ON cache_log.cache_path = cache.cache_path " -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size AND downloaded_size < 20 * 1024 * 1024; " -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size AND formatReadableSize(downloaded_size) not in ('20.00 MiB', '40.00 MiB'); " -all=$($CLICKHOUSE_CLIENT -nm -q " +all=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_size < file_size AND file_segment_range_end + 1 != file_size; ") @@ -112,7 +112,7 @@ else echo "FAIL" fi -count2=$($CLICKHOUSE_CLIENT -nm -q " +count2=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size diff --git a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh index 5a37d51233d..239485ab8dd 100755 --- a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh +++ b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " DROP TABLE IF EXISTS test_s3; CREATE TABLE test_s3 (a UInt64, b UInt64) @@ -25,7 +25,7 @@ do query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" - RES=$(${CLICKHOUSE_CLIENT} -nm --query " + RES=$(${CLICKHOUSE_CLIENT} -m --query " SELECT ProfileEvents['DiskConnectionsPreserved'] > 0 FROM system.query_log WHERE type = 'QueryFinish' @@ -41,7 +41,7 @@ done while true do - query_id=$(${CLICKHOUSE_CLIENT} -nq " + query_id=$(${CLICKHOUSE_CLIENT} -q " create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n; set insert_keeper_fault_injection_probability=0; insert into mut values (1, 2, 3), (10, 20, 30); @@ -60,7 +60,7 @@ do ) limit 1 settings max_threads=1; " 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" - RES=$(${CLICKHOUSE_CLIENT} -nm --query " + RES=$(${CLICKHOUSE_CLIENT} -m --query " SELECT ProfileEvents['StorageConnectionsPreserved'] > 0 FROM system.query_log WHERE type = 'QueryFinish' diff --git a/tests/queries/0_stateless/02801_backup_native_copy.sh b/tests/queries/0_stateless/02801_backup_native_copy.sh index b8ee97a7c7d..bc485f903bc 100755 --- a/tests/queries/0_stateless/02801_backup_native_copy.sh +++ b/tests/queries/0_stateless/02801_backup_native_copy.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk'; insert into data select * from numbers(10); @@ -16,28 +16,28 @@ $CLICKHOUSE_CLIENT -nm -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_no_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " diff --git a/tests/queries/0_stateless/02803_backup_tmp_files.sh b/tests/queries/0_stateless/02803_backup_tmp_files.sh index d86beae4923..2208d3d32ef 100755 --- a/tests/queries/0_stateless/02803_backup_tmp_files.sh +++ b/tests/queries/0_stateless/02803_backup_tmp_files.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk'; insert into data select * from numbers(10); diff --git a/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh b/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh index 63fa60bd548..4d7ab4063be 100755 --- a/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh +++ b/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() @@ -17,17 +17,17 @@ SETTINGS disk = disk(name = 's3_disk', type = cache, max_size = '100Ki', path = disk_name="${CLICKHOUSE_TEST_UNIQUE_NAME}" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = ${CLICKHOUSE_TEST_UNIQUE_NAME}, disk = s3_disk); """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ diff --git a/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh b/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh index 8a4a2e906b0..a7dfa035a22 100755 --- a/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh +++ b/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh @@ -9,7 +9,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="${CLICKHOUSE_TEST_UNIQUE_NAME}" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() @@ -22,29 +22,29 @@ query_id=$RANDOM $CLICKHOUSE_CLIENT --query_id "$query_id" --query "SELECT * FROM test FORMAT Null SETTINGS enable_filesystem_cache_log = 1" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY kek; """ 2>&1 | grep -q "Invalid cache key hex: kek" && echo "OK" || echo "FAIL" ${CLICKHOUSE_CLIENT} -q " system flush logs" -key=$($CLICKHOUSE_CLIENT -nm --query """ +key=$($CLICKHOUSE_CLIENT -m --query """ SELECT key FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -offset=$($CLICKHOUSE_CLIENT -nm --query """ +offset=$($CLICKHOUSE_CLIENT -m --query """ SELECT offset FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key' AND file_segment_range_begin = $offset; """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY $key OFFSET $offset; """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key' AND file_segment_range_begin = $offset; """ @@ -54,18 +54,18 @@ $CLICKHOUSE_CLIENT --query_id "$query_id" --query "SELECT * FROM test FORMAT Nul ${CLICKHOUSE_CLIENT} -q " system flush logs" -key=$($CLICKHOUSE_CLIENT -nm --query """ +key=$($CLICKHOUSE_CLIENT -m --query """ SELECT key FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key'; """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY $key """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key'; """ diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh index a2b1a953e24..ecd73bcc8a0 100755 --- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh +++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS data; DROP TABLE IF EXISTS data_1; DROP TABLE IF EXISTS data_2; diff --git a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh index 16ac095312c..e6a1d547945 100755 --- a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh +++ b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh @@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple(); insert into data select * from numbers(10); diff --git a/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh b/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh index 4650415c202..830e78e1e9f 100755 --- a/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh +++ b/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh @@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, disk='s3_disk'; -- reading 1e6*8 bytes with 1M bandwith it should take (8-1)/1=7 seconds @@ -15,7 +15,7 @@ $CLICKHOUSE_CLIENT -nm -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --query_id "$query_id" -q "backup table data to S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data/backup2') SETTINGS allow_s3_native_copy=1" --max_backup_bandwidth=1M > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT 'native_copy', @@ -26,7 +26,7 @@ $CLICKHOUSE_CLIENT -nm -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --query_id "$query_id" -q "backup table data to S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data/backup3') SETTINGS allow_s3_native_copy=0" --max_backup_bandwidth=1M > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT 'no_native_copy', diff --git a/tests/queries/0_stateless/02867_storage_set_tsan.sh b/tests/queries/0_stateless/02867_storage_set_tsan.sh index 81ae5f0bda8..f6b684c0a55 100755 --- a/tests/queries/0_stateless/02867_storage_set_tsan.sh +++ b/tests/queries/0_stateless/02867_storage_set_tsan.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -mn -q """ +$CLICKHOUSE_CLIENT -m -q """ DROP TABLE IF EXISTS t1_02867; CREATE TABLE t1_02867 (x UInt64) ENGINE=Set(); """ @@ -39,4 +39,4 @@ repeat_truncate_insert & sleep 10 -$CLICKHOUSE_CLIENT -mn -q "DROP TABLE IF EXISTS t1_02867;" +$CLICKHOUSE_CLIENT -m -q "DROP TABLE IF EXISTS t1_02867;" diff --git a/tests/queries/0_stateless/02900_union_schema_inference_mode.sh b/tests/queries/0_stateless/02900_union_schema_inference_mode.sh index a0fdb5276e0..a091645d1e3 100755 --- a/tests/queries/0_stateless/02900_union_schema_inference_mode.sh +++ b/tests/queries/0_stateless/02900_union_schema_inference_mode.sh @@ -10,14 +10,14 @@ echo '{"a" : 1, "obj" : {"f1" : 1, "f2" : "2020-01-01"}}' > $CLICKHOUSE_TEST_UNI echo '{"b" : 2, "obj" : {"f3" : 2, "f2" : "Some string"}}' > $CLICKHOUSE_TEST_UNIQUE_NAME/data2.jsonl echo '{"c" : "hello"}' > $CLICKHOUSE_TEST_UNIQUE_NAME/data3.jsonl -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl') order by tuple(*) format JSONEachRow; select schema_inference_mode, splitByChar('/', source)[-1] as file, schema from system.schema_inference_cache order by file; " -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data3.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); @@ -25,14 +25,14 @@ desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); cd $CLICKHOUSE_TEST_UNIQUE_NAME/ && tar -cf archive.tar data1.jsonl data2.jsonl data3.jsonl && cd .. -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl') order by tuple(*) format JSONEachRow; select schema_inference_mode, splitByChar('/', source)[-1] as file, schema from system.schema_inference_cache order by file; " -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data3.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); @@ -41,7 +41,7 @@ desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); echo 'Error' > $CLICKHOUSE_TEST_UNIQUE_NAME/data4.jsonl $CLICKHOUSE_LOCAL -q "desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3,4}.jsonl') settings schema_inference_mode='union'" 2>&1 | grep -c -F "CANNOT_EXTRACT_TABLE_STRUCTURE" -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{2,3}.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3,4}.jsonl'); diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index 81ba59fc591..91eeb22c19e 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -67,7 +67,7 @@ curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT sum(is_leader wait; -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " SYSTEM FLUSH LOGS; -- Check that number of ZK request is less then a half of (total replicas * concurrency) diff --git a/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh b/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh index d1cbc54d294..2eccade5c81 100755 --- a/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh +++ b/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh @@ -8,7 +8,7 @@ CURDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists r1; drop table if exists r2; @@ -64,7 +64,7 @@ function insert_duplicates() { wait - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " system sync replica r1; system sync replica r2; " @@ -84,7 +84,7 @@ function loop() do while ! insert_duplicates do - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " truncate table r1; truncate table r2; system sync replica r1; @@ -137,8 +137,8 @@ function list_keeper_nodes() { list_keeper_nodes "${table_shared_id}" -$CLICKHOUSE_CLIENT -nm -q "drop table r1;" --allow_repeated_settings --send_logs_level="error" & -$CLICKHOUSE_CLIENT -nm -q "drop table r2;" --allow_repeated_settings --send_logs_level="error" & +$CLICKHOUSE_CLIENT -m -q "drop table r1;" --allow_repeated_settings --send_logs_level="error" & +$CLICKHOUSE_CLIENT -m -q "drop table r2;" --allow_repeated_settings --send_logs_level="error" & wait list_keeper_nodes "${table_shared_id}" diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh index 2b92a113e91..057f76e63d0 100755 --- a/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh +++ b/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh @@ -10,11 +10,11 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --session_timezone Etc/UTC"`" -$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" +$CLICKHOUSE_CLIENT -q "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" # Basic refreshing. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view a refresh after 2 second engine Memory @@ -23,41 +23,41 @@ $CLICKHOUSE_CLIENT -nq " select '<1: created view>', view, remaining_dependencies, exception, last_refresh_result in ('Unknown', 'Finished') from refreshes; show create a;" # Wait for any refresh. (xargs trims the string and turns \t and \n into spaces) -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" == 'Unknown' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" == 'Unknown' ] do sleep 0.5 done -start_time="`$CLICKHOUSE_CLIENT -nq "select reinterpret(now64(), 'Int64')"`" +start_time="`$CLICKHOUSE_CLIENT -q "select reinterpret(now64(), 'Int64')"`" # Check table contents. -$CLICKHOUSE_CLIENT -nq "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" +$CLICKHOUSE_CLIENT -q "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" # Wait for table contents to change. -res1="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`" +res1="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values'`" while : do - res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" + res2="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values -- $LINENO'`" [ "$res2" == "$res1" ] || break sleep 0.5 done # Wait for another change. while : do - res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" + res3="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values -- $LINENO'`" [ "$res3" == "$res2" ] || break sleep 0.5 done # Check that the two changes were at least 1 second apart, in particular that we're not refreshing # like crazy. This is potentially flaky, but we need at least one test that uses non-mocked timer # to make sure the clock+timer code works at all. If it turns out flaky, increase refresh period above. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<3: time difference at least>', min2(reinterpret(now64(), 'Int64') - $start_time, 1000); select '<4: next refresh in>', next_refresh_time-last_refresh_time from refreshes;" # Create a source table from which views will read. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create table src (x Int8) engine Memory as select 1;" # Switch to fake clock, change refresh schedule, change query. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view a set fake time '2050-01-01 00:00:01'; system wait view a; system refresh view a; @@ -68,19 +68,19 @@ $CLICKHOUSE_CLIENT -nq " select '<4.5: altered>', status, last_refresh_result, next_refresh_time from refreshes; show create a;" # Advance time to trigger the refresh. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<5: no refresh>', count() from a; system test view a set fake time '2052-02-03 04:05:06';" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_time from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<6: refreshed>', * from a; select '<7: refreshed>', status, last_refresh_result, next_refresh_time from refreshes;" # Create a dependent view, refresh it once. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view b refresh every 2 year depends on a (y Int32) engine MergeTree order by y empty as select x*10 as y from a; show create b; system test view b set fake time '2052-11-11 11:11:11'; @@ -88,89 +88,89 @@ $CLICKHOUSE_CLIENT -nq " system wait view b; select '<7.5: created dependent>', last_refresh_time from refreshes where view = 'b';" # Next refresh shouldn't start until the dependency refreshes. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<8: refreshed>', * from b; select '<9: refreshed>', view, status, last_refresh_result, next_refresh_time from refreshes; system test view b set fake time '2054-01-24 23:22:21';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status, next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ] do sleep 0.5 done # Drop the source table, check that refresh fails and doesn't leave a temp table behind. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<9.2: dropping>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase(); drop table src; system refresh view a;" -$CLICKHOUSE_CLIENT -nq "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -q " select '<9.4: dropped>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase();" # Create the source table again, check that refresh succeeds (in particular that tables are looked # up by name rather than uuid). -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<10: creating>', view, status, remaining_dependencies, next_refresh_time from refreshes; create table src (x Int16) engine Memory as select 2; system test view a set fake time '2054-01-01 00:00:01';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] do sleep 0.5 done # Both tables should've refreshed. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<11: chain-refreshed a>', * from a; select '<12: chain-refreshed b>', * from b; select '<13: chain-refreshed>', view, status, remaining_dependencies, last_refresh_result, last_refresh_time, next_refresh_time, exception == '' from refreshes;" # Make the dependent table run ahead by one refresh cycle, make sure it waits for the dependency to # catch up to the same cycle. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view b set fake time '2059-01-01 00:00:00'; system refresh view b;" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2060-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2060-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view b set fake time '2061-01-01 00:00:00'; system test view a set fake time '2057-01-01 00:00:00';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<14: waiting for next cycle>', view, status, remaining_dependencies, next_refresh_time from refreshes; truncate src; insert into src values (3); system test view a set fake time '2060-02-02 02:02:02';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<15: chain-refreshed a>', * from a; select '<16: chain-refreshed b>', * from b; select '<17: chain-refreshed>', view, status, next_refresh_time from refreshes;" # Get to WaitingForDependencies state and remove the depencency. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view b set fake time '2062-03-03 03:03:03'" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " alter table b modify refresh every 2 year" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<18: removed dependency>', view, status, remaining_dependencies, last_refresh_time,next_refresh_time, refresh_count from refreshes where view = 'b'; show create b;" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table src; drop table a; drop table b; diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh index 50a905576d5..2d00d61f253 100755 --- a/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh +++ b/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh @@ -12,29 +12,29 @@ CLICKHOUSE_LOG_COMMENT= CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --allow_materialized_view_with_bad_select=0 --session_timezone Etc/UTC"`" -$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" +$CLICKHOUSE_CLIENT -q "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" # Select from a table that doesn't exist, get an exception. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create table src (x Int8) engine Memory as select 1; create materialized view c refresh every 1 second (x Int64) engine Memory empty as select * from src; drop table src;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes where view = 'c' -- $LINENO" | xargs`" != 'Error' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes where view = 'c' -- $LINENO" | xargs`" != 'Error' ] do sleep 0.5 done # Check exception, create src, expect successful refresh. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<19: exception>', exception ilike '%UNKNOWN_TABLE%' ? '1' : exception from refreshes where view = 'c'; create table src (x Int64) engine Memory as select 1; system refresh view c;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done # Rename table. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<20: unexception>', * from c; rename table c to d; select '<21: rename>', * from d; @@ -42,130 +42,130 @@ $CLICKHOUSE_CLIENT -nq " # Do various things during a refresh. # First make a nonempty view. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table d; truncate src; insert into src values (1); create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x empty as select x + sleepEachRow(1) as x from src settings max_block_size = 1;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done # Stop refreshes. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<23: simple refresh>', * from e; system stop view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] do sleep 0.5 done # Make refreshes slow, wait for a slow refresh to start. (We stopped refreshes first to make sure # we wait for a slow refresh, not a previous fast one.) -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " insert into src select * from numbers(1000) settings max_block_size=1; system start view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] do sleep 0.5 done # Rename. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " rename table e to f; select '<24: rename during refresh>', * from f; select '<25: rename during refresh>', view, status from refreshes where view = 'f'; alter table f modify refresh after 10 year;" # Cancel. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system cancel view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Scheduled' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Scheduled' ] do sleep 0.5 done # Check that another refresh doesn't immediately start after the cancelled one. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<27: cancelled>', view, status, last_refresh_result from refreshes where view = 'f'; system refresh view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Running' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Running' ] do sleep 0.5 done # Drop. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table f; select '<28: drop during refresh>', view, status from refreshes; select '<28: drop during refresh>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase()" # Try OFFSET and RANDOMIZE FOR. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view g refresh every 1 week offset 3 day 4 hour randomize for 4 day 1 hour (x Int64) engine Memory empty as select 42 as x; show create g; system test view g set fake time '2050-02-03 15:30:13';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] +while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " with '2050-02-10 04:00:00'::DateTime as expected select '<29: randomize>', abs(next_refresh_time::Int64 - expected::Int64) <= 3600*(24*4+1), next_refresh_time != expected from refreshes;" # Send data 'TO' an existing table. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table g; create table dest (x Int64) engine MergeTree order by x; truncate src; insert into src values (1); create materialized view h refresh every 1 second to dest empty as select x*10 as x from src; show create h;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<30: to existing table>', * from dest; insert into src values (2);" -while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest -- $LINENO" | xargs`" != '2' ] +while [ "`$CLICKHOUSE_CLIENT -q "select count() from dest -- $LINENO" | xargs`" != '2' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<31: to existing table>', * from dest; drop table dest; drop table h;" # Retries. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view h2 refresh after 1 year settings refresh_retries = 10 (x Int64) engine Memory as select x*10 + throwIf(x % 2 == 0) as x from src;" -$CLICKHOUSE_CLIENT -nq "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -q " select '<31.5: will retry>', last_refresh_result, retry > 0 from refreshes; create table src2 (x Int8) engine Memory; insert into src2 values (1); exchange tables src and src2; drop table src2;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result, retry from refreshes -- $LINENO" | xargs`" != 'Finished 0' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result, retry from refreshes -- $LINENO" | xargs`" != 'Finished 0' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<31.6: did retry>', x from h2; drop table h2" # EMPTY -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view i refresh after 1 year engine Memory empty as select number as x from numbers(2); create materialized view j refresh after 1 year engine Memory as select number as x from numbers(2);" -while [ "`$CLICKHOUSE_CLIENT -nq "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] +while [ "`$CLICKHOUSE_CLIENT -q "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<32: empty>', view, status, last_refresh_result, retry from refreshes order by view; drop table i; drop table j;" # APPEND -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view k refresh every 10 year append (x Int64) engine Memory empty as select x*10 as x from src; select '<33: append>', * from k; system refresh view k; @@ -177,7 +177,7 @@ $CLICKHOUSE_CLIENT -nq " system wait view k; select '<35: append>', * from k order by x;" # ALTER to non-APPEND -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " alter table k modify refresh every 10 year; system wait view k; system refresh view k; @@ -187,7 +187,7 @@ $CLICKHOUSE_CLIENT -nq " truncate table src;" # APPEND + TO + regular materialized view reading from it. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create table mid (x Int64) engine MergeTree order by x; create materialized view l refresh every 10 year append to mid empty as select x*10 as x from src; create materialized view m (x Int64) engine Memory as select x*10 as x from mid; @@ -204,19 +204,19 @@ $CLICKHOUSE_CLIENT -nq " drop table mid;" # Failing to create inner table. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view n refresh every 1 second (x Int64) engine MergeTree as select 1 as x from numbers(2);" 2>/dev/null || echo "creating MergeTree without ORDER BY failed, as expected" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view n refresh every 1 second (x Int64) engine MergeTree order by x as select 1 as x from numbers(2); drop table n;" # Reading from table that doesn't exist yet. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view o refresh every 1 second (x Int64) engine Memory as select x from nonexist; -- { serverError UNKNOWN_TABLE } create materialized view o (x Int64) engine Memory as select x from nonexist; -- { serverError UNKNOWN_TABLE } create materialized view o (x Int64) engine Memory as select x from nope.nonexist; -- { serverError UNKNOWN_DATABASE } create materialized view o refresh every 1 second (x Int64) engine Memory as select x from nope.nonexist settings allow_materialized_view_with_bad_select = 1; drop table o;" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table refreshes;" diff --git a/tests/queries/0_stateless/02941_variant_type_1.sh b/tests/queries/0_stateless/02941_variant_type_1.sh index c12bece6d54..ef1d1deff4e 100755 --- a/tests/queries/0_stateless/02941_variant_type_1.sh +++ b/tests/queries/0_stateless/02941_variant_type_1.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test1_insert() { echo "test1 insert" - $CH_CLIENT -nmq "insert into test select number, NULL from numbers(3); + $CH_CLIENT -mq "insert into test select number, NULL from numbers(3); insert into test select number + 3, number from numbers(3); insert into test select number + 6, ('str_' || toString(number))::Variant(String) from numbers(3); insert into test select number + 9, ('lc_str_' || toString(number))::LowCardinality(String) from numbers(3); @@ -21,7 +21,7 @@ insert into test select number + 15, range(number + 1)::Array(UInt64) from numbe function test1_select() { echo "test1 select" - $CH_CLIENT -nmq "select v from test order by id; + $CH_CLIENT -mq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; @@ -36,7 +36,7 @@ select v.\`Array(UInt64)\`.size0 from test order by id;" function test2_insert() { echo "test2 insert" - $CH_CLIENT -nmq "insert into test select number, NULL from numbers(3); + $CH_CLIENT -mq "insert into test select number, NULL from numbers(3); insert into test select number + 3, number % 2 ? NULL : number from numbers(3); insert into test select number + 6, number % 2 ? NULL : ('str_' || toString(number))::Variant(String) from numbers(3); insert into test select number + 9, number % 2 ? CAST(NULL, 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') : CAST(('lc_str_' || toString(number))::LowCardinality(String), 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') from numbers(3); @@ -47,7 +47,7 @@ insert into test select number + 15, number % 2 ? CAST(NULL, 'Variant(String, UI function test2_select() { echo "test2 select" - $CH_CLIENT -nmq "select v from test order by id; + $CH_CLIENT -mq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; @@ -68,7 +68,7 @@ function test3_insert() function test3_select() { echo "test3 select" - $CH_CLIENT -nmq "select v from test order by id; + $CH_CLIENT -mq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; diff --git a/tests/queries/0_stateless/02941_variant_type_2.sh b/tests/queries/0_stateless/02941_variant_type_2.sh index e93dfac8510..7017edac525 100755 --- a/tests/queries/0_stateless/02941_variant_type_2.sh +++ b/tests/queries/0_stateless/02941_variant_type_2.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test4_insert() { echo "test4 insert" - $CH_CLIENT -nmq "insert into test select number, NULL from numbers(100000); + $CH_CLIENT -mq "insert into test select number, NULL from numbers(100000); insert into test select number + 100000, number from numbers(100000); insert into test select number + 200000, ('str_' || toString(number))::Variant(String) from numbers(100000); insert into test select number + 300000, ('lc_str_' || toString(number))::LowCardinality(String) from numbers(100000); @@ -21,7 +21,7 @@ insert into test select number + 500000, range(number % 20 + 1)::Array(UInt64) f function test4_select { echo "test4 select" - $CH_CLIENT -nmq "select v from test format Null; + $CH_CLIENT -mq "select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; select count() from test where isNotNull(v.String); diff --git a/tests/queries/0_stateless/02941_variant_type_3.sh b/tests/queries/0_stateless/02941_variant_type_3.sh index cc0fde5b689..5d923a47e9b 100755 --- a/tests/queries/0_stateless/02941_variant_type_3.sh +++ b/tests/queries/0_stateless/02941_variant_type_3.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test5_insert() { echo "test5 insert" - $CH_CLIENT -nmq " + $CH_CLIENT -mq " insert into test select number, NULL from numbers(200000); insert into test select number + 200000, number % 2 ? NULL : number from numbers(200000); insert into test select number + 400000, number % 2 ? NULL : ('str_' || toString(number))::Variant(String) from numbers(200000); @@ -22,7 +22,7 @@ insert into test select number + 1000000, number % 2 ? CAST(NULL, 'Variant(Strin function test5_select() { echo "test5 select" - $CH_CLIENT -nmq " + $CH_CLIENT -mq " select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; diff --git a/tests/queries/0_stateless/02941_variant_type_4.sh b/tests/queries/0_stateless/02941_variant_type_4.sh index 93a1770d05e..88f6a475d46 100755 --- a/tests/queries/0_stateless/02941_variant_type_4.sh +++ b/tests/queries/0_stateless/02941_variant_type_4.sh @@ -17,7 +17,7 @@ function test6_insert() function test6_select() { echo "test6 select" - $CH_CLIENT -nmq "select v from test format Null; + $CH_CLIENT -mq "select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; select count() from test where isNotNull(v.String); diff --git a/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh b/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh index cb099bb59ae..f9e84dc2b50 100755 --- a/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh +++ b/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh @@ -10,7 +10,7 @@ disk_name="s3_cache_02944" $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a String) engine=MergeTree() ORDER BY tuple() SETTINGS disk = '$disk_name'; INSERT INTO test SELECT randomString(100); @@ -33,7 +33,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -47,7 +47,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -63,7 +63,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -77,7 +77,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" diff --git a/tests/queries/0_stateless/02960_polygon_bound_bug.sh b/tests/queries/0_stateless/02960_polygon_bound_bug.sh index 0c3db01a77c..1e9be36da55 100755 --- a/tests/queries/0_stateless/02960_polygon_bound_bug.sh +++ b/tests/queries/0_stateless/02960_polygon_bound_bug.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_LOCAL -nm -q "CREATE TABLE test_table (geom MultiPolygon) engine=MergeTree ORDER BY geom; +$CLICKHOUSE_LOCAL -m -q "CREATE TABLE test_table (geom MultiPolygon) engine=MergeTree ORDER BY geom; INSERT INTO test_table SELECT * FROM file('$CURDIR/data_parquet/02960_polygon_bound_bug.parquet', Parquet); CREATE DICTIONARY test_dict (geom MultiPolygon) PRIMARY KEY geom SOURCE (CLICKHOUSE(TABLE 'test_table')) LIFETIME(MIN 0 MAX 0) LAYOUT(POLYGON(STORE_POLYGON_KEY_COLUMN 1)); SELECT dictHas(test_dict,(174.84729269276494,-36.99524960275426));" diff --git a/tests/queries/0_stateless/02961_storage_config_volume_priority.sh b/tests/queries/0_stateless/02961_storage_config_volume_priority.sh index 145b921a750..1bd86ee0c75 100755 --- a/tests/queries/0_stateless/02961_storage_config_volume_priority.sh +++ b/tests/queries/0_stateless/02961_storage_config_volume_priority.sh @@ -24,7 +24,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='error'; SYSTEM RELOAD CONFIG" 2>&1 | grep -c 'volume_priority values must be unique across the policy' @@ -40,7 +40,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='error'; SYSTEM RELOAD CONFIG" 2>&1 | grep -c 'volume_priority values must cover the range from 1 to N (lowest priority specified) without gaps' diff --git a/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh b/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh index 24fe964b824..3b57941452d 100755 --- a/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh +++ b/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="02963_remote_read_bug" -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, s String) @@ -22,7 +22,7 @@ OPTIMIZE TABLE test FINAL; query_id=$(random_str 10) -$CLICKHOUSE_CLIENT -nm --query_id "$query_id" --query " +$CLICKHOUSE_CLIENT -m --query_id "$query_id" --query " WITH RANDOM_SET AS ( SELECT rand32() % 10000 FROM numbers(100) ) @@ -37,7 +37,7 @@ SETTINGS merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem = 1, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem = 1; " -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " SYSTEM FLUSH LOGS; -- This threshold was determined experimentally - before the fix this ratio had values around 50K diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh index 1089eb4051f..b4271c3d29b 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh index 7a0e2d9bfdb..ed13bf3321b 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh index c4b44ce11c5..9abe870f4e4 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02969_auto_format_detection.sh b/tests/queries/0_stateless/02969_auto_format_detection.sh index 88d6575e499..ab21dbb3df8 100755 --- a/tests/queries/0_stateless/02969_auto_format_detection.sh +++ b/tests/queries/0_stateless/02969_auto_format_detection.sh @@ -24,12 +24,12 @@ $CLICKHOUSE_LOCAL -q "select * from generateRandom('a UInt64, b String, c Array( $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE', auto, 'a UInt64, b String, c Array(UInt64), d Tuple(a UInt64, b String)')" -$CLICKHOUSE_LOCAL -nmq " +$CLICKHOUSE_LOCAL -mq " desc file('$DATA_FILE'); desc file('$DATA_FILE'); " -$CLICKHOUSE_LOCAL -nmq " +$CLICKHOUSE_LOCAL -mq " desc file('$DATA_FILE', JSONEachRow); desc file('$DATA_FILE'); " @@ -39,7 +39,7 @@ $CLICKHOUSE_LOCAL -q "select * from generateRandom('a UInt64, b String, c Array( $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE.{1,2}')" $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE.{1,2}') settings schema_inference_mode='union'" 2>&1 | grep -c "CANNOT_DETECT_FORMAT" -$CLICKHOUSE_LOCAL -nmq " +$CLICKHOUSE_LOCAL -mq " desc file('$DATA_FILE.2'); desc file('$DATA_FILE.{1,2}'); " diff --git a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh index d543f7195a9..a46f0018ad3 100755 --- a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh +++ b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh @@ -19,7 +19,7 @@ $CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -q "create database $n CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT/--database=$CLICKHOUSE_DATABASE/--database=$new_database} CLICKHOUSE_DATABASE="$new_database" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by key; insert into data values (1); @@ -29,7 +29,7 @@ $CLICKHOUSE_CLIENT -nm -q " # suppress output $CLICKHOUSE_CLIENT -q "backup table data to S3('http://localhost:11111/test/s3_plain/backups/$CLICKHOUSE_DATABASE', 'test', 'testtest')" > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table data; attach table data (key Int) engine=MergeTree() order by key settings diff --git a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh index eec05c81344..522fdd6096a 100755 --- a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh +++ b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh @@ -18,7 +18,7 @@ $CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -q "create database $n CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT/--database=$CLICKHOUSE_DATABASE/--database=$new_database} CLICKHOUSE_DATABASE="$new_database" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data_read; drop table if exists data_write; @@ -33,7 +33,7 @@ $CLICKHOUSE_CLIENT -nm -q " # suppress output $CLICKHOUSE_CLIENT -q "backup table data_read to S3('http://localhost:11111/test/s3_plain/backups/$CLICKHOUSE_DATABASE', 'test', 'testtest')" > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table data_read; attach table data_read (key Int) engine=ReplicatedMergeTree('/tables/{database}/data', 'read') order by key settings @@ -57,7 +57,7 @@ echo "Files before DETACH TABLE" # sed to match any part, since in case of fault injection part name may not be all_0_0_0 but all_1_1_0 clickhouse-disks -C "$config" --disk s3_plain_disk --query "list --recursive $path" | tail -n+2 | sed 's/all_[^_]*_[^_]*_0/all_X_X_X/g' -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " detach table data_read; detach table data_write; " diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh index ef4866cb69e..fca8b7a5e97 100755 --- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh index dd4b1f7cec0..08d5ae3a6b2 100755 --- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh index 33386c76edb..678b1db4d68 100755 --- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh index b66ef83abf2..606fda6d4fd 100755 --- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh index f9e1838f491..b571631dc8f 100755 --- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh index 698e70d4064..e4358011e50 100755 --- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_s3_plain_rewritable.sh b/tests/queries/0_stateless/03008_s3_plain_rewritable.sh index 8eea7940774..c13674b9bd1 100755 --- a/tests/queries/0_stateless/03008_s3_plain_rewritable.sh +++ b/tests/queries/0_stateless/03008_s3_plain_rewritable.sh @@ -9,7 +9,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query "drop table if exists test_s3_mt" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " create table test_s3_mt (a Int32, b Int64, c Int64) engine = MergeTree() partition by intDiv(a, 1000) order by tuple(a, b) settings disk = disk( name = 03008_s3_plain_rewritable, @@ -19,7 +19,7 @@ settings disk = disk( secret_access_key = clickhouse); " -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " insert into test_s3_mt (*) values (1, 2, 0), (2, 2, 2), (3, 1, 9), (4, 7, 7), (5, 10, 2), (6, 12, 5); insert into test_s3_mt (*) select number, number, number from numbers_mt(10000); select count(*) from test_s3_mt; @@ -31,13 +31,13 @@ ${CLICKHOUSE_CLIENT} --query "optimize table test_s3_mt final" ${CLICKHOUSE_CLIENT} -m --query " alter table test_s3_mt add projection test_s3_mt_projection (select * order by b)" 2>&1 | grep -Fq "SUPPORT_IS_DISABLED" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " alter table test_s3_mt update c = 0 where a % 2 = 1; alter table test_s3_mt add column d Int64 after c; alter table test_s3_mt drop column c; " 2>&1 | grep -Fq "SUPPORT_IS_DISABLED" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " detach table test_s3_mt; attach table test_s3_mt; " diff --git a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh index cba5317fcfa..278ad6f0654 100755 --- a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh +++ b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="s3_cache" -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a String) engine=MergeTree() ORDER BY tuple() SETTINGS disk = '$disk_name'; INSERT INTO test SELECT randomString(1000); @@ -26,7 +26,7 @@ sed -i "s|$prev_max_size<\/max_size>|$new_max_size<\/max_siz # echo $prev_max_size # echo $new_max_size -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" @@ -36,7 +36,7 @@ $CLICKHOUSE_CLIENT --query "SELECT current_size <= max_size FROM system.filesyst sed -i "s|$new_max_size<\/max_size>|$prev_max_size<\/max_size>|" $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" diff --git a/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh index 9ea86105a3a..8183b636549 100755 --- a/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from (select sumMerge(sum) as sum from test group by id, _part) group by sum order by sum, count()" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from (select sumMerge(sum) as sum from test group by id, _part) group by sum order by sum, count()" $CH_CLIENT -q "drop table test" diff --git a/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh index 9a2a6dd957c..ddafa9db210 100755 --- a/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh @@ -16,7 +16,7 @@ function test() $CH_CLIENT -q "insert into test select number, -1, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh index 0199035a3df..b1046ca08ba 100755 --- a/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh index e2ea5bc3466..f3693a2e7b1 100755 --- a/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from test group by sum order by sum, count()" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from test group by sum order by sum, count()" $CH_CLIENT -q "drop table test" diff --git a/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh index 43607cf95a8..bba92a2ee06 100755 --- a/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "insert into test select number, -1, number >= 75000 ? 2 : 1, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03151_unload_index_race.sh b/tests/queries/0_stateless/03151_unload_index_race.sh index 7e9dfa7cddc..4cdf06abae2 100755 --- a/tests/queries/0_stateless/03151_unload_index_race.sh +++ b/tests/queries/0_stateless/03151_unload_index_race.sh @@ -42,8 +42,8 @@ function thread_alter_settings() { local TIMELIMIT=$((SECONDS+$1)) while [ $SECONDS -lt "$TIMELIMIT" ]; do - $CLICKHOUSE_CLIENT -n --query "ALTER TABLE t MODIFY SETTING primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns=0.$RANDOM" - $CLICKHOUSE_CLIENT -n --query "SYSTEM UNLOAD PRIMARY KEY t" + $CLICKHOUSE_CLIENT --query "ALTER TABLE t MODIFY SETTING primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns=0.$RANDOM" + $CLICKHOUSE_CLIENT --query "SYSTEM UNLOAD PRIMARY KEY t" sleep 0.0$RANDOM done } @@ -52,7 +52,7 @@ function thread_query_table() { local TIMELIMIT=$((SECONDS+$1)) while [ $SECONDS -lt "$TIMELIMIT" ]; do - COUNT=$($CLICKHOUSE_CLIENT -n --query "SELECT count() FROM t where not ignore(*);") + COUNT=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM t where not ignore(*);") if [ "$COUNT" -ne "2000" ]; then echo "$COUNT" fi diff --git a/tests/queries/0_stateless/03172_error_log_table_not_empty.sh b/tests/queries/0_stateless/03172_error_log_table_not_empty.sh index 22a2fd82c64..d1afafa77a5 100755 --- a/tests/queries/0_stateless/03172_error_log_table_not_empty.sh +++ b/tests/queries/0_stateless/03172_error_log_table_not_empty.sh @@ -15,7 +15,7 @@ errors_222=$($CLICKHOUSE_CLIENT -q "SELECT sum(value) FROM system.error_log WHER errors_333=$($CLICKHOUSE_CLIENT -q "SELECT sum(value) FROM system.error_log WHERE code = 333") # Throw three random errors: 111, 222 and 333 and wait for more than collect_interval_milliseconds to ensure system.error_log is flushed -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT throwIf(true, 'error_log', toInt16(111)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 111 } SELECT throwIf(true, 'error_log', toInt16(222)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 222 } SELECT throwIf(true, 'error_log', toInt16(333)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 333 } @@ -24,14 +24,14 @@ SYSTEM FLUSH LOGS; " # Check that the three random errors are propagated -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT sum(value) > $errors_111 FROM system.error_log WHERE code = 111; SELECT sum(value) > $errors_222 FROM system.error_log WHERE code = 222; SELECT sum(value) > $errors_333 FROM system.error_log WHERE code = 333; " # Ensure that if we throw them again, they're still propagated -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT throwIf(true, 'error_log', toInt16(111)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 111 } SELECT throwIf(true, 'error_log', toInt16(222)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 222 } SELECT throwIf(true, 'error_log', toInt16(333)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 333 } @@ -39,7 +39,7 @@ SELECT sleep(2) format NULL; SYSTEM FLUSH LOGS; " -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT sum(value) > $(($errors_111+1)) FROM system.error_log WHERE code = 111; SELECT sum(value) > $(($errors_222+1)) FROM system.error_log WHERE code = 222; SELECT sum(value) > $(($errors_333+1)) FROM system.error_log WHERE code = 333; diff --git a/tests/queries/0_stateless/03203_hive_style_partitioning.sh b/tests/queries/0_stateless/03203_hive_style_partitioning.sh index 60e8a6e9faa..fdd0fcd8ec5 100755 --- a/tests/queries/0_stateless/03203_hive_style_partitioning.sh +++ b/tests/queries/0_stateless/03203_hive_style_partitioning.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE FILE HIVE PARTITIONING'" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM file('$CURDIR/data_hive/partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -22,20 +22,20 @@ SELECT toTypeName(array), toTypeName(float) FROM file('$CURDIR/data_hive/partiti SELECT count(*) FROM file('$CURDIR/data_hive/partitioning/number=42/date=2020-01-01/sample.parquet') WHERE number = 42; """ -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT identifier FROM file('$CURDIR/data_hive/partitioning/identifier=*/email.csv') LIMIT 2; SELECT a FROM file('$CURDIR/data_hive/partitioning/a=b/a=b/sample.parquet') LIMIT 1; """ -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM file('$CURDIR/data_hive/partitioning/column0=Elizabeth/column0=Elizabeth1/sample.parquet') LIMIT 10; """ 2>&1 | grep -c "INCORRECT_DATA" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 0; SELECT *, non_existing_column FROM file('$CURDIR/data_hive/partitioning/non_existing_column=Elizabeth/sample.parquet') LIMIT 10; @@ -45,14 +45,14 @@ SELECT *, non_existing_column FROM file('$CURDIR/data_hive/partitioning/non_exis $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE URL PARTITIONING'" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM url('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; SELECT *, non_existing_column FROM url('http://localhost:11111/test/hive_partitioning/non_existing_column=Elizabeth/sample.parquet') LIMIT 10;""" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 0; SELECT *, _column0 FROM url('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -62,7 +62,7 @@ SELECT *, _column0 FROM url('http://localhost:11111/test/hive_partitioning/colum $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE S3 PARTITIONING'" -$CLICKHOUSE_CLIENT -n -q """ +$CLICKHOUSE_CLIENT -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -71,7 +71,7 @@ SELECT *, non_existing_column FROM s3('http://localhost:11111/test/hive_partitio SELECT *, column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=*/sample.parquet') WHERE column0 = 'Elizabeth' LIMIT 10; """ -$CLICKHOUSE_CLIENT -n -q """ +$CLICKHOUSE_CLIENT -q """ set use_hive_partitioning = 0; SELECT *, _column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -79,7 +79,7 @@ SELECT *, _column0 FROM s3('http://localhost:11111/test/hive_partitioning/column $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE S3CLUSTER PARTITIONING'" -$CLICKHOUSE_CLIENT -n -q """ +$CLICKHOUSE_CLIENT -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM s3Cluster(test_cluster_one_shard_three_replicas_localhost, 'http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; diff --git a/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh index e0c8f08e695..6bfe82a91a6 100755 --- a/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh +++ b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # In this test we restore from "/tests/queries/0_stateless/backups/mt_250_parts.zip" backup_name="$($CURDIR/helpers/install_predefined_backup.sh mt_250_parts.zip)" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " DROP TABLE IF EXISTS manyparts; CREATE TABLE manyparts (x Int64) ENGINE=MergeTree ORDER BY tuple() SETTINGS merge_tree_clear_old_temporary_directories_interval_seconds=1, temporary_directories_lifetime=1; " @@ -16,7 +16,7 @@ CREATE TABLE manyparts (x Int64) ENGINE=MergeTree ORDER BY tuple() SETTINGS merg # RESTORE must protect its temporary directories from removing. ${CLICKHOUSE_CLIENT} --query "RESTORE TABLE default.mt_250_parts AS manyparts FROM Disk('backups', '${backup_name}') SETTINGS allow_different_table_def=true" | grep -o "RESTORED" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " SELECT count(), sum(x) FROM manyparts; DROP TABLE manyparts; " From 7ff244d7a3aec0c39bb56ea5d7bb3de57d7ee26e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 7 Sep 2024 20:46:56 +0200 Subject: [PATCH 239/289] Update docs/en/sql-reference/functions/string-replace-functions.md Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com> --- docs/en/sql-reference/functions/string-replace-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index 283d41d8b73..4417656fc80 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -22,7 +22,7 @@ overlay(s, replace, offset[, length]) - `s`: A string type [String](../data-types/string.md). - `replace`: A string type [String](../data-types/string.md). -- `offset`: An integer type [Int](../data-types/int-uint.md). If `offset` is negative, it is counted from the end of the string `s`. +- `offset`: An integer type [Int](../data-types/int-uint.md) (1-based). If `offset` is negative, it is counted from the end of the string `s`. - `length`: Optional. An integer type [Int](../data-types/int-uint.md). `length` specifies the length of the snippet within the input string `s` to be replaced. If `length` is not specified, the number of bytes removed from `s` equals the length of `replace`; otherwise `length` bytes are removed. **Returned value** From 654f8b98de5337474798b60c2708b31cbc0cb0b8 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 7 Sep 2024 20:47:03 +0200 Subject: [PATCH 240/289] Update docs/en/sql-reference/functions/string-replace-functions.md Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com> --- docs/en/sql-reference/functions/string-replace-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index 4417656fc80..3f50cd24f93 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -72,7 +72,7 @@ overlayUTF8(s, replace, offset[, length]) - `s`: A string type [String](../data-types/string.md). - `replace`: A string type [String](../data-types/string.md). -- `offset`: An integer type [Int](../data-types/int-uint.md). If `offset` is negative, it is counted from the end of the input string `s`. +- `offset`: An integer type [Int](../data-types/int-uint.md) (1-based). If `offset` is negative, it is counted from the end of the input string `s`. - `length`: Optional. An integer type [Int](../data-types/int-uint.md). `length` specifies the length of the snippet within the input string `s` to be replaced. If `length` is not specified, the number of characters removed from `s` equals the length of `replace`; otherwise `length` characters are removed. **Returned value** From 33866fb5bd8c682abcd47593aa8d717f2611a3c0 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 8 Sep 2024 08:38:05 +0000 Subject: [PATCH 241/289] Fix config.h for v3.7.1 --- contrib/libarchive-cmake/config.h | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/contrib/libarchive-cmake/config.h b/contrib/libarchive-cmake/config.h index 0b0cab47a52..6e6870e5d70 100644 --- a/contrib/libarchive-cmake/config.h +++ b/contrib/libarchive-cmake/config.h @@ -334,13 +334,13 @@ typedef uint64_t uintmax_t; /* #undef ARCHIVE_XATTR_LINUX */ /* Version number of bsdcpio */ -#define BSDCPIO_VERSION_STRING "3.7.0" +#define BSDCPIO_VERSION_STRING "3.7.1" /* Version number of bsdtar */ -#define BSDTAR_VERSION_STRING "3.7.0" +#define BSDTAR_VERSION_STRING "3.7.1" /* Version number of bsdcat */ -#define BSDCAT_VERSION_STRING "3.7.0" +#define BSDCAT_VERSION_STRING "3.7.1" /* Define to 1 if you have the `acl_create_entry' function. */ /* #undef HAVE_ACL_CREATE_ENTRY */ @@ -642,6 +642,9 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `getgrnam_r' function. */ #define HAVE_GETGRNAM_R 1 +/* Define to 1 if you have the `getline' function. */ +#define HAVE_GETLINE 1 + /* Define to 1 if platform uses `optreset` to reset `getopt` */ #define HAVE_GETOPT_OPTRESET 1 @@ -1273,13 +1276,13 @@ typedef uint64_t uintmax_t; /* #undef HAVE__MKGMTIME */ /* Define as const if the declaration of iconv() needs const. */ -#define ICONV_CONST +#define ICONV_CONST /* Version number of libarchive as a single integer */ -#define LIBARCHIVE_VERSION_NUMBER "3007000" +#define LIBARCHIVE_VERSION_NUMBER "3007001" /* Version number of libarchive */ -#define LIBARCHIVE_VERSION_STRING "3.7.0" +#define LIBARCHIVE_VERSION_STRING "3.7.1" /* Define to 1 if `lstat' dereferences a symlink specified with a trailing slash. */ @@ -1333,7 +1336,7 @@ typedef uint64_t uintmax_t; #endif /* SAFE_TO_DEFINE_EXTENSIONS */ /* Version number of package */ -#define VERSION "3.7.0" +#define VERSION "3.7.1" /* Number of bits in a file offset, on hosts where this is settable. */ /* #undef _FILE_OFFSET_BITS */ From 483dd7eebea0cd002a8c2db0fa364437728b3081 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 8 Sep 2024 08:45:13 +0000 Subject: [PATCH 242/289] Bump to v3.7.2 --- contrib/libarchive | 2 +- contrib/libarchive-cmake/config.h | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/contrib/libarchive b/contrib/libarchive index 0c21691b177..6468cd1f5c9 160000 --- a/contrib/libarchive +++ b/contrib/libarchive @@ -1 +1 @@ -Subproject commit 0c21691b177fac5f4cceca2a1ff2ddfa5d60f51c +Subproject commit 6468cd1f5c9b76e2c3b10cdd6938faf6b82823b6 diff --git a/contrib/libarchive-cmake/config.h b/contrib/libarchive-cmake/config.h index 6e6870e5d70..ad5d83168dc 100644 --- a/contrib/libarchive-cmake/config.h +++ b/contrib/libarchive-cmake/config.h @@ -334,13 +334,16 @@ typedef uint64_t uintmax_t; /* #undef ARCHIVE_XATTR_LINUX */ /* Version number of bsdcpio */ -#define BSDCPIO_VERSION_STRING "3.7.1" +#define BSDCPIO_VERSION_STRING "3.7.2" /* Version number of bsdtar */ -#define BSDTAR_VERSION_STRING "3.7.1" +#define BSDTAR_VERSION_STRING "3.7.2" /* Version number of bsdcat */ -#define BSDCAT_VERSION_STRING "3.7.1" +#define BSDCAT_VERSION_STRING "3.7.2" + +/* Version number of bsdunzip */ +#define BSDUNZIP_VERSION_STRING "3.7.2" /* Define to 1 if you have the `acl_create_entry' function. */ /* #undef HAVE_ACL_CREATE_ENTRY */ @@ -645,9 +648,6 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `getline' function. */ #define HAVE_GETLINE 1 -/* Define to 1 if platform uses `optreset` to reset `getopt` */ -#define HAVE_GETOPT_OPTRESET 1 - /* Define to 1 if you have the `getpid' function. */ #define HAVE_GETPID 1 @@ -1032,6 +1032,12 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `strrchr' function. */ #define HAVE_STRRCHR 1 +/* Define to 1 if the system has the type `struct statfs'. */ +/* #undef HAVE_STRUCT_STATFS */ + +/* Define to 1 if `f_iosize' is a member of `struct statfs'. */ +/* #undef HAVE_STRUCT_STATFS_F_IOSIZE */ + /* Define to 1 if `f_namemax' is a member of `struct statfs'. */ /* #undef HAVE_STRUCT_STATFS_F_NAMEMAX */ @@ -1279,10 +1285,10 @@ typedef uint64_t uintmax_t; #define ICONV_CONST /* Version number of libarchive as a single integer */ -#define LIBARCHIVE_VERSION_NUMBER "3007001" +#define LIBARCHIVE_VERSION_NUMBER "3007002" /* Version number of libarchive */ -#define LIBARCHIVE_VERSION_STRING "3.7.1" +#define LIBARCHIVE_VERSION_STRING "3.7.2" /* Define to 1 if `lstat' dereferences a symlink specified with a trailing slash. */ @@ -1336,7 +1342,7 @@ typedef uint64_t uintmax_t; #endif /* SAFE_TO_DEFINE_EXTENSIONS */ /* Version number of package */ -#define VERSION "3.7.1" +#define VERSION "3.7.2" /* Number of bits in a file offset, on hosts where this is settable. */ /* #undef _FILE_OFFSET_BITS */ From 2bbede2d939c6929cfa8dcb9e75c323269d56fff Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 7 Sep 2024 16:21:06 +0000 Subject: [PATCH 243/289] Remove superfluous --multiquery/-n --- .../00115_shard_in_incomplete_result.sh | 2 +- ...ard_memory_tracker_and_exception_safety.sh | 2 +- ...rage_file_and_clickhouse-local_app_long.sh | 6 ++-- tests/queries/0_stateless/00505_secure.sh | 2 +- .../0_stateless/00531_client_ignore_error.sh | 6 ++-- .../0_stateless/00534_client_ignore_error.sh | 6 ++-- .../0_stateless/00686_client_exit_code.sh | 2 +- .../00705_drop_create_merge_tree.sh | 4 +-- ...731_long_merge_tree_select_opened_files.sh | 2 +- .../00838_system_tables_drop_table_race.sh | 4 +-- ...ong_concurrent_select_and_drop_deadlock.sh | 2 +- .../queries/0_stateless/00900_long_parquet.sh | 28 +++++++++---------- .../0_stateless/00900_long_parquet_decimal.sh | 16 +++++------ .../00921_datetime64_compatibility_long.sh | 2 +- ...ices_mutation_replicated_zookeeper_long.sh | 2 +- .../00991_system_parts_race_condition_long.sh | 2 +- ...tem_parts_race_condition_zookeeper_long.sh | 8 +++--- .../0_stateless/01014_lazy_database_basic.sh | 12 ++++---- ...rrent_recreate_reattach_and_show_tables.sh | 2 +- ...8_ddl_dictionaries_concurrent_requrests.sh | 8 +++--- .../01019_alter_materialized_view_atomic.sh | 2 +- .../0_stateless/01035_avg_weighted_long.sh | 2 +- .../0_stateless/01053_ssd_dictionary.sh | 2 +- .../01079_bad_alters_zookeeper_long.sh | 2 +- .../01111_create_drop_replicated_db_stress.sh | 2 +- .../0_stateless/01114_database_atomic.sh | 6 ++-- .../queries/0_stateless/01119_session_log.sh | 2 +- ...1213_alter_rename_column_zookeeper_long.sh | 2 +- .../0_stateless/01238_http_memory_tracking.sh | 2 +- .../01280_ssd_complex_key_dictionary.sh | 12 ++++---- ..._recreate_reattach_and_show_tables_long.sh | 2 +- .../01305_replica_create_drop_zookeeper.sh | 2 +- ...20_create_sync_race_condition_zookeeper.sh | 2 +- .../0_stateless/01395_limit_more_cases.sh | 2 +- .../01395_limit_more_cases_random.sh | 2 +- .../01412_cache_dictionary_race.sh | 4 +-- ...01454_storagememory_data_race_challenge.sh | 2 +- ...check_many_parallel_quorum_inserts_long.sh | 6 ++-- ...1509_check_parallel_quorum_inserts_long.sh | 8 +++--- .../01563_distributed_query_finish.sh | 4 +-- .../01732_race_condition_storage_join_long.sh | 10 +++---- ...est_postgresql_protocol_with_row_policy.sh | 6 ++-- .../01810_max_part_removal_threads_long.sh | 8 +++--- .../01889_postgresql_protocol_null_fields.sh | 2 +- .../01900_kill_mutation_parallel_long.sh | 10 +++---- ...nt_ttl_and_normal_merges_zookeeper_long.sh | 2 +- .../1_stateful/00177_memory_bound_merging.sh | 8 +++--- ...o_seek_avoiding_when_reading_from_cache.sh | 2 +- 48 files changed, 117 insertions(+), 117 deletions(-) diff --git a/tests/queries/0_stateless/00115_shard_in_incomplete_result.sh b/tests/queries/0_stateless/00115_shard_in_incomplete_result.sh index 4916721764c..1efe529ac24 100755 --- a/tests/queries/0_stateless/00115_shard_in_incomplete_result.sh +++ b/tests/queries/0_stateless/00115_shard_in_incomplete_result.sh @@ -15,6 +15,6 @@ $CLICKHOUSE_CLIENT --query=" INSERT INTO users VALUES (1321770221388956068); "; -for _ in {1..10}; do seq 1 10 | sed "s/.*/SELECT count() FROM (SELECT * FROM remote('127.0.0.{2,3}', ${CLICKHOUSE_DATABASE}, users) WHERE UserID IN (SELECT arrayJoin([1468013291393583084, 1321770221388956068])));/" | $CLICKHOUSE_CLIENT -n | grep -vE '^4$' && echo 'Fail!' && break; echo -n '.'; done; echo +for _ in {1..10}; do seq 1 10 | sed "s/.*/SELECT count() FROM (SELECT * FROM remote('127.0.0.{2,3}', ${CLICKHOUSE_DATABASE}, users) WHERE UserID IN (SELECT arrayJoin([1468013291393583084, 1321770221388956068])));/" | $CLICKHOUSE_CLIENT | grep -vE '^4$' && echo 'Fail!' && break; echo -n '.'; done; echo $CLICKHOUSE_CLIENT --query="DROP TABLE users;"; diff --git a/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.sh b/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.sh index d57efaa1f0e..f0c0354ab33 100755 --- a/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.sh +++ b/tests/queries/0_stateless/00133_long_shard_memory_tracker_and_exception_safety.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" DROP TABLE IF EXISTS numbers_100k; CREATE VIEW numbers_100k AS SELECT * FROM system.numbers LIMIT 100000; "; diff --git a/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app_long.sh b/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app_long.sh index b59aae83e81..b6cc270994f 100755 --- a/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app_long.sh +++ b/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app_long.sh @@ -48,13 +48,13 @@ pack_unpack_compare "SELECT name, is_aggregate FROM system.functions" "name Stri echo # Check settings are passed correctly ${CLICKHOUSE_LOCAL} --max_rows_in_distinct=33 -q "SELECT name, value FROM system.settings WHERE name = 'max_rows_in_distinct'" -${CLICKHOUSE_LOCAL} -n -q "SET max_rows_in_distinct=33; SELECT name, value FROM system.settings WHERE name = 'max_rows_in_distinct'" +${CLICKHOUSE_LOCAL} -q "SET max_rows_in_distinct=33; SELECT name, value FROM system.settings WHERE name = 'max_rows_in_distinct'" ${CLICKHOUSE_LOCAL} --max_bytes_before_external_group_by=1 --max_block_size=10 -q "SELECT sum(ignore(*)) FROM (SELECT number, count() FROM numbers(1000) GROUP BY number)" echo # Check exta options, we expect zero exit code and no stderr output -(${CLICKHOUSE_LOCAL} --ignore-error -n --echo -q "SELECT nothing_to_do();SELECT 42;" 2>/dev/null || echo "Wrong RC") +(${CLICKHOUSE_LOCAL} --ignore-error --echo -q "SELECT nothing_to_do();SELECT 42;" 2>/dev/null || echo "Wrong RC") echo -${CLICKHOUSE_LOCAL} -n -q "CREATE TABLE sophisticated_default +${CLICKHOUSE_LOCAL} -q "CREATE TABLE sophisticated_default ( a UInt8 DEFAULT 3, b UInt8 ALIAS a + 5, diff --git a/tests/queries/0_stateless/00505_secure.sh b/tests/queries/0_stateless/00505_secure.sh index eaa50bce6b1..eed0d3bf5c6 100755 --- a/tests/queries/0_stateless/00505_secure.sh +++ b/tests/queries/0_stateless/00505_secure.sh @@ -23,7 +23,7 @@ $CLICKHOUSE_CLIENT_SECURE -q "SELECT 4;" # TODO: can test only on unchanged port. Possible solutions: generate config or pass shard port via command line if [[ "$CLICKHOUSE_PORT_TCP_SECURE" = "$CLICKHOUSE_PORT_TCP_SECURE" ]]; then - cat "$CURDIR"/00505_distributed_secure.data | $CLICKHOUSE_CLIENT_SECURE -n -m + cat "$CURDIR"/00505_distributed_secure.data | $CLICKHOUSE_CLIENT_SECURE -m else tail -n 13 "$CURDIR"/00505_secure.reference fi diff --git a/tests/queries/0_stateless/00531_client_ignore_error.sh b/tests/queries/0_stateless/00531_client_ignore_error.sh index d3215e1beac..553cb9fa897 100755 --- a/tests/queries/0_stateless/00531_client_ignore_error.sh +++ b/tests/queries/0_stateless/00531_client_ignore_error.sh @@ -5,8 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -echo "SELECT 1; SELECT 2; SELECT CAST(); SELECT ';'; SELECT 3;SELECT CAST();SELECT 4;" | $CLICKHOUSE_CLIENT -n --ignore-error 2>/dev/null -echo "SELECT CAST();" | $CLICKHOUSE_CLIENT -n --ignore-error 2>/dev/null -echo "SELECT 5;" | $CLICKHOUSE_CLIENT -n --ignore-error +echo "SELECT 1; SELECT 2; SELECT CAST(); SELECT ';'; SELECT 3;SELECT CAST();SELECT 4;" | $CLICKHOUSE_CLIENT --ignore-error 2>/dev/null +echo "SELECT CAST();" | $CLICKHOUSE_CLIENT --ignore-error 2>/dev/null +echo "SELECT 5;" | $CLICKHOUSE_CLIENT --ignore-error #$CLICKHOUSE_CLIENT -q "SELECT 'Still alive'" diff --git a/tests/queries/0_stateless/00534_client_ignore_error.sh b/tests/queries/0_stateless/00534_client_ignore_error.sh index d3215e1beac..553cb9fa897 100755 --- a/tests/queries/0_stateless/00534_client_ignore_error.sh +++ b/tests/queries/0_stateless/00534_client_ignore_error.sh @@ -5,8 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -echo "SELECT 1; SELECT 2; SELECT CAST(); SELECT ';'; SELECT 3;SELECT CAST();SELECT 4;" | $CLICKHOUSE_CLIENT -n --ignore-error 2>/dev/null -echo "SELECT CAST();" | $CLICKHOUSE_CLIENT -n --ignore-error 2>/dev/null -echo "SELECT 5;" | $CLICKHOUSE_CLIENT -n --ignore-error +echo "SELECT 1; SELECT 2; SELECT CAST(); SELECT ';'; SELECT 3;SELECT CAST();SELECT 4;" | $CLICKHOUSE_CLIENT --ignore-error 2>/dev/null +echo "SELECT CAST();" | $CLICKHOUSE_CLIENT --ignore-error 2>/dev/null +echo "SELECT 5;" | $CLICKHOUSE_CLIENT --ignore-error #$CLICKHOUSE_CLIENT -q "SELECT 'Still alive'" diff --git a/tests/queries/0_stateless/00686_client_exit_code.sh b/tests/queries/0_stateless/00686_client_exit_code.sh index 9586ddbd0a3..e348f93f30d 100755 --- a/tests/queries/0_stateless/00686_client_exit_code.sh +++ b/tests/queries/0_stateless/00686_client_exit_code.sh @@ -8,5 +8,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=./mergetree_mutations.lib . "$CURDIR"/mergetree_mutations.lib -echo "INSERT INTO test FORMAT CSV" | ${CLICKHOUSE_CLIENT} -n 2>/dev/null +echo "INSERT INTO test FORMAT CSV" | ${CLICKHOUSE_CLIENT} 2>/dev/null echo $? diff --git a/tests/queries/0_stateless/00705_drop_create_merge_tree.sh b/tests/queries/0_stateless/00705_drop_create_merge_tree.sh index ea8b9d02e49..fd002668696 100755 --- a/tests/queries/0_stateless/00705_drop_create_merge_tree.sh +++ b/tests/queries/0_stateless/00705_drop_create_merge_tree.sh @@ -5,8 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -yes 'CREATE TABLE IF NOT EXISTS table (x UInt8) ENGINE = MergeTree ORDER BY tuple();' | head -n 1000 | $CLICKHOUSE_CLIENT --multiquery & -yes 'DROP TABLE IF EXISTS table;' | head -n 1000 | $CLICKHOUSE_CLIENT --multiquery & +yes 'CREATE TABLE IF NOT EXISTS table (x UInt8) ENGINE = MergeTree ORDER BY tuple();' | head -n 1000 | $CLICKHOUSE_CLIENT & +yes 'DROP TABLE IF EXISTS table;' | head -n 1000 | $CLICKHOUSE_CLIENT & wait ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table" diff --git a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh index 5a4fd901f8d..285fd3945f9 100755 --- a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh +++ b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh @@ -12,7 +12,7 @@ settings="--log_queries=1 --log_query_threads=1 --log_profile_events=1 --log_que # Test insert logging on each block and checkPacket() method -$CLICKHOUSE_CLIENT $settings -n -q " +$CLICKHOUSE_CLIENT $settings -q " DROP TABLE IF EXISTS merge_tree_table; CREATE TABLE merge_tree_table (id UInt64, date Date, uid UInt32) ENGINE = MergeTree(date, id, 8192);" diff --git a/tests/queries/0_stateless/00838_system_tables_drop_table_race.sh b/tests/queries/0_stateless/00838_system_tables_drop_table_race.sh index da4d3b05987..a29d2e5bc71 100755 --- a/tests/queries/0_stateless/00838_system_tables_drop_table_race.sh +++ b/tests/queries/0_stateless/00838_system_tables_drop_table_race.sh @@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS table" -seq 1 100 | sed -r -e "s/.+/CREATE TABLE table (x UInt8) ENGINE = MergeTree ORDER BY x; DROP TABLE table;/" | $CLICKHOUSE_CLIENT -n & -seq 1 100 | sed -r -e "s/.+/SELECT * FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' LIMIT 1000000, 1;/" | $CLICKHOUSE_CLIENT -n 2>/dev/null & +seq 1 100 | sed -r -e "s/.+/CREATE TABLE table (x UInt8) ENGINE = MergeTree ORDER BY x; DROP TABLE table;/" | $CLICKHOUSE_CLIENT & +seq 1 100 | sed -r -e "s/.+/SELECT * FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' LIMIT 1000000, 1;/" | $CLICKHOUSE_CLIENT 2>/dev/null & wait diff --git a/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh b/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh index f0bc52ee356..cb774116356 100755 --- a/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh +++ b/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh @@ -27,7 +27,7 @@ function thread_drop_create() while [ $SECONDS -lt "$TIMELIMIT" ] && [ $it -lt 100 ]; do it=$((it+1)) - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " drop table if exists view_00840; create view view_00840 as select count(*),database,table from system.columns group by database,table; " diff --git a/tests/queries/0_stateless/00900_long_parquet.sh b/tests/queries/0_stateless/00900_long_parquet.sh index 07d2f24e446..86a0d013078 100755 --- a/tests/queries/0_stateless/00900_long_parquet.sh +++ b/tests/queries/0_stateless/00900_long_parquet.sh @@ -8,11 +8,11 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" DROP TABLE IF EXISTS contributors; CREATE TABLE contributors (name String) ENGINE = Memory;" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.contributors ORDER BY name DESC FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO contributors FORMAT Parquet" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" -- random results SELECT * FROM contributors LIMIT 10 FORMAT Null; DROP TABLE contributors; @@ -21,30 +21,30 @@ ${CLICKHOUSE_CLIENT} -n --query=" CREATE TABLE parquet_numbers (number UInt64) ENGINE = Memory;" # less than default block size (65k) ${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.numbers LIMIT 10000 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_numbers FORMAT Parquet" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT * FROM parquet_numbers ORDER BY number DESC LIMIT 10; TRUNCATE TABLE parquet_numbers;" # More than default block size ${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.numbers LIMIT 100000 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_numbers FORMAT Parquet" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT * FROM parquet_numbers ORDER BY number DESC LIMIT 10; TRUNCATE TABLE parquet_numbers;" ${CLICKHOUSE_CLIENT} --max_block_size=2 --query="SELECT * FROM system.numbers LIMIT 3 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_numbers FORMAT Parquet" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT * FROM parquet_numbers ORDER BY number DESC LIMIT 10; TRUNCATE TABLE parquet_numbers;" ${CLICKHOUSE_CLIENT} --max_block_size=1 --query="SELECT * FROM system.numbers LIMIT 1000 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_numbers FORMAT Parquet" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT * FROM parquet_numbers ORDER BY number DESC LIMIT 10; DROP TABLE parquet_numbers; DROP TABLE IF EXISTS parquet_events; CREATE TABLE parquet_events (event String, value UInt64, description String) ENGINE = Memory;" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.events FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_events FORMAT Parquet" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT event, description FROM parquet_events WHERE event IN ('ContextLock', 'Query') ORDER BY event; DROP TABLE parquet_events; @@ -78,7 +78,7 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types2 ORDER BY int8 FORMAT echo diff: diff "${CLICKHOUSE_TMP}"/parquet_all_types_1.dump "${CLICKHOUSE_TMP}"/parquet_all_types_2.dump -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" TRUNCATE TABLE parquet_types2; INSERT INTO parquet_types3 values ( 79, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str01', 'fstr1', '2003-03-04', '2004-05-06', toDateTime64('2004-05-06 07:08:09.012', 9));" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types3 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types2 FORMAT Parquet" @@ -88,7 +88,7 @@ ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types4 values ( 80, ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types4 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types2 FORMAT Parquet" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types1 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types4 FORMAT Parquet" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT 'dest:'; SELECT * FROM parquet_types2 ORDER BY int8; SELECT 'min:'; @@ -106,7 +106,7 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types5 ORDER BY int8 FORMAT ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types5 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types6 FORMAT Parquet" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types1 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types6 FORMAT Parquet" echo dest from null: -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT * FROM parquet_types6 ORDER BY int8; DROP TABLE parquet_types5; @@ -126,7 +126,7 @@ ${CLICKHOUSE_CLIENT} -n --query=" INSERT INTO parquet_arrays VALUES (2, [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []);" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_arrays FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_arrays FORMAT Parquet" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT * FROM parquet_arrays ORDER BY id; DROP TABLE parquet_arrays; @@ -135,7 +135,7 @@ ${CLICKHOUSE_CLIENT} -n --query=" CREATE TABLE parquet_nullable_arrays (id UInt32, a1 Array(Nullable(UInt32)), a2 Array(Nullable(String)), a3 Array(Nullable(Decimal(4, 2)))) engine=Memory(); INSERT INTO parquet_nullable_arrays VALUES (1, [1, Null, 2], [Null, 'Some string', Null], [0.001, Null, 42.42]), (2, [Null], [Null], [Null]), (3, [], [], []);" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_nullable_arrays FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_nullable_arrays FORMAT Parquet" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT * FROM parquet_nullable_arrays ORDER BY id; DROP TABLE parquet_nullable_arrays; @@ -143,7 +143,7 @@ ${CLICKHOUSE_CLIENT} -n --query=" CREATE TABLE parquet_nested_arrays (a1 Array(Array(Array(UInt32))), a2 Array(Array(Array(String))), a3 Array(Array(Nullable(UInt32))), a4 Array(Array(Nullable(String)))) engine=Memory(); INSERT INTO parquet_nested_arrays VALUES ([[[1,2,3], [1,2,3]], [[1,2,3]], [[], [1,2,3]]], [[['Some string', 'Some string'], []], [['Some string']], [[]]], [[Null, 1, 2], [Null], [1, 2], []], [['Some string', Null, 'Some string'], [Null], []]);" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_nested_arrays FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_nested_arrays FORMAT Parquet" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT * FROM parquet_nested_arrays; DROP TABLE parquet_nested_arrays; @@ -151,6 +151,6 @@ ${CLICKHOUSE_CLIENT} -n --query=" CREATE TABLE parquet_decimal (d1 Decimal32(4), d2 Decimal64(8), d3 Decimal128(16), d4 Decimal256(32)) ENGINE = Memory; INSERT INTO TABLE parquet_decimal VALUES (0.123, 0.123123123, 0.123123123123, 0.123123123123123123);" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_decimal FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_decimal FORMAT Arrow" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT * FROM parquet_decimal; DROP TABLE parquet_decimal;" diff --git a/tests/queries/0_stateless/00900_long_parquet_decimal.sh b/tests/queries/0_stateless/00900_long_parquet_decimal.sh index a819dcbcdc3..14e8fdcc038 100755 --- a/tests/queries/0_stateless/00900_long_parquet_decimal.sh +++ b/tests/queries/0_stateless/00900_long_parquet_decimal.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" DROP TABLE IF EXISTS decimal; DROP TABLE IF EXISTS decimal2; @@ -26,7 +26,7 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal ORDER BY a, b, c, d, e, f, g ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal2 ORDER BY a, b, c, d, e, f, g, h, i, j;" > "${CLICKHOUSE_TMP}"/parquet_decimal0_2.dump echo diff0: diff "${CLICKHOUSE_TMP}"/parquet_decimal0_1.dump "${CLICKHOUSE_TMP}"/parquet_decimal0_2.dump -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" DROP TABLE IF EXISTS decimal; DROP TABLE IF EXISTS decimal2; @@ -61,7 +61,7 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal ORDER BY a, b, c, d, e, f, g ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal2 ORDER BY a, b, c, d, e, f, g, h, i, j;" > "${CLICKHOUSE_TMP}"/parquet_decimal1_2.dump echo diff1: diff "${CLICKHOUSE_TMP}"/parquet_decimal1_1.dump "${CLICKHOUSE_TMP}"/parquet_decimal1_2.dump -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" DROP TABLE IF EXISTS decimal; DROP TABLE IF EXISTS decimal2; @@ -75,7 +75,7 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal ORDER BY a, b, c, d, e, f, g ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal2 ORDER BY a, b, c, d, e, f, g, h, i, j;" > "${CLICKHOUSE_TMP}"/parquet_decimal2_2.dump echo diff2: diff "${CLICKHOUSE_TMP}"/parquet_decimal2_1.dump "${CLICKHOUSE_TMP}"/parquet_decimal2_2.dump -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" DROP TABLE IF EXISTS decimal; DROP TABLE IF EXISTS decimal2; @@ -86,7 +86,7 @@ ${CLICKHOUSE_CLIENT} -n --query=" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal ORDER BY a, b, c, d FORMAT Parquet;" > "${CLICKHOUSE_TMP}"/parquet_decimal3_1.parquet ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal ORDER BY a, b, c, d FORMAT Parquet;" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO decimal2 FORMAT Parquet" 2> /dev/null echo nothing: -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT * FROM decimal2 ORDER BY a, b, c, d; TRUNCATE TABLE decimal2; @@ -94,7 +94,7 @@ ${CLICKHOUSE_CLIENT} -n --query=" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal ORDER BY a, b, c, d FORMAT Parquet;" > "${CLICKHOUSE_TMP}"/parquet_decimal3_2.parquet ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal ORDER BY a, b, c, d FORMAT Parquet;" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO decimal2 FORMAT Parquet" echo nulls: -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT * FROM decimal2 ORDER BY a, b, c, d; TRUNCATE TABLE decimal2; @@ -104,7 +104,7 @@ ${CLICKHOUSE_CLIENT} -n --query=" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal ORDER BY a, b, c, d FORMAT Parquet;" > "${CLICKHOUSE_TMP}"/parquet_decimal3_3.parquet ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal ORDER BY a, b, c, d FORMAT Parquet;" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO decimal2 FORMAT Parquet" -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" SELECT 'full orig:'; SELECT * FROM decimal ORDER BY a, b, c, d; SELECT 'full inserted:'; @@ -115,6 +115,6 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM decimal2 ORDER BY a, b, c, d;" > "${ echo diff3: diff "${CLICKHOUSE_TMP}"/parquet_decimal3_1.dump "${CLICKHOUSE_TMP}"/parquet_decimal3_2.dump -${CLICKHOUSE_CLIENT} -n --query=" +${CLICKHOUSE_CLIENT} --query=" DROP TABLE IF EXISTS decimal; DROP TABLE IF EXISTS decimal2;" diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh b/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh index d310a2c3612..152d5a847b7 100755 --- a/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh +++ b/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh @@ -13,5 +13,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # ${CURDIR}/00921_datetime64_compatibility.python python3 "${CURDIR}"/00921_datetime64_compatibility_long.python \ - | ${CLICKHOUSE_CLIENT} --ignore-error -nm --calculate_text_stack_trace 0 --log-level 'error' 2>&1 \ + | ${CLICKHOUSE_CLIENT} --ignore-error -m --calculate_text_stack_trace 0 --log-level 'error' 2>&1 \ | grep -v -e 'Received exception .*$' -e '^(query: ' | sed 's/^\(Code: [0-9]\+\).*$/\1/g' diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh index 686dd7f6df0..1d35daf9f2e 100755 --- a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh +++ b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh @@ -11,7 +11,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS indices_mutaions1;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS indices_mutaions2;" -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" CREATE TABLE indices_mutaions1 ( u64 UInt64, diff --git a/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh b/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh index 8243c6bde62..df330b82c80 100755 --- a/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh +++ b/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh @@ -22,7 +22,7 @@ function thread1() function thread2() { - while true; do $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table ADD COLUMN h String '0'; ALTER TABLE alter_table MODIFY COLUMN h UInt64; ALTER TABLE alter_table DROP COLUMN h;"; done + while true; do $CLICKHOUSE_CLIENT --query "ALTER TABLE alter_table ADD COLUMN h String '0'; ALTER TABLE alter_table MODIFY COLUMN h UInt64; ALTER TABLE alter_table DROP COLUMN h;"; done } function thread3() diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh index 02a739ece4a..d5c0248e2b3 100755 --- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh +++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh @@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS alter_table0; DROP TABLE IF EXISTS alter_table1; @@ -31,7 +31,7 @@ function thread1() function thread2() { - while true; do $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table0 ADD COLUMN h String DEFAULT '0'; ALTER TABLE alter_table0 MODIFY COLUMN h UInt64; ALTER TABLE alter_table0 DROP COLUMN h;"; done + while true; do $CLICKHOUSE_CLIENT --query "ALTER TABLE alter_table0 ADD COLUMN h String DEFAULT '0'; ALTER TABLE alter_table0 MODIFY COLUMN h UInt64; ALTER TABLE alter_table0 DROP COLUMN h;"; done } function thread3() @@ -87,6 +87,6 @@ check_replication_consistency "alter_table" "count(), sum(a), sum(b), round(sum( $CLICKHOUSE_CLIENT -q "SELECT table, lost_part_count FROM system.replicas WHERE database=currentDatabase() AND lost_part_count!=0"; -$CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table0;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & -$CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table1;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & +$CLICKHOUSE_CLIENT -q "DROP TABLE alter_table0;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & +$CLICKHOUSE_CLIENT -q "DROP TABLE alter_table1;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & wait diff --git a/tests/queries/0_stateless/01014_lazy_database_basic.sh b/tests/queries/0_stateless/01014_lazy_database_basic.sh index ea7603b2519..55d18a7c527 100755 --- a/tests/queries/0_stateless/01014_lazy_database_basic.sh +++ b/tests/queries/0_stateless/01014_lazy_database_basic.sh @@ -5,9 +5,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} -n -q "DROP DATABASE IF EXISTS testlazy" +${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS testlazy" -${CLICKHOUSE_CLIENT} -n -q " +${CLICKHOUSE_CLIENT} -q " CREATE DATABASE testlazy ENGINE = Lazy(1); CREATE TABLE testlazy.log (a UInt64, b UInt64) ENGINE = Log; CREATE TABLE testlazy.slog (a UInt64, b UInt64) ENGINE = StripeLog; @@ -30,7 +30,7 @@ ${CLICKHOUSE_CLIENT} -q " sleep 1.5 -${CLICKHOUSE_CLIENT} -n -q " +${CLICKHOUSE_CLIENT} -q " SELECT * FROM testlazy.log LIMIT 0; -- drop testlazy.log from cache RENAME TABLE testlazy.log TO testlazy.log2; SELECT database, name FROM system.tables WHERE database = 'testlazy'; @@ -44,7 +44,7 @@ ${CLICKHOUSE_CLIENT} -q " sleep 1.5 -${CLICKHOUSE_CLIENT} -n -q " +${CLICKHOUSE_CLIENT} -q " INSERT INTO testlazy.log2 VALUES (1, 1); INSERT INTO testlazy.slog VALUES (2, 2); INSERT INTO testlazy.tlog VALUES (3, 3); @@ -55,14 +55,14 @@ ${CLICKHOUSE_CLIENT} -n -q " sleep 1.5 -${CLICKHOUSE_CLIENT} -n -q " +${CLICKHOUSE_CLIENT} -q " SELECT * FROM testlazy.log2 LIMIT 0; -- drop testlazy.log2 from cache DROP TABLE testlazy.log2; " sleep 1.5 -${CLICKHOUSE_CLIENT} -n -q " +${CLICKHOUSE_CLIENT} -q " SELECT * FROM testlazy.slog; SELECT * FROM testlazy.tlog; " diff --git a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh index e4b3a31b13f..ff2c0b8821e 100755 --- a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh +++ b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh @@ -83,7 +83,7 @@ export -f recreate_lazy_func4; export -f show_tables_func; -${CLICKHOUSE_CLIENT} -n -q " +${CLICKHOUSE_CLIENT} -q " DROP DATABASE IF EXISTS $CURR_DATABASE; CREATE DATABASE $CURR_DATABASE ENGINE = Lazy(1); " diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh b/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh index 872b0a7c1a1..82221eb06e9 100755 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -q " DROP DATABASE IF EXISTS database_for_dict; DROP TABLE IF EXISTS table_for_dict1; DROP TABLE IF EXISTS table_for_dict2; @@ -44,7 +44,7 @@ function thread3() function thread4() { - while true; do $CLICKHOUSE_CLIENT -n -q " + while true; do $CLICKHOUSE_CLIENT -q " SELECT * FROM database_for_dict.dict1 FORMAT Null; SELECT * FROM database_for_dict.dict2 FORMAT Null; " ||: ; done @@ -52,7 +52,7 @@ function thread4() function thread5() { - while true; do $CLICKHOUSE_CLIENT -n -q " + while true; do $CLICKHOUSE_CLIENT -q " SELECT dictGetString('database_for_dict.dict1', 'value_column', toUInt64(number)) from numbers(1000) FROM FORMAT Null; SELECT dictGetString('database_for_dict.dict2', 'value_column', toUInt64(number)) from numbers(1000) FROM FORMAT Null; " ||: ; done @@ -117,7 +117,7 @@ $CLICKHOUSE_CLIENT -q "SELECT 'Still alive'" $CLICKHOUSE_CLIENT -q "ATTACH DICTIONARY IF NOT EXISTS database_for_dict.dict1" $CLICKHOUSE_CLIENT -q "ATTACH DICTIONARY IF NOT EXISTS database_for_dict.dict2" -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -q " DROP DATABASE database_for_dict; DROP TABLE table_for_dict1; DROP TABLE table_for_dict2; diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh index 4bd21fcee02..eb12a76eb62 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh +++ b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT --multiquery <&1 \ | grep -c 'Code: 43. DB::Exception: .* DB::Exception:.* Types .* are non-conforming as arguments for aggregate function avgWeighted' diff --git a/tests/queries/0_stateless/01053_ssd_dictionary.sh b/tests/queries/0_stateless/01053_ssd_dictionary.sh index fdd40940ab5..7dd3fa4657a 100755 --- a/tests/queries/0_stateless/01053_ssd_dictionary.sh +++ b/tests/queries/0_stateless/01053_ssd_dictionary.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -n --query=" +$CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 --query=" DROP DATABASE IF EXISTS 01053_db; CREATE DATABASE 01053_db; diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper_long.sh b/tests/queries/0_stateless/01079_bad_alters_zookeeper_long.sh index 39e65af039b..22f8e5269bd 100755 --- a/tests/queries/0_stateless/01079_bad_alters_zookeeper_long.sh +++ b/tests/queries/0_stateless/01079_bad_alters_zookeeper_long.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_bad_alters"; -$CLICKHOUSE_CLIENT -n --query "CREATE TABLE table_for_bad_alters ( +$CLICKHOUSE_CLIENT --query "CREATE TABLE table_for_bad_alters ( key UInt64, value1 UInt8, value2 String diff --git a/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh b/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh index 619b6e91d11..9101b9faa3d 100755 --- a/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh +++ b/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh @@ -30,7 +30,7 @@ function drop_db() database=$($CLICKHOUSE_CLIENT -q "select name from system.databases where name like '${CLICKHOUSE_DATABASE}%' order by rand() limit 1") if [[ "$database" == "$CLICKHOUSE_DATABASE" ]]; then continue; fi if [ -z "$database" ]; then continue; fi - $CLICKHOUSE_CLIENT -n --query \ + $CLICKHOUSE_CLIENT --query \ "drop database if exists $database" 2>&1| grep -Fa "Exception: " sleep 0.$RANDOM done diff --git a/tests/queries/0_stateless/01114_database_atomic.sh b/tests/queries/0_stateless/01114_database_atomic.sh index 5eebb558575..5fe85136d05 100755 --- a/tests/queries/0_stateless/01114_database_atomic.sh +++ b/tests/queries/0_stateless/01114_database_atomic.sh @@ -31,7 +31,7 @@ $CLICKHOUSE_CLIENT -q "SELECT name, splitByChar('/', metadata_path)[-2] as uuid_path, ((splitByChar('/', metadata_path)[-3] as metadata) = substr(uuid_path, 1, 3)) OR metadata='metadata' FROM system.databases WHERE name LIKE '${CLICKHOUSE_DATABASE}_%'" | sed "s/$uuid_db_1/00001114-1000-4000-8000-000000000001/g" | sed "s/$uuid_db_2/00001114-1000-4000-8000-000000000002/g" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " CREATE TABLE ${DATABASE_1}.mt_tmp (n UInt64) ENGINE=MergeTree() ORDER BY tuple(); INSERT INTO ${DATABASE_1}.mt_tmp SELECT * FROM numbers(100); CREATE TABLE ${DATABASE_3}.mt (n UInt64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY (n % 5); @@ -65,7 +65,7 @@ while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE que sleep 0.1 done -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " RENAME TABLE ${DATABASE_1}.mt TO ${DATABASE_1}.mt_tmp; RENAME TABLE ${DATABASE_1}.mt_tmp TO ${DATABASE_2}.mt_tmp; EXCHANGE TABLES ${DATABASE_2}.mt AND ${DATABASE_2}.mt_tmp; @@ -79,7 +79,7 @@ uuid_mt1=$($CLICKHOUSE_CLIENT -q "SELECT uuid FROM system.tables WHERE database= $CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE ${DATABASE_1}.mt" | sed "s/$uuid_mt1/00001114-0000-4000-8000-000000000001/g" $CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE ${DATABASE_2}.mt" | sed "s/$explicit_uuid/00001114-0000-4000-8000-000000000002/g" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP TABLE ${DATABASE_1}.mt SETTINGS database_atomic_wait_for_drop_and_detach_synchronously=0; CREATE TABLE ${DATABASE_1}.mt (s String) ENGINE=Log(); INSERT INTO ${DATABASE_1}.mt SELECT 's' || toString(number) FROM numbers(5); diff --git a/tests/queries/0_stateless/01119_session_log.sh b/tests/queries/0_stateless/01119_session_log.sh index 2d17b545276..61bb7cf3ea8 100755 --- a/tests/queries/0_stateless/01119_session_log.sh +++ b/tests/queries/0_stateless/01119_session_log.sh @@ -14,7 +14,7 @@ and interface in ('HTTP', 'TCP', 'TCP_Interserver') and (user != 'default' or (a=1 and b=1)) -- FIXME: we should not write uninitialized address and port (but we do sometimes) and event_time >= now() - interval 5 minute" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " select * from remote('127.0.0.2', system, one, 'default', ''); select * from remote('127.0.0.2', system, one, 'default', 'wrong password'); -- { serverError AUTHENTICATION_FAILED } select * from remote('127.0.0.2', system, one, 'nonexistsnt_user_1119', ''); -- { serverError AUTHENTICATION_FAILED } diff --git a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper_long.sh b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper_long.sh index a255c1db30e..8afb0c18462 100755 --- a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper_long.sh +++ b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper_long.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_rename_replicated" -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " CREATE TABLE table_for_rename_replicated ( date Date, diff --git a/tests/queries/0_stateless/01238_http_memory_tracking.sh b/tests/queries/0_stateless/01238_http_memory_tracking.sh index ce1310cf302..f88c8fb47c6 100755 --- a/tests/queries/0_stateless/01238_http_memory_tracking.sh +++ b/tests/queries/0_stateless/01238_http_memory_tracking.sh @@ -14,7 +14,7 @@ ${CLICKHOUSE_CLIENT} --format Null -q "CREATE USER $MISTER_USER" # This is needed to keep at least one running query for user for the time of test. # (1k http queries takes ~1 second, let's run for 5x more to avoid flaps) -${CLICKHOUSE_CLIENT} --user ${MISTER_USER} --function_sleep_max_microseconds_per_block 5000000 --format Null -n <<<'SELECT sleepEachRow(1) FROM numbers(5)' & +${CLICKHOUSE_CLIENT} --user ${MISTER_USER} --function_sleep_max_microseconds_per_block 5000000 --format Null <<<'SELECT sleepEachRow(1) FROM numbers(5)' & # ignore "yes: standard output: Broken pipe" yes 'SELECT 1' 2>/dev/null | { diff --git a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh index 60a65b9a253..c1ec812875c 100755 --- a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh +++ b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" DROP DATABASE IF EXISTS 01280_db; CREATE DATABASE 01280_db; DROP TABLE IF EXISTS 01280_db.table_for_dict; @@ -39,9 +39,9 @@ $CLICKHOUSE_CLIENT -n --query=" LIFETIME(MIN 1000 MAX 2000) LAYOUT(COMPLEX_KEY_SSD_CACHE(FILE_SIZE 8192 PATH '$USER_FILES_PATH/0d'));" -$CLICKHOUSE_CLIENT -nq "SELECT dictHas('01280_db.ssd_dict', 'a', tuple('1')); -- { serverError 43 }" +$CLICKHOUSE_CLIENT -q "SELECT dictHas('01280_db.ssd_dict', 'a', tuple('1')); -- { serverError 43 }" -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" SELECT 'TEST_SMALL'; SELECT 'VALUE FROM RAM BUFFER'; SELECT dictGetUInt64('01280_db.ssd_dict', 'a', tuple('1', toInt32(3))); @@ -63,9 +63,9 @@ $CLICKHOUSE_CLIENT -n --query=" SELECT dictGetInt32('01280_db.ssd_dict', 'b', tuple('10', toInt32(-20))); SELECT dictGetString('01280_db.ssd_dict', 'c', tuple('10', toInt32(-20)));" -$CLICKHOUSE_CLIENT -nq "SELECT dictGetUInt64('01280_db.ssd_dict', 'a', tuple(toInt32(3))); -- { serverError 53 }" +$CLICKHOUSE_CLIENT -q "SELECT dictGetUInt64('01280_db.ssd_dict', 'a', tuple(toInt32(3))); -- { serverError 53 }" -$CLICKHOUSE_CLIENT -n --query="DROP DICTIONARY 01280_db.ssd_dict; +$CLICKHOUSE_CLIENT --query="DROP DICTIONARY 01280_db.ssd_dict; DROP TABLE IF EXISTS 01280_db.keys_table; CREATE TABLE 01280_db.keys_table ( @@ -122,4 +122,4 @@ $CLICKHOUSE_CLIENT -n --query="DROP DICTIONARY 01280_db.ssd_dict; DROP DICTIONARY IF EXISTS database_for_dict.ssd_dict; DROP TABLE IF EXISTS database_for_dict.keys_table;" -$CLICKHOUSE_CLIENT -n --query="DROP DATABASE IF EXISTS 01280_db;" +$CLICKHOUSE_CLIENT --query="DROP DATABASE IF EXISTS 01280_db;" diff --git a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh index 21f46a34514..00619f13173 100755 --- a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh +++ b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh @@ -85,7 +85,7 @@ export -f recreate_lazy_func4; export -f test_func; -${CLICKHOUSE_CLIENT} -n -q " +${CLICKHOUSE_CLIENT} -q " DROP DATABASE IF EXISTS $CURR_DATABASE; CREATE DATABASE $CURR_DATABASE ENGINE = Lazy(1); " diff --git a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh index 1d2d4516b9c..6ff6644f11e 100755 --- a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh +++ b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh @@ -10,7 +10,7 @@ set -e function thread() { while true; do - $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1 SYNC; + $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_table_$1 SYNC; CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now| were removed by another replica|Removing leftovers from table|Another replica was suddenly created|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time|^\(query: ' done diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh index 1d5f5d54853..cc96a37a0ce 100755 --- a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh @@ -17,7 +17,7 @@ function thread1() { local TIMELIMIT=$((SECONDS+$1)) while [ $SECONDS -lt "$TIMELIMIT" ]; do - $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r') ORDER BY x; DROP TABLE test_01320.r;" + $CLICKHOUSE_CLIENT --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r') ORDER BY x; DROP TABLE test_01320.r;" done } diff --git a/tests/queries/0_stateless/01395_limit_more_cases.sh b/tests/queries/0_stateless/01395_limit_more_cases.sh index 9709bd74f26..6be8a91f0c7 100755 --- a/tests/queries/0_stateless/01395_limit_more_cases.sh +++ b/tests/queries/0_stateless/01395_limit_more_cases.sh @@ -20,4 +20,4 @@ for OFFSET in {0..15}; do FROM (SELECT * FROM numbers($SIZE) LIMIT $OFFSET, $LIMIT); " done -done | $CLICKHOUSE_CLIENT -n --max_block_size 5 +done | $CLICKHOUSE_CLIENT --max_block_size 5 diff --git a/tests/queries/0_stateless/01395_limit_more_cases_random.sh b/tests/queries/0_stateless/01395_limit_more_cases_random.sh index c2f6b060aab..bb942906e63 100755 --- a/tests/queries/0_stateless/01395_limit_more_cases_random.sh +++ b/tests/queries/0_stateless/01395_limit_more_cases_random.sh @@ -19,4 +19,4 @@ for _ in $(seq $ITERATIONS); do throwIf((c != 0 OR first != 0 OR last != 0) AND (c != last - first + 1)) FROM (SELECT * FROM numbers($SIZE) LIMIT $OFFSET, $LIMIT); " -done | $CLICKHOUSE_CLIENT -n --max_block_size $(($RANDOM % 20 + 1)) | uniq +done | $CLICKHOUSE_CLIENT --max_block_size $(($RANDOM % 20 + 1)) | uniq diff --git a/tests/queries/0_stateless/01412_cache_dictionary_race.sh b/tests/queries/0_stateless/01412_cache_dictionary_race.sh index 36295ca01ea..b0c73cf742f 100755 --- a/tests/queries/0_stateless/01412_cache_dictionary_race.sh +++ b/tests/queries/0_stateless/01412_cache_dictionary_race.sh @@ -10,7 +10,7 @@ $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ordinary_db" $CLICKHOUSE_CLIENT --query "CREATE DATABASE ordinary_db" -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -q " CREATE DICTIONARY ordinary_db.dict1 ( @@ -35,7 +35,7 @@ function dict_get_thread() function drop_create_table_thread() { while true; do - $CLICKHOUSE_CLIENT -n --query "CREATE TABLE ordinary_db.table_for_dict_real ( + $CLICKHOUSE_CLIENT --query "CREATE TABLE ordinary_db.table_for_dict_real ( key_column UInt64, second_column UInt8, third_column String diff --git a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh index ec9c5134059..950afea9086 100755 --- a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh +++ b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh @@ -23,7 +23,7 @@ function f { function g { local TIMELIMIT=$((SECONDS+$1)) for _ in $(seq 1 100); do - $CLICKHOUSE_CLIENT -n -q " + $CLICKHOUSE_CLIENT -q " INSERT INTO mem SELECT number FROM numbers(1000000); INSERT INTO mem SELECT number FROM numbers(1000000); INSERT INTO mem SELECT number FROM numbers(1000000); diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh index 22cd6fb8127..0e6ab287146 100755 --- a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh +++ b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh @@ -11,7 +11,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) NUM_REPLICAS=6 for i in $(seq 1 $NUM_REPLICAS); do - $CLICKHOUSE_CLIENT -n -q " + $CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS r$i SYNC; CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/parallel_quorum_many', 'r$i') ORDER BY x; " @@ -39,12 +39,12 @@ done wait for i in $(seq 1 $NUM_REPLICAS); do - $CLICKHOUSE_CLIENT -n -q " + $CLICKHOUSE_CLIENT -q " SYSTEM SYNC REPLICA r$i; SELECT count(), min(x), max(x), sum(x) FROM r$i; " done for i in $(seq 1 $NUM_REPLICAS); do - $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS r$i SYNC;" + $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS r$i SYNC;" done diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.sh b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.sh index 1589f17c752..3f4210f9bb0 100755 --- a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.sh +++ b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.sh @@ -12,13 +12,13 @@ NUM_REPLICAS=2 NUM_INSERTS=5 for i in $(seq 1 $NUM_REPLICAS); do - $CLICKHOUSE_CLIENT -n -q " + $CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS r$i; CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/parallel_quorum', 'r$i') ORDER BY x; " done -$CLICKHOUSE_CLIENT -n -q "SYSTEM STOP REPLICATION QUEUES r2;" +$CLICKHOUSE_CLIENT -q "SYSTEM STOP REPLICATION QUEUES r2;" function thread { $CLICKHOUSE_CLIENT --insert_quorum 2 --insert_quorum_parallel 1 --query "INSERT INTO r1 SELECT $1" @@ -28,12 +28,12 @@ for i in $(seq 1 $NUM_INSERTS); do thread $i & done -$CLICKHOUSE_CLIENT -n -q "SYSTEM START REPLICATION QUEUES r2;" +$CLICKHOUSE_CLIENT -q "SYSTEM START REPLICATION QUEUES r2;" wait for i in $(seq 1 $NUM_REPLICAS); do - $CLICKHOUSE_CLIENT -n -q " + $CLICKHOUSE_CLIENT -q " SELECT count(), min(x), max(x), sum(x) FROM r$i; DROP TABLE IF EXISTS r$i; " diff --git a/tests/queries/0_stateless/01563_distributed_query_finish.sh b/tests/queries/0_stateless/01563_distributed_query_finish.sh index e3c5928f108..50d194b1f15 100755 --- a/tests/queries/0_stateless/01563_distributed_query_finish.sh +++ b/tests/queries/0_stateless/01563_distributed_query_finish.sh @@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm < 0 @@ -22,7 +22,7 @@ check_replicas_read_in_order() { # at some point we had a bug in this logic (see https://github.com/ClickHouse/ClickHouse/pull/45892#issue-1566140414) test1() { query_id="query_id_memory_bound_merging_$RANDOM$RANDOM" - $CLICKHOUSE_CLIENT --query_id="$query_id" -nq " + $CLICKHOUSE_CLIENT --query_id="$query_id" -q " SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; SELECT URL, EventDate, max(URL) @@ -39,7 +39,7 @@ test1() { # at some point we had a bug in this logic (see https://github.com/ClickHouse/ClickHouse/pull/45892#issue-1566140414) test2() { query_id="query_id_memory_bound_merging_$RANDOM$RANDOM" - $CLICKHOUSE_CLIENT --query_id="$query_id" -nq " + $CLICKHOUSE_CLIENT --query_id="$query_id" -q " SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; SELECT URL, EventDate, max(URL) @@ -53,7 +53,7 @@ test2() { } test3() { - $CLICKHOUSE_CLIENT -nq " + $CLICKHOUSE_CLIENT -q " SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; SET max_threads = 16, prefer_localhost_replica = 1, read_in_order_two_level_merge_threshold = 1000, query_plan_aggregation_in_order = 1, distributed_aggregation_memory_efficient = 1; diff --git a/tests/queries/1_stateful/00180_no_seek_avoiding_when_reading_from_cache.sh b/tests/queries/1_stateful/00180_no_seek_avoiding_when_reading_from_cache.sh index f9fea2c1dad..bf44f2d7ce7 100755 --- a/tests/queries/1_stateful/00180_no_seek_avoiding_when_reading_from_cache.sh +++ b/tests/queries/1_stateful/00180_no_seek_avoiding_when_reading_from_cache.sh @@ -23,7 +23,7 @@ $CLICKHOUSE_CLIENT -q "SELECT * FROM hits_s3_sampled WHERE URL LIKE '%google%' O query_id=02906_read_from_cache_$RANDOM $CLICKHOUSE_CLIENT --query_id ${query_id} -q "SELECT * FROM hits_s3_sampled WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10 FORMAT Null SETTINGS filesystem_cache_reserve_space_wait_lock_timeout_milliseconds=2000" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " SYSTEM FLUSH LOGS; -- AsynchronousReaderIgnoredBytes = 0: no seek-avoiding happened From f1c43472d80bb1f04f0d0dfac87d985a4f33bfd5 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Mon, 9 Sep 2024 11:35:54 +0800 Subject: [PATCH 244/289] change as request --- src/Functions/array/array.cpp | 44 ++++++++++++++++++----------------- src/Functions/map.cpp | 4 ++-- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index f193acafe06..57d24b185e4 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -100,14 +100,14 @@ public: if (num_elements == 1) out_data.insertRangeFrom(*column_ptrs[0], 0, input_rows_count); else - executeAny(column_ptrs, out_data, input_rows_count); + execute(column_ptrs, out_data, input_rows_count); return out; } + private: - bool executeAny(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + bool execute(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const { - return false // NOLINT - || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + return executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) @@ -123,14 +123,7 @@ private: || executeFixedString(columns, out_data, input_rows_count) || executeGeneric(columns, out_data, input_rows_count); } - bool executeGeneric(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const - { - for (size_t i = 0; i < input_rows_count; ++i) - for (const auto * column : columns) - out_data.insertFrom(*column, i); - return true; - } - + template bool executeNumber(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const { @@ -178,18 +171,18 @@ private: out_chars.resize_exact(total_bytes); out_offsets.resize_exact(input_rows_count * columns.size()); - size_t curr_out_offset = 0; + size_t cur_out_offset = 0; for (size_t row_i = 0; row_i < input_rows_count; ++row_i) { const size_t base = row_i * columns.size(); for (size_t col_i = 0; col_i < columns.size(); ++col_i) { StringRef ref = concrete_columns[col_i]->getDataAt(row_i); - memcpySmallAllowReadWriteOverflow15(&out_chars[curr_out_offset], ref.data, ref.size); - out_chars[curr_out_offset + ref.size] = 0; + memcpySmallAllowReadWriteOverflow15(&out_chars[cur_out_offset], ref.data, ref.size); + out_chars[cur_out_offset + ref.size] = 0; - curr_out_offset += ref.size + 1; - out_offsets[base + col_i] = curr_out_offset; + cur_out_offset += ref.size + 1; + out_offsets[base + col_i] = cur_out_offset; } } return true; @@ -211,7 +204,7 @@ private: auto & out_chars = concrete_out_data.getChars(); const size_t n = concrete_out_data.getN(); - size_t total_bytes = concrete_out_data.getN() * columns.size() * input_rows_count; + size_t total_bytes = n * columns.size() * input_rows_count; out_chars.resize_exact(total_bytes); size_t curr_out_offset = 0; @@ -244,8 +237,8 @@ private: ColumnNullable & concrete_out_data = assert_cast(out_data); auto & out_null_map = concrete_out_data.getNullMapColumn(); auto & out_nested_column = concrete_out_data.getNestedColumn(); - executeAny(null_maps, out_null_map, input_rows_count); - executeAny(nested_columns, out_nested_column, input_rows_count); + execute(null_maps, out_null_map, input_rows_count); + execute(nested_columns, out_nested_column, input_rows_count); return true; } @@ -264,11 +257,20 @@ private: const ColumnTuple * concrete_column = assert_cast(columns[j]); elem_columns[j] = &concrete_column->getColumn(i); } - executeAny(elem_columns, concrete_out_data->getColumn(i), input_rows_count); + execute(elem_columns, concrete_out_data->getColumn(i), input_rows_count); } return true; } + bool executeGeneric(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + for (size_t i = 0; i < input_rows_count; ++i) + for (const auto * column : columns) + out_data.insertFrom(*column, i); + return true; + } + + String getName() const override { return name; diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index d9643087b11..14672cfd568 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -38,7 +38,7 @@ public: static constexpr auto name = "map"; explicit FunctionMap(ContextPtr context_) - : context(std::move(context_)) + : context(context_) , use_variant_as_common_type( context->getSettingsRef().allow_experimental_variant_type && context->getSettingsRef().use_variant_as_common_type) , function_array(FunctionFactory::instance().get("array", context)) @@ -48,7 +48,7 @@ public: static FunctionPtr create(ContextPtr context) { - return std::make_shared(std::move(context)); + return std::make_shared(context); } String getName() const override From 774ec08fafc851f6c71a922dd0dde4a3f1f0d41d Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Mon, 9 Sep 2024 11:37:27 +0800 Subject: [PATCH 245/289] fix style --- src/Functions/array/array.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index 57d24b185e4..d2aedd57f99 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -123,7 +123,6 @@ private: || executeFixedString(columns, out_data, input_rows_count) || executeGeneric(columns, out_data, input_rows_count); } - template bool executeNumber(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const { From eb800d9e3918387c243c0480ac9df37190a91dc7 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Mon, 9 Sep 2024 11:45:39 +0800 Subject: [PATCH 246/289] Rename statistics of type count_min to countmin --- .../engines/table-engines/mergetree-family/mergetree.md | 8 ++++---- src/Storages/Statistics/Statistics.cpp | 2 +- src/Storages/Statistics/StatisticsCountMinSketch.cpp | 4 ++-- src/Storages/StatisticsDescription.cpp | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 74f1295ef96..317d1cca61e 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -1003,16 +1003,16 @@ They can be used for prewhere optimization only if we enable `set allow_statisti [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) sketches which provide an estimation how many distinct values a column contains. -- `count_min` +- `CountMin` - [Count-min](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch) sketches which provide an approximate count of the frequency of each value in a column. + [CountMin](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch) sketches which provide an approximate count of the frequency of each value in a column. ### Supported Data Types {#supported-data-types} | | (U)Int* | Float* | Decimal(*) | Date* | Boolean | Enum* | (Fixed)String | |-----------|---------|--------|------------|-------|---------|-------|------------------| -| count_min | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| CountMin | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | | MinMax | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✗ | | TDigest | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✗ | | Uniq | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | @@ -1022,7 +1022,7 @@ They can be used for prewhere optimization only if we enable `set allow_statisti | | Equality filters (==) | Range filters (>, >=, <, <=) | |-----------|-----------------------|------------------------------| -| count_min | ✔ | ✗ | +| CountMin | ✔ | ✗ | | MinMax | ✗ | ✔ | | TDigest | ✗ | ✔ | | Uniq | ✔ | ✗ | diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp index 795963bd55d..0557530515f 100644 --- a/src/Storages/Statistics/Statistics.cpp +++ b/src/Storages/Statistics/Statistics.cpp @@ -248,7 +248,7 @@ ColumnStatisticsPtr MergeTreeStatisticsFactory::get(const ColumnDescription & co { auto it = creators.find(type); if (it == creators.end()) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistic type '{}'. Available types: 'count_min', 'minmax', 'tdigest' and 'uniq'", type); + throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistic type '{}'. Available types: 'countmin', 'minmax', 'tdigest' and 'uniq'", type); auto stat_ptr = (it->second)(desc, column_desc.type); column_stat->stats[type] = stat_ptr; } diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.cpp b/src/Storages/Statistics/StatisticsCountMinSketch.cpp index 1a2459c230d..f477181ec2d 100644 --- a/src/Storages/Statistics/StatisticsCountMinSketch.cpp +++ b/src/Storages/Statistics/StatisticsCountMinSketch.cpp @@ -49,7 +49,7 @@ Float64 StatisticsCountMinSketch::estimateEqual(const Field & val) const if (isStringOrFixedString(data_type)) return sketch.get_estimate(val.safeGet()); - throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'count_min' does not support estimate data type of {}", data_type->getName()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'countmin' does not support estimate data type of {}", data_type->getName()); } void StatisticsCountMinSketch::update(const ColumnPtr & column) @@ -88,7 +88,7 @@ void countMinSketchStatisticsValidator(const SingleStatisticsDescription & /*des DataTypePtr inner_data_type = removeNullable(data_type); inner_data_type = removeLowCardinalityAndNullable(inner_data_type); if (!inner_data_type->isValueRepresentedByNumber() && !isStringOrFixedString(inner_data_type)) - throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'count_min' does not support type {}", data_type->getName()); + throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'countmin' does not support type {}", data_type->getName()); } StatisticsPtr countMinSketchStatisticsCreator(const SingleStatisticsDescription & description, const DataTypePtr & data_type) diff --git a/src/Storages/StatisticsDescription.cpp b/src/Storages/StatisticsDescription.cpp index 4e0d901d0c7..ac7fa8898de 100644 --- a/src/Storages/StatisticsDescription.cpp +++ b/src/Storages/StatisticsDescription.cpp @@ -48,11 +48,11 @@ static StatisticsType stringToStatisticsType(String type) return StatisticsType::TDigest; if (type == "uniq") return StatisticsType::Uniq; - if (type == "count_min") + if (type == "countmin") return StatisticsType::CountMinSketch; if (type == "minmax") return StatisticsType::MinMax; - throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are 'count_min', 'minmax', 'tdigest' and 'uniq'.", type); + throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are 'countmin', 'minmax', 'tdigest' and 'uniq'.", type); } String SingleStatisticsDescription::getTypeName() const @@ -64,11 +64,11 @@ String SingleStatisticsDescription::getTypeName() const case StatisticsType::Uniq: return "Uniq"; case StatisticsType::CountMinSketch: - return "count_min"; + return "countmin"; case StatisticsType::MinMax: return "minmax"; default: - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are 'count_min', 'minmax', 'tdigest' and 'uniq'.", type); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are 'countmin', 'minmax', 'tdigest' and 'uniq'.", type); } } From 077b8239f55163ae25f901924f5a93997d084256 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Mon, 9 Sep 2024 11:52:58 +0800 Subject: [PATCH 247/289] Rename in tests --- ...atistics_create_materialize_drop.reference | 4 +- ...864_statistics_create_materialize_drop.sql | 10 +-- .../0_stateless/02864_statistics_ddl.sql | 56 ++++++++-------- .../02864_statistics_predicates.sql | 64 +++++++++---------- 4 files changed, 67 insertions(+), 67 deletions(-) diff --git a/tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference index 4ca2c5e5f9b..1843964377d 100644 --- a/tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference +++ b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.reference @@ -1,5 +1,5 @@ Test create statistics: -CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64) STATISTICS(tdigest, uniq, count_min, minmax),\n `b` LowCardinality(Nullable(String)) STATISTICS(uniq, count_min),\n `c` LowCardinality(Nullable(Int64)) STATISTICS(tdigest, uniq, count_min, minmax),\n `d` DateTime STATISTICS(tdigest, uniq, count_min, minmax),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 +CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64) STATISTICS(tdigest, uniq, countmin, minmax),\n `b` LowCardinality(Nullable(String)) STATISTICS(uniq, countmin),\n `c` LowCardinality(Nullable(Int64)) STATISTICS(tdigest, uniq, countmin, minmax),\n `d` DateTime STATISTICS(tdigest, uniq, countmin, minmax),\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 Test materialize and drop statistics: -CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64),\n `b` LowCardinality(Nullable(String)) STATISTICS(uniq, count_min),\n `c` LowCardinality(Nullable(Int64)),\n `d` DateTime,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 +CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64),\n `b` LowCardinality(Nullable(String)) STATISTICS(uniq, countmin),\n `c` LowCardinality(Nullable(Int64)),\n `d` DateTime,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 CREATE TABLE default.tab\n(\n `a` LowCardinality(Int64),\n `b` LowCardinality(Nullable(String)),\n `c` LowCardinality(Nullable(Int64)),\n `d` DateTime,\n `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql index 90a57c99624..249e3c84a51 100644 --- a/tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql +++ b/tests/queries/0_stateless/02864_statistics_create_materialize_drop.sql @@ -12,10 +12,10 @@ SELECT 'Test create statistics:'; CREATE TABLE tab ( - a LowCardinality(Int64) STATISTICS(count_min, minmax, tdigest, uniq), - b LowCardinality(Nullable(String)) STATISTICS(count_min, uniq), - c LowCardinality(Nullable(Int64)) STATISTICS(count_min, minmax, tdigest, uniq), - d DateTime STATISTICS(count_min, minmax, tdigest, uniq), + a LowCardinality(Int64) STATISTICS(countmin, minmax, tdigest, uniq), + b LowCardinality(Nullable(String)) STATISTICS(countmin, uniq), + c LowCardinality(Nullable(Int64)) STATISTICS(countmin, minmax, tdigest, uniq), + d DateTime STATISTICS(countmin, minmax, tdigest, uniq), pk String, ) Engine = MergeTree() ORDER BY pk; @@ -25,7 +25,7 @@ SHOW CREATE TABLE tab; SELECT 'Test materialize and drop statistics:'; ALTER TABLE tab DROP STATISTICS a, b, c, d; -ALTER TABLE tab ADD STATISTICS b TYPE count_min, uniq; +ALTER TABLE tab ADD STATISTICS b TYPE countmin, uniq; ALTER TABLE tab MATERIALIZE STATISTICS b; SHOW CREATE TABLE tab; diff --git a/tests/queries/0_stateless/02864_statistics_ddl.sql b/tests/queries/0_stateless/02864_statistics_ddl.sql index bcaaa9e7b61..5b2c5cebc1d 100644 --- a/tests/queries/0_stateless/02864_statistics_ddl.sql +++ b/tests/queries/0_stateless/02864_statistics_ddl.sql @@ -1,5 +1,5 @@ -- Tags: no-fasttest --- no-fasttest: 'count_min' sketches need a 3rd party library +-- no-fasttest: 'countmin' sketches need a 3rd party library -- Tests that DDL statements which create / drop / materialize statistics @@ -71,29 +71,29 @@ CREATE TABLE tab (col Map(UInt64, UInt64) STATISTICS(uniq)) Engine = MergeTree() CREATE TABLE tab (col UUID STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } CREATE TABLE tab (col IPv6 STATISTICS(uniq)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } --- count_min requires data_type.isValueRepresentedByInteger or data_type = (Fixed)String +-- countmin requires data_type.isValueRepresentedByInteger or data_type = (Fixed)String -- These types work: -CREATE TABLE tab (col UInt8 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col UInt256 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col Float32 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col Decimal32(3) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col Date STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col Date32 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col DateTime STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col DateTime64 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col Enum('hello', 'world') STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col IPv4 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col Nullable(UInt8) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col LowCardinality(UInt8) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col LowCardinality(Nullable(UInt8)) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col String STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -CREATE TABLE tab (col FixedString(1) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col UInt8 STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col UInt256 STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Float32 STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Decimal32(3) STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Date STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Date32 STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col DateTime STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col DateTime64 STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Enum('hello', 'world') STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col IPv4 STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col Nullable(UInt8) STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col LowCardinality(UInt8) STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col LowCardinality(Nullable(UInt8)) STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col String STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; +CREATE TABLE tab (col FixedString(1) STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); DROP TABLE tab; -- These types don't work: -CREATE TABLE tab (col Array(Float64) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } -CREATE TABLE tab (col Tuple(Float64, Float64) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } -CREATE TABLE tab (col Map(UInt64, UInt64) STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } -CREATE TABLE tab (col UUID STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } -CREATE TABLE tab (col IPv6 STATISTICS(count_min)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +CREATE TABLE tab (col Array(Float64) STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +CREATE TABLE tab (col Tuple(Float64, Float64) STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +CREATE TABLE tab (col Map(UInt64, UInt64) STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +CREATE TABLE tab (col UUID STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } +CREATE TABLE tab (col IPv6 STATISTICS(countmin)) Engine = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_STATISTICS } -- minmax requires data_type.isValueRepresentedByInteger -- These types work: @@ -187,17 +187,17 @@ ALTER TABLE tab MODIFY STATISTICS a TYPE tdigest; -- { serverError ILLEGAL_STATI -- uniq -- Works: ALTER TABLE tab ADD STATISTICS f64 TYPE uniq; ALTER TABLE tab DROP STATISTICS f64; -ALTER TABLE tab MODIFY STATISTICS f64 TYPE count_min; ALTER TABLE tab DROP STATISTICS f64; +ALTER TABLE tab MODIFY STATISTICS f64 TYPE countmin; ALTER TABLE tab DROP STATISTICS f64; -- Doesn't work: ALTER TABLE tab ADD STATISTICS a TYPE uniq; -- { serverError ILLEGAL_STATISTICS } ALTER TABLE tab MODIFY STATISTICS a TYPE uniq; -- { serverError ILLEGAL_STATISTICS } --- count_min +-- countmin -- Works: -ALTER TABLE tab ADD STATISTICS f64 TYPE count_min; ALTER TABLE tab DROP STATISTICS f64; -ALTER TABLE tab MODIFY STATISTICS f64 TYPE count_min; ALTER TABLE tab DROP STATISTICS f64; +ALTER TABLE tab ADD STATISTICS f64 TYPE countmin; ALTER TABLE tab DROP STATISTICS f64; +ALTER TABLE tab MODIFY STATISTICS f64 TYPE countmin; ALTER TABLE tab DROP STATISTICS f64; -- Doesn't work: -ALTER TABLE tab ADD STATISTICS a TYPE count_min; -- { serverError ILLEGAL_STATISTICS } -ALTER TABLE tab MODIFY STATISTICS a TYPE count_min; -- { serverError ILLEGAL_STATISTICS } +ALTER TABLE tab ADD STATISTICS a TYPE countmin; -- { serverError ILLEGAL_STATISTICS } +ALTER TABLE tab MODIFY STATISTICS a TYPE countmin; -- { serverError ILLEGAL_STATISTICS } -- minmax -- Works: ALTER TABLE tab ADD STATISTICS f64 TYPE minmax; ALTER TABLE tab DROP STATISTICS f64; diff --git a/tests/queries/0_stateless/02864_statistics_predicates.sql b/tests/queries/0_stateless/02864_statistics_predicates.sql index 473a7bc95ad..d7afba12c1d 100644 --- a/tests/queries/0_stateless/02864_statistics_predicates.sql +++ b/tests/queries/0_stateless/02864_statistics_predicates.sql @@ -1,5 +1,5 @@ -- Tags: no-fasttest --- no-fasttest: 'count_min' sketches need a 3rd party library +-- no-fasttest: 'countmin' sketches need a 3rd party library -- Tests the cross product of all predicates with all right-hand sides on all data types and all statistics types. @@ -13,27 +13,27 @@ CREATE TABLE tab u64 UInt64, u64_tdigest UInt64 STATISTICS(tdigest), u64_minmax UInt64 STATISTICS(minmax), - u64_count_min UInt64 STATISTICS(count_min), + u64_countmin UInt64 STATISTICS(countmin), u64_uniq UInt64 STATISTICS(uniq), f64 Float64, f64_tdigest Float64 STATISTICS(tdigest), f64_minmax Float64 STATISTICS(minmax), - f64_count_min Float64 STATISTICS(count_min), + f64_countmin Float64 STATISTICS(countmin), f64_uniq Float64 STATISTICS(uniq), dt DateTime, dt_tdigest DateTime STATISTICS(tdigest), dt_minmax DateTime STATISTICS(minmax), - dt_count_min DateTime STATISTICS(count_min), + dt_countmin DateTime STATISTICS(countmin), dt_uniq DateTime STATISTICS(uniq), b Bool, b_tdigest Bool STATISTICS(tdigest), b_minmax Bool STATISTICS(minmax), - b_count_min Bool STATISTICS(count_min), + b_countmin Bool STATISTICS(countmin), b_uniq Bool STATISTICS(uniq), s String, -- s_tdigest String STATISTICS(tdigest), -- not supported by tdigest -- s_minmax String STATISTICS(minmax), -- not supported by minmax - s_count_min String STATISTICS(count_min), + s_countmin String STATISTICS(countmin), s_uniq String STATISTICS(uniq) ) Engine = MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; @@ -72,25 +72,25 @@ SELECT 'u64 and ='; SELECT count(*) FROM tab WHERE u64 = 7; SELECT count(*) FROM tab WHERE u64_tdigest = 7; SELECT count(*) FROM tab WHERE u64_minmax = 7; -SELECT count(*) FROM tab WHERE u64_count_min = 7; +SELECT count(*) FROM tab WHERE u64_countmin = 7; SELECT count(*) FROM tab WHERE u64_uniq = 7; SELECT count(*) FROM tab WHERE u64 = 7.7; SELECT count(*) FROM tab WHERE u64_tdigest = 7.7; SELECT count(*) FROM tab WHERE u64_minmax = 7.7; -SELECT count(*) FROM tab WHERE u64_count_min = 7.7; +SELECT count(*) FROM tab WHERE u64_countmin = 7.7; SELECT count(*) FROM tab WHERE u64_uniq = 7.7; SELECT count(*) FROM tab WHERE u64 = '7'; SELECT count(*) FROM tab WHERE u64_tdigest = '7'; SELECT count(*) FROM tab WHERE u64_minmax = '7'; -SELECT count(*) FROM tab WHERE u64_count_min = '7'; +SELECT count(*) FROM tab WHERE u64_countmin = '7'; SELECT count(*) FROM tab WHERE u64_uniq = '7'; SELECT count(*) FROM tab WHERE u64 = '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_tdigest = '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_minmax = '7.7'; -- { serverError TYPE_MISMATCH } -SELECT count(*) FROM tab WHERE u64_count_min = '7.7'; -- { serverError TYPE_MISMATCH } +SELECT count(*) FROM tab WHERE u64_countmin = '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_uniq = '7.7'; -- { serverError TYPE_MISMATCH } SELECT 'u64 and <'; @@ -98,25 +98,25 @@ SELECT 'u64 and <'; SELECT count(*) FROM tab WHERE u64 < 7; SELECT count(*) FROM tab WHERE u64_tdigest < 7; SELECT count(*) FROM tab WHERE u64_minmax < 7; -SELECT count(*) FROM tab WHERE u64_count_min < 7; +SELECT count(*) FROM tab WHERE u64_countmin < 7; SELECT count(*) FROM tab WHERE u64_uniq < 7; SELECT count(*) FROM tab WHERE u64 < 7.7; SELECT count(*) FROM tab WHERE u64_tdigest < 7.7; SELECT count(*) FROM tab WHERE u64_minmax < 7.7; -SELECT count(*) FROM tab WHERE u64_count_min < 7.7; +SELECT count(*) FROM tab WHERE u64_countmin < 7.7; SELECT count(*) FROM tab WHERE u64_uniq < 7.7; SELECT count(*) FROM tab WHERE u64 < '7'; SELECT count(*) FROM tab WHERE u64_tdigest < '7'; SELECT count(*) FROM tab WHERE u64_minmax < '7'; -SELECT count(*) FROM tab WHERE u64_count_min < '7'; +SELECT count(*) FROM tab WHERE u64_countmin < '7'; SELECT count(*) FROM tab WHERE u64_uniq < '7'; SELECT count(*) FROM tab WHERE u64 < '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_tdigest < '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_minmax < '7.7'; -- { serverError TYPE_MISMATCH } -SELECT count(*) FROM tab WHERE u64_count_min < '7.7'; -- { serverError TYPE_MISMATCH } +SELECT count(*) FROM tab WHERE u64_countmin < '7.7'; -- { serverError TYPE_MISMATCH } SELECT count(*) FROM tab WHERE u64_uniq < '7.7'; -- { serverError TYPE_MISMATCH } -- f64 ---------------------------------------------------- @@ -126,25 +126,25 @@ SELECT 'f64 and ='; SELECT count(*) FROM tab WHERE f64 = 7; SELECT count(*) FROM tab WHERE f64_tdigest = 7; SELECT count(*) FROM tab WHERE f64_minmax = 7; -SELECT count(*) FROM tab WHERE f64_count_min = 7; +SELECT count(*) FROM tab WHERE f64_countmin = 7; SELECT count(*) FROM tab WHERE f64_uniq = 7; SELECT count(*) FROM tab WHERE f64 = 7.7; SELECT count(*) FROM tab WHERE f64_tdigest = 7.7; SELECT count(*) FROM tab WHERE f64_minmax = 7.7; -SELECT count(*) FROM tab WHERE f64_count_min = 7.7; +SELECT count(*) FROM tab WHERE f64_countmin = 7.7; SELECT count(*) FROM tab WHERE f64_uniq = 7.7; SELECT count(*) FROM tab WHERE f64 = '7'; SELECT count(*) FROM tab WHERE f64_tdigest = '7'; SELECT count(*) FROM tab WHERE f64_minmax = '7'; -SELECT count(*) FROM tab WHERE f64_count_min = '7'; +SELECT count(*) FROM tab WHERE f64_countmin = '7'; SELECT count(*) FROM tab WHERE f64_uniq = '7'; SELECT count(*) FROM tab WHERE f64 = '7.7'; SELECT count(*) FROM tab WHERE f64_tdigest = '7.7'; SELECT count(*) FROM tab WHERE f64_minmax = '7.7'; -SELECT count(*) FROM tab WHERE f64_count_min = '7.7'; +SELECT count(*) FROM tab WHERE f64_countmin = '7.7'; SELECT count(*) FROM tab WHERE f64_uniq = '7.7'; SELECT 'f64 and <'; @@ -152,25 +152,25 @@ SELECT 'f64 and <'; SELECT count(*) FROM tab WHERE f64 < 7; SELECT count(*) FROM tab WHERE f64_tdigest < 7; SELECT count(*) FROM tab WHERE f64_minmax < 7; -SELECT count(*) FROM tab WHERE f64_count_min < 7; +SELECT count(*) FROM tab WHERE f64_countmin < 7; SELECT count(*) FROM tab WHERE f64_uniq < 7; SELECT count(*) FROM tab WHERE f64 < 7.7; SELECT count(*) FROM tab WHERE f64_tdigest < 7.7; SELECT count(*) FROM tab WHERE f64_minmax < 7.7; -SELECT count(*) FROM tab WHERE f64_count_min < 7.7; +SELECT count(*) FROM tab WHERE f64_countmin < 7.7; SELECT count(*) FROM tab WHERE f64_uniq < 7.7; SELECT count(*) FROM tab WHERE f64 < '7'; SELECT count(*) FROM tab WHERE f64_tdigest < '7'; SELECT count(*) FROM tab WHERE f64_minmax < '7'; -SELECT count(*) FROM tab WHERE f64_count_min < '7'; +SELECT count(*) FROM tab WHERE f64_countmin < '7'; SELECT count(*) FROM tab WHERE f64_uniq < '7'; SELECT count(*) FROM tab WHERE f64 < '7.7'; SELECT count(*) FROM tab WHERE f64_tdigest < '7.7'; SELECT count(*) FROM tab WHERE f64_minmax < '7.7'; -SELECT count(*) FROM tab WHERE f64_count_min < '7.7'; +SELECT count(*) FROM tab WHERE f64_countmin < '7.7'; SELECT count(*) FROM tab WHERE f64_uniq < '7.7'; -- dt ---------------------------------------------------- @@ -180,13 +180,13 @@ SELECT 'dt and ='; SELECT count(*) FROM tab WHERE dt = '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_tdigest = '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_minmax = '2024-08-08 11:12:13'; -SELECT count(*) FROM tab WHERE dt_count_min = '2024-08-08 11:12:13'; +SELECT count(*) FROM tab WHERE dt_countmin = '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_uniq = '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt = 7; SELECT count(*) FROM tab WHERE dt_tdigest = 7; SELECT count(*) FROM tab WHERE dt_minmax = 7; -SELECT count(*) FROM tab WHERE dt_count_min = 7; +SELECT count(*) FROM tab WHERE dt_countmin = 7; SELECT count(*) FROM tab WHERE dt_uniq = 7; SELECT 'dt and <'; @@ -194,13 +194,13 @@ SELECT 'dt and <'; SELECT count(*) FROM tab WHERE dt < '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_tdigest < '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_minmax < '2024-08-08 11:12:13'; -SELECT count(*) FROM tab WHERE dt_count_min < '2024-08-08 11:12:13'; +SELECT count(*) FROM tab WHERE dt_countmin < '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt_uniq < '2024-08-08 11:12:13'; SELECT count(*) FROM tab WHERE dt < 7; SELECT count(*) FROM tab WHERE dt_tdigest < 7; SELECT count(*) FROM tab WHERE dt_minmax < 7; -SELECT count(*) FROM tab WHERE dt_count_min < 7; +SELECT count(*) FROM tab WHERE dt_countmin < 7; SELECT count(*) FROM tab WHERE dt_uniq < 7; -- b ---------------------------------------------------- @@ -210,25 +210,25 @@ SELECT 'b and ='; SELECT count(*) FROM tab WHERE b = true; SELECT count(*) FROM tab WHERE b_tdigest = true; SELECT count(*) FROM tab WHERE b_minmax = true; -SELECT count(*) FROM tab WHERE b_count_min = true; +SELECT count(*) FROM tab WHERE b_countmin = true; SELECT count(*) FROM tab WHERE b_uniq = true; SELECT count(*) FROM tab WHERE b = 'true'; SELECT count(*) FROM tab WHERE b_tdigest = 'true'; SELECT count(*) FROM tab WHERE b_minmax = 'true'; -SELECT count(*) FROM tab WHERE b_count_min = 'true'; +SELECT count(*) FROM tab WHERE b_countmin = 'true'; SELECT count(*) FROM tab WHERE b_uniq = 'true'; SELECT count(*) FROM tab WHERE b = 1; SELECT count(*) FROM tab WHERE b_tdigest = 1; SELECT count(*) FROM tab WHERE b_minmax = 1; -SELECT count(*) FROM tab WHERE b_count_min = 1; +SELECT count(*) FROM tab WHERE b_countmin = 1; SELECT count(*) FROM tab WHERE b_uniq = 1; SELECT count(*) FROM tab WHERE b = 1.1; SELECT count(*) FROM tab WHERE b_tdigest = 1.1; SELECT count(*) FROM tab WHERE b_minmax = 1.1; -SELECT count(*) FROM tab WHERE b_count_min = 1.1; +SELECT count(*) FROM tab WHERE b_countmin = 1.1; SELECT count(*) FROM tab WHERE b_uniq = 1.1; -- s ---------------------------------------------------- @@ -238,13 +238,13 @@ SELECT 's and ='; SELECT count(*) FROM tab WHERE s = 7; -- { serverError NO_COMMON_TYPE } -- SELECT count(*) FROM tab WHERE s_tdigest = 7; -- not supported -- SELECT count(*) FROM tab WHERE s_minmax = 7; -- not supported -SELECT count(*) FROM tab WHERE s_count_min = 7; -- { serverError NO_COMMON_TYPE } +SELECT count(*) FROM tab WHERE s_countmin = 7; -- { serverError NO_COMMON_TYPE } SELECT count(*) FROM tab WHERE s_uniq = 7; -- { serverError NO_COMMON_TYPE } SELECT count(*) FROM tab WHERE s = '7'; -- SELECT count(*) FROM tab WHERE s_tdigest = '7'; -- not supported -- SELECT count(*) FROM tab WHERE s_minmax = '7'; -- not supported -SELECT count(*) FROM tab WHERE s_count_min = '7'; +SELECT count(*) FROM tab WHERE s_countmin = '7'; SELECT count(*) FROM tab WHERE s_uniq = '7'; DROP TABLE tab; From c5f8802ac8e43efd51441cf4dbbf1ccf2381af9a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 9 Sep 2024 08:04:53 +0000 Subject: [PATCH 248/289] Docs: Mention FROM-before-SELECT --- docs/en/sql-reference/statements/select/from.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/statements/select/from.md b/docs/en/sql-reference/statements/select/from.md index 7a6e2ab054c..f319e7b1357 100644 --- a/docs/en/sql-reference/statements/select/from.md +++ b/docs/en/sql-reference/statements/select/from.md @@ -15,7 +15,14 @@ The `FROM` clause specifies the source to read data from: Subquery is another `SELECT` query that may be specified in parenthesis inside `FROM` clause. -`FROM` clause can contain multiple data sources, separated by commas, which is equivalent of performing [CROSS JOIN](../../../sql-reference/statements/select/join.md) on them. +The `FROM` can contain multiple data sources, separated by commas, which is equivalent of performing [CROSS JOIN](../../../sql-reference/statements/select/join.md) on them. + +`FROM` can optionally appear before a `SELECT` clause. This is a ClickHouse-specific extension of standard SQL which makes `SELECT` statements easier to read. Example: + +```sql +FROM table +SELECT * +``` ## FINAL Modifier @@ -45,19 +52,19 @@ As an alternative to using `FINAL`, it is sometimes possible to use different qu ### Example Usage -**Using the `FINAL` keyword** +Using the `FINAL` keyword ```sql SELECT x, y FROM mytable FINAL WHERE x > 1; ``` -**Using `FINAL` as a query-level setting** +Using `FINAL` as a query-level setting ```sql SELECT x, y FROM mytable WHERE x > 1 SETTINGS final = 1; ``` -**Using `FINAL` as a session-level setting** +Using `FINAL` as a session-level setting ```sql SET final = 1; From 6539cbd1ce1997ca14c65131502cda2e0daf7676 Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Mon, 9 Sep 2024 17:34:15 +0800 Subject: [PATCH 249/289] Add CountMin to dictionary file and a little fixup --- .../table-engines/mergetree-family/mergetree.md | 17 +++++++++++++++++ .../aspell-ignore/en/aspell-dict.txt | 1 + 2 files changed, 18 insertions(+) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 317d1cca61e..d17208ab099 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -991,6 +991,23 @@ They can be used for prewhere optimization only if we enable `set allow_statisti ### Available Types of Column Statistics {#available-types-of-column-statistics} +Names of statistics types are case-insensitive, i.e. both + +``` sql +CREATE TABLE tab +( +a LowCardinality(Int64) STATISTICS(minmax) +) [...] +``` +and +``` sql +CREATE TABLE tab +( +a LowCardinality(Int64) STATISTICS(MiNmAx) +) [...] +``` +work. + - `MinMax` The minimum and maximum column value which allows to estimate the selectivity of range filters on numeric columns. diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index d10db5f0d3d..a51132e6d7f 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -120,6 +120,7 @@ CMPLNT CMake CMakeLists CODECS +CountMin COVID CPUFrequencyMHz CPUs From 3363fe33888b2c1f994ea4157b7f323a518e89b6 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 9 Sep 2024 10:49:53 +0000 Subject: [PATCH 250/289] Docs: Document per-logger level overriding --- .../settings.md | 71 ++++++++++++------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 9fce83a0dc4..ccc8cf017ca 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -1463,26 +1463,29 @@ Examples: ## logger {#logger} -Logging settings. +The location and format of log messages. Keys: -- `level` – Logging level. Acceptable values: `trace`, `debug`, `information`, `warning`, `error`. -- `log` – The log file. Contains all the entries according to `level`. -- `errorlog` – Error log file. -- `size` – Size of the file. Applies to `log` and `errorlog`. Once the file reaches `size`, ClickHouse archives and renames it, and creates a new log file in its place. -- `count` – The number of archived log files that ClickHouse stores. -- `console` – Send `log` and `errorlog` to the console instead of file. To enable, set to `1` or `true`. -- `console_log_level` – Logging level for console. Default to `level`. -- `use_syslog` - Log to syslog as well. -- `syslog_level` - Logging level for logging to syslog. -- `stream_compress` – Compress `log` and `errorlog` with `lz4` stream compression. To enable, set to `1` or `true`. -- `formatting` – Specify log format to be printed in console log (currently only `json` supported). +- `level` – Log level. Acceptable values: `none` (turn logging off), `fatal`, `critical`, `error`, `warning`, `notice`, `information`, + `debug`, `trace`, `test` +- `log` – The path to the log file. +- `errorlog` – The path to the error log file. +- `size` – Rotation policy: Maximum size of the log files in bytes. Once the log file size exceeds this threshold, it is renamed and archived, and a new log file is created. +- `count` – Rotation policy: How many historical log files Clickhouse are kept at most. +- `stream_compress` – Compress log messages using LZ4. Set to `1` or `true` to enable. +- `console` – Do not write log messages to log files, instead print them in the console. Set to `1` or `true` to enable. Default is + `1` if Clickhouse does not run in daemon mode, `0` otherwise. +- `console_log_level` – Log level for console output. Defaults to `level`. +- `formatting` – Log format for console output. Currently, only `json` is supported). +- `use_syslog` - Also forward log output to syslog. +- `syslog_level` - Log level for logging to syslog. -Both log and error log file names (only file names, not directories) support date and time format specifiers. +**Log format specifiers** -**Format specifiers** -Using the following format specifiers, you can define a pattern for the resulting file name. “Example” column shows possible results for `2023-07-06 18:32:07`. +File names in `log` and `errorLog` paths support below format specifiers for the resulting file name (the directory part does not support them). + +Column “Example” shows the output at `2023-07-06 18:32:07`. | Specifier | Description | Example | |-------------|---------------------------------------------------------------------------------------------------------------------|--------------------------| @@ -1537,18 +1540,37 @@ Using the following format specifiers, you can define a pattern for the resultin ``` -Writing to the console can be configured. Config example: +To print log messages only in the console: ``` xml information - 1 + true + +``` + +**Per-level Overrides** + +The log level of individual log names can be overridden. For example, to mute all messages of loggers "Backup" and "RBAC". + +```xml + + + + Backup + none + + + RBAC + none + + ``` ### syslog -Writing to the syslog is also supported. Config example: +To write log messages additionally to syslog: ``` xml @@ -1562,14 +1584,12 @@ Writing to the syslog is also supported. Config example: ``` -Keys for syslog: +Keys for ``: -- use_syslog — Required setting if you want to write to the syslog. -- address — The host\[:port\] of syslogd. If omitted, the local daemon is used. -- hostname — Optional. The name of the host that logs are sent from. -- facility — [The syslog facility keyword](https://en.wikipedia.org/wiki/Syslog#Facility) in uppercase letters with the “LOG_” prefix: (`LOG_USER`, `LOG_DAEMON`, `LOG_LOCAL3`, and so on). - Default value: `LOG_USER` if `address` is specified, `LOG_DAEMON` otherwise. -- format – Message format. Possible values: `bsd` and `syslog.` +- `address` — The address of syslog in format `host\[:port\]`. If omitted, the local daemon is used. +- `hostname` — The name of the host from which logs are send. Optional. +- `facility` — The syslog [facility keyword](https://en.wikipedia.org/wiki/Syslog#Facility). Must be specified uppercase with a “LOG_” prefix, e.g. `LOG_USER`, `LOG_DAEMON`, `LOG_LOCAL3`, etc. Default value: `LOG_USER` if `address` is specified, `LOG_DAEMON` otherwise. +- `format` – Log message format. Possible values: `bsd` and `syslog.` ### Log formats @@ -1588,6 +1608,7 @@ You can specify the log format that will be outputted in the console log. Curren "source_line": "192" } ``` + To enable JSON logging support, use the following snippet: ```xml From c4720d9728ba0e264da4834dea7fcaad6b2a8322 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 9 Sep 2024 11:18:42 +0000 Subject: [PATCH 251/289] Minor doc fixups --- .../mergetree-family/mergetree.md | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index d17208ab099..0bbee5f86f3 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -991,48 +991,39 @@ They can be used for prewhere optimization only if we enable `set allow_statisti ### Available Types of Column Statistics {#available-types-of-column-statistics} -Names of statistics types are case-insensitive, i.e. both - -``` sql -CREATE TABLE tab -( -a LowCardinality(Int64) STATISTICS(minmax) -) [...] -``` -and -``` sql -CREATE TABLE tab -( -a LowCardinality(Int64) STATISTICS(MiNmAx) -) [...] -``` -work. - - `MinMax` The minimum and maximum column value which allows to estimate the selectivity of range filters on numeric columns. + Syntax: `minmax` + - `TDigest` [TDigest](https://github.com/tdunning/t-digest) sketches which allow to compute approximate percentiles (e.g. the 90th percentile) for numeric columns. + Syntax: `tdigest` + - `Uniq` [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) sketches which provide an estimation how many distinct values a column contains. + Syntax: `uniq` + - `CountMin` [CountMin](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch) sketches which provide an approximate count of the frequency of each value in a column. + Syntax `countmin` + ### Supported Data Types {#supported-data-types} -| | (U)Int* | Float* | Decimal(*) | Date* | Boolean | Enum* | (Fixed)String | -|-----------|---------|--------|------------|-------|---------|-------|------------------| -| CountMin | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | -| MinMax | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✗ | -| TDigest | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✗ | -| Uniq | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| | (U)Int*, Float*, Decimal(*), Date*, Boolean, Enum* | String or FixedString | +|-----------|----------------------------------------------------|-----------------------| +| CountMin | ✔ | ✔ | +| MinMax | ✔ | ✗ | +| TDigest | ✔ | ✗ | +| Uniq | ✔ | ✔ | ### Supported Operations {#supported-operations} From 43d057ed0f60a637133f5ebd19071d7a859ff892 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 9 Sep 2024 13:04:45 +0100 Subject: [PATCH 252/289] impl --- src/Interpreters/ProcessList.cpp | 2 +- src/Interpreters/QueryPriorities.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 6cb50b310ad..f8a808f6c68 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -258,7 +258,7 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q query_context, query_, client_info, - priorities.insert(static_cast(settings.priority)), + priorities.insert(settings.priority), std::move(thread_group), query_kind, settings, diff --git a/src/Interpreters/QueryPriorities.h b/src/Interpreters/QueryPriorities.h index 9e18e7bcff3..7601c7ba6eb 100644 --- a/src/Interpreters/QueryPriorities.h +++ b/src/Interpreters/QueryPriorities.h @@ -31,7 +31,7 @@ namespace DB class QueryPriorities { public: - using Priority = int; + using Priority = size_t; private: friend struct Handle; From 942d0fab4a7f5583992dedc63a31db6b0f1f39ee Mon Sep 17 00:00:00 2001 From: imddba Date: Mon, 9 Sep 2024 20:24:16 +0800 Subject: [PATCH 253/289] spelling fixes --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 14b1d90c319..89f63f73928 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -151,6 +151,7 @@ ChannelID Cidr Ciphertext CityHash +CKibana Clangd ClickBench ClickCat @@ -247,6 +248,7 @@ DoubleDelta Doxygen Durre ECMA +ElasticSearch ETag Ecto EdgeAngle @@ -471,6 +473,7 @@ KeeperMap KeeperOutstandingRequests Kerberos Khanna +Kibana KittenHouse Klickhouse Kolmogorov @@ -1366,6 +1369,7 @@ cipherList ciphertext cityHash cityhash +ckibana ckman clangd cli From 66572aa029122c433acd743fdf77c4585cee7d8d Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Fri, 6 Sep 2024 16:13:08 +0200 Subject: [PATCH 254/289] Fix 24.8 setting compatibility `rows_before_aggregation` https://github.com/ClickHouse/ClickHouse/pull/66084 --- src/Core/SettingsChangesHistory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 258065dcfd4..ed1b1d4f177 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -81,7 +81,7 @@ static std::initializer_list Date: Mon, 9 Sep 2024 15:30:19 +0200 Subject: [PATCH 255/289] Renamed ApplyMergeStep into MergePartsStep --- src/Storages/MergeTree/MergeTask.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 398a9472456..3ca909a2d09 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1321,10 +1321,10 @@ bool MergeTask::execute() /// Apply merge strategy (Ordinary, Colapsing, Aggregating, etc) to the stream -class ApplyMergeStep : public ITransformingStep +class MergePartsStep : public ITransformingStep { public: - ApplyMergeStep( + MergePartsStep( const DataStream & input_stream_, const SortDescription & sort_description_, const Names partition_key_columns_, @@ -1347,7 +1347,7 @@ public: , time_of_merge(time_of_merge_) {} - String getName() const override { return "ApplyMergePolicy"; } + String getName() const override { return "MergeParts"; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*pipelineSettings*/) override { @@ -1651,7 +1651,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const if (global_ctx->cleanup && !data_settings->allow_experimental_replacing_merge_with_cleanup) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental merges with CLEANUP are not allowed"); - auto merge_step = std::make_unique( + auto merge_step = std::make_unique( merge_parts_query_plan.getCurrentDataStream(), sort_description, partition_key_columns, From 8c1f434b1ac2c9fbb83561a43a6ee10f20d81974 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 9 Sep 2024 15:31:43 +0200 Subject: [PATCH 256/289] Do column materialization using ActionsDAG::addMaterializingOutputActions instead of a special step --- src/Storages/MergeTree/MergeTask.cpp | 42 +--------------------------- 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 3ca909a2d09..33cdff10b6a 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1456,45 +1456,6 @@ private: const time_t time_of_merge{0}; }; - -class MaterializingStep : public ITransformingStep -{ -public: - explicit MaterializingStep( - const DataStream & input_stream_) - : ITransformingStep(input_stream_, input_stream_.header, getTraits()) - {} - - String getName() const override { return "Materializing"; } - - void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override - { - pipeline.addTransform(std::make_shared(input_streams.front().header)); - } - - void updateOutputStream() override - { - output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); - } - -private: - static Traits getTraits() - { - return ITransformingStep::Traits - { - { - .returns_single_stream = true, - .preserves_number_of_streams = true, - .preserves_sorting = true, - }, - { - .preserves_number_of_rows = true, - } - }; - } -}; - - class TTLStep : public ITransformingStep { public: @@ -1709,12 +1670,11 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const if (!global_ctx->merging_skip_indexes.empty()) { auto indices_expression_dag = global_ctx->merging_skip_indexes.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), global_ctx->data->getContext())->getActionsDAG().clone(); + indices_expression_dag.addMaterializingOutputActions(); /// Const columns cannot be written without materialization. auto calculate_indices_expression_step = std::make_unique( merge_parts_query_plan.getCurrentDataStream(), std::move(indices_expression_dag)); merge_parts_query_plan.addStep(std::move(calculate_indices_expression_step)); - /// TODO: what is the purpose of MaterializingTransform in the original code? - merge_parts_query_plan.addStep(std::make_unique(merge_parts_query_plan.getCurrentDataStream())); } if (!subqueries.empty()) From e6e79c3c4ada9f2eb82e6e9c63c736663cfb8e02 Mon Sep 17 00:00:00 2001 From: Pablo Marcos Date: Mon, 9 Sep 2024 11:36:18 +0000 Subject: [PATCH 257/289] Fix minor typos --- src/Client/ConnectionPoolWithFailover.h | 2 +- src/Client/HedgedConnectionsFactory.cpp | 2 +- src/Core/Defines.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index 308644ebbdb..19232e0ee7c 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -98,7 +98,7 @@ public: std::vector getShuffledPools(const Settings & settings, GetPriorityFunc priority_func = {}, bool use_slowdown_count = false); - size_t getMaxErrorCup() const { return Base::max_error_cap; } + size_t getMaxErrorCap() const { return Base::max_error_cap; } void updateSharedError(std::vector & shuffled_pools) { diff --git a/src/Client/HedgedConnectionsFactory.cpp b/src/Client/HedgedConnectionsFactory.cpp index be7397b0fad..df63a124539 100644 --- a/src/Client/HedgedConnectionsFactory.cpp +++ b/src/Client/HedgedConnectionsFactory.cpp @@ -327,7 +327,7 @@ HedgedConnectionsFactory::State HedgedConnectionsFactory::processFinishedConnect ShuffledPool & shuffled_pool = shuffled_pools[index]; LOG_INFO(log, "Connection failed at try №{}, reason: {}", (shuffled_pool.error_count + 1), fail_message); - shuffled_pool.error_count = std::min(pool->getMaxErrorCup(), shuffled_pool.error_count + 1); + shuffled_pool.error_count = std::min(pool->getMaxErrorCap(), shuffled_pool.error_count + 1); shuffled_pool.slowdown_count = 0; if (shuffled_pool.error_count >= max_tries) diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 6df335a9c8f..3341669fed2 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -40,7 +40,7 @@ static constexpr auto SHOW_CHARS_ON_SYNTAX_ERROR = ptrdiff_t(160); /// each period reduces the error counter by 2 times /// too short a period can cause errors to disappear immediately after creation. static constexpr auto DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD = 60; -/// replica error max cap, this is to prevent replica from accumulating too many errors and taking to long to recover. +/// replica error max cap, this is to prevent replica from accumulating too many errors and taking too long to recover. static constexpr auto DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT = 1000; /// The boundary on which the blocks for asynchronous file operations should be aligned. From 67b57eb89fb8724c542cc197a7371c3e60009566 Mon Sep 17 00:00:00 2001 From: Pablo Marcos Date: Mon, 9 Sep 2024 13:55:14 +0000 Subject: [PATCH 258/289] Fix undefined behavior if all connection tries fail For some reason, getManyCheckedForInsert was calling getManyImpl with skip_unavailable_endpoints=nullptr, which resulted in getManyImpl using the `skip_unavailable_shards` settings. Since this is true by default, min_entries was set to 0. Having min_entries set to 0 while using PoolMode::GET_ONE was strange, to say the least. There was an edge case where if all connection attempts failed and min_entries was 0, it was returning an empty vector. That was not considered to be possible by the caller and it was getting the front entry of an empty vector, causing undefined behavior. Conclusion: set `skip_unavailable_endpoints=false` explicitly in getManyCheckedForInsert so that min_entries=1. In case all connections fail, an exception will be thrown. Also, add some defensive programming to ensure we never ever try to get the front element of an empty vector. --- src/Client/ConnectionPoolWithFailover.cpp | 2 +- src/Client/ConnectionPoolWithFailover.h | 2 +- src/Common/PoolWithFailoverBase.h | 8 +++++++- src/Storages/Distributed/DistributedAsyncInsertBatch.cpp | 6 ++---- .../Distributed/DistributedAsyncInsertDirectoryQueue.cpp | 3 +-- src/Storages/Distributed/DistributedSink.cpp | 3 +-- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index fb895d17763..a5c14dc9957 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -168,7 +168,7 @@ std::vector ConnectionPoolWithFailover::g { return tryGetEntry(pool, timeouts, fail_message, settings, &table_to_check, /*async_callback=*/ {}); }; return getManyImpl(settings, pool_mode, try_get_entry, - /*skip_unavailable_endpoints=*/ std::nullopt, + /*skip_unavailable_endpoints=*/ false, /// skip_unavailable_endpoints is used to get the min number of entries, and we need at least one /*priority_func=*/ {}, settings.distributed_insert_skip_read_only_replicas); } diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index 19232e0ee7c..6db52140854 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -42,7 +42,7 @@ public: size_t max_error_cap = DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT); using Entry = IConnectionPool::Entry; - using PoolWithFailoverBase::checkTryResultIsValid; + using PoolWithFailoverBase::getValidTryResult; /** Allocates connection to work. */ Entry get(const ConnectionTimeouts & timeouts) override; diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index 53a746c316e..a0da72071f9 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -122,12 +122,18 @@ public: return result.entry.isNull() || !result.is_usable || (skip_read_only_replicas && result.is_readonly); } - void checkTryResultIsValid(const TryResult & result, bool skip_read_only_replicas) const + TryResult getValidTryResult(const std::vector & results, bool skip_read_only_replicas) const { + if (results.empty()) + throw DB::Exception(DB::ErrorCodes::ALL_CONNECTION_TRIES_FAILED, "Cannot get any valid connection. Probably all connection tries failed"); + + auto result = results.front(); if (isTryResultInvalid(result, skip_read_only_replicas)) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Got an invalid connection result: entry.isNull {}, is_usable {}, is_up_to_date {}, delay {}, is_readonly {}, skip_read_only_replicas {}", result.entry.isNull(), result.is_usable, result.is_up_to_date, result.delay, result.is_readonly, skip_read_only_replicas); + + return result; } size_t getPoolSize() const { return nested_pools.size(); } diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp index 2db2bdf3981..625c64128e7 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp @@ -242,8 +242,7 @@ void DistributedAsyncInsertBatch::sendBatch(const SettingsChanges & settings_cha auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName()); - auto result = results.front(); - parent.pool->checkTryResultIsValid(result, insert_settings.distributed_insert_skip_read_only_replicas); + auto result = parent.pool->getValidTryResult(results, insert_settings.distributed_insert_skip_read_only_replicas); connection = std::move(result.entry); compression_expected = connection->getCompression() == Protocol::Compression::Enable; @@ -302,8 +301,7 @@ void DistributedAsyncInsertBatch::sendSeparateFiles(const SettingsChanges & sett auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName()); - auto result = results.front(); - parent.pool->checkTryResultIsValid(result, insert_settings.distributed_insert_skip_read_only_replicas); + auto result = parent.pool->getValidTryResult(results, insert_settings.distributed_insert_skip_read_only_replicas); auto connection = std::move(result.entry); bool compression_expected = connection->getCompression() == Protocol::Compression::Enable; diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp index 2400de4c07c..7f368102dfd 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -415,8 +415,7 @@ void DistributedAsyncInsertDirectoryQueue::processFile(std::string & file_path, auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName()); - auto result = results.front(); - pool->checkTryResultIsValid(result, insert_settings.distributed_insert_skip_read_only_replicas); + auto result = pool->getValidTryResult(results, insert_settings.distributed_insert_skip_read_only_replicas); auto connection = std::move(result.entry); LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)", diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 39f75fd7d56..f01ea10065c 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -377,8 +377,7 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si /// NOTE: INSERT will also take into account max_replica_delay_for_distributed_queries /// (anyway fallback_to_stale_replicas_for_distributed_queries=true by default) auto results = shard_info.pool->getManyCheckedForInsert(timeouts, settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName()); - auto result = results.front(); - shard_info.pool->checkTryResultIsValid(result, settings.distributed_insert_skip_read_only_replicas); + auto result = shard_info.pool->getValidTryResult(results, settings.distributed_insert_skip_read_only_replicas); job.connection_entry = std::move(result.entry); } else From 4da1e10ac66059b47a89e69327ea79d487e2847f Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 9 Sep 2024 16:01:00 +0200 Subject: [PATCH 259/289] Move sorting key calculation step outside the loop --- src/Storages/MergeTree/MergeTask.cpp | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 33cdff10b6a..9a1e749734c 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1540,8 +1540,6 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const global_ctx->horizontal_stage_progress = std::make_unique( ctx->column_sizes ? ctx->column_sizes->keyColumnsWeight() : 1.0); - auto sorting_key_expression_dag = global_ctx->metadata_snapshot->getSortingKey().expression->getActionsDAG().clone(); - /// Read from all parts std::vector plans; for (size_t i = 0; i < global_ctx->future_part->parts.size(); ++i) @@ -1566,15 +1564,6 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const global_ctx->context, ctx->log); - if (global_ctx->metadata_snapshot->hasSortingKey()) - { - /// Calculate sorting key expressions so that they are available for merge sorting. - auto calculate_sorting_key_expression_step = std::make_unique( - plan_for_part->getCurrentDataStream(), - sorting_key_expression_dag.clone()); /// TODO: can we avoid cloning here? - plan_for_part->addStep(std::move(calculate_sorting_key_expression_step)); - } - plans.emplace_back(std::move(plan_for_part)); } @@ -1591,6 +1580,16 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const merge_parts_query_plan.unitePlans(std::move(union_step), std::move(plans)); } + if (global_ctx->metadata_snapshot->hasSortingKey()) + { + /// Calculate sorting key expressions so that they are available for merge sorting. + auto sorting_key_expression_dag = global_ctx->metadata_snapshot->getSortingKey().expression->getActionsDAG().clone(); + auto calculate_sorting_key_expression_step = std::make_unique( + merge_parts_query_plan.getCurrentDataStream(), + std::move(sorting_key_expression_dag)); + merge_parts_query_plan.addStep(std::move(calculate_sorting_key_expression_step)); + } + /// Merge { Names sort_columns = global_ctx->metadata_snapshot->getSortingKeyColumns(); From f7dee100309a0c47d6daaad19164b4e968fd3b8d Mon Sep 17 00:00:00 2001 From: Pablo Marcos Date: Mon, 9 Sep 2024 14:38:06 +0000 Subject: [PATCH 260/289] Make a more assertive exception text --- src/Common/PoolWithFailoverBase.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index a0da72071f9..989ffd888f8 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -125,7 +125,7 @@ public: TryResult getValidTryResult(const std::vector & results, bool skip_read_only_replicas) const { if (results.empty()) - throw DB::Exception(DB::ErrorCodes::ALL_CONNECTION_TRIES_FAILED, "Cannot get any valid connection. Probably all connection tries failed"); + throw DB::Exception(DB::ErrorCodes::ALL_CONNECTION_TRIES_FAILED, "Cannot get any valid connection because all connection tries failed"); auto result = results.front(); if (isTryResultInvalid(result, skip_read_only_replicas)) From 72dbc8205bbc1711a39d127590dfa12b3d05c96a Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 9 Sep 2024 15:12:17 +0000 Subject: [PATCH 261/289] Fix unit tests --- src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp | 9 +++++++++ src/IO/WriteBufferFromS3.cpp | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp index cbcfe0bdb97..229b36a05f6 100644 --- a/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp @@ -251,7 +251,16 @@ void WriteBufferFromAzureBlobStorage::allocateBuffer() /// First buffer was already allocated in BufferWithOwnMemory constructor with buffer size provided in constructor. /// It will be reallocated in subsequent nextImpl calls up to the desired buffer size from buffer_allocation_policy. if (buffer_allocation_policy->getBufferNumber() == 1) + { + /// Reduce memory size if initial size was larger then desired size from buffer_allocation_policy. + /// Usually it doesn't happen but we have it in unit tests. + if (memory.size() > buffer_allocation_policy->getBufferSize()) + { + memory.resize(buffer_allocation_policy->getBufferSize()); + WriteBuffer::set(memory.data(), memory.size()); + } return; + } auto size = buffer_allocation_policy->getBufferSize(); memory = Memory(size); diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 424708cf2c8..3a4ab3ee882 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -355,7 +355,16 @@ void WriteBufferFromS3::allocateBuffer() /// First buffer was already allocated in BufferWithOwnMemory constructor with provided in constructor buffer size. /// It will be reallocated in subsequent nextImpl calls up to the desired buffer size from buffer_allocation_policy. if (buffer_allocation_policy->getBufferNumber() == 1) + { + /// Reduce memory size if initial size was larger then desired size from buffer_allocation_policy. + /// Usually it doesn't happen but we have it in unit tests. + if (memory.size() > buffer_allocation_policy->getBufferSize()) + { + memory.resize(buffer_allocation_policy->getBufferSize()); + WriteBuffer::set(memory.data(), memory.size()); + } return; + } memory = Memory(buffer_allocation_policy->getBufferSize()); WriteBuffer::set(memory.data(), memory.size()); From d2ee805d8f9bf9d72ab6ddca536c6b180d7b6aa0 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 9 Sep 2024 17:32:22 +0200 Subject: [PATCH 262/289] Revert "Remove obsolete `--multiquery` parameter (follow-up to #63898), pt. V" --- tests/docker_scripts/stateful_runner.sh | 2 +- tests/docker_scripts/stress_runner.sh | 2 +- tests/integration/helpers/client.py | 1 + tests/performance/scripts/compare.sh | 12 +-- ...006_client_test_hint_no_such_error_name.sh | 2 +- .../0_stateless/02030_rocksdb_race_long.sh | 6 +- .../02151_hash_table_sizes_stats_joins.sh | 2 +- .../02229_client_stop_multiquery_in_SIGINT.sh | 4 +- ...41_filesystem_cache_on_write_operations.sh | 8 +- .../0_stateless/02242_delete_user_race.sh | 2 +- .../0_stateless/02243_drop_user_grant_race.sh | 4 +- .../02266_protobuf_format_google_wrappers.sh | 16 ++-- .../02286_drop_filesystem_cache.sh | 6 +- .../02313_filesystem_cache_seeks.sh | 2 +- .../02340_parts_refcnt_mergetree.sh | 6 +- .../0_stateless/02344_describe_cache.sh | 4 +- tests/queries/0_stateless/02352_rwlock.sh | 4 +- .../0_stateless/02380_insert_mv_race.sh | 4 +- ...entelemetry_insert_on_distributed_table.sh | 8 +- .../02419_keeper_map_primary_key.sh | 2 +- .../02443_detach_attach_partition.sh | 2 +- .../0_stateless/02473_optimize_old_parts.sh | 8 +- .../0_stateless/02477_s3_request_throttler.sh | 2 +- ...02494_zero_copy_projection_cancel_fetch.sh | 4 +- .../02504_regexp_dictionary_ua_parser.sh | 8 +- .../02504_regexp_dictionary_yaml_source.sh | 18 ++-- .../02515_cleanup_async_insert_block_ids.sh | 6 +- .../02585_query_status_deadlock.sh | 2 +- ...de_client_info_initial_query_start_time.sh | 14 ++-- ...rofile_events_from_query_log_and_client.sh | 16 ++-- .../02676_optimize_old_parts_replicated.sh | 8 +- .../02686_postgres_protocol_decimal_256.sh | 2 +- .../0_stateless/02700_s3_part_INT_MAX.sh | 2 +- .../0_stateless/02725_start_stop_fetches.sh | 2 +- .../02731_zero_objects_in_metadata.sh | 6 +- tests/queries/0_stateless/02766_prql.sh | 2 +- .../0_stateless/02770_async_buffer_ignore.sh | 4 +- .../02789_filesystem_cache_alignment.sh | 18 ++-- ...89_reading_from_s3_with_connection_pool.sh | 8 +- .../0_stateless/02801_backup_native_copy.sh | 10 +-- .../0_stateless/02803_backup_tmp_files.sh | 2 +- ...2808_custom_disk_with_user_defined_name.sh | 8 +- .../02808_filesystem_cache_drop_query.sh | 22 ++--- ...ackup_use_same_password_for_base_backup.sh | 2 +- ...use_same_s3_credentials_for_base_backup.sh | 2 +- .../02844_max_backup_bandwidth_s3.sh | 6 +- .../0_stateless/02867_storage_set_tsan.sh | 4 +- .../02900_union_schema_inference_mode.sh | 10 +-- .../02908_many_requests_to_system_replicas.sh | 2 +- .../02922_deduplication_with_zero_copy.sh | 10 +-- .../02932_refreshable_materialized_views_1.sh | 72 ++++++++-------- .../02932_refreshable_materialized_views_2.sh | 82 +++++++++---------- .../0_stateless/02941_variant_type_1.sh | 10 +-- .../0_stateless/02941_variant_type_2.sh | 4 +- .../0_stateless/02941_variant_type_3.sh | 4 +- .../0_stateless/02941_variant_type_4.sh | 2 +- ...ynamically_change_filesystem_cache_size.sh | 10 +-- .../0_stateless/02960_polygon_bound_bug.sh | 2 +- .../02961_storage_config_volume_priority.sh | 4 +- ...02963_remote_read_small_buffer_size_bug.sh | 6 +- ...allel_replicas_join_algo_and_analyzer_1.sh | 2 +- ...allel_replicas_join_algo_and_analyzer_2.sh | 2 +- ...allel_replicas_join_algo_and_analyzer_3.sh | 2 +- .../02969_auto_format_detection.sh | 6 +- .../02980_s3_plain_DROP_TABLE_MergeTree.sh | 4 +- ...s3_plain_DROP_TABLE_ReplicatedMergeTree.sh | 6 +- ...ion_insert_several_blocks_nonreplicated.sh | 2 +- ...cation_insert_several_blocks_replicated.sh | 2 +- ..._generates_several_blocks_nonreplicated.sh | 2 +- ..._mv_generates_several_blocks_replicated.sh | 2 +- ...several_mv_into_one_table_nonreplicated.sh | 2 +- ...on_several_mv_into_one_table_replicated.sh | 2 +- .../0_stateless/03008_s3_plain_rewritable.sh | 8 +- ...2_dynamically_resize_filesystem_cache_2.sh | 6 +- .../03039_dynamic_aggregating_merge_tree.sh | 2 +- .../03039_dynamic_collapsing_merge_tree.sh | 2 +- .../03039_dynamic_replacing_merge_tree.sh | 2 +- .../03039_dynamic_summing_merge_tree.sh | 2 +- ...dynamic_versioned_collapsing_merge_tree.sh | 2 +- .../0_stateless/03151_unload_index_race.sh | 6 +- .../03172_error_log_table_not_empty.sh | 8 +- .../03203_hive_style_partitioning.sh | 18 ++-- ...kup_and_clear_old_temporary_directories.sh | 4 +- 83 files changed, 299 insertions(+), 298 deletions(-) diff --git a/tests/docker_scripts/stateful_runner.sh b/tests/docker_scripts/stateful_runner.sh index defc24781a1..86f6a299ad3 100755 --- a/tests/docker_scripts/stateful_runner.sh +++ b/tests/docker_scripts/stateful_runner.sh @@ -105,7 +105,7 @@ setup_logs_replication clickhouse-client --query "SHOW DATABASES" clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client < /repo/tests/docker_scripts/create.sql +clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql clickhouse-client --query "SHOW TABLES FROM datasets" if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index af93c93f202..039c60c8e4e 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -62,7 +62,7 @@ start_server setup_logs_replication clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client < /repo/tests/docker_scripts/create.sql +clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql clickhouse-client --query "SHOW TABLES FROM datasets" clickhouse-client --query "CREATE DATABASE IF NOT EXISTS test" diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index 97405860784..8ba7b342020 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -89,6 +89,7 @@ class Client: command = self.command[:] if stdin is None: + command += ["--multiquery"] stdin = sql else: command += ["--query", sql] diff --git a/tests/performance/scripts/compare.sh b/tests/performance/scripts/compare.sh index c3566c51a16..da7bbf77a28 100755 --- a/tests/performance/scripts/compare.sh +++ b/tests/performance/scripts/compare.sh @@ -427,7 +427,7 @@ do done # for each query run, prepare array of metrics from query log -clickhouse-local --query " +clickhouse-local --multiquery --query " create view query_runs as select * from file('analyze/query-runs.tsv', TSV, 'test text, query_index int, query_id text, version UInt8, time float'); @@ -582,7 +582,7 @@ numactl --cpunodebind=all --membind=all numactl --show # If the available memory falls below 2 * size, GNU parallel will suspend some of the running jobs. numactl --cpunodebind=all --membind=all parallel -v --joblog analyze/parallel-log.txt --memsuspend 15G --null < analyze/commands.txt 2>> analyze/errors.log -clickhouse-local --query " +clickhouse-local --multiquery --query " -- Join the metric names back to the metric statistics we've calculated, and make -- a denormalized table of them -- statistics for all metrics for all queries. -- The WITH, ARRAY JOIN and CROSS JOIN do not like each other: @@ -680,7 +680,7 @@ rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.ts cat analyze/errors.log >> report/errors.log ||: cat profile-errors.log >> report/errors.log ||: -clickhouse-local --query " +clickhouse-local --multiquery --query " create view query_display_names as select * from file('analyze/query-display-names.tsv', TSV, 'test text, query_index int, query_display_name text') @@ -981,7 +981,7 @@ create table all_query_metrics_tsv engine File(TSV, 'report/all-query-metrics.ts for version in {right,left} do rm -rf data - clickhouse-local --query " + clickhouse-local --multiquery --query " create view query_profiles as with 0 as left, 1 as right select * from file('analyze/query-profiles.tsv', TSV, @@ -1151,7 +1151,7 @@ function report_metrics rm -rf metrics ||: mkdir metrics -clickhouse-local --query " +clickhouse-local --multiquery --query " create view right_async_metric_log as select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes) ; @@ -1211,7 +1211,7 @@ function upload_results # Prepare info for the CI checks table. rm -f ci-checks.tsv - clickhouse-local --query " + clickhouse-local --multiquery --query " create view queries as select * from file('report/queries.tsv', TSVWithNamesAndTypes); create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv') diff --git a/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh b/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh index 7bf8100a53d..972ff3ba73f 100755 --- a/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh +++ b/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh @@ -5,4 +5,4 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*' +$CLICKHOUSE_CLIENT -n -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*' diff --git a/tests/queries/0_stateless/02030_rocksdb_race_long.sh b/tests/queries/0_stateless/02030_rocksdb_race_long.sh index 7d329f78d98..da31861991c 100755 --- a/tests/queries/0_stateless/02030_rocksdb_race_long.sh +++ b/tests/queries/0_stateless/02030_rocksdb_race_long.sh @@ -12,14 +12,14 @@ echo " DROP TABLE IF EXISTS rocksdb_race; CREATE TABLE rocksdb_race (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key); INSERT INTO rocksdb_race SELECT '1_' || toString(number), number FROM numbers(100000); -" | $CLICKHOUSE_CLIENT +" | $CLICKHOUSE_CLIENT -n function read_stat_thread() { while true; do echo " SELECT * FROM system.rocksdb FORMAT Null; - " | $CLICKHOUSE_CLIENT + " | $CLICKHOUSE_CLIENT -n done } @@ -29,7 +29,7 @@ function truncate_thread() sleep 3s; echo " TRUNCATE TABLE rocksdb_race; - " | $CLICKHOUSE_CLIENT + " | $CLICKHOUSE_CLIENT -n done } diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh b/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh index a9b5c461c5a..007dae6e427 100755 --- a/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh +++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh @@ -12,7 +12,7 @@ opts=( --join_algorithm='parallel_hash' ) -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " CREATE TABLE t1(a UInt32, b UInt32) ENGINE=MergeTree ORDER BY (); INSERT INTO t1 SELECT number, number FROM numbers_mt(1e6); diff --git a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh index b23164f8354..e5d00bc1a1c 100755 --- a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh +++ b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh @@ -5,12 +5,12 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -timeout -s INT 3s $CLICKHOUSE_CLIENT --max_block_size 1 -m -q " +timeout -s INT 3s $CLICKHOUSE_CLIENT --max_block_size 1 -nm -q " SELECT sleep(1) FROM numbers(100) FORMAT Null; SELECT 'FAIL'; " -timeout -s INT 3s $CLICKHOUSE_LOCAL --max_block_size 1 -m -q " +timeout -s INT 3s $CLICKHOUSE_LOCAL --max_block_size 1 -nm -q " SELECT sleep(1) FROM numbers(100) FORMAT Null; SELECT 'FAIL'; " diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh index 8a44b65e6d1..f8e7b7e7e72 100755 --- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh +++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh @@ -16,7 +16,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --query "SYSTEM DROP FILESYSTEM CACHE" - $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -37,7 +37,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)" - $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -70,7 +70,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100, 200)" - $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -109,7 +109,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --query "SYSTEM FLUSH LOGS" - $CLICKHOUSE_CLIENT --query "SELECT + $CLICKHOUSE_CLIENT -n --query "SELECT query, ProfileEvents['RemoteFSReadBytes'] > 0 as remote_fs_read FROM system.query_log diff --git a/tests/queries/0_stateless/02242_delete_user_race.sh b/tests/queries/0_stateless/02242_delete_user_race.sh index 3cb79d69955..2af54276469 100755 --- a/tests/queries/0_stateless/02242_delete_user_race.sh +++ b/tests/queries/0_stateless/02242_delete_user_race.sh @@ -15,7 +15,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " DROP ROLE IF EXISTS test_role_02242; CREATE ROLE test_role_02242; " diff --git a/tests/queries/0_stateless/02243_drop_user_grant_race.sh b/tests/queries/0_stateless/02243_drop_user_grant_race.sh index 1972ecade06..4dce8e8124c 100755 --- a/tests/queries/0_stateless/02243_drop_user_grant_race.sh +++ b/tests/queries/0_stateless/02243_drop_user_grant_race.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " DROP ROLE IF EXISTS test_role_02244; CREATE ROLE test_role_02244; DROP USER IF EXISTS kek_02243; @@ -37,4 +37,4 @@ $CLICKHOUSE_CLIENT --user kek_02243 -q "SELECT * FROM test" 2>&1| grep -Fa "Exce $CLICKHOUSE_CLIENT -q "DROP ROLE IF EXISTS test_role_02243" $CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS test_user_02243" -$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS kek_02243" +$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS kek_02243" \ No newline at end of file diff --git a/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh b/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh index eb1109aaac1..ae2a2351c6b 100755 --- a/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh +++ b/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh @@ -44,7 +44,7 @@ protobuf_info() { fi } -$CLICKHOUSE_CLIENT --query " +$CLICKHOUSE_CLIENT -n --query " DROP TABLE IF EXISTS $MAIN_TABLE; DROP TABLE IF EXISTS $ROUNDTRIP_TABLE; DROP TABLE IF EXISTS $COMPATIBILITY_TABLE; @@ -78,14 +78,14 @@ echo $SET_OUTPUT echo echo "Insert $INITIAL_INSERT_VALUES into table (Nullable(String), Int32):" -$CLICKHOUSE_CLIENT --query " +$CLICKHOUSE_CLIENT -n --query " INSERT INTO $MAIN_TABLE VALUES $INITIAL_INSERT_VALUES; SELECT * FROM $MAIN_TABLE; " echo echo "Protobuf representation of the second row:" -$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $MAIN_TABLE WHERE ref = 2 LIMIT 1 $(protobuf_info output ProtobufSingle Message)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $MAIN_TABLE WHERE ref = 2 LIMIT 1 $(protobuf_info output ProtobufSingle Message)" > "$BINARY_FILE_PATH" hexdump -C $BINARY_FILE_PATH echo @@ -101,12 +101,12 @@ hexdump -C $MESSAGE_FILE_PATH echo echo "Insert proto message into table (Nullable(String), Int32):" -$CLICKHOUSE_CLIENT --query "$SET_INPUT INSERT INTO $ROUNDTRIP_TABLE $(protobuf_info input Protobuf Message)" < "$MESSAGE_FILE_PATH" +$CLICKHOUSE_CLIENT -n --query "$SET_INPUT INSERT INTO $ROUNDTRIP_TABLE $(protobuf_info input Protobuf Message)" < "$MESSAGE_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM $ROUNDTRIP_TABLE" echo echo "Proto output of the table using Google wrapper:" -$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $ROUNDTRIP_TABLE $(protobuf_info output Protobuf Message)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $ROUNDTRIP_TABLE $(protobuf_info output Protobuf Message)" > "$BINARY_FILE_PATH" hexdump -C $BINARY_FILE_PATH echo @@ -124,14 +124,14 @@ echo echo "Insert $MULTI_WRAPPER_VALUES and reinsert using Google wrappers into:" echo "Table (Nullable(Int32), Nullable(Int32), Int32):" $CLICKHOUSE_CLIENT --query "INSERT INTO $MULTI_TABLE VALUES $MULTI_WRAPPER_VALUES" -$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $MULTI_TABLE $(protobuf_info output Protobuf MessageMultiWrapper)" > "$BINARY_FILE_PATH" -$CLICKHOUSE_CLIENT --query "$SET_INPUT INSERT INTO $MULTI_TABLE $(protobuf_info input Protobuf MessageMultiWrapper)" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $MULTI_TABLE $(protobuf_info output Protobuf MessageMultiWrapper)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT -n --query "$SET_INPUT INSERT INTO $MULTI_TABLE $(protobuf_info input Protobuf MessageMultiWrapper)" < "$BINARY_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM $MULTI_TABLE" rm "$BINARY_FILE_PATH" rm "$MESSAGE_FILE_PATH" -$CLICKHOUSE_CLIENT --query " +$CLICKHOUSE_CLIENT -n --query " DROP TABLE $MAIN_TABLE; DROP TABLE $ROUNDTRIP_TABLE; DROP TABLE $COMPATIBILITY_TABLE; diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh index 672bf4d068b..dab777fcc31 100755 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh @@ -11,7 +11,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do echo "Using storage policy: $STORAGE_POLICY" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02286" - $CLICKHOUSE_CLIENT --query "CREATE TABLE test_02286 (key UInt32, value String) + $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_02286 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='$STORAGE_POLICY', min_bytes_for_wide_part = 10485760" @@ -38,7 +38,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --query "SELECT * FROM test_02286 FORMAT Null" $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache" - $CLICKHOUSE_CLIENT --query "SELECT count() + $CLICKHOUSE_CLIENT -n --query "SELECT count() FROM ( SELECT arrayJoin(cache_paths) AS cache_path, @@ -54,7 +54,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache" $CLICKHOUSE_CLIENT --query "SELECT cache_path FROM system.filesystem_cache" - $CLICKHOUSE_CLIENT --query "SELECT cache_path, local_path + $CLICKHOUSE_CLIENT -n --query "SELECT cache_path, local_path FROM ( SELECT arrayJoin(cache_paths) AS cache_path, diff --git a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh index 64fae06f220..b7adde6fcbb 100755 --- a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh +++ b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh @@ -23,7 +23,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; d ORDER BY tuple() SETTINGS storage_policy = '$STORAGE_POLICY'" > /dev/null - $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=0 --query "INSERT INTO test_02313 + $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=0 -n --query "INSERT INTO test_02313 SELECT * FROM generateRandom('id Int32, val String') LIMIT 100000" diff --git a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh index f252b9304cd..bd018018789 100755 --- a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh +++ b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh @@ -9,7 +9,7 @@ function check_refcnt_for_table() { local table=$1 && shift - $CLICKHOUSE_CLIENT -m -q " + $CLICKHOUSE_CLIENT -nm -q " system stop merges $table; -- cleanup thread may hold the parts lock system stop cleanup $table; @@ -66,14 +66,14 @@ function check_refcnt_for_table() # NOTE: index_granularity=1 to cancel ASAP -$CLICKHOUSE_CLIENT -mq " +$CLICKHOUSE_CLIENT -nmq " drop table if exists data_02340; create table data_02340 (key Int, part Int) engine=MergeTree() partition by part order by key settings index_granularity=1; " || exit 1 check_refcnt_for_table data_02340 $CLICKHOUSE_CLIENT -q "drop table data_02340 sync" -$CLICKHOUSE_CLIENT -mq " +$CLICKHOUSE_CLIENT -nmq " drop table if exists data_02340_rep sync; create table data_02340_rep (key Int, part Int) engine=ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') partition by part order by key settings index_granularity=1; " || exit 1 diff --git a/tests/queries/0_stateless/02344_describe_cache.sh b/tests/queries/0_stateless/02344_describe_cache.sh index 4cc4c415f3f..c5373b4d7e3 100755 --- a/tests/queries/0_stateless/02344_describe_cache.sh +++ b/tests/queries/0_stateless/02344_describe_cache.sh @@ -7,14 +7,14 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="02344_describe_cache_test" -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = '$disk_name', disk = 's3_disk', load_metadata_asynchronously = 0); """ -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ diff --git a/tests/queries/0_stateless/02352_rwlock.sh b/tests/queries/0_stateless/02352_rwlock.sh index 7f02b3ee935..b4a77e0b08a 100755 --- a/tests/queries/0_stateless/02352_rwlock.sh +++ b/tests/queries/0_stateless/02352_rwlock.sh @@ -24,7 +24,7 @@ function wait_query_by_id_started() # wait for query to be started while [ "$($CLICKHOUSE_CLIENT "$@" -q "select count() from system.processes where query_id = '$query_id'")" -ne 1 ]; do if [ "$( - $CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 -m -q " + $CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 -nm -q " system flush logs; select count() from system.query_log @@ -52,7 +52,7 @@ $CLICKHOUSE_CLIENT -q "CREATE DATABASE ${CLICKHOUSE_DATABASE}_ordinary Engine=Or # debug build on CI, so if this will happen, then DROP query will be # finished instantly, and to avoid flakiness we will retry in this case while :; do - $CLICKHOUSE_CLIENT -m -q " + $CLICKHOUSE_CLIENT -nm -q " DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}_ordinary.data_02352; CREATE TABLE ${CLICKHOUSE_DATABASE}_ordinary.data_02352 (key Int) Engine=Null(); " diff --git a/tests/queries/0_stateless/02380_insert_mv_race.sh b/tests/queries/0_stateless/02380_insert_mv_race.sh index a5f05365180..725c7eacce6 100755 --- a/tests/queries/0_stateless/02380_insert_mv_race.sh +++ b/tests/queries/0_stateless/02380_insert_mv_race.sh @@ -9,13 +9,13 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q "ATTACH TABLE mv" |& { +$CLICKHOUSE_CLIENT -nm -q "ATTACH TABLE mv" |& { # CANNOT_GET_CREATE_TABLE_QUERY -- ATTACH TABLE IF EXISTS # TABLE_ALREADY_EXISTS -- ATTACH TABLE IF NOT EXISTS grep -F -m1 Exception | grep -v -e CANNOT_GET_CREATE_TABLE_QUERY -e TABLE_ALREADY_EXISTS } -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " DROP TABLE IF EXISTS null; CREATE TABLE null (key Int) ENGINE = Null; DROP TABLE IF EXISTS mv; diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh index 9e8b854375b..c692c37f27f 100755 --- a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh +++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh @@ -27,7 +27,7 @@ function insert() function check_span() { -${CLICKHOUSE_CLIENT} -q " +${CLICKHOUSE_CLIENT} -nq " SYSTEM FLUSH LOGS; SELECT operation_name, @@ -50,7 +50,7 @@ ${CLICKHOUSE_CLIENT} -q " # $2 - value of distributed_foreground_insert function check_span_kind() { -${CLICKHOUSE_CLIENT} -q " +${CLICKHOUSE_CLIENT} -nq " SYSTEM FLUSH LOGS; SELECT count() @@ -65,7 +65,7 @@ ${CLICKHOUSE_CLIENT} -q " # # Prepare tables for tests # -${CLICKHOUSE_CLIENT} -q " +${CLICKHOUSE_CLIENT} -nq " DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.dist_opentelemetry; DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.local_opentelemetry; @@ -122,7 +122,7 @@ check_span_kind $trace_id 'CLIENT' # # Cleanup # -${CLICKHOUSE_CLIENT} -q " +${CLICKHOUSE_CLIENT} -nq " DROP TABLE ${CLICKHOUSE_DATABASE}.dist_opentelemetry; DROP TABLE ${CLICKHOUSE_DATABASE}.local_opentelemetry; " diff --git a/tests/queries/0_stateless/02419_keeper_map_primary_key.sh b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh index 6a23f666f36..c43c5bb6408 100755 --- a/tests/queries/0_stateless/02419_keeper_map_primary_key.sh +++ b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh @@ -9,7 +9,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS 02419_test SYNC;" test_primary_key() { - $CLICKHOUSE_CLIENT -m -q " + $CLICKHOUSE_CLIENT -nm -q " CREATE TABLE 02419_test (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 3) PRIMARY KEY($1); INSERT INTO 02419_test VALUES (1, 1.1), (2, 2.2); SELECT value FROM 02419_test WHERE key = 1; diff --git a/tests/queries/0_stateless/02443_detach_attach_partition.sh b/tests/queries/0_stateless/02443_detach_attach_partition.sh index 4a5377a952a..6a47b7d8d61 100755 --- a/tests/queries/0_stateless/02443_detach_attach_partition.sh +++ b/tests/queries/0_stateless/02443_detach_attach_partition.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/replication.lib -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS alter_table0; DROP TABLE IF EXISTS alter_table1; diff --git a/tests/queries/0_stateless/02473_optimize_old_parts.sh b/tests/queries/0_stateless/02473_optimize_old_parts.sh index 3a4e6145f12..b563bc31b39 100755 --- a/tests/queries/0_stateless/02473_optimize_old_parts.sh +++ b/tests/queries/0_stateless/02473_optimize_old_parts.sh @@ -21,7 +21,7 @@ wait_for_number_of_parts() { echo "$res" } -$CLICKHOUSE_CLIENT -mq " +$CLICKHOUSE_CLIENT -nmq " DROP TABLE IF EXISTS test_without_merge; DROP TABLE IF EXISTS test_with_merge; @@ -34,7 +34,7 @@ INSERT INTO test_without_merge SELECT 3;" wait_for_number_of_parts 'test_without_merge' 1 10 -$CLICKHOUSE_CLIENT -mq " +$CLICKHOUSE_CLIENT -nmq " DROP TABLE test_without_merge; SELECT 'With merge any part range'; @@ -47,7 +47,7 @@ INSERT INTO test_with_merge SELECT 3;" wait_for_number_of_parts 'test_with_merge' 1 100 -$CLICKHOUSE_CLIENT -mq " +$CLICKHOUSE_CLIENT -nmq " DROP TABLE test_with_merge; SELECT 'With merge partition only'; @@ -60,7 +60,7 @@ INSERT INTO test_with_merge SELECT 3;" wait_for_number_of_parts 'test_with_merge' 1 100 -$CLICKHOUSE_CLIENT -mq " +$CLICKHOUSE_CLIENT -nmq " SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_with_merge' AND active; diff --git a/tests/queries/0_stateless/02477_s3_request_throttler.sh b/tests/queries/0_stateless/02477_s3_request_throttler.sh index 5e6185e7304..c74cb598d42 100755 --- a/tests/queries/0_stateless/02477_s3_request_throttler.sh +++ b/tests/queries/0_stateless/02477_s3_request_throttler.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " -- Limit S3 PUT request per second rate SET s3_max_put_rps = 2; SET s3_max_put_burst = 1; diff --git a/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh b/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh index 122684abe9c..b72c3eb56c7 100755 --- a/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh +++ b/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT --query " +$CLICKHOUSE_CLIENT -n --query " DROP TABLE IF EXISTS wikistat1 SYNC; DROP TABLE IF EXISTS wikistat2 SYNC; " @@ -60,7 +60,7 @@ wait $CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat1 WHERE NOT ignore(*)" $CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat2 WHERE NOT ignore(*)" -$CLICKHOUSE_CLIENT --query " +$CLICKHOUSE_CLIENT -n --query " DROP TABLE IF EXISTS wikistat1 SYNC; DROP TABLE IF EXISTS wikistat2 SYNC; " diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh index 77c8049bd58..e389cf410e8 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh @@ -11,7 +11,7 @@ cp $CURDIR/data_ua_parser/os.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ cp $CURDIR/data_ua_parser/browser.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ cp $CURDIR/data_ua_parser/device.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ -$CLICKHOUSE_CLIENT --query=" +$CLICKHOUSE_CLIENT -n --query=" drop dictionary if exists regexp_os; drop dictionary if exists regexp_browser; drop dictionary if exists regexp_device; @@ -61,10 +61,10 @@ create table user_agents Engine = Log(); " -$CLICKHOUSE_CLIENT --query=" +$CLICKHOUSE_CLIENT -n --query=" insert into user_agents select ua from input('ua String') FORMAT LineAsString" < $CURDIR/data_ua_parser/useragents.txt -$CLICKHOUSE_CLIENT --query=" +$CLICKHOUSE_CLIENT -n --query=" select ua, device, concat(tupleElement(browser, 1), ' ', tupleElement(browser, 2), '.', tupleElement(browser, 3)) as browser , concat(tupleElement(os, 1), ' ', tupleElement(os, 2), '.', tupleElement(os, 3), '.', tupleElement(os, 4)) as os @@ -74,7 +74,7 @@ from ( dictGet('regexp_device', 'device_replacement', ua) device from user_agents) order by ua; " -$CLICKHOUSE_CLIENT --query=" +$CLICKHOUSE_CLIENT -n --query=" drop dictionary if exists regexp_os; drop dictionary if exists regexp_browser; drop dictionary if exists regexp_device; diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh index dec03bcaaa0..68a87a14320 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh @@ -27,7 +27,7 @@ cat > "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < /dev/null & diff --git a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh b/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh index 9f743581e1a..3b0d2309784 100755 --- a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh +++ b/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh @@ -13,7 +13,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function get_query_id() { random_str 10; } -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table if exists buf; drop table if exists dist; drop table if exists data; @@ -31,7 +31,7 @@ query_id="$(get_query_id)" # test, since we care about the difference between NOW() and there should # not be any significant difference. $CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist" -$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; " @@ -42,25 +42,25 @@ query_id="$(get_query_id)" # this query (and all subsequent) should reuse the previous connection (at least most of the time) $CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist" -$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; " echo "INSERT" query_id="$(get_query_id)" -$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -m -q " +$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -nm -q " insert into dist_dist values (1),(2); select * from data; " sleep 1 -$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "system flush distributed dist_dist" +$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist_dist" sleep 1 -$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "system flush distributed dist" +$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist" echo "CHECK" -$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " select * from data order by key; system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; diff --git a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh index 5015521ca4d..ff534a6a2e6 100755 --- a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh +++ b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh echo "INSERT TO S3" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " INSERT INTO TABLE FUNCTION s3('http://localhost:11111/test/profile_events.csv', 'test', 'testtest', 'CSV', 'number UInt64') SELECT number FROM numbers(1000000) SETTINGS s3_max_single_part_upload_size = 10, s3_truncate_on_insert = 1; " 2>&1 | $CLICKHOUSE_LOCAL -q " WITH '(\\w+): (\\d+)' AS pattern, @@ -30,7 +30,7 @@ SELECT * FROM ( " echo "CHECK WITH query_log" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " SYSTEM FLUSH LOGS; SELECT type, 'S3CreateMultipartUpload', ProfileEvents['S3CreateMultipartUpload'], @@ -45,7 +45,7 @@ ORDER BY query_start_time DESC; " echo "CREATE" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " DROP TABLE IF EXISTS times; CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t SETTINGS @@ -56,29 +56,29 @@ CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t " echo "INSERT" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " INSERT INTO times SELECT now() + INTERVAL 1 day SETTINGS optimize_on_insert = 0; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "READ" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " SELECT '1', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "INSERT and READ INSERT" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " INSERT INTO times SELECT now() + INTERVAL 2 day SETTINGS optimize_on_insert = 0; SELECT '2', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1; INSERT INTO times SELECT now() + INTERVAL 3 day SETTINGS optimize_on_insert = 0; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "DROP" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " DROP TABLE times; " echo "CHECK with query_log" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " SYSTEM FLUSH LOGS; SELECT type, query, diff --git a/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh b/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh index 84be5959344..c1f28f9f079 100755 --- a/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh +++ b/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh @@ -21,7 +21,7 @@ wait_for_number_of_parts() { echo "$res" } -$CLICKHOUSE_CLIENT -mq " +$CLICKHOUSE_CLIENT -nmq " DROP TABLE IF EXISTS test_without_merge; DROP TABLE IF EXISTS test_replicated; @@ -34,7 +34,7 @@ INSERT INTO test_without_merge SELECT 3;" wait_for_number_of_parts 'test_without_merge' 1 10 -$CLICKHOUSE_CLIENT -mq " +$CLICKHOUSE_CLIENT -nmq " DROP TABLE test_without_merge; SELECT 'With merge replicated any part range'; @@ -47,7 +47,7 @@ INSERT INTO test_replicated SELECT 3;" wait_for_number_of_parts 'test_replicated' 1 100 -$CLICKHOUSE_CLIENT -mq " +$CLICKHOUSE_CLIENT -nmq " DROP TABLE test_replicated; SELECT 'With merge replicated partition only'; @@ -60,7 +60,7 @@ INSERT INTO test_replicated SELECT 3;" wait_for_number_of_parts 'test_replicated' 1 100 -$CLICKHOUSE_CLIENT -mq " +$CLICKHOUSE_CLIENT -nmq " SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_replicated' AND active; diff --git a/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh b/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh index a540a0981b3..2a94f940327 100755 --- a/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh +++ b/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh @@ -9,6 +9,6 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) echo " DROP USER IF EXISTS postgresql_user; CREATE USER postgresql_user HOST IP '127.0.0.1' IDENTIFIED WITH no_password; -" | $CLICKHOUSE_CLIENT +" | $CLICKHOUSE_CLIENT -n psql --host localhost --port ${CLICKHOUSE_PORT_POSTGRESQL} ${CLICKHOUSE_DATABASE} --user postgresql_user -c "SELECT 1.23::Decimal256(70) AS test;" diff --git a/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh b/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh index dcd83f9fec3..cfb38c60615 100755 --- a/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh +++ b/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh @@ -12,7 +12,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # too slow with this. # # Unfortunately, the test has to buffer it in memory. -$CLICKHOUSE_CLIENT --max_memory_usage 16G -m -q " +$CLICKHOUSE_CLIENT --max_memory_usage 16G -nm -q " INSERT INTO FUNCTION s3('http://localhost:11111/test/$CLICKHOUSE_DATABASE/test_INT_MAX.tsv', '', '', 'TSV') SELECT repeat('a', 1024) FROM numbers((pow(2, 30) * 2) / 1024) SETTINGS s3_max_single_part_upload_size = '5Gi'; diff --git a/tests/queries/0_stateless/02725_start_stop_fetches.sh b/tests/queries/0_stateless/02725_start_stop_fetches.sh index 604d0774b83..c9922455d94 100755 --- a/tests/queries/0_stateless/02725_start_stop_fetches.sh +++ b/tests/queries/0_stateless/02725_start_stop_fetches.sh @@ -10,7 +10,7 @@ set -e NUM_REPLICAS=5 for i in $(seq 1 $NUM_REPLICAS); do - $CLICKHOUSE_CLIENT -q " + $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS r$i SYNC; CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x SETTINGS replicated_deduplication_window = 1, allow_remote_fs_zero_copy_replication = 1; " diff --git a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh index 6e40495157b..78659b70129 100755 --- a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh +++ b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) for DISK in s3_disk s3_cache do - ${CLICKHOUSE_CLIENT} --query " + ${CLICKHOUSE_CLIENT} -n --query " DROP TABLE IF EXISTS test; CREATE TABLE test (id Int32, empty Array(Int32)) ENGINE=MergeTree ORDER BY id @@ -17,13 +17,13 @@ do SELECT * FROM test; " - ${CLICKHOUSE_CLIENT} --query " + ${CLICKHOUSE_CLIENT} -n --query " BACKUP TABLE test TO Disk('backups', 'test_s3_backup'); DROP TABLE test; RESTORE TABLE test FROM Disk('backups', 'test_s3_backup'); " &>/dev/null - ${CLICKHOUSE_CLIENT} --query " + ${CLICKHOUSE_CLIENT} -n --query " SELECT * FROM test; SELECT empty FROM test; " diff --git a/tests/queries/0_stateless/02766_prql.sh b/tests/queries/0_stateless/02766_prql.sh index d1e4dca070c..85b1167027c 100755 --- a/tests/queries/0_stateless/02766_prql.sh +++ b/tests/queries/0_stateless/02766_prql.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -n -q " CREATE TEMPORARY TABLE IF NOT EXISTS aboba ( user_id UInt32, diff --git a/tests/queries/0_stateless/02770_async_buffer_ignore.sh b/tests/queries/0_stateless/02770_async_buffer_ignore.sh index af9f18b232f..37f002767d6 100755 --- a/tests/queries/0_stateless/02770_async_buffer_ignore.sh +++ b/tests/queries/0_stateless/02770_async_buffer_ignore.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} -m --query " +${CLICKHOUSE_CLIENT} -nm --query " DROP TABLE IF EXISTS test_s3; CREATE TABLE test_s3 (a UInt64, b UInt64) @@ -17,7 +17,7 @@ INSERT INTO test_s3 SELECT number, number FROM numbers(1000000); query="SELECT sum(b) FROM test_s3 WHERE a >= 100000 AND a <= 102000" query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CLIENT} -m --query " +${CLICKHOUSE_CLIENT} -nm --query " SELECT ProfileEvents['S3ReadRequestsCount'], ProfileEvents['ReadBufferFromS3Bytes'], diff --git a/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh b/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh index 53d2832c589..c69c635f6ed 100755 --- a/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh +++ b/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() @@ -22,7 +22,7 @@ INSERT INTO test SELECT number, randomString(100) FROM numbers(1000000); " QUERY_ID=$RANDOM -$CLICKHOUSE_CLIENT --query_id "$QUERY_ID" -m -q " +$CLICKHOUSE_CLIENT --query_id "$QUERY_ID" -nm -q " SET enable_filesystem_cache_log = 1; SYSTEM DROP FILESYSTEM CACHE; SELECT * FROM test WHERE NOT ignore() LIMIT 1 FORMAT Null; @@ -49,14 +49,14 @@ WHERE query_id = '$QUERY_ID' " # File segments cannot be less that 20Mi, # except for last file segment in a file or if file size is less. -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size AND file_segment_size < 20 * 1024 * 1024; " -all=$($CLICKHOUSE_CLIENT -m -q " +all=$($CLICKHOUSE_CLIENT -nm -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size; ") @@ -68,7 +68,7 @@ else echo "FAIL" fi -count=$($CLICKHOUSE_CLIENT -m -q " +count=$($CLICKHOUSE_CLIENT -nm -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size @@ -87,21 +87,21 @@ FROM (SELECT * FROM ($query)) AS cache_log INNER JOIN system.filesystem_cache AS cache ON cache_log.cache_path = cache.cache_path " -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size AND downloaded_size < 20 * 1024 * 1024; " -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size AND formatReadableSize(downloaded_size) not in ('20.00 MiB', '40.00 MiB'); " -all=$($CLICKHOUSE_CLIENT -m -q " +all=$($CLICKHOUSE_CLIENT -nm -q " SELECT count() FROM ($query2) WHERE file_segment_size < file_size AND file_segment_range_end + 1 != file_size; ") @@ -112,7 +112,7 @@ else echo "FAIL" fi -count2=$($CLICKHOUSE_CLIENT -m -q " +count2=$($CLICKHOUSE_CLIENT -nm -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size diff --git a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh index 239485ab8dd..5a37d51233d 100755 --- a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh +++ b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} -m --query " +${CLICKHOUSE_CLIENT} -nm --query " DROP TABLE IF EXISTS test_s3; CREATE TABLE test_s3 (a UInt64, b UInt64) @@ -25,7 +25,7 @@ do query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" - RES=$(${CLICKHOUSE_CLIENT} -m --query " + RES=$(${CLICKHOUSE_CLIENT} -nm --query " SELECT ProfileEvents['DiskConnectionsPreserved'] > 0 FROM system.query_log WHERE type = 'QueryFinish' @@ -41,7 +41,7 @@ done while true do - query_id=$(${CLICKHOUSE_CLIENT} -q " + query_id=$(${CLICKHOUSE_CLIENT} -nq " create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n; set insert_keeper_fault_injection_probability=0; insert into mut values (1, 2, 3), (10, 20, 30); @@ -60,7 +60,7 @@ do ) limit 1 settings max_threads=1; " 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" - RES=$(${CLICKHOUSE_CLIENT} -m --query " + RES=$(${CLICKHOUSE_CLIENT} -nm --query " SELECT ProfileEvents['StorageConnectionsPreserved'] > 0 FROM system.query_log WHERE type = 'QueryFinish' diff --git a/tests/queries/0_stateless/02801_backup_native_copy.sh b/tests/queries/0_stateless/02801_backup_native_copy.sh index bc485f903bc..b8ee97a7c7d 100755 --- a/tests/queries/0_stateless/02801_backup_native_copy.sh +++ b/tests/queries/0_stateless/02801_backup_native_copy.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk'; insert into data select * from numbers(10); @@ -16,28 +16,28 @@ $CLICKHOUSE_CLIENT -m -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true" -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false" -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true" -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_no_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false" -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " diff --git a/tests/queries/0_stateless/02803_backup_tmp_files.sh b/tests/queries/0_stateless/02803_backup_tmp_files.sh index 2208d3d32ef..d86beae4923 100755 --- a/tests/queries/0_stateless/02803_backup_tmp_files.sh +++ b/tests/queries/0_stateless/02803_backup_tmp_files.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk'; insert into data select * from numbers(10); diff --git a/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh b/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh index 4d7ab4063be..63fa60bd548 100755 --- a/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh +++ b/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() @@ -17,17 +17,17 @@ SETTINGS disk = disk(name = 's3_disk', type = cache, max_size = '100Ki', path = disk_name="${CLICKHOUSE_TEST_UNIQUE_NAME}" -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = ${CLICKHOUSE_TEST_UNIQUE_NAME}, disk = s3_disk); """ -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ diff --git a/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh b/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh index a7dfa035a22..8a4a2e906b0 100755 --- a/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh +++ b/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh @@ -9,7 +9,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="${CLICKHOUSE_TEST_UNIQUE_NAME}" -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() @@ -22,29 +22,29 @@ query_id=$RANDOM $CLICKHOUSE_CLIENT --query_id "$query_id" --query "SELECT * FROM test FORMAT Null SETTINGS enable_filesystem_cache_log = 1" -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY kek; """ 2>&1 | grep -q "Invalid cache key hex: kek" && echo "OK" || echo "FAIL" ${CLICKHOUSE_CLIENT} -q " system flush logs" -key=$($CLICKHOUSE_CLIENT -m --query """ +key=$($CLICKHOUSE_CLIENT -nm --query """ SELECT key FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -offset=$($CLICKHOUSE_CLIENT -m --query """ +offset=$($CLICKHOUSE_CLIENT -nm --query """ SELECT offset FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key' AND file_segment_range_begin = $offset; """ -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY $key OFFSET $offset; """ -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key' AND file_segment_range_begin = $offset; """ @@ -54,18 +54,18 @@ $CLICKHOUSE_CLIENT --query_id "$query_id" --query "SELECT * FROM test FORMAT Nul ${CLICKHOUSE_CLIENT} -q " system flush logs" -key=$($CLICKHOUSE_CLIENT -m --query """ +key=$($CLICKHOUSE_CLIENT -nm --query """ SELECT key FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key'; """ -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY $key """ -$CLICKHOUSE_CLIENT -m --query """ +$CLICKHOUSE_CLIENT -nm --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key'; """ diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh index ecd73bcc8a0..a2b1a953e24 100755 --- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh +++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " DROP TABLE IF EXISTS data; DROP TABLE IF EXISTS data_1; DROP TABLE IF EXISTS data_2; diff --git a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh index e6a1d547945..16ac095312c 100755 --- a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh +++ b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh @@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple(); insert into data select * from numbers(10); diff --git a/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh b/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh index 830e78e1e9f..4650415c202 100755 --- a/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh +++ b/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh @@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table if exists data; create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, disk='s3_disk'; -- reading 1e6*8 bytes with 1M bandwith it should take (8-1)/1=7 seconds @@ -15,7 +15,7 @@ $CLICKHOUSE_CLIENT -m -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --query_id "$query_id" -q "backup table data to S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data/backup2') SETTINGS allow_s3_native_copy=1" --max_backup_bandwidth=1M > /dev/null -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " SYSTEM FLUSH LOGS; SELECT 'native_copy', @@ -26,7 +26,7 @@ $CLICKHOUSE_CLIENT -m -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --query_id "$query_id" -q "backup table data to S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data/backup3') SETTINGS allow_s3_native_copy=0" --max_backup_bandwidth=1M > /dev/null -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " SYSTEM FLUSH LOGS; SELECT 'no_native_copy', diff --git a/tests/queries/0_stateless/02867_storage_set_tsan.sh b/tests/queries/0_stateless/02867_storage_set_tsan.sh index f6b684c0a55..81ae5f0bda8 100755 --- a/tests/queries/0_stateless/02867_storage_set_tsan.sh +++ b/tests/queries/0_stateless/02867_storage_set_tsan.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q """ +$CLICKHOUSE_CLIENT -mn -q """ DROP TABLE IF EXISTS t1_02867; CREATE TABLE t1_02867 (x UInt64) ENGINE=Set(); """ @@ -39,4 +39,4 @@ repeat_truncate_insert & sleep 10 -$CLICKHOUSE_CLIENT -m -q "DROP TABLE IF EXISTS t1_02867;" +$CLICKHOUSE_CLIENT -mn -q "DROP TABLE IF EXISTS t1_02867;" diff --git a/tests/queries/0_stateless/02900_union_schema_inference_mode.sh b/tests/queries/0_stateless/02900_union_schema_inference_mode.sh index a091645d1e3..a0fdb5276e0 100755 --- a/tests/queries/0_stateless/02900_union_schema_inference_mode.sh +++ b/tests/queries/0_stateless/02900_union_schema_inference_mode.sh @@ -10,14 +10,14 @@ echo '{"a" : 1, "obj" : {"f1" : 1, "f2" : "2020-01-01"}}' > $CLICKHOUSE_TEST_UNI echo '{"b" : 2, "obj" : {"f3" : 2, "f2" : "Some string"}}' > $CLICKHOUSE_TEST_UNIQUE_NAME/data2.jsonl echo '{"c" : "hello"}' > $CLICKHOUSE_TEST_UNIQUE_NAME/data3.jsonl -$CLICKHOUSE_LOCAL -m -q " +$CLICKHOUSE_LOCAL -nm -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl') order by tuple(*) format JSONEachRow; select schema_inference_mode, splitByChar('/', source)[-1] as file, schema from system.schema_inference_cache order by file; " -$CLICKHOUSE_LOCAL -m -q " +$CLICKHOUSE_LOCAL -nm -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data3.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); @@ -25,14 +25,14 @@ desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); cd $CLICKHOUSE_TEST_UNIQUE_NAME/ && tar -cf archive.tar data1.jsonl data2.jsonl data3.jsonl && cd .. -$CLICKHOUSE_LOCAL -m -q " +$CLICKHOUSE_LOCAL -nm -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl') order by tuple(*) format JSONEachRow; select schema_inference_mode, splitByChar('/', source)[-1] as file, schema from system.schema_inference_cache order by file; " -$CLICKHOUSE_LOCAL -m -q " +$CLICKHOUSE_LOCAL -nm -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data3.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); @@ -41,7 +41,7 @@ desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); echo 'Error' > $CLICKHOUSE_TEST_UNIQUE_NAME/data4.jsonl $CLICKHOUSE_LOCAL -q "desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3,4}.jsonl') settings schema_inference_mode='union'" 2>&1 | grep -c -F "CANNOT_EXTRACT_TABLE_STRUCTURE" -$CLICKHOUSE_LOCAL -m -q " +$CLICKHOUSE_LOCAL -nm -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{2,3}.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3,4}.jsonl'); diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index 91eeb22c19e..81ba59fc591 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -67,7 +67,7 @@ curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT sum(is_leader wait; -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " SYSTEM FLUSH LOGS; -- Check that number of ZK request is less then a half of (total replicas * concurrency) diff --git a/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh b/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh index 2eccade5c81..d1cbc54d294 100755 --- a/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh +++ b/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh @@ -8,7 +8,7 @@ CURDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table if exists r1; drop table if exists r2; @@ -64,7 +64,7 @@ function insert_duplicates() { wait - $CLICKHOUSE_CLIENT -m -q " + $CLICKHOUSE_CLIENT -nm -q " system sync replica r1; system sync replica r2; " @@ -84,7 +84,7 @@ function loop() do while ! insert_duplicates do - $CLICKHOUSE_CLIENT -m -q " + $CLICKHOUSE_CLIENT -nm -q " truncate table r1; truncate table r2; system sync replica r1; @@ -137,8 +137,8 @@ function list_keeper_nodes() { list_keeper_nodes "${table_shared_id}" -$CLICKHOUSE_CLIENT -m -q "drop table r1;" --allow_repeated_settings --send_logs_level="error" & -$CLICKHOUSE_CLIENT -m -q "drop table r2;" --allow_repeated_settings --send_logs_level="error" & +$CLICKHOUSE_CLIENT -nm -q "drop table r1;" --allow_repeated_settings --send_logs_level="error" & +$CLICKHOUSE_CLIENT -nm -q "drop table r2;" --allow_repeated_settings --send_logs_level="error" & wait list_keeper_nodes "${table_shared_id}" diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh index 057f76e63d0..2b92a113e91 100755 --- a/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh +++ b/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh @@ -10,11 +10,11 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --session_timezone Etc/UTC"`" -$CLICKHOUSE_CLIENT -q "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" +$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" # Basic refreshing. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create materialized view a refresh after 2 second engine Memory @@ -23,41 +23,41 @@ $CLICKHOUSE_CLIENT -q " select '<1: created view>', view, remaining_dependencies, exception, last_refresh_result in ('Unknown', 'Finished') from refreshes; show create a;" # Wait for any refresh. (xargs trims the string and turns \t and \n into spaces) -while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" == 'Unknown' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" == 'Unknown' ] do sleep 0.5 done -start_time="`$CLICKHOUSE_CLIENT -q "select reinterpret(now64(), 'Int64')"`" +start_time="`$CLICKHOUSE_CLIENT -nq "select reinterpret(now64(), 'Int64')"`" # Check table contents. -$CLICKHOUSE_CLIENT -q "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" +$CLICKHOUSE_CLIENT -nq "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" # Wait for table contents to change. -res1="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values'`" +res1="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`" while : do - res2="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values -- $LINENO'`" + res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" [ "$res2" == "$res1" ] || break sleep 0.5 done # Wait for another change. while : do - res3="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values -- $LINENO'`" + res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" [ "$res3" == "$res2" ] || break sleep 0.5 done # Check that the two changes were at least 1 second apart, in particular that we're not refreshing # like crazy. This is potentially flaky, but we need at least one test that uses non-mocked timer # to make sure the clock+timer code works at all. If it turns out flaky, increase refresh period above. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<3: time difference at least>', min2(reinterpret(now64(), 'Int64') - $start_time, 1000); select '<4: next refresh in>', next_refresh_time-last_refresh_time from refreshes;" # Create a source table from which views will read. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create table src (x Int8) engine Memory as select 1;" # Switch to fake clock, change refresh schedule, change query. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " system test view a set fake time '2050-01-01 00:00:01'; system wait view a; system refresh view a; @@ -68,19 +68,19 @@ $CLICKHOUSE_CLIENT -q " select '<4.5: altered>', status, last_refresh_result, next_refresh_time from refreshes; show create a;" # Advance time to trigger the refresh. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<5: no refresh>', count() from a; system test view a set fake time '2052-02-03 04:05:06';" -while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_time from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<6: refreshed>', * from a; select '<7: refreshed>', status, last_refresh_result, next_refresh_time from refreshes;" # Create a dependent view, refresh it once. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create materialized view b refresh every 2 year depends on a (y Int32) engine MergeTree order by y empty as select x*10 as y from a; show create b; system test view b set fake time '2052-11-11 11:11:11'; @@ -88,89 +88,89 @@ $CLICKHOUSE_CLIENT -q " system wait view b; select '<7.5: created dependent>', last_refresh_time from refreshes where view = 'b';" # Next refresh shouldn't start until the dependency refreshes. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<8: refreshed>', * from b; select '<9: refreshed>', view, status, last_refresh_result, next_refresh_time from refreshes; system test view b set fake time '2054-01-24 23:22:21';" -while [ "`$CLICKHOUSE_CLIENT -q "select status, next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ] do sleep 0.5 done # Drop the source table, check that refresh fails and doesn't leave a temp table behind. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<9.2: dropping>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase(); drop table src; system refresh view a;" -$CLICKHOUSE_CLIENT -q "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -nq " select '<9.4: dropped>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase();" # Create the source table again, check that refresh succeeds (in particular that tables are looked # up by name rather than uuid). -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<10: creating>', view, status, remaining_dependencies, next_refresh_time from refreshes; create table src (x Int16) engine Memory as select 2; system test view a set fake time '2054-01-01 00:00:01';" -while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] do sleep 0.5 done # Both tables should've refreshed. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<11: chain-refreshed a>', * from a; select '<12: chain-refreshed b>', * from b; select '<13: chain-refreshed>', view, status, remaining_dependencies, last_refresh_result, last_refresh_time, next_refresh_time, exception == '' from refreshes;" # Make the dependent table run ahead by one refresh cycle, make sure it waits for the dependency to # catch up to the same cycle. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " system test view b set fake time '2059-01-01 00:00:00'; system refresh view b;" -while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2060-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2060-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " system test view b set fake time '2061-01-01 00:00:00'; system test view a set fake time '2057-01-01 00:00:00';" -while [ "`$CLICKHOUSE_CLIENT -q "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<14: waiting for next cycle>', view, status, remaining_dependencies, next_refresh_time from refreshes; truncate src; insert into src values (3); system test view a set fake time '2060-02-02 02:02:02';" -while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<15: chain-refreshed a>', * from a; select '<16: chain-refreshed b>', * from b; select '<17: chain-refreshed>', view, status, next_refresh_time from refreshes;" # Get to WaitingForDependencies state and remove the depencency. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " system test view b set fake time '2062-03-03 03:03:03'" -while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " alter table b modify refresh every 2 year" -while [ "`$CLICKHOUSE_CLIENT -q "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<18: removed dependency>', view, status, remaining_dependencies, last_refresh_time,next_refresh_time, refresh_count from refreshes where view = 'b'; show create b;" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " drop table src; drop table a; drop table b; diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh index 2d00d61f253..50a905576d5 100755 --- a/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh +++ b/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh @@ -12,29 +12,29 @@ CLICKHOUSE_LOG_COMMENT= CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --allow_materialized_view_with_bad_select=0 --session_timezone Etc/UTC"`" -$CLICKHOUSE_CLIENT -q "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" +$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" # Select from a table that doesn't exist, get an exception. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create table src (x Int8) engine Memory as select 1; create materialized view c refresh every 1 second (x Int64) engine Memory empty as select * from src; drop table src;" -while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes where view = 'c' -- $LINENO" | xargs`" != 'Error' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes where view = 'c' -- $LINENO" | xargs`" != 'Error' ] do sleep 0.5 done # Check exception, create src, expect successful refresh. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<19: exception>', exception ilike '%UNKNOWN_TABLE%' ? '1' : exception from refreshes where view = 'c'; create table src (x Int64) engine Memory as select 1; system refresh view c;" -while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done # Rename table. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<20: unexception>', * from c; rename table c to d; select '<21: rename>', * from d; @@ -42,130 +42,130 @@ $CLICKHOUSE_CLIENT -q " # Do various things during a refresh. # First make a nonempty view. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " drop table d; truncate src; insert into src values (1); create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x empty as select x + sleepEachRow(1) as x from src settings max_block_size = 1;" -while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done # Stop refreshes. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<23: simple refresh>', * from e; system stop view e;" -while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] do sleep 0.5 done # Make refreshes slow, wait for a slow refresh to start. (We stopped refreshes first to make sure # we wait for a slow refresh, not a previous fast one.) -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " insert into src select * from numbers(1000) settings max_block_size=1; system start view e;" -while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] do sleep 0.5 done # Rename. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " rename table e to f; select '<24: rename during refresh>', * from f; select '<25: rename during refresh>', view, status from refreshes where view = 'f'; alter table f modify refresh after 10 year;" # Cancel. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " system cancel view f;" -while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Scheduled' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Scheduled' ] do sleep 0.5 done # Check that another refresh doesn't immediately start after the cancelled one. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<27: cancelled>', view, status, last_refresh_result from refreshes where view = 'f'; system refresh view f;" -while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Running' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Running' ] do sleep 0.5 done # Drop. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " drop table f; select '<28: drop during refresh>', view, status from refreshes; select '<28: drop during refresh>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase()" # Try OFFSET and RANDOMIZE FOR. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create materialized view g refresh every 1 week offset 3 day 4 hour randomize for 4 day 1 hour (x Int64) engine Memory empty as select 42 as x; show create g; system test view g set fake time '2050-02-03 15:30:13';" -while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " with '2050-02-10 04:00:00'::DateTime as expected select '<29: randomize>', abs(next_refresh_time::Int64 - expected::Int64) <= 3600*(24*4+1), next_refresh_time != expected from refreshes;" # Send data 'TO' an existing table. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " drop table g; create table dest (x Int64) engine MergeTree order by x; truncate src; insert into src values (1); create materialized view h refresh every 1 second to dest empty as select x*10 as x from src; show create h;" -while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<30: to existing table>', * from dest; insert into src values (2);" -while [ "`$CLICKHOUSE_CLIENT -q "select count() from dest -- $LINENO" | xargs`" != '2' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest -- $LINENO" | xargs`" != '2' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<31: to existing table>', * from dest; drop table dest; drop table h;" # Retries. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create materialized view h2 refresh after 1 year settings refresh_retries = 10 (x Int64) engine Memory as select x*10 + throwIf(x % 2 == 0) as x from src;" -$CLICKHOUSE_CLIENT -q "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -nq " select '<31.5: will retry>', last_refresh_result, retry > 0 from refreshes; create table src2 (x Int8) engine Memory; insert into src2 values (1); exchange tables src and src2; drop table src2;" -while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result, retry from refreshes -- $LINENO" | xargs`" != 'Finished 0' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result, retry from refreshes -- $LINENO" | xargs`" != 'Finished 0' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<31.6: did retry>', x from h2; drop table h2" # EMPTY -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create materialized view i refresh after 1 year engine Memory empty as select number as x from numbers(2); create materialized view j refresh after 1 year engine Memory as select number as x from numbers(2);" -while [ "`$CLICKHOUSE_CLIENT -q "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " select '<32: empty>', view, status, last_refresh_result, retry from refreshes order by view; drop table i; drop table j;" # APPEND -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create materialized view k refresh every 10 year append (x Int64) engine Memory empty as select x*10 as x from src; select '<33: append>', * from k; system refresh view k; @@ -177,7 +177,7 @@ $CLICKHOUSE_CLIENT -q " system wait view k; select '<35: append>', * from k order by x;" # ALTER to non-APPEND -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " alter table k modify refresh every 10 year; system wait view k; system refresh view k; @@ -187,7 +187,7 @@ $CLICKHOUSE_CLIENT -q " truncate table src;" # APPEND + TO + regular materialized view reading from it. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create table mid (x Int64) engine MergeTree order by x; create materialized view l refresh every 10 year append to mid empty as select x*10 as x from src; create materialized view m (x Int64) engine Memory as select x*10 as x from mid; @@ -204,19 +204,19 @@ $CLICKHOUSE_CLIENT -q " drop table mid;" # Failing to create inner table. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create materialized view n refresh every 1 second (x Int64) engine MergeTree as select 1 as x from numbers(2);" 2>/dev/null || echo "creating MergeTree without ORDER BY failed, as expected" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create materialized view n refresh every 1 second (x Int64) engine MergeTree order by x as select 1 as x from numbers(2); drop table n;" # Reading from table that doesn't exist yet. -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " create materialized view o refresh every 1 second (x Int64) engine Memory as select x from nonexist; -- { serverError UNKNOWN_TABLE } create materialized view o (x Int64) engine Memory as select x from nonexist; -- { serverError UNKNOWN_TABLE } create materialized view o (x Int64) engine Memory as select x from nope.nonexist; -- { serverError UNKNOWN_DATABASE } create materialized view o refresh every 1 second (x Int64) engine Memory as select x from nope.nonexist settings allow_materialized_view_with_bad_select = 1; drop table o;" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT -nq " drop table refreshes;" diff --git a/tests/queries/0_stateless/02941_variant_type_1.sh b/tests/queries/0_stateless/02941_variant_type_1.sh index ef1d1deff4e..c12bece6d54 100755 --- a/tests/queries/0_stateless/02941_variant_type_1.sh +++ b/tests/queries/0_stateless/02941_variant_type_1.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test1_insert() { echo "test1 insert" - $CH_CLIENT -mq "insert into test select number, NULL from numbers(3); + $CH_CLIENT -nmq "insert into test select number, NULL from numbers(3); insert into test select number + 3, number from numbers(3); insert into test select number + 6, ('str_' || toString(number))::Variant(String) from numbers(3); insert into test select number + 9, ('lc_str_' || toString(number))::LowCardinality(String) from numbers(3); @@ -21,7 +21,7 @@ insert into test select number + 15, range(number + 1)::Array(UInt64) from numbe function test1_select() { echo "test1 select" - $CH_CLIENT -mq "select v from test order by id; + $CH_CLIENT -nmq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; @@ -36,7 +36,7 @@ select v.\`Array(UInt64)\`.size0 from test order by id;" function test2_insert() { echo "test2 insert" - $CH_CLIENT -mq "insert into test select number, NULL from numbers(3); + $CH_CLIENT -nmq "insert into test select number, NULL from numbers(3); insert into test select number + 3, number % 2 ? NULL : number from numbers(3); insert into test select number + 6, number % 2 ? NULL : ('str_' || toString(number))::Variant(String) from numbers(3); insert into test select number + 9, number % 2 ? CAST(NULL, 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') : CAST(('lc_str_' || toString(number))::LowCardinality(String), 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') from numbers(3); @@ -47,7 +47,7 @@ insert into test select number + 15, number % 2 ? CAST(NULL, 'Variant(String, UI function test2_select() { echo "test2 select" - $CH_CLIENT -mq "select v from test order by id; + $CH_CLIENT -nmq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; @@ -68,7 +68,7 @@ function test3_insert() function test3_select() { echo "test3 select" - $CH_CLIENT -mq "select v from test order by id; + $CH_CLIENT -nmq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; diff --git a/tests/queries/0_stateless/02941_variant_type_2.sh b/tests/queries/0_stateless/02941_variant_type_2.sh index 7017edac525..e93dfac8510 100755 --- a/tests/queries/0_stateless/02941_variant_type_2.sh +++ b/tests/queries/0_stateless/02941_variant_type_2.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test4_insert() { echo "test4 insert" - $CH_CLIENT -mq "insert into test select number, NULL from numbers(100000); + $CH_CLIENT -nmq "insert into test select number, NULL from numbers(100000); insert into test select number + 100000, number from numbers(100000); insert into test select number + 200000, ('str_' || toString(number))::Variant(String) from numbers(100000); insert into test select number + 300000, ('lc_str_' || toString(number))::LowCardinality(String) from numbers(100000); @@ -21,7 +21,7 @@ insert into test select number + 500000, range(number % 20 + 1)::Array(UInt64) f function test4_select { echo "test4 select" - $CH_CLIENT -mq "select v from test format Null; + $CH_CLIENT -nmq "select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; select count() from test where isNotNull(v.String); diff --git a/tests/queries/0_stateless/02941_variant_type_3.sh b/tests/queries/0_stateless/02941_variant_type_3.sh index 5d923a47e9b..cc0fde5b689 100755 --- a/tests/queries/0_stateless/02941_variant_type_3.sh +++ b/tests/queries/0_stateless/02941_variant_type_3.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test5_insert() { echo "test5 insert" - $CH_CLIENT -mq " + $CH_CLIENT -nmq " insert into test select number, NULL from numbers(200000); insert into test select number + 200000, number % 2 ? NULL : number from numbers(200000); insert into test select number + 400000, number % 2 ? NULL : ('str_' || toString(number))::Variant(String) from numbers(200000); @@ -22,7 +22,7 @@ insert into test select number + 1000000, number % 2 ? CAST(NULL, 'Variant(Strin function test5_select() { echo "test5 select" - $CH_CLIENT -mq " + $CH_CLIENT -nmq " select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; diff --git a/tests/queries/0_stateless/02941_variant_type_4.sh b/tests/queries/0_stateless/02941_variant_type_4.sh index 88f6a475d46..93a1770d05e 100755 --- a/tests/queries/0_stateless/02941_variant_type_4.sh +++ b/tests/queries/0_stateless/02941_variant_type_4.sh @@ -17,7 +17,7 @@ function test6_insert() function test6_select() { echo "test6 select" - $CH_CLIENT -mq "select v from test format Null; + $CH_CLIENT -nmq "select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; select count() from test where isNotNull(v.String); diff --git a/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh b/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh index f9e84dc2b50..cb099bb59ae 100755 --- a/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh +++ b/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh @@ -10,7 +10,7 @@ disk_name="s3_cache_02944" $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a String) engine=MergeTree() ORDER BY tuple() SETTINGS disk = '$disk_name'; INSERT INTO test SELECT randomString(100); @@ -33,7 +33,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -47,7 +47,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -63,7 +63,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -77,7 +77,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" diff --git a/tests/queries/0_stateless/02960_polygon_bound_bug.sh b/tests/queries/0_stateless/02960_polygon_bound_bug.sh index 1e9be36da55..0c3db01a77c 100755 --- a/tests/queries/0_stateless/02960_polygon_bound_bug.sh +++ b/tests/queries/0_stateless/02960_polygon_bound_bug.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_LOCAL -m -q "CREATE TABLE test_table (geom MultiPolygon) engine=MergeTree ORDER BY geom; +$CLICKHOUSE_LOCAL -nm -q "CREATE TABLE test_table (geom MultiPolygon) engine=MergeTree ORDER BY geom; INSERT INTO test_table SELECT * FROM file('$CURDIR/data_parquet/02960_polygon_bound_bug.parquet', Parquet); CREATE DICTIONARY test_dict (geom MultiPolygon) PRIMARY KEY geom SOURCE (CLICKHOUSE(TABLE 'test_table')) LIFETIME(MIN 0 MAX 0) LAYOUT(POLYGON(STORE_POLYGON_KEY_COLUMN 1)); SELECT dictHas(test_dict,(174.84729269276494,-36.99524960275426));" diff --git a/tests/queries/0_stateless/02961_storage_config_volume_priority.sh b/tests/queries/0_stateless/02961_storage_config_volume_priority.sh index 1bd86ee0c75..145b921a750 100755 --- a/tests/queries/0_stateless/02961_storage_config_volume_priority.sh +++ b/tests/queries/0_stateless/02961_storage_config_volume_priority.sh @@ -24,7 +24,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " set send_logs_level='error'; SYSTEM RELOAD CONFIG" 2>&1 | grep -c 'volume_priority values must be unique across the policy' @@ -40,7 +40,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " set send_logs_level='error'; SYSTEM RELOAD CONFIG" 2>&1 | grep -c 'volume_priority values must cover the range from 1 to N (lowest priority specified) without gaps' diff --git a/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh b/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh index 3b57941452d..24fe964b824 100755 --- a/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh +++ b/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="02963_remote_read_bug" -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, s String) @@ -22,7 +22,7 @@ OPTIMIZE TABLE test FINAL; query_id=$(random_str 10) -$CLICKHOUSE_CLIENT -m --query_id "$query_id" --query " +$CLICKHOUSE_CLIENT -nm --query_id "$query_id" --query " WITH RANDOM_SET AS ( SELECT rand32() % 10000 FROM numbers(100) ) @@ -37,7 +37,7 @@ SETTINGS merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem = 1, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem = 1; " -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " SYSTEM FLUSH LOGS; -- This threshold was determined experimentally - before the fix this ratio had values around 50K diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh index b4271c3d29b..1089eb4051f 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh index ed13bf3321b..7a0e2d9bfdb 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh index 9abe870f4e4..c4b44ce11c5 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02969_auto_format_detection.sh b/tests/queries/0_stateless/02969_auto_format_detection.sh index ab21dbb3df8..88d6575e499 100755 --- a/tests/queries/0_stateless/02969_auto_format_detection.sh +++ b/tests/queries/0_stateless/02969_auto_format_detection.sh @@ -24,12 +24,12 @@ $CLICKHOUSE_LOCAL -q "select * from generateRandom('a UInt64, b String, c Array( $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE', auto, 'a UInt64, b String, c Array(UInt64), d Tuple(a UInt64, b String)')" -$CLICKHOUSE_LOCAL -mq " +$CLICKHOUSE_LOCAL -nmq " desc file('$DATA_FILE'); desc file('$DATA_FILE'); " -$CLICKHOUSE_LOCAL -mq " +$CLICKHOUSE_LOCAL -nmq " desc file('$DATA_FILE', JSONEachRow); desc file('$DATA_FILE'); " @@ -39,7 +39,7 @@ $CLICKHOUSE_LOCAL -q "select * from generateRandom('a UInt64, b String, c Array( $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE.{1,2}')" $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE.{1,2}') settings schema_inference_mode='union'" 2>&1 | grep -c "CANNOT_DETECT_FORMAT" -$CLICKHOUSE_LOCAL -mq " +$CLICKHOUSE_LOCAL -nmq " desc file('$DATA_FILE.2'); desc file('$DATA_FILE.{1,2}'); " diff --git a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh index a46f0018ad3..d543f7195a9 100755 --- a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh +++ b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh @@ -19,7 +19,7 @@ $CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -q "create database $n CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT/--database=$CLICKHOUSE_DATABASE/--database=$new_database} CLICKHOUSE_DATABASE="$new_database" -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by key; insert into data values (1); @@ -29,7 +29,7 @@ $CLICKHOUSE_CLIENT -m -q " # suppress output $CLICKHOUSE_CLIENT -q "backup table data to S3('http://localhost:11111/test/s3_plain/backups/$CLICKHOUSE_DATABASE', 'test', 'testtest')" > /dev/null -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table data; attach table data (key Int) engine=MergeTree() order by key settings diff --git a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh index 522fdd6096a..eec05c81344 100755 --- a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh +++ b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh @@ -18,7 +18,7 @@ $CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -q "create database $n CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT/--database=$CLICKHOUSE_DATABASE/--database=$new_database} CLICKHOUSE_DATABASE="$new_database" -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table if exists data_read; drop table if exists data_write; @@ -33,7 +33,7 @@ $CLICKHOUSE_CLIENT -m -q " # suppress output $CLICKHOUSE_CLIENT -q "backup table data_read to S3('http://localhost:11111/test/s3_plain/backups/$CLICKHOUSE_DATABASE', 'test', 'testtest')" > /dev/null -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " drop table data_read; attach table data_read (key Int) engine=ReplicatedMergeTree('/tables/{database}/data', 'read') order by key settings @@ -57,7 +57,7 @@ echo "Files before DETACH TABLE" # sed to match any part, since in case of fault injection part name may not be all_0_0_0 but all_1_1_0 clickhouse-disks -C "$config" --disk s3_plain_disk --query "list --recursive $path" | tail -n+2 | sed 's/all_[^_]*_[^_]*_0/all_X_X_X/g' -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -nm -q " detach table data_read; detach table data_write; " diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh index fca8b7a5e97..ef4866cb69e 100755 --- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh index 08d5ae3a6b2..dd4b1f7cec0 100755 --- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh index 678b1db4d68..33386c76edb 100755 --- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh index 606fda6d4fd..b66ef83abf2 100755 --- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh index b571631dc8f..f9e1838f491 100755 --- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh index e4358011e50..698e70d4064 100755 --- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_s3_plain_rewritable.sh b/tests/queries/0_stateless/03008_s3_plain_rewritable.sh index c13674b9bd1..8eea7940774 100755 --- a/tests/queries/0_stateless/03008_s3_plain_rewritable.sh +++ b/tests/queries/0_stateless/03008_s3_plain_rewritable.sh @@ -9,7 +9,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query "drop table if exists test_s3_mt" -${CLICKHOUSE_CLIENT} -m --query " +${CLICKHOUSE_CLIENT} -nm --query " create table test_s3_mt (a Int32, b Int64, c Int64) engine = MergeTree() partition by intDiv(a, 1000) order by tuple(a, b) settings disk = disk( name = 03008_s3_plain_rewritable, @@ -19,7 +19,7 @@ settings disk = disk( secret_access_key = clickhouse); " -${CLICKHOUSE_CLIENT} -m --query " +${CLICKHOUSE_CLIENT} -nm --query " insert into test_s3_mt (*) values (1, 2, 0), (2, 2, 2), (3, 1, 9), (4, 7, 7), (5, 10, 2), (6, 12, 5); insert into test_s3_mt (*) select number, number, number from numbers_mt(10000); select count(*) from test_s3_mt; @@ -31,13 +31,13 @@ ${CLICKHOUSE_CLIENT} --query "optimize table test_s3_mt final" ${CLICKHOUSE_CLIENT} -m --query " alter table test_s3_mt add projection test_s3_mt_projection (select * order by b)" 2>&1 | grep -Fq "SUPPORT_IS_DISABLED" -${CLICKHOUSE_CLIENT} -m --query " +${CLICKHOUSE_CLIENT} -nm --query " alter table test_s3_mt update c = 0 where a % 2 = 1; alter table test_s3_mt add column d Int64 after c; alter table test_s3_mt drop column c; " 2>&1 | grep -Fq "SUPPORT_IS_DISABLED" -${CLICKHOUSE_CLIENT} -m --query " +${CLICKHOUSE_CLIENT} -nm --query " detach table test_s3_mt; attach table test_s3_mt; " diff --git a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh index 278ad6f0654..cba5317fcfa 100755 --- a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh +++ b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="s3_cache" -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a String) engine=MergeTree() ORDER BY tuple() SETTINGS disk = '$disk_name'; INSERT INTO test SELECT randomString(1000); @@ -26,7 +26,7 @@ sed -i "s|$prev_max_size<\/max_size>|$new_max_size<\/max_siz # echo $prev_max_size # echo $new_max_size -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" @@ -36,7 +36,7 @@ $CLICKHOUSE_CLIENT --query "SELECT current_size <= max_size FROM system.filesyst sed -i "s|$new_max_size<\/max_size>|$prev_max_size<\/max_size>|" $config_path -$CLICKHOUSE_CLIENT -m --query " +$CLICKHOUSE_CLIENT -nm --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" diff --git a/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh index 8183b636549..9ea86105a3a 100755 --- a/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from (select sumMerge(sum) as sum from test group by id, _part) group by sum order by sum, count()" - $CH_CLIENT -m -q "system start merges test; optimize table test final" + $CH_CLIENT -nm -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from (select sumMerge(sum) as sum from test group by id, _part) group by sum order by sum, count()" $CH_CLIENT -q "drop table test" diff --git a/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh index ddafa9db210..9a2a6dd957c 100755 --- a/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh @@ -16,7 +16,7 @@ function test() $CH_CLIENT -q "insert into test select number, -1, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -m -q "system start merges test; optimize table test final" + $CH_CLIENT -nm -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh index b1046ca08ba..0199035a3df 100755 --- a/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -m -q "system start merges test; optimize table test final" + $CH_CLIENT -nm -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh index f3693a2e7b1..e2ea5bc3466 100755 --- a/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from test group by sum order by sum, count()" - $CH_CLIENT -m -q "system start merges test; optimize table test final" + $CH_CLIENT -nm -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from test group by sum order by sum, count()" $CH_CLIENT -q "drop table test" diff --git a/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh index bba92a2ee06..43607cf95a8 100755 --- a/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "insert into test select number, -1, number >= 75000 ? 2 : 1, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -m -q "system start merges test; optimize table test final" + $CH_CLIENT -nm -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03151_unload_index_race.sh b/tests/queries/0_stateless/03151_unload_index_race.sh index 4cdf06abae2..7e9dfa7cddc 100755 --- a/tests/queries/0_stateless/03151_unload_index_race.sh +++ b/tests/queries/0_stateless/03151_unload_index_race.sh @@ -42,8 +42,8 @@ function thread_alter_settings() { local TIMELIMIT=$((SECONDS+$1)) while [ $SECONDS -lt "$TIMELIMIT" ]; do - $CLICKHOUSE_CLIENT --query "ALTER TABLE t MODIFY SETTING primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns=0.$RANDOM" - $CLICKHOUSE_CLIENT --query "SYSTEM UNLOAD PRIMARY KEY t" + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE t MODIFY SETTING primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns=0.$RANDOM" + $CLICKHOUSE_CLIENT -n --query "SYSTEM UNLOAD PRIMARY KEY t" sleep 0.0$RANDOM done } @@ -52,7 +52,7 @@ function thread_query_table() { local TIMELIMIT=$((SECONDS+$1)) while [ $SECONDS -lt "$TIMELIMIT" ]; do - COUNT=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM t where not ignore(*);") + COUNT=$($CLICKHOUSE_CLIENT -n --query "SELECT count() FROM t where not ignore(*);") if [ "$COUNT" -ne "2000" ]; then echo "$COUNT" fi diff --git a/tests/queries/0_stateless/03172_error_log_table_not_empty.sh b/tests/queries/0_stateless/03172_error_log_table_not_empty.sh index d1afafa77a5..22a2fd82c64 100755 --- a/tests/queries/0_stateless/03172_error_log_table_not_empty.sh +++ b/tests/queries/0_stateless/03172_error_log_table_not_empty.sh @@ -15,7 +15,7 @@ errors_222=$($CLICKHOUSE_CLIENT -q "SELECT sum(value) FROM system.error_log WHER errors_333=$($CLICKHOUSE_CLIENT -q "SELECT sum(value) FROM system.error_log WHERE code = 333") # Throw three random errors: 111, 222 and 333 and wait for more than collect_interval_milliseconds to ensure system.error_log is flushed -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -mn -q " SELECT throwIf(true, 'error_log', toInt16(111)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 111 } SELECT throwIf(true, 'error_log', toInt16(222)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 222 } SELECT throwIf(true, 'error_log', toInt16(333)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 333 } @@ -24,14 +24,14 @@ SYSTEM FLUSH LOGS; " # Check that the three random errors are propagated -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -mn -q " SELECT sum(value) > $errors_111 FROM system.error_log WHERE code = 111; SELECT sum(value) > $errors_222 FROM system.error_log WHERE code = 222; SELECT sum(value) > $errors_333 FROM system.error_log WHERE code = 333; " # Ensure that if we throw them again, they're still propagated -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -mn -q " SELECT throwIf(true, 'error_log', toInt16(111)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 111 } SELECT throwIf(true, 'error_log', toInt16(222)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 222 } SELECT throwIf(true, 'error_log', toInt16(333)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 333 } @@ -39,7 +39,7 @@ SELECT sleep(2) format NULL; SYSTEM FLUSH LOGS; " -$CLICKHOUSE_CLIENT -m -q " +$CLICKHOUSE_CLIENT -mn -q " SELECT sum(value) > $(($errors_111+1)) FROM system.error_log WHERE code = 111; SELECT sum(value) > $(($errors_222+1)) FROM system.error_log WHERE code = 222; SELECT sum(value) > $(($errors_333+1)) FROM system.error_log WHERE code = 333; diff --git a/tests/queries/0_stateless/03203_hive_style_partitioning.sh b/tests/queries/0_stateless/03203_hive_style_partitioning.sh index fdd0fcd8ec5..60e8a6e9faa 100755 --- a/tests/queries/0_stateless/03203_hive_style_partitioning.sh +++ b/tests/queries/0_stateless/03203_hive_style_partitioning.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE FILE HIVE PARTITIONING'" -$CLICKHOUSE_LOCAL -q """ +$CLICKHOUSE_LOCAL -n -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM file('$CURDIR/data_hive/partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -22,20 +22,20 @@ SELECT toTypeName(array), toTypeName(float) FROM file('$CURDIR/data_hive/partiti SELECT count(*) FROM file('$CURDIR/data_hive/partitioning/number=42/date=2020-01-01/sample.parquet') WHERE number = 42; """ -$CLICKHOUSE_LOCAL -q """ +$CLICKHOUSE_LOCAL -n -q """ set use_hive_partitioning = 1; SELECT identifier FROM file('$CURDIR/data_hive/partitioning/identifier=*/email.csv') LIMIT 2; SELECT a FROM file('$CURDIR/data_hive/partitioning/a=b/a=b/sample.parquet') LIMIT 1; """ -$CLICKHOUSE_LOCAL -q """ +$CLICKHOUSE_LOCAL -n -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM file('$CURDIR/data_hive/partitioning/column0=Elizabeth/column0=Elizabeth1/sample.parquet') LIMIT 10; """ 2>&1 | grep -c "INCORRECT_DATA" -$CLICKHOUSE_LOCAL -q """ +$CLICKHOUSE_LOCAL -n -q """ set use_hive_partitioning = 0; SELECT *, non_existing_column FROM file('$CURDIR/data_hive/partitioning/non_existing_column=Elizabeth/sample.parquet') LIMIT 10; @@ -45,14 +45,14 @@ SELECT *, non_existing_column FROM file('$CURDIR/data_hive/partitioning/non_exis $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE URL PARTITIONING'" -$CLICKHOUSE_LOCAL -q """ +$CLICKHOUSE_LOCAL -n -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM url('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; SELECT *, non_existing_column FROM url('http://localhost:11111/test/hive_partitioning/non_existing_column=Elizabeth/sample.parquet') LIMIT 10;""" -$CLICKHOUSE_LOCAL -q """ +$CLICKHOUSE_LOCAL -n -q """ set use_hive_partitioning = 0; SELECT *, _column0 FROM url('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -62,7 +62,7 @@ SELECT *, _column0 FROM url('http://localhost:11111/test/hive_partitioning/colum $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE S3 PARTITIONING'" -$CLICKHOUSE_CLIENT -q """ +$CLICKHOUSE_CLIENT -n -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -71,7 +71,7 @@ SELECT *, non_existing_column FROM s3('http://localhost:11111/test/hive_partitio SELECT *, column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=*/sample.parquet') WHERE column0 = 'Elizabeth' LIMIT 10; """ -$CLICKHOUSE_CLIENT -q """ +$CLICKHOUSE_CLIENT -n -q """ set use_hive_partitioning = 0; SELECT *, _column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -79,7 +79,7 @@ SELECT *, _column0 FROM s3('http://localhost:11111/test/hive_partitioning/column $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE S3CLUSTER PARTITIONING'" -$CLICKHOUSE_CLIENT -q """ +$CLICKHOUSE_CLIENT -n -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM s3Cluster(test_cluster_one_shard_three_replicas_localhost, 'http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; diff --git a/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh index 6bfe82a91a6..e0c8f08e695 100755 --- a/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh +++ b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # In this test we restore from "/tests/queries/0_stateless/backups/mt_250_parts.zip" backup_name="$($CURDIR/helpers/install_predefined_backup.sh mt_250_parts.zip)" -${CLICKHOUSE_CLIENT} -m --query " +${CLICKHOUSE_CLIENT} -nm --query " DROP TABLE IF EXISTS manyparts; CREATE TABLE manyparts (x Int64) ENGINE=MergeTree ORDER BY tuple() SETTINGS merge_tree_clear_old_temporary_directories_interval_seconds=1, temporary_directories_lifetime=1; " @@ -16,7 +16,7 @@ CREATE TABLE manyparts (x Int64) ENGINE=MergeTree ORDER BY tuple() SETTINGS merg # RESTORE must protect its temporary directories from removing. ${CLICKHOUSE_CLIENT} --query "RESTORE TABLE default.mt_250_parts AS manyparts FROM Disk('backups', '${backup_name}') SETTINGS allow_different_table_def=true" | grep -o "RESTORED" -${CLICKHOUSE_CLIENT} -m --query " +${CLICKHOUSE_CLIENT} -nm --query " SELECT count(), sum(x) FROM manyparts; DROP TABLE manyparts; " From 47589d48980a790a76e2163e88200213c6de2fee Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 9 Sep 2024 19:47:10 +0100 Subject: [PATCH 263/289] impl --- src/IO/S3/URI.cpp | 2 +- src/IO/tests/gtest_s3_uri.cpp | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 9c80b377661..73bbba055d0 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -36,7 +36,7 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax) /// Case when bucket name represented in domain name of S3 URL. /// E.g. (https://bucket-name.s3.region.amazonaws.com/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access - static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3express[\-a-z0-9]+|s3|cos|obs|oss|eos)([.\-][a-z0-9\-.:]+))"); + static const RE2 virtual_hosted_style_pattern(R"(([^.]+)\.(s3express[\-a-z0-9]+|s3|cos|obs|.*oss[^\/]*|eos)([.\-][a-z0-9\-.:]+))"); /// Case when AWS Private Link Interface is being used /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/bucket-name/key) diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index c0bf7fcb28a..abe80db7ba5 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -204,6 +204,14 @@ TEST(S3UriTest, validPatterns) ASSERT_EQ("", uri.version_id); ASSERT_EQ(true, uri.is_virtual_hosted_style); } + { + S3::URI uri("https://bucket-test.cn-beijing-internal.oss-data-acc.aliyuncs.com/cc-2zeh496zqm0g6e09g"); + ASSERT_EQ("https://cn-beijing-internal.oss-data-acc.aliyuncs.com", uri.endpoint); + ASSERT_EQ("bucket-test", uri.bucket); + ASSERT_EQ("cc-2zeh496zqm0g6e09g", uri.key); + ASSERT_EQ("", uri.version_id); + ASSERT_EQ(true, uri.is_virtual_hosted_style); + } } TEST(S3UriTest, versionIdChecks) From e252bdc30d7b10e257ff63b0288dc808ed717d53 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 8 Sep 2024 12:57:37 +0000 Subject: [PATCH 264/289] Bump to v3.7.3 --- contrib/libarchive | 2 +- contrib/libarchive-cmake/CMakeLists.txt | 8 +++---- contrib/libarchive-cmake/config.h | 28 ++++++++++++++++--------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/contrib/libarchive b/contrib/libarchive index 6468cd1f5c9..4fcc02d906c 160000 --- a/contrib/libarchive +++ b/contrib/libarchive @@ -1 +1 @@ -Subproject commit 6468cd1f5c9b76e2c3b10cdd6938faf6b82823b6 +Subproject commit 4fcc02d906cca4b9e21a78a833f1142a2689ec52 diff --git a/contrib/libarchive-cmake/CMakeLists.txt b/contrib/libarchive-cmake/CMakeLists.txt index e89770da5f6..aa6dd9638b6 100644 --- a/contrib/libarchive-cmake/CMakeLists.txt +++ b/contrib/libarchive-cmake/CMakeLists.txt @@ -1,6 +1,6 @@ set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/libarchive") -set(SRCS +set(SRCS "${LIBRARY_DIR}/libarchive/archive_acl.c" "${LIBRARY_DIR}/libarchive/archive_blake2sp_ref.c" "${LIBRARY_DIR}/libarchive/archive_blake2s_ref.c" @@ -135,7 +135,7 @@ set(SRCS ) add_library(_libarchive ${SRCS}) -target_include_directories(_libarchive PUBLIC +target_include_directories(_libarchive PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} "${LIBRARY_DIR}/libarchive" ) @@ -157,7 +157,7 @@ if (TARGET ch_contrib::zlib) endif() if (TARGET ch_contrib::zstd) - target_compile_definitions(_libarchive PUBLIC HAVE_ZSTD_H=1 HAVE_LIBZSTD=1 HAVE_LIBZSTD_COMPRESSOR=1) + target_compile_definitions(_libarchive PUBLIC HAVE_ZSTD_H=1 HAVE_LIBZSTD=1 HAVE_ZSTD_compressStream=1) target_link_libraries(_libarchive PRIVATE ch_contrib::zstd) endif() @@ -179,4 +179,4 @@ if (OS_LINUX) ) endif() -add_library(ch_contrib::libarchive ALIAS _libarchive) \ No newline at end of file +add_library(ch_contrib::libarchive ALIAS _libarchive) diff --git a/contrib/libarchive-cmake/config.h b/contrib/libarchive-cmake/config.h index ad5d83168dc..86a797b575d 100644 --- a/contrib/libarchive-cmake/config.h +++ b/contrib/libarchive-cmake/config.h @@ -334,16 +334,16 @@ typedef uint64_t uintmax_t; /* #undef ARCHIVE_XATTR_LINUX */ /* Version number of bsdcpio */ -#define BSDCPIO_VERSION_STRING "3.7.2" +#define BSDCPIO_VERSION_STRING "3.7.3" /* Version number of bsdtar */ -#define BSDTAR_VERSION_STRING "3.7.2" +#define BSDTAR_VERSION_STRING "3.7.3" /* Version number of bsdcat */ -#define BSDCAT_VERSION_STRING "3.7.2" +#define BSDCAT_VERSION_STRING "3.7.3" /* Version number of bsdunzip */ -#define BSDUNZIP_VERSION_STRING "3.7.2" +#define BSDUNZIP_VERSION_STRING "3.7.3" /* Define to 1 if you have the `acl_create_entry' function. */ /* #undef HAVE_ACL_CREATE_ENTRY */ @@ -753,6 +753,12 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `pcreposix' library (-lpcreposix). */ /* #undef HAVE_LIBPCREPOSIX */ +/* Define to 1 if you have the `pcre2-8' library (-lpcre2-8). */ +/* #undef HAVE_LIBPCRE2 */ + +/* Define to 1 if you have the `pcreposix' library (-lpcre2posix). */ +/* #undef HAVE_LIBPCRE2POSIX */ + /* Define to 1 if you have the `xml2' library (-lxml2). */ #define HAVE_LIBXML2 1 @@ -768,9 +774,8 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `zstd' library (-lzstd). */ /* #undef HAVE_LIBZSTD */ -/* Define to 1 if you have the `zstd' library (-lzstd) with compression - support. */ -/* #undef HAVE_LIBZSTD_COMPRESSOR */ +/* Define to 1 if you have the ZSTD_compressStream function. */ +/* #undef HAVE_ZSTD_compressStream */ /* Define to 1 if you have the header file. */ #define HAVE_LIMITS_H 1 @@ -926,6 +931,9 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the header file. */ /* #undef HAVE_PCREPOSIX_H */ +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PCRE2POSIX_H */ + /* Define to 1 if you have the `pipe' function. */ #define HAVE_PIPE 1 @@ -1285,10 +1293,10 @@ typedef uint64_t uintmax_t; #define ICONV_CONST /* Version number of libarchive as a single integer */ -#define LIBARCHIVE_VERSION_NUMBER "3007002" +#define LIBARCHIVE_VERSION_NUMBER "3007003" /* Version number of libarchive */ -#define LIBARCHIVE_VERSION_STRING "3.7.2" +#define LIBARCHIVE_VERSION_STRING "3.7.3" /* Define to 1 if `lstat' dereferences a symlink specified with a trailing slash. */ @@ -1342,7 +1350,7 @@ typedef uint64_t uintmax_t; #endif /* SAFE_TO_DEFINE_EXTENSIONS */ /* Version number of package */ -#define VERSION "3.7.2" +#define VERSION "3.7.3" /* Number of bits in a file offset, on hosts where this is settable. */ /* #undef _FILE_OFFSET_BITS */ From 6464d47d34134d9134b37e404e48a1182695d47b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 9 Sep 2024 20:29:41 +0000 Subject: [PATCH 265/289] Bump to v3.7.4 --- contrib/libarchive | 2 +- contrib/libarchive-cmake/config.h | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/contrib/libarchive b/contrib/libarchive index 4fcc02d906c..313aa1fa10b 160000 --- a/contrib/libarchive +++ b/contrib/libarchive @@ -1 +1 @@ -Subproject commit 4fcc02d906cca4b9e21a78a833f1142a2689ec52 +Subproject commit 313aa1fa10b657de791e3202c168a6c833bc3543 diff --git a/contrib/libarchive-cmake/config.h b/contrib/libarchive-cmake/config.h index 86a797b575d..9696cfb112d 100644 --- a/contrib/libarchive-cmake/config.h +++ b/contrib/libarchive-cmake/config.h @@ -334,16 +334,16 @@ typedef uint64_t uintmax_t; /* #undef ARCHIVE_XATTR_LINUX */ /* Version number of bsdcpio */ -#define BSDCPIO_VERSION_STRING "3.7.3" +#define BSDCPIO_VERSION_STRING "3.7.4" /* Version number of bsdtar */ -#define BSDTAR_VERSION_STRING "3.7.3" +#define BSDTAR_VERSION_STRING "3.7.4" /* Version number of bsdcat */ -#define BSDCAT_VERSION_STRING "3.7.3" +#define BSDCAT_VERSION_STRING "3.7.4" /* Version number of bsdunzip */ -#define BSDUNZIP_VERSION_STRING "3.7.3" +#define BSDUNZIP_VERSION_STRING "3.7.4" /* Define to 1 if you have the `acl_create_entry' function. */ /* #undef HAVE_ACL_CREATE_ENTRY */ @@ -1094,6 +1094,9 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `symlink' function. */ #define HAVE_SYMLINK 1 +/* Define to 1 if you have the `sysconf' function. */ +#define HAVE_SYSCONF 1 + /* Define to 1 if you have the header file. */ /* #undef HAVE_SYS_ACL_H */ @@ -1293,10 +1296,10 @@ typedef uint64_t uintmax_t; #define ICONV_CONST /* Version number of libarchive as a single integer */ -#define LIBARCHIVE_VERSION_NUMBER "3007003" +#define LIBARCHIVE_VERSION_NUMBER "3007004" /* Version number of libarchive */ -#define LIBARCHIVE_VERSION_STRING "3.7.3" +#define LIBARCHIVE_VERSION_STRING "3.7.4" /* Define to 1 if `lstat' dereferences a symlink specified with a trailing slash. */ @@ -1350,7 +1353,7 @@ typedef uint64_t uintmax_t; #endif /* SAFE_TO_DEFINE_EXTENSIONS */ /* Version number of package */ -#define VERSION "3.7.3" +#define VERSION "3.7.4" /* Number of bits in a file offset, on hosts where this is settable. */ /* #undef _FILE_OFFSET_BITS */ From 49ea016a3d73467afa85b1558b245eaac180c696 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 10 Sep 2024 10:19:23 +0100 Subject: [PATCH 266/289] impl --- tests/queries/0_stateless/01603_read_with_backoff_bug.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01603_read_with_backoff_bug.sql b/tests/queries/0_stateless/01603_read_with_backoff_bug.sql index 8a6fa9b7845..212a18b5799 100644 --- a/tests/queries/0_stateless/01603_read_with_backoff_bug.sql +++ b/tests/queries/0_stateless/01603_read_with_backoff_bug.sql @@ -11,7 +11,7 @@ create table t (x UInt64, s String) engine = MergeTree order by x SETTINGS index INSERT INTO t SELECT number, if(number < (8129 * 1024), arrayStringConcat(arrayMap(x -> toString(x), range(number % 128)), ' '), '') -FROM numbers_mt((8129 * 1024) * 3) settings max_insert_threads=8, max_rows_to_read=0; +FROM numbers_mt((8129 * 1024) * 3) settings max_insert_threads=8, max_rows_to_read=0, max_memory_usage='10Gi'; -- optimize table t final; From 8b1f21e7dd491bbf56bd3ebdf1f00bee1a78d7fe Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 10 Sep 2024 12:18:53 +0200 Subject: [PATCH 267/289] Hide exported logs statements in GH logs --- docker/test/base/setup_export_logs.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh index e544397dd0c..b007c3a3aa0 100755 --- a/docker/test/base/setup_export_logs.sh +++ b/docker/test/base/setup_export_logs.sh @@ -184,7 +184,12 @@ function setup_logs_replication /^TTL /d ') - echo -e "Creating remote destination table ${table}_${hash} with statement:\n${statement}" >&2 + echo -e "Creating remote destination table ${table}_${hash} with statement:" >&2 + echo "::group::${table}" + cat >&2 < Date: Tue, 10 Sep 2024 12:56:19 +0200 Subject: [PATCH 268/289] Add a hostname to log; we need something to grep remote query_log --- docker/test/base/setup_export_logs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh index b007c3a3aa0..6924f9c79dd 100755 --- a/docker/test/base/setup_export_logs.sh +++ b/docker/test/base/setup_export_logs.sh @@ -124,6 +124,8 @@ function setup_logs_replication check_logs_credentials || return 0 __set_connection_args + echo "My hostname is ${HOSTNAME}" + echo 'Create all configured system logs' clickhouse-client --query "SYSTEM FLUSH LOGS" From 89dd3188bb937db9dfc8b755e8a0a96f3c51adb0 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 10 Sep 2024 13:35:18 +0200 Subject: [PATCH 269/289] add a setting to disallow DETACH PERMANENTLY in Replicated --- src/Core/ServerSettings.h | 1 + src/Databases/DatabaseReplicated.cpp | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 79173503f28..5b2aaf5407a 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -148,6 +148,7 @@ namespace DB M(Bool, storage_metadata_write_full_object_key, false, "Write disk metadata files with VERSION_FULL_OBJECT_KEY format", 0) \ M(UInt64, max_materialized_views_count_for_table, 0, "A limit on the number of materialized views attached to a table.", 0) \ M(UInt32, max_database_replicated_create_table_thread_pool_size, 1, "The number of threads to create tables during replica recovery in DatabaseReplicated. Zero means number of threads equal number of cores.", 0) \ + M(Bool, database_replicated_allow_detach_permanently, true, "Allow detaching tables permanently in Replicated databases", 0) \ M(Bool, format_alter_operations_with_parentheses, false, "If enabled, each operation in alter queries will be surrounded with parentheses in formatted queries to make them less ambiguous.", 0) \ M(String, default_replica_path, "/clickhouse/tables/{uuid}/{shard}", "The path to the table in ZooKeeper", 0) \ M(String, default_replica_name, "{replica}", "The replica name in ZooKeeper", 0) \ diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 8e3378bcc12..a9009e57dde 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -63,6 +63,7 @@ namespace ErrorCodes extern const int NO_ACTIVE_REPLICAS; extern const int CANNOT_GET_REPLICATED_DATABASE_SNAPSHOT; extern const int CANNOT_RESTORE_TABLE; + extern const int SUPPORT_IS_DISABLED; } static constexpr const char * REPLICATED_DATABASE_MARK = "DatabaseReplicated"; @@ -1693,6 +1694,9 @@ void DatabaseReplicated::detachTablePermanently(ContextPtr local_context, const { waitDatabaseStarted(); + if (!local_context->getServerSettings().database_replicated_allow_detach_permanently) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Support for DETACH TABLE PERMANENTLY is disabled"); + auto txn = local_context->getZooKeeperMetadataTransaction(); assert(!ddl_worker->isCurrentlyActive() || txn); if (txn && txn->isInitialQuery()) From 41aaf075370a01dcfeda649a5c072e7746a48cb1 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 10 Sep 2024 14:17:57 +0200 Subject: [PATCH 270/289] Test materialize() on sparse columns --- .../01780_column_sparse_materialize.reference | 50 ++++++++++++++++++ .../01780_column_sparse_materialize.sql | 52 +++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 tests/queries/0_stateless/01780_column_sparse_materialize.reference create mode 100644 tests/queries/0_stateless/01780_column_sparse_materialize.sql diff --git a/tests/queries/0_stateless/01780_column_sparse_materialize.reference b/tests/queries/0_stateless/01780_column_sparse_materialize.reference new file mode 100644 index 00000000000..660cfabff33 --- /dev/null +++ b/tests/queries/0_stateless/01780_column_sparse_materialize.reference @@ -0,0 +1,50 @@ +-- { echoOn } + +SELECT dumpColumnStructure(id) FROM sparse_t; +UInt64, Sparse(size = 2, UInt64(size = 2), UInt64(size = 1)) +UInt64, Sparse(size = 2, UInt64(size = 2), UInt64(size = 1)) +SELECT dumpColumnStructure(materialize(id)) FROM sparse_t; +UInt64, UInt64(size = 2) +UInt64, UInt64(size = 2) +SELECT dumpColumnStructure(u) FROM sparse_t; +UInt64, Sparse(size = 2, UInt64(size = 1), UInt64(size = 0)) +UInt64, Sparse(size = 2, UInt64(size = 1), UInt64(size = 0)) +SELECT dumpColumnStructure(materialize(u)) FROM sparse_t; +UInt64, UInt64(size = 2) +UInt64, UInt64(size = 2) +SELECT dumpColumnStructure(s) FROM sparse_t; +String, Sparse(size = 2, String(size = 2), UInt64(size = 1)) +String, Sparse(size = 2, String(size = 2), UInt64(size = 1)) +SELECT dumpColumnStructure(materialize(s)) FROM sparse_t; +String, String(size = 2) +String, String(size = 2) +SELECT dumpColumnStructure(arr1) FROM sparse_t; +Array(String), Array(size = 2, UInt64(size = 2), String(size = 1)) +Array(String), Array(size = 2, UInt64(size = 2), String(size = 1)) +SELECT dumpColumnStructure(materialize(arr1)) FROM sparse_t; +Array(String), Array(size = 2, UInt64(size = 2), String(size = 1)) +Array(String), Array(size = 2, UInt64(size = 2), String(size = 1)) +SELECT dumpColumnStructure(arr2) FROM sparse_t; +Array(UInt64), Array(size = 2, UInt64(size = 2), UInt64(size = 1)) +Array(UInt64), Array(size = 2, UInt64(size = 2), UInt64(size = 1)) +SELECT dumpColumnStructure(materialize(arr2)) FROM sparse_t; +Array(UInt64), Array(size = 2, UInt64(size = 2), UInt64(size = 1)) +Array(UInt64), Array(size = 2, UInt64(size = 2), UInt64(size = 1)) +SELECT dumpColumnStructure(t) FROM sparse_t; +Tuple(a UInt64, s String), Tuple(size = 2, Sparse(size = 2, UInt64(size = 1), UInt64(size = 0)), Sparse(size = 2, String(size = 1), UInt64(size = 0))) +Tuple(a UInt64, s String), Tuple(size = 2, Sparse(size = 2, UInt64(size = 1), UInt64(size = 0)), Sparse(size = 2, String(size = 1), UInt64(size = 0))) +SELECT dumpColumnStructure(materialize(t)) FROM sparse_t; +Tuple(a UInt64, s String), Tuple(size = 2, UInt64(size = 2), String(size = 2)) +Tuple(a UInt64, s String), Tuple(size = 2, UInt64(size = 2), String(size = 2)) +SELECT dumpColumnStructure(t.a) FROM sparse_t; +UInt64, Sparse(size = 2, UInt64(size = 1), UInt64(size = 0)) +UInt64, Sparse(size = 2, UInt64(size = 1), UInt64(size = 0)) +SELECT dumpColumnStructure(materialize(t.a)) FROM sparse_t; +UInt64, UInt64(size = 2) +UInt64, UInt64(size = 2) +SELECT dumpColumnStructure(t.s) FROM sparse_t; +String, Sparse(size = 2, String(size = 1), UInt64(size = 0)) +String, Sparse(size = 2, String(size = 1), UInt64(size = 0)) +SELECT dumpColumnStructure(materialize(t.s)) FROM sparse_t; +String, String(size = 2) +String, String(size = 2) diff --git a/tests/queries/0_stateless/01780_column_sparse_materialize.sql b/tests/queries/0_stateless/01780_column_sparse_materialize.sql new file mode 100644 index 00000000000..a53ea140f0f --- /dev/null +++ b/tests/queries/0_stateless/01780_column_sparse_materialize.sql @@ -0,0 +1,52 @@ +DROP TABLE IF EXISTS sparse_t; + +CREATE TABLE sparse_t ( + id UInt64, + u UInt64, + s String, + arr1 Array(String), + arr2 Array(UInt64), + t Tuple(a UInt64, s String)) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.1; + +INSERT INTO sparse_t SELECT + number, + if (number % 2 = 0, number, 0), + if (number % 2 = 0, toString(number), ''), + if (number % 2 = 0, [''], []), + if (number % 2 = 0, [0], []), + (if (number % 2 = 0, number, 0), '') +FROM numbers(2); + +-- { echoOn } + +SELECT dumpColumnStructure(id) FROM sparse_t; +SELECT dumpColumnStructure(materialize(id)) FROM sparse_t; + +SELECT dumpColumnStructure(u) FROM sparse_t; +SELECT dumpColumnStructure(materialize(u)) FROM sparse_t; + +SELECT dumpColumnStructure(s) FROM sparse_t; +SELECT dumpColumnStructure(materialize(s)) FROM sparse_t; + +SELECT dumpColumnStructure(arr1) FROM sparse_t; +SELECT dumpColumnStructure(materialize(arr1)) FROM sparse_t; + +SELECT dumpColumnStructure(arr2) FROM sparse_t; +SELECT dumpColumnStructure(materialize(arr2)) FROM sparse_t; + +SELECT dumpColumnStructure(t) FROM sparse_t; +SELECT dumpColumnStructure(materialize(t)) FROM sparse_t; + +SELECT dumpColumnStructure(t.a) FROM sparse_t; +SELECT dumpColumnStructure(materialize(t.a)) FROM sparse_t; + +SELECT dumpColumnStructure(t.s) FROM sparse_t; +SELECT dumpColumnStructure(materialize(t.s)) FROM sparse_t; + +-- { echoOff } + + +DROP TABLE IF EXISTS sparse_t +; From a01229bba2ce4ee7a3a02301205e2e4bcc47a2f8 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 10 Sep 2024 14:18:37 +0200 Subject: [PATCH 271/289] Convert sparse columns to full in materialize() --- src/Functions/materialize.h | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Functions/materialize.h b/src/Functions/materialize.h index 41994509745..571391faba7 100644 --- a/src/Functions/materialize.h +++ b/src/Functions/materialize.h @@ -2,7 +2,7 @@ #include #include #include -#include +#include namespace DB { @@ -18,11 +18,6 @@ public: return std::make_shared(); } - bool useDefaultImplementationForNulls() const override - { - return false; - } - /// Get the function name. String getName() const override { @@ -34,8 +29,16 @@ public: return true; } + bool useDefaultImplementationForNulls() const override { return false; } + + bool useDefaultImplementationForNothing() const override { return false; } + + bool useDefaultImplementationForConstants() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } + bool useDefaultImplementationForSparseColumns() const override { return false; } + bool isSuitableForConstantFolding() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } @@ -52,7 +55,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { - return arguments[0].column->convertToFullColumnIfConst(); + return recursiveRemoveSparse(arguments[0].column->convertToFullColumnIfConst()); } bool hasInformationAboutMonotonicity() const override { return true; } From 30cf6ee711d7cf02890e347328e3e3559625df28 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 10 Sep 2024 14:30:07 +0200 Subject: [PATCH 272/289] Update 03232_pr_not_ready_set.sql --- tests/queries/0_stateless/03232_pr_not_ready_set.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/03232_pr_not_ready_set.sql b/tests/queries/0_stateless/03232_pr_not_ready_set.sql index 1a724085903..3b2d5d28cfb 100644 --- a/tests/queries/0_stateless/03232_pr_not_ready_set.sql +++ b/tests/queries/0_stateless/03232_pr_not_ready_set.sql @@ -1,3 +1,4 @@ +SYSTEM FLUSH LOGS; SELECT is_initial_query, count() AS c, From 6719112b02e67ecc24346ac81fdb504f97e047a9 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 10 Sep 2024 15:08:58 +0200 Subject: [PATCH 273/289] Add azure-queue.md --- .../table-engines/integrations/azure-queue.md | 72 +++++++++++++++++++ .../engines/table-engines/integrations/s3.md | 2 +- .../table-engines/integrations/s3queue.md | 31 ++++---- 3 files changed, 88 insertions(+), 17 deletions(-) create mode 100644 docs/en/engines/table-engines/integrations/azure-queue.md diff --git a/docs/en/engines/table-engines/integrations/azure-queue.md b/docs/en/engines/table-engines/integrations/azure-queue.md new file mode 100644 index 00000000000..7c3d510ddcc --- /dev/null +++ b/docs/en/engines/table-engines/integrations/azure-queue.md @@ -0,0 +1,72 @@ +--- +slug: /en/engines/table-engines/integrations/azure-queue +sidebar_position: 181 +sidebar_label: AzureQueue +--- + +# AzureQueue Table Engine + +This engine provides an integration with [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs) ecosystem, allowing streaming data import. + +## Create Table {#creating-a-table} + +``` sql +CREATE TABLE test (name String, value UInt32) + ENGINE = AzureQueue(...) + [SETTINGS] + [mode = '',] + [after_processing = 'keep',] + [keeper_path = '',] + ... +``` + +**Engine parameters** + +`AzureQueue` parameters are the same as `AzureBlobStorage` table engine supports. See parameters section [here](../../../engines/table-engines/integrations/azureBlobStorage.md). + +**Example** + +```sql +CREATE TABLE s3queue_engine_table (name String, value UInt32) +ENGINE=AzureQueue('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/data/') +SETTINGS + mode = 'unordered' +``` + +## Settings {#settings} + +The set of supported settings is the same as for `S3Queue` table engine, but without `s3queue_` prefix. See [full list of settings settings](../../../engines/table-engines/integrations/s3queue.md#settings). + +## Description {#description} + +`SELECT` is not particularly useful for streaming import (except for debugging), because each file can be imported only once. It is more practical to create real-time threads using [materialized views](../../../sql-reference/statements/create/view.md). To do this: + +1. Use the engine to create a table for consuming from specified path in S3 and consider it a data stream. +2. Create a table with the desired structure. +3. Create a materialized view that converts data from the engine and puts it into a previously created table. + +When the `MATERIALIZED VIEW` joins the engine, it starts collecting data in the background. + +Example: + +``` sql + CREATE TABLE s3queue_engine_table (name String, value UInt32) + ENGINE=AzureQueue('', 'CSV', 'gzip') + SETTINGS + mode = 'unordered'; + + CREATE TABLE stats (name String, value UInt32) + ENGINE = MergeTree() ORDER BY name; + + CREATE MATERIALIZED VIEW consumer TO stats + AS SELECT name, value FROM s3queue_engine_table; + + SELECT * FROM stats ORDER BY name; +``` + +## Virtual columns {#virtual-columns} + +- `_path` — Path to the file. +- `_file` — Name of the file. + +For more information about virtual columns see [here](../../../engines/table-engines/index.md#table_engines-virtual_columns). diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index 48a08dfa499..f02d0563491 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -35,7 +35,7 @@ CREATE TABLE s3_engine_table (name String, value UInt32) [SETTINGS ...] ``` -### Engine parameters +### Engine parameters {#parameters} - `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `**`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [below](#wildcards-in-path). - `NOSIGN` - If this keyword is provided in place of credentials, all the requests will not be signed. diff --git a/docs/en/engines/table-engines/integrations/s3queue.md b/docs/en/engines/table-engines/integrations/s3queue.md index 06325fa15fb..f1957cfe1ce 100644 --- a/docs/en/engines/table-engines/integrations/s3queue.md +++ b/docs/en/engines/table-engines/integrations/s3queue.md @@ -5,6 +5,7 @@ sidebar_label: S3Queue --- # S3Queue Table Engine + This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ecosystem and allows streaming import. This engine is similar to the [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) engines, but provides S3-specific features. ## Create Table {#creating-a-table} @@ -16,27 +17,25 @@ CREATE TABLE s3_queue_engine_table (name String, value UInt32) [mode = '',] [after_processing = 'keep',] [keeper_path = '',] - [s3queue_loading_retries = 0,] - [s3queue_processing_threads_num = 1,] - [s3queue_enable_logging_to_s3queue_log = 0,] - [s3queue_polling_min_timeout_ms = 1000,] - [s3queue_polling_max_timeout_ms = 10000,] - [s3queue_polling_backoff_ms = 0,] - [s3queue_tracked_file_ttl_sec = 0,] - [s3queue_tracked_files_limit = 1000,] - [s3queue_cleanup_interval_min_ms = 10000,] - [s3queue_cleanup_interval_max_ms = 30000,] + [loading_retries = 0,] + [processing_threads_num = 1,] + [enable_logging_to_s3queue_log = 0,] + [polling_min_timeout_ms = 1000,] + [polling_max_timeout_ms = 10000,] + [polling_backoff_ms = 0,] + [tracked_file_ttl_sec = 0,] + [tracked_files_limit = 1000,] + [cleanup_interval_min_ms = 10000,] + [cleanup_interval_max_ms = 30000,] ``` -Starting with `24.7` settings without `s3queue_` prefix are also supported. +:::warning +Before `24.7`, it is required to use `s3queue_` prefix for all settings apart from `mode`, `after_processing` and `keeper_path`. +::: **Engine parameters** -- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `**`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [below](#wildcards-in-path). -- `NOSIGN` - If this keyword is provided in place of credentials, all the requests will not be signed. -- `format` — The [format](../../../interfaces/formats.md#formats) of the file. -- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3). -- `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will autodetect compression by file extension. +`S3Queue` parameters are the same as `S3` table engine supports. See parameters section [here](../../../engines/table-engines/integrations/s3.md#parameters). **Example** From acbeaa6d3339b7206c6f9e822445952fe664a807 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 10 Sep 2024 16:14:53 +0100 Subject: [PATCH 274/289] fix --- tests/integration/test_disks_app_func/test.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_disks_app_func/test.py b/tests/integration/test_disks_app_func/test.py index 56ea5c8846a..a4b2399e117 100644 --- a/tests/integration/test_disks_app_func/test.py +++ b/tests/integration/test_disks_app_func/test.py @@ -13,8 +13,20 @@ def started_cluster(): main_configs=["config.xml"], with_minio=True, ) - cluster.start() + + # local disk requires its `path` directory to exist. + # the two paths below belong to `test1` and `test2` disks + node = cluster.instances["disks_app_test"] + for path in ["path1", "path2"]: + node.exec_in_container( + [ + "bash", + "-c", + f"mkdir -p /var/lib/clickhouse/{path}", + ] + ) + yield cluster finally: From 04dcf73e8fb98fbcdee2ead46e7177c961e1b5f6 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 10 Sep 2024 15:35:59 +0000 Subject: [PATCH 275/289] Fix crash in sqidDecode --- src/Functions/sqid.cpp | 2 +- tests/queries/0_stateless/02933_sqid.reference | 1 + tests/queries/0_stateless/02933_sqid.sql | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Functions/sqid.cpp b/src/Functions/sqid.cpp index 0e133590b84..32434eb5e73 100644 --- a/src/Functions/sqid.cpp +++ b/src/Functions/sqid.cpp @@ -124,7 +124,7 @@ public: std::string_view sqid = col_non_const->getDataAt(i).toView(); std::vector integers = sqids.decode(String(sqid)); res_nested_data.insert(integers.begin(), integers.end()); - res_offsets_data.push_back(integers.size()); + res_offsets_data.push_back(i == 0 ? integers.size() : res_offsets_data.back() + integers.size()); } } else diff --git a/tests/queries/0_stateless/02933_sqid.reference b/tests/queries/0_stateless/02933_sqid.reference index a559bacb0ac..4597e2347e3 100644 --- a/tests/queries/0_stateless/02933_sqid.reference +++ b/tests/queries/0_stateless/02933_sqid.reference @@ -13,5 +13,6 @@ Td1EnWQo [1,2,3,4] XMbT -- invalid sqid [] +-- bug 69450 -- alias XMbT diff --git a/tests/queries/0_stateless/02933_sqid.sql b/tests/queries/0_stateless/02933_sqid.sql index 81d4b2bc35c..822fe33df51 100644 --- a/tests/queries/0_stateless/02933_sqid.sql +++ b/tests/queries/0_stateless/02933_sqid.sql @@ -25,5 +25,12 @@ SELECT sqidEncode(toNullable(materialize(1)), toLowCardinality(materialize(2))); SELECT '-- invalid sqid'; SELECT sqidDecode('invalid sqid'); +SELECT '-- bug 69450'; +DROP TABLE IF EXISTS tab; +CREATE TABLE tab (id String) ENGINE = MergeTree ORDER BY id; +INSERT INTO tab SELECT * FROM generateRandom() LIMIT 1000000; +SELECT sqidDecode(id) FROM tab FORMAT Null; +DROP TABLE tab; + SELECT '-- alias'; SELECT sqid(1, 2); From d43264c44e80f1f51604d266e5b1085462552aff Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 10 Sep 2024 17:41:52 +0200 Subject: [PATCH 276/289] Quick fix for s3queue problem --- .../StorageObjectStorageQueue.cpp | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 9452ce81e9e..55a1d43b26b 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -64,9 +64,7 @@ namespace void checkAndAdjustSettings( ObjectStorageQueueSettings & queue_settings, - ASTStorage * engine_args, - bool is_attach, - const LoggerPtr & log) + bool is_attach) { if (!is_attach && !queue_settings.mode.changed) { @@ -85,16 +83,6 @@ namespace "Setting `cleanup_interval_min_ms` ({}) must be less or equal to `cleanup_interval_max_ms` ({})", queue_settings.cleanup_interval_min_ms, queue_settings.cleanup_interval_max_ms); } - - if (!is_attach && !queue_settings.processing_threads_num.changed) - { - queue_settings.processing_threads_num = std::max(getNumberOfPhysicalCPUCores(), 16); - engine_args->settings->as()->changes.insertSetting( - "processing_threads_num", - queue_settings.processing_threads_num.value); - - LOG_TRACE(log, "Set `processing_threads_num` to {}", queue_settings.processing_threads_num); - } } std::shared_ptr getQueueLog(const ObjectStoragePtr & storage, const ContextPtr & context, const ObjectStorageQueueSettings & table_settings) @@ -154,7 +142,7 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( throw Exception(ErrorCodes::BAD_QUERY_PARAMETER, "ObjectStorageQueue url must either end with '/' or contain globs"); } - checkAndAdjustSettings(*queue_settings, engine_args, mode > LoadingStrictnessLevel::CREATE, log); + checkAndAdjustSettings(*queue_settings, mode > LoadingStrictnessLevel::CREATE); object_storage = configuration->createObjectStorage(context_, /* is_readonly */true); FormatFactory::instance().checkFormatName(configuration->format); From d8e670297bcdc1850260a1d60d4ee39b2a895a4d Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:42:27 +0200 Subject: [PATCH 277/289] groupConcat consistency --- .../AggregateFunctionGroupConcat.cpp | 13 +++++++++++-- .../03235_groupArray_returns_string.reference | 1 + .../0_stateless/03235_groupArray_returns_string.sql | 10 ++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03235_groupArray_returns_string.reference create mode 100644 tests/queries/0_stateless/03235_groupArray_returns_string.sql diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp index 636ac80e350..8fb0b645096 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp @@ -116,15 +116,17 @@ class GroupConcatImpl final SerializationPtr serialization; UInt64 limit; const String delimiter; + const DataTypePtr type; public: GroupConcatImpl(const DataTypePtr & data_type_, const Array & parameters_, UInt64 limit_, const String & delimiter_) : IAggregateFunctionDataHelper, GroupConcatImpl>( {data_type_}, parameters_, std::make_shared()) - , serialization(this->argument_types[0]->getDefaultSerialization()) , limit(limit_) , delimiter(delimiter_) + , type(data_type_) { + serialization = isFixedString(type) ? std::make_shared()->getDefaultSerialization() : this->argument_types[0]->getDefaultSerialization(); } String getName() const override { return name; } @@ -140,7 +142,14 @@ public: if (cur_data.data_size != 0) cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); - cur_data.insert(columns[0], serialization, row_num, arena); + if (isFixedString(type)) + { + ColumnWithTypeAndName col = {columns[0]->getPtr(), type, "column"}; + const auto & col_str = castColumn(col, std::make_shared()); + cur_data.insert(col_str.get(), serialization, row_num, arena); + } + else + cur_data.insert(columns[0], serialization, row_num, arena); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override diff --git a/tests/queries/0_stateless/03235_groupArray_returns_string.reference b/tests/queries/0_stateless/03235_groupArray_returns_string.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/03235_groupArray_returns_string.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/03235_groupArray_returns_string.sql b/tests/queries/0_stateless/03235_groupArray_returns_string.sql new file mode 100644 index 00000000000..618ec6f839b --- /dev/null +++ b/tests/queries/0_stateless/03235_groupArray_returns_string.sql @@ -0,0 +1,10 @@ +CREATE TABLE t (st FixedString(54)) ENGINE=MergeTree ORDER BY (); + +INSERT INTO t VALUES +('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRTUVWXYZ'), +('\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0'), +('IIIIIIIIII\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0'); + +WITH (SELECT groupConcat(',')(st) FROM t) AS a, + (SELECT groupConcat(',')(st :: String) FROM t) AS b +SELECT equals(a, b); From f588e3c31bc2f097f168646f73ed078259265659 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:43:50 +0200 Subject: [PATCH 278/289] rename tests --- ...ng.reference => 03235_groupArray_string_consistency.reference} | 0 ...returns_string.sql => 03235_groupArray_string_consistency.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{03235_groupArray_returns_string.reference => 03235_groupArray_string_consistency.reference} (100%) rename tests/queries/0_stateless/{03235_groupArray_returns_string.sql => 03235_groupArray_string_consistency.sql} (100%) diff --git a/tests/queries/0_stateless/03235_groupArray_returns_string.reference b/tests/queries/0_stateless/03235_groupArray_string_consistency.reference similarity index 100% rename from tests/queries/0_stateless/03235_groupArray_returns_string.reference rename to tests/queries/0_stateless/03235_groupArray_string_consistency.reference diff --git a/tests/queries/0_stateless/03235_groupArray_returns_string.sql b/tests/queries/0_stateless/03235_groupArray_string_consistency.sql similarity index 100% rename from tests/queries/0_stateless/03235_groupArray_returns_string.sql rename to tests/queries/0_stateless/03235_groupArray_string_consistency.sql From 080193cfc36909ab9a1016b70edc007c8dd3540f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 10 Sep 2024 15:46:21 +0000 Subject: [PATCH 279/289] 14% more aesthetic code --- src/Functions/sqid.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/sqid.cpp b/src/Functions/sqid.cpp index 32434eb5e73..074a34bd083 100644 --- a/src/Functions/sqid.cpp +++ b/src/Functions/sqid.cpp @@ -124,7 +124,7 @@ public: std::string_view sqid = col_non_const->getDataAt(i).toView(); std::vector integers = sqids.decode(String(sqid)); res_nested_data.insert(integers.begin(), integers.end()); - res_offsets_data.push_back(i == 0 ? integers.size() : res_offsets_data.back() + integers.size()); + res_offsets_data.push_back(res_offsets_data.back() + integers.size()); } } else From 7727b30f5cd7482a79e25ab3148f29380065aeec Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 10 Sep 2024 18:06:32 +0200 Subject: [PATCH 280/289] Attempt to fix EAGAIN error --- docker/test/base/setup_export_logs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh index 6924f9c79dd..3df9655701c 100755 --- a/docker/test/base/setup_export_logs.sh +++ b/docker/test/base/setup_export_logs.sh @@ -188,9 +188,9 @@ function setup_logs_replication echo -e "Creating remote destination table ${table}_${hash} with statement:" >&2 echo "::group::${table}" - cat >&2 < Date: Tue, 10 Sep 2024 18:12:56 +0200 Subject: [PATCH 281/289] Update docs/en/engines/table-engines/integrations/azure-queue.md Co-authored-by: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> --- docs/en/engines/table-engines/integrations/azure-queue.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/azure-queue.md b/docs/en/engines/table-engines/integrations/azure-queue.md index 7c3d510ddcc..7e19bc92973 100644 --- a/docs/en/engines/table-engines/integrations/azure-queue.md +++ b/docs/en/engines/table-engines/integrations/azure-queue.md @@ -50,7 +50,7 @@ When the `MATERIALIZED VIEW` joins the engine, it starts collecting data in the Example: ``` sql - CREATE TABLE s3queue_engine_table (name String, value UInt32) + CREATE TABLE azure_queue_engine_table (name String, value UInt32) ENGINE=AzureQueue('', 'CSV', 'gzip') SETTINGS mode = 'unordered'; @@ -59,7 +59,7 @@ Example: ENGINE = MergeTree() ORDER BY name; CREATE MATERIALIZED VIEW consumer TO stats - AS SELECT name, value FROM s3queue_engine_table; + AS SELECT name, value FROM azure_queue_engine_table; SELECT * FROM stats ORDER BY name; ``` From b7e863b0e9b1891c5e4adaec61b5d51db6fc0fcd Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:13:04 +0200 Subject: [PATCH 282/289] Update docs/en/engines/table-engines/integrations/azure-queue.md Co-authored-by: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> --- docs/en/engines/table-engines/integrations/azure-queue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/integrations/azure-queue.md b/docs/en/engines/table-engines/integrations/azure-queue.md index 7e19bc92973..b5259336a8b 100644 --- a/docs/en/engines/table-engines/integrations/azure-queue.md +++ b/docs/en/engines/table-engines/integrations/azure-queue.md @@ -27,7 +27,7 @@ CREATE TABLE test (name String, value UInt32) **Example** ```sql -CREATE TABLE s3queue_engine_table (name String, value UInt32) +CREATE TABLE azure_queue_engine_table (name String, value UInt32) ENGINE=AzureQueue('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/data/') SETTINGS mode = 'unordered' From 6a6d26aeeda7c6d480dbaf73658567ed494ca67b Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:16:27 +0200 Subject: [PATCH 283/289] Update StorageObjectStorageQueue.cpp --- src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 55a1d43b26b..c1ef37e1a48 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -118,7 +118,7 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( const String & comment, ContextPtr context_, std::optional format_settings_, - ASTStorage * engine_args, + ASTStorage * /* engine_args */, LoadingStrictnessLevel mode) : IStorage(table_id_) , WithContext(context_) From 2f15fcd23fb069a69d2f1a0caee36ae33f39fe45 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 10 Sep 2024 20:57:03 +0200 Subject: [PATCH 284/289] Test with sparse serialization, vertical merge and skip indices --- .../03175_sparse_and_skip_index.reference | 4 ++ .../03175_sparse_and_skip_index.sql | 45 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 tests/queries/0_stateless/03175_sparse_and_skip_index.reference create mode 100644 tests/queries/0_stateless/03175_sparse_and_skip_index.sql diff --git a/tests/queries/0_stateless/03175_sparse_and_skip_index.reference b/tests/queries/0_stateless/03175_sparse_and_skip_index.reference new file mode 100644 index 00000000000..619e98a152a --- /dev/null +++ b/tests/queries/0_stateless/03175_sparse_and_skip_index.reference @@ -0,0 +1,4 @@ +key Sparse +value Sparse +1000 +1 diff --git a/tests/queries/0_stateless/03175_sparse_and_skip_index.sql b/tests/queries/0_stateless/03175_sparse_and_skip_index.sql new file mode 100644 index 00000000000..4de6d1ac6df --- /dev/null +++ b/tests/queries/0_stateless/03175_sparse_and_skip_index.sql @@ -0,0 +1,45 @@ +DROP TABLE IF EXISTS t_bloom_filter; +CREATE TABLE t_bloom_filter( + key UInt64, + value UInt64, + + INDEX key_bf key TYPE bloom_filter(0.01) GRANULARITY 2147483648, -- bloom filter on sorting key column + INDEX value_bf value TYPE bloom_filter(0.01) GRANULARITY 2147483648 -- bloom filter on no-sorting column +) ENGINE=MergeTree ORDER BY key +SETTINGS + -- settings to trigger sparse serialization and vertical merge + ratio_of_defaults_for_sparse_serialization = 0.0 + ,vertical_merge_algorithm_min_rows_to_activate = 1 + ,vertical_merge_algorithm_min_columns_to_activate = 1 + ,allow_vertical_merges_from_compact_to_wide_parts = 1 + ,min_bytes_for_wide_part=0 +; + +SYSTEM STOP MERGES t_bloom_filter; + +-- Create at least one part +INSERT INTO t_bloom_filter +SELECT + number % 100 as key, -- 100 unique keys + rand() % 100 as value -- 100 unique values +FROM numbers(50_000); + +-- And another part +INSERT INTO t_bloom_filter +SELECT + number % 100 as key, -- 100 unique keys + rand() % 100 as value -- 100 unique values +FROM numbers(50_000, 50_000); + +SYSTEM START MERGES t_bloom_filter; + +-- Merge everything into a single part +OPTIMIZE TABLE t_bloom_filter FINAL; + +-- Check sparse serialization +SELECT column, serialization_kind FROM system.parts_columns WHERE database = currentDatabase() AND table = 't_bloom_filter' AND active ORDER BY column; + +SELECT COUNT() FROM t_bloom_filter WHERE key = 1; + +-- Check bloom filter non-zero size +SELECT COUNT() FROM system.parts WHERE database = currentDatabase() AND table = 't_bloom_filter' AND secondary_indices_uncompressed_bytes > 200 AND active; From d4aa06524b4e65e2bca4ff851ad54dd4468e5cc0 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 10 Sep 2024 20:57:55 +0200 Subject: [PATCH 285/289] Add materialization when building indices in vertical merge --- src/Storages/MergeTree/MergeTask.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 9a1e749734c..5c993504245 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -952,6 +952,7 @@ MergeTask::VerticalMergeRuntimeContext::PreparedColumnPipeline MergeTask::Vertic indexes_to_recalc = MergeTreeIndexFactory::instance().getMany(indexes_it->second); auto indices_expression_dag = indexes_it->second.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), global_ctx->data->getContext())->getActionsDAG().clone(); + indices_expression_dag.addMaterializingOutputActions(); /// Const columns cannot be written without materialization. auto calculate_indices_expression_step = std::make_unique( merge_column_query_plan.getCurrentDataStream(), std::move(indices_expression_dag)); From 3921f910f5d0d7e69581a2cc7033213809af48aa Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 11 Sep 2024 10:33:23 +0200 Subject: [PATCH 286/289] Another attempt to address EAGAIN "Resource unavailable" --- docker/test/base/setup_export_logs.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh index 3df9655701c..7ec1c31f04a 100755 --- a/docker/test/base/setup_export_logs.sh +++ b/docker/test/base/setup_export_logs.sh @@ -187,10 +187,15 @@ function setup_logs_replication ') echo -e "Creating remote destination table ${table}_${hash} with statement:" >&2 + echo "::group::${table}" # there's the only way big "$statement" can be printed without causing EAGAIN error # cat: write error: Resource temporarily unavailable - echo "$statement" | cat + statement_print="${statement}" + if [ "${#statement_print}" -gt 4000 ]; then + statement_print="${statement::1999}\n…\n${statement:${#statement}-1999}" + fi + echo "$statement_print" echo "::endgroup::" echo "$statement" | clickhouse-client --database_replicated_initial_query_timeout_sec=10 \ From c1830bc041a067d1fae1e8971091c453614eaca7 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 11 Sep 2024 13:08:58 +0200 Subject: [PATCH 287/289] Escape the `\n` in statements --- docker/test/base/setup_export_logs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh index 7ec1c31f04a..a39f96867be 100755 --- a/docker/test/base/setup_export_logs.sh +++ b/docker/test/base/setup_export_logs.sh @@ -195,7 +195,7 @@ function setup_logs_replication if [ "${#statement_print}" -gt 4000 ]; then statement_print="${statement::1999}\n…\n${statement:${#statement}-1999}" fi - echo "$statement_print" + echo -e "$statement_print" echo "::endgroup::" echo "$statement" | clickhouse-client --database_replicated_initial_query_timeout_sec=10 \ From 6f9d1631e26bbb0263962c6d50c65e9ee2be561a Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:11:29 +0200 Subject: [PATCH 288/289] Update aspell-ignore --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 796df6f1042..3467f21c812 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -24,6 +24,7 @@ Aggregatefunction AggregatingMergeTree AggregatorThreads AggregatorThreadsActive +AzureQueue Akka AlertManager Alexey From 9e335abe1735eeed100c4b964bf24220b6086bf9 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 11 Sep 2024 13:05:24 +0000 Subject: [PATCH 289/289] CMake: Add comment about ICU data files --- contrib/icu-cmake/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt index afaa189701d..5714fef8347 100644 --- a/contrib/icu-cmake/CMakeLists.txt +++ b/contrib/icu-cmake/CMakeLists.txt @@ -481,6 +481,11 @@ if (ARCH_S390X) else() set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt75l_dat.S" ) endif() +# ^^ you might be confused how for different little endian platforms (x86, ARM) the same assembly files can be used. +# These files are indeed assembly but they only contain data ('.long' directive), which makes them portable accross CPUs. +# Only the endianness and the character set (ASCII, EBCDIC) makes a difference, also see +# https://unicode-org.github.io/icu/userguide/icu_data/#sharing-icu-data-between-platforms, 'Sharing ICU Data Between Platforms') +# (and as an experiment, try re-generating the data files on x86 vs. ARM, ... you'll get exactly the same files) set(ICUDATA_SOURCES "${ICUDATA_SOURCE_FILE}"