From 0dbd569b50828003dc6a55af56717dab92054d8f Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 12 Mar 2024 20:00:04 +0100 Subject: [PATCH 01/72] Minor change --- src/Storages/MergeTree/MergeTreePartInfo.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index 5fbb5d70bf3..918acc78e8f 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -101,9 +101,8 @@ struct MergeTreePartInfo bool isFakeDropRangePart() const { - /// Another max level was previously used for REPLACE/MOVE PARTITION - auto another_max_level = std::numeric_limits::max(); - return level == MergeTreePartInfo::MAX_LEVEL || level == another_max_level; + /// LEGACY_MAX_LEVEL was previously used for REPLACE/MOVE PARTITION + return level == MergeTreePartInfo::MAX_LEVEL || level == MergeTreePartInfo::LEGACY_MAX_LEVEL; } String getPartNameAndCheckFormat(MergeTreeDataFormatVersion format_version) const; From d84e272ed8687f8d13f0286ac37b93db3f874715 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 30 Apr 2024 13:15:23 +0000 Subject: [PATCH 02/72] Add changeDate functions --- src/Functions/changeDate.cpp | 342 ++++++++++++++++++ .../0_stateless/02982_changeDate.reference | 36 ++ .../queries/0_stateless/02982_changeDate.sql | 42 +++ 3 files changed, 420 insertions(+) create mode 100644 src/Functions/changeDate.cpp create mode 100644 tests/queries/0_stateless/02982_changeDate.reference create mode 100644 tests/queries/0_stateless/02982_changeDate.sql diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp new file mode 100644 index 00000000000..14f6887af47 --- /dev/null +++ b/src/Functions/changeDate.cpp @@ -0,0 +1,342 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Common/DateLUTImpl.h" +#include +#include +#include "Columns/IColumn.h" +#include "DataTypes/IDataType.h" +#include "base/DayNum.h" + +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; +} + +namespace +{ + +enum ChangeDateFunctionsNames +{ + CHANGE_YEAR = 0, + CHANGE_MONTH = 1, + CHANGE_DAY = 2, + CHANGE_HOUR = 3, + CHANGE_MINUTE = 4, + CHANGE_SECOND = 5 +}; + +template +class FunctionChangeDate : public IFunction +{ +public: + static constexpr auto name = Traits::Name; + + static constexpr std::array mandatory_argument_names = {"date", "new_value"}; + + String getName() const override { return name; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + size_t getNumberOfArguments() const override { return mandatory_argument_names.size(); } + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + FunctionArgumentDescriptors args{ + {mandatory_argument_names[0], &isDateOrDate32OrDateTimeOrDateTime64, nullptr, "Date"}, + {mandatory_argument_names[1], &isNumber, nullptr, "Number"} + }; + validateFunctionArgumentTypes(*this, arguments, args); + + if (Traits::EnumName >= 3) + { + if (isDate(arguments[0].type)) + return std::make_shared(); + if (isDate32(arguments[0].type)) + return std::make_shared(DataTypeDateTime64::default_scale); + } + + return arguments[0].type; + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + const auto & input_type = arguments[0].type; + if (isDate(input_type)) + { + if (Traits::EnumName >= 3) + return execute(arguments, input_type, result_type, input_rows_count); + return execute(arguments, input_type, result_type, input_rows_count); + } + if (isDate32(input_type)) + { + if (Traits::EnumName >= 3) + return executeDate32ToDateTime64(arguments, result_type, input_rows_count); + return execute(arguments, input_type, result_type, input_rows_count); + } + if (isDateTime(input_type)) + return execute(arguments, input_type, result_type, input_rows_count); + return executeDateTime64(arguments, result_type, input_rows_count); + } + + + template + ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & input_type, const DataTypePtr & result_type, size_t input_rows_count) const + { + const auto & date_lut = DateLUT::instance(); + + auto result_column = ResultDataType::ColumnType::create(input_rows_count); + auto & result_data = result_column->getData(); + + auto input_column = castColumn(arguments[0], std::make_shared()); + input_column = input_column->convertToFullColumnIfConst(); + const auto & input_column_data = typeid_cast(*input_column).getData(); + + auto new_value_column = castColumn(arguments[1], std::make_shared()); + new_value_column = new_value_column->convertToFullColumnIfConst(); + const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + + for (size_t i = 0; i < input_rows_count; ++i) + { + Int64 time; + if (isDateOrDate32(input_type)) + time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1000'000; + else + time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); + + if (isDateOrDate32(result_type)) + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + else + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + } + + return result_column; + } + + ColumnPtr executeDate32ToDateTime64(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const + { + const auto & date_lut = DateLUT::instance(); + + auto result_column = ColumnDateTime64::create(input_rows_count, DataTypeDateTime64::default_scale); + auto & result_data = result_column->getData(); + + auto input_column = arguments[0].column->convertToFullColumnIfConst(); + const auto & input_column_data = typeid_cast(*input_column).getData(); + + auto new_value_column = castColumn(arguments[1], std::make_shared()); + new_value_column = new_value_column->convertToFullColumnIfConst(); + const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + + for (size_t i = 0; i < input_rows_count; ++i) + { + Int64 time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1000'000; + + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1000, 0); + } + + return result_column; + } + + ColumnPtr executeDateTime64(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const + { + auto result_column = ColumnDateTime64::create(input_rows_count, DataTypeDateTime64::default_scale); + auto & result_data = result_column->getData(); + + auto input_column = arguments[0].column->convertToFullColumnIfConst(); + const auto & input_column_data = typeid_cast(*input_column).getData(); + + auto new_value_column = castColumn(arguments[1], std::make_shared()); + new_value_column = new_value_column->convertToFullColumnIfConst(); + const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + + const auto scale = typeid_cast(*result_type).getScale(); + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 deg = 1; + for (size_t i = 0; i < scale; ++i) { + deg *= 10; + } + + for (size_t i = 0; i < input_rows_count; ++i) + { + Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); + Int64 fraction = input_column_data[i] % deg; + + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, deg, fraction); + } + + return result_column; + } + + Int64 getChangedDate(Int64 time, Float32 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 deg = 0, Int64 fraction = 0) const + { + auto year = time / 10'000'000'000; + auto month = (time % 10'000'000'000) / 100'000'000; + auto day = (time % 100'000'000) / 1000'000; + auto hours = (time % 1000'000) / 10'000; + auto minutes = (time % 10'000) / 100; + auto seconds = time % 100; + + Int64 min_date, max_date; + Int16 min_year, max_year; + if (isDate(result_type)) + { + min_date = date_lut.makeDayNum(1970, 1, 1); + max_date = date_lut.makeDayNum(2149, 6, 6); + min_year = 1970; + max_year = 2149; + } + else if (isDate32(result_type)) + { + min_date = date_lut.makeDayNum(1900, 1, 1); + max_date = date_lut.makeDayNum(2299, 12, 31); + min_year = 1900; + max_year = 2299; + } + else if (isDateTime(result_type)) + { + min_date = 0; + max_date = 0x0ffffffffll; + min_year = 1970; + max_year = 2106; + } + else + { + min_date = date_lut.makeDateTime(1900, 1, 1, 0,0 , 0) * deg; + max_date = date_lut.makeDateTime(2299, 12, 31, 23, 59, 59) * deg + (deg - 1); + min_year = 1900; + max_year = 2299; + } + + Int8 fl = 0; + + switch (Traits::EnumName) + { + case CHANGE_YEAR: + if (new_value < min_year) + fl = 1; + else if (new_value > max_year) + fl = 2; + year = static_cast(new_value); + break; + case CHANGE_MONTH: + if (new_value < 1 || new_value > 12) + fl = 1; + month = static_cast(new_value); + break; + case CHANGE_DAY: + if (new_value < 1 || new_value > 31) + fl = 1; + day = static_cast(new_value); + break; + case CHANGE_HOUR: + if (new_value < 0 || new_value > 23) + fl = 1; + hours = static_cast(new_value); + break; + case CHANGE_MINUTE: + if (new_value < 0 || new_value > 59) + fl = 1; + minutes = static_cast(new_value); + break; + case CHANGE_SECOND: + if (new_value < 0 || new_value > 59) + fl = 1; + seconds = static_cast(new_value); + break; + } + + if (fl == 1) + return min_date; + + if (fl == 2) + return max_date; + + Int64 result; + if (isDateOrDate32(result_type)) + result = date_lut.makeDayNum(year, month, day); + else + { + if (isDateTime(result_type)) + result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); + else + result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; + } + + + if (result > max_date) + return max_date; + + return result; + } +}; + + +struct ChangeYearTraits +{ + static constexpr auto Name = "changeYear"; + static constexpr auto EnumName = CHANGE_YEAR; +}; + +struct ChangeMonthTraits +{ + static constexpr auto Name = "changeMonth"; + static constexpr auto EnumName = CHANGE_MONTH; +}; + +struct ChangeDayTraits +{ + static constexpr auto Name = "changeDay"; + static constexpr auto EnumName = CHANGE_DAY; +}; + +struct ChangeHourTraits +{ + static constexpr auto Name = "changeHour"; + static constexpr auto EnumName = CHANGE_HOUR; +}; + +struct ChangeMinuteTraits +{ + static constexpr auto Name = "changeMinute"; + static constexpr auto EnumName = CHANGE_MINUTE; +}; + +struct ChangeSecondTraits +{ + static constexpr auto Name = "changeSecond"; + static constexpr auto EnumName = CHANGE_SECOND; +}; + + +} + +REGISTER_FUNCTION(ChangeDate) +{ + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); +} + +} diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference new file mode 100644 index 00000000000..d7d4edf4b43 --- /dev/null +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -0,0 +1,36 @@ +2000-01-01 +2001-01-01 +2002-01-01 11:22:33 +2003-01-01 11:22:33.4444 +1970-02-01 +1970-03-01 +1970-04-01 11:22:33 +1970-05-01 11:22:33.4444 +1970-01-02 +1970-01-03 +1970-01-04 11:22:33 +1970-01-05 11:22:33.4444 +1970-01-01 12:00:00 +1970-01-01 13:00:00.000 +1970-01-01 14:22:33 +1970-01-01 15:22:33.4444 +1970-01-01 00:23:00 +1970-01-01 00:24:00.000 +1970-01-01 11:25:33 +1970-01-01 11:26:33.4444 +1970-01-01 00:00:34 +1970-01-01 00:00:35.000 +1970-01-01 11:22:36 +1970-01-01 11:22:37.4444 +1970-01-01 +2149-06-06 +2149-06-06 +1970-01-01 +1970-01-01 +1970-01-01 +1970-01-01 00:00:00 +1970-01-01 00:00:00 +1970-01-01 00:00:00 +1970-01-01 00:00:00 +1970-01-01 00:00:00 +1970-01-01 00:00:00 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql new file mode 100644 index 00000000000..fbfe3771b33 --- /dev/null +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -0,0 +1,42 @@ +SELECT changeYear(makeDate(1970, 01, 01), 2000); +SELECT changeYear(makeDate32(1970, 01, 01), 2001); +SELECT changeYear(makeDateTime(1970, 01, 01, 11, 22, 33), 2002); +SELECT changeYear(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 2003); + +SELECT changeMonth(makeDate(1970, 01, 01), 02); +SELECT changeMonth(makeDate32(1970, 01, 01), 03); +SELECT changeMonth(makeDateTime(1970, 01, 01, 11, 22, 33), 04); +SELECT changeMonth(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 05); + +SELECT changeDay(makeDate(1970, 01, 01), 02); +SELECT changeDay(makeDate32(1970, 01, 01), 03); +SELECT changeDay(makeDateTime(1970, 01, 01, 11, 22, 33), 04); +SELECT changeDay(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 05); + +SELECT changeHour(makeDate(1970, 01, 01), 12); +SELECT changeHour(makeDate32(1970, 01, 01), 13); +SELECT changeHour(makeDateTime(1970, 01, 01, 11, 22, 33), 14); +SELECT changeHour(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 15); + +SELECT changeMinute(makeDate(1970, 01, 01), 23); +SELECT changeMinute(makeDate32(1970, 01, 01), 24); +SELECT changeMinute(makeDateTime(1970, 01, 01, 11, 22, 33), 25); +SELECT changeMinute(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 26); + +SELECT changeSecond(makeDate(1970, 01, 01), 34); +SELECT changeSecond(makeDate32(1970, 01, 01), 35); +SELECT changeSecond(makeDateTime(1970, 01, 01, 11, 22, 33), 36); +SELECT changeSecond(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 37); + +SELECT changeYear(makeDate(2000, 01, 01), 1969.0); +SELECT changeYear(makeDate(2000, 06, 07), 2149.0); +SELECT changeMonth(makeDate(2149, 01, 01), 07); +SELECT changeMonth(makeDate(2000, 06, 07), 13); +SELECT changeDay(makeDate(2000, 01, 01), 0); +SELECT changeDay(makeDate(2000, 06, 07), 32); +SELECT changeHour(makeDate(2000, 01, 01), -1); +SELECT changeHour(makeDate(2000, 06, 07), 24); +SELECT changeMinute(makeDate(2000, 01, 01), -1); +SELECT changeMinute(makeDate(2000, 06, 07), 60); +SELECT changeSecond(makeDate(2000, 01, 01), -1); +SELECT changeSecond(makeDate(2000, 06, 07), 60); \ No newline at end of file From 8f11262666132a9a940c2a10e67c5be8f0151c3a Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Tue, 30 Apr 2024 13:36:36 +0000 Subject: [PATCH 03/72] ome style changes --- src/Functions/changeDate.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 14f6887af47..4edc31384c3 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -25,10 +25,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int ARGUMENT_OUT_OF_BOUND; -} namespace { @@ -171,7 +167,8 @@ public: const auto scale = typeid_cast(*result_type).getScale(); const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 deg = 1; - for (size_t i = 0; i < scale; ++i) { + for (size_t i = 0; i < scale; ++i) + { deg *= 10; } @@ -266,7 +263,7 @@ public: if (fl == 1) return min_date; - + if (fl == 2) return max_date; @@ -280,7 +277,7 @@ public: else result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; } - + if (result > max_date) return max_date; From 32f267999da8e9edd68c843431d7a1ae26c7250e Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Sat, 4 May 2024 21:28:32 +0000 Subject: [PATCH 04/72] changeDate implementation fixes --- src/DataTypes/DataTypeDate.h | 1 + src/DataTypes/DataTypeDate32.h | 1 + src/DataTypes/DataTypeDateTime.h | 1 + src/Functions/changeDate.cpp | 296 +++++++++++++++---------------- 4 files changed, 143 insertions(+), 156 deletions(-) diff --git a/src/DataTypes/DataTypeDate.h b/src/DataTypes/DataTypeDate.h index 0e08b9ba2ca..72b7ef2509f 100644 --- a/src/DataTypes/DataTypeDate.h +++ b/src/DataTypes/DataTypeDate.h @@ -10,6 +10,7 @@ class DataTypeDate final : public DataTypeNumberBase { public: static constexpr auto family_name = "Date"; + static constexpr auto type_id = TypeIndex::Date; TypeIndex getTypeId() const override { return TypeIndex::Date; } TypeIndex getColumnType() const override { return TypeIndex::UInt16; } diff --git a/src/DataTypes/DataTypeDate32.h b/src/DataTypes/DataTypeDate32.h index 02e818f10df..052bb39fc31 100644 --- a/src/DataTypes/DataTypeDate32.h +++ b/src/DataTypes/DataTypeDate32.h @@ -10,6 +10,7 @@ class DataTypeDate32 final : public DataTypeNumberBase { public: static constexpr auto family_name = "Date32"; + static constexpr auto type_id = TypeIndex::Date32; TypeIndex getTypeId() const override { return TypeIndex::Date32; } TypeIndex getColumnType() const override { return TypeIndex::Int32; } diff --git a/src/DataTypes/DataTypeDateTime.h b/src/DataTypes/DataTypeDateTime.h index 5519240dee1..3b1212d910d 100644 --- a/src/DataTypes/DataTypeDateTime.h +++ b/src/DataTypes/DataTypeDateTime.h @@ -36,6 +36,7 @@ public: explicit DataTypeDateTime(const TimezoneMixin & time_zone); static constexpr auto family_name = "DateTime"; + static constexpr auto type_id = TypeIndex::DateTime; const char * getFamilyName() const override { return family_name; } String doGetName() const override; diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 4edc31384c3..8a5d0a87ca2 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -10,26 +10,23 @@ #include #include #include -#include #include - #include "Common/DateLUTImpl.h" #include #include #include "Columns/IColumn.h" #include "DataTypes/IDataType.h" -#include "base/DayNum.h" - + #include #include - + namespace DB { - + namespace { - -enum ChangeDateFunctionsNames + +enum class ChangeDateFunctionsNames { CHANGE_YEAR = 0, CHANGE_MONTH = 1, @@ -38,160 +35,161 @@ enum ChangeDateFunctionsNames CHANGE_MINUTE = 4, CHANGE_SECOND = 5 }; - + +constexpr bool isTimeChange(const ChangeDateFunctionsNames & type) +{ + return type == ChangeDateFunctionsNames::CHANGE_HOUR || + type == ChangeDateFunctionsNames::CHANGE_MINUTE || + type == ChangeDateFunctionsNames::CHANGE_SECOND; +} + +template +constexpr bool isDate32() +{ + return DataType::type_id == TypeIndex::Date32; +} + +template +constexpr bool isDateTime64() +{ + return DataType::type_id == TypeIndex::DateTime64; +} + template class FunctionChangeDate : public IFunction { public: static constexpr auto name = Traits::Name; - + static constexpr std::array mandatory_argument_names = {"date", "new_value"}; - + String getName() const override { return name; } - + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - + size_t getNumberOfArguments() const override { return mandatory_argument_names.size(); } - + static FunctionPtr create(ContextPtr) { return std::make_shared(); } - + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { FunctionArgumentDescriptors args{ - {mandatory_argument_names[0], &isDateOrDate32OrDateTimeOrDateTime64, nullptr, "Date"}, + {mandatory_argument_names[0], &isDateOrDate32OrDateTimeOrDateTime64, nullptr, "Date(32) or DateTime(64)"}, {mandatory_argument_names[1], &isNumber, nullptr, "Number"} }; validateFunctionArgumentTypes(*this, arguments, args); - - if (Traits::EnumName >= 3) + + if (isTimeChange(Traits::EnumName)) { if (isDate(arguments[0].type)) return std::make_shared(); if (isDate32(arguments[0].type)) return std::make_shared(DataTypeDateTime64::default_scale); } - + return arguments[0].type; } - + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const auto & input_type = arguments[0].type; if (isDate(input_type)) { - if (Traits::EnumName >= 3) + if (isTimeChange(Traits::EnumName)) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDate32(input_type)) { - if (Traits::EnumName >= 3) - return executeDate32ToDateTime64(arguments, result_type, input_rows_count); + if (isTimeChange(Traits::EnumName)) + return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDateTime(input_type)) return execute(arguments, input_type, result_type, input_rows_count); - return executeDateTime64(arguments, result_type, input_rows_count); + return execute(arguments, input_type, result_type, input_rows_count); } - - + + template ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & input_type, const DataTypePtr & result_type, size_t input_rows_count) const { - const auto & date_lut = DateLUT::instance(); + bool is_const = (isColumnConst(*arguments[0].column) && isColumnConst(*arguments[1].column)); + size_t result_rows_count = (is_const ? 1 : input_rows_count); - auto result_column = ResultDataType::ColumnType::create(input_rows_count); + typename ResultDataType::ColumnType::MutablePtr result_column; + if constexpr (isDateTime64()) + { + auto scale = DataTypeDateTime64::default_scale; + if constexpr (isDateTime64()) + scale = typeid_cast(*result_type).getScale(); + result_column = ResultDataType::ColumnType::create(result_rows_count, scale); + } + else + result_column = ResultDataType::ColumnType::create(result_rows_count); + auto & result_data = result_column->getData(); - - auto input_column = castColumn(arguments[0], std::make_shared()); - input_column = input_column->convertToFullColumnIfConst(); + + auto input_column = arguments[0].column->convertToFullIfNeeded(); const auto & input_column_data = typeid_cast(*input_column).getData(); - - auto new_value_column = castColumn(arguments[1], std::make_shared()); - new_value_column = new_value_column->convertToFullColumnIfConst(); - const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); - - for (size_t i = 0; i < input_rows_count; ++i) + + auto new_value_column = castColumn(arguments[1], std::make_shared()); + new_value_column = new_value_column->convertToFullIfNeeded(); + const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + + for (size_t i = 0; i < result_rows_count; ++i) { - Int64 time; - if (isDateOrDate32(input_type)) - time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1000'000; + if constexpr (isDateTime64()) + { + const auto scale = typeid_cast(*result_type).getScale(); + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 deg = 1; + for (size_t j = 0; j < scale; ++j) + deg *= 10; + + Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); + Int64 fraction = input_column_data[i] % deg; + + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, deg, fraction); + } + else if constexpr (isDate32() && isDateTime64()) + { + const auto & date_lut = DateLUT::instance(); + Int64 time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; + + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1'000, 0); + } else - time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - - if (isDateOrDate32(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); - else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + { + const auto & date_lut = DateLUT::instance(); + Int64 time; + if (isDateOrDate32(input_type)) + time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; + else + time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); + + if (isDateOrDate32(result_type)) + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + else + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1, 0)); + } } + if (is_const) + return ColumnConst::create(std::move(result_column), input_rows_count); + return result_column; } - - ColumnPtr executeDate32ToDateTime64(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const - { - const auto & date_lut = DateLUT::instance(); - - auto result_column = ColumnDateTime64::create(input_rows_count, DataTypeDateTime64::default_scale); - auto & result_data = result_column->getData(); - - auto input_column = arguments[0].column->convertToFullColumnIfConst(); - const auto & input_column_data = typeid_cast(*input_column).getData(); - - auto new_value_column = castColumn(arguments[1], std::make_shared()); - new_value_column = new_value_column->convertToFullColumnIfConst(); - const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); - - for (size_t i = 0; i < input_rows_count; ++i) - { - Int64 time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1000'000; - - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1000, 0); - } - - return result_column; - } - - ColumnPtr executeDateTime64(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const - { - auto result_column = ColumnDateTime64::create(input_rows_count, DataTypeDateTime64::default_scale); - auto & result_data = result_column->getData(); - - auto input_column = arguments[0].column->convertToFullColumnIfConst(); - const auto & input_column_data = typeid_cast(*input_column).getData(); - - auto new_value_column = castColumn(arguments[1], std::make_shared()); - new_value_column = new_value_column->convertToFullColumnIfConst(); - const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); - - const auto scale = typeid_cast(*result_type).getScale(); - const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - Int64 deg = 1; - for (size_t i = 0; i < scale; ++i) - { - deg *= 10; - } - - for (size_t i = 0; i < input_rows_count; ++i) - { - Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); - Int64 fraction = input_column_data[i] % deg; - - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, deg, fraction); - } - - return result_column; - } - - Int64 getChangedDate(Int64 time, Float32 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 deg = 0, Int64 fraction = 0) const + + Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 deg = 0, Int64 fraction = 0) const { auto year = time / 10'000'000'000; auto month = (time % 10'000'000'000) / 100'000'000; - auto day = (time % 100'000'000) / 1000'000; - auto hours = (time % 1000'000) / 10'000; + auto day = (time % 100'000'000) / 1'000'000; + auto hours = (time % 1'000'000) / 10'000; auto minutes = (time % 10'000) / 100; auto seconds = time % 100; - + Int64 min_date, max_date; Int16 min_year, max_year; if (isDate(result_type)) @@ -217,115 +215,101 @@ public: } else { - min_date = date_lut.makeDateTime(1900, 1, 1, 0,0 , 0) * deg; + min_date = date_lut.makeDateTime(1900, 1, 1, 0, 0, 0) * deg; max_date = date_lut.makeDateTime(2299, 12, 31, 23, 59, 59) * deg + (deg - 1); min_year = 1900; max_year = 2299; } - - Int8 fl = 0; - + switch (Traits::EnumName) { - case CHANGE_YEAR: + case ChangeDateFunctionsNames::CHANGE_YEAR: if (new_value < min_year) - fl = 1; + return min_date; else if (new_value > max_year) - fl = 2; + return max_date; year = static_cast(new_value); break; - case CHANGE_MONTH: + case ChangeDateFunctionsNames::CHANGE_MONTH: if (new_value < 1 || new_value > 12) - fl = 1; + return min_date; month = static_cast(new_value); break; - case CHANGE_DAY: + case ChangeDateFunctionsNames::CHANGE_DAY: if (new_value < 1 || new_value > 31) - fl = 1; + return min_date; day = static_cast(new_value); break; - case CHANGE_HOUR: + case ChangeDateFunctionsNames::CHANGE_HOUR: if (new_value < 0 || new_value > 23) - fl = 1; + return min_date; hours = static_cast(new_value); break; - case CHANGE_MINUTE: + case ChangeDateFunctionsNames::CHANGE_MINUTE: if (new_value < 0 || new_value > 59) - fl = 1; + return min_date; minutes = static_cast(new_value); break; - case CHANGE_SECOND: + case ChangeDateFunctionsNames::CHANGE_SECOND: if (new_value < 0 || new_value > 59) - fl = 1; + return min_date; seconds = static_cast(new_value); break; } - - if (fl == 1) - return min_date; - - if (fl == 2) - return max_date; - + Int64 result; if (isDateOrDate32(result_type)) result = date_lut.makeDayNum(year, month, day); else - { - if (isDateTime(result_type)) - result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); - else - result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; - } - - + result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; + if (result > max_date) return max_date; - + return result; } }; - - + + struct ChangeYearTraits { static constexpr auto Name = "changeYear"; - static constexpr auto EnumName = CHANGE_YEAR; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_YEAR; }; - + struct ChangeMonthTraits { static constexpr auto Name = "changeMonth"; - static constexpr auto EnumName = CHANGE_MONTH; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MONTH; }; - + struct ChangeDayTraits { static constexpr auto Name = "changeDay"; - static constexpr auto EnumName = CHANGE_DAY; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_DAY; }; - + struct ChangeHourTraits { static constexpr auto Name = "changeHour"; - static constexpr auto EnumName = CHANGE_HOUR; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_HOUR; }; - + struct ChangeMinuteTraits { static constexpr auto Name = "changeMinute"; - static constexpr auto EnumName = CHANGE_MINUTE; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MINUTE; }; - + struct ChangeSecondTraits { static constexpr auto Name = "changeSecond"; - static constexpr auto EnumName = CHANGE_SECOND; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_SECOND; }; - - + + } - + REGISTER_FUNCTION(ChangeDate) { factory.registerFunction>(); @@ -335,5 +319,5 @@ REGISTER_FUNCTION(ChangeDate) factory.registerFunction>(); factory.registerFunction>(); } - + } From fb6c931262c106bf3e076949fc47a28cf06181ba Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Sat, 4 May 2024 22:49:10 +0000 Subject: [PATCH 05/72] Style fixes --- src/Functions/changeDate.cpp | 108 ++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 8a5d0a87ca2..ed6b3255cfc 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -12,20 +12,27 @@ #include #include #include "Common/DateLUTImpl.h" +#include "Common/Exception.h" #include #include #include "Columns/IColumn.h" #include "DataTypes/IDataType.h" - + #include #include - + namespace DB { - + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + namespace { - + enum class ChangeDateFunctionsNames { CHANGE_YEAR = 0, @@ -35,50 +42,50 @@ enum class ChangeDateFunctionsNames CHANGE_MINUTE = 4, CHANGE_SECOND = 5 }; - + constexpr bool isTimeChange(const ChangeDateFunctionsNames & type) { return type == ChangeDateFunctionsNames::CHANGE_HOUR || type == ChangeDateFunctionsNames::CHANGE_MINUTE || type == ChangeDateFunctionsNames::CHANGE_SECOND; } - + template constexpr bool isDate32() { return DataType::type_id == TypeIndex::Date32; } - + template constexpr bool isDateTime64() { return DataType::type_id == TypeIndex::DateTime64; } - + template class FunctionChangeDate : public IFunction { public: static constexpr auto name = Traits::Name; - + static constexpr std::array mandatory_argument_names = {"date", "new_value"}; - + String getName() const override { return name; } - + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - + size_t getNumberOfArguments() const override { return mandatory_argument_names.size(); } - + static FunctionPtr create(ContextPtr) { return std::make_shared(); } - + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - FunctionArgumentDescriptors args{ - {mandatory_argument_names[0], &isDateOrDate32OrDateTimeOrDateTime64, nullptr, "Date(32) or DateTime(64)"}, - {mandatory_argument_names[1], &isNumber, nullptr, "Number"} - }; - validateFunctionArgumentTypes(*this, arguments, args); - + if (arguments.size() != 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires 2 parameters: date, new_value. Passed {}.", getName(), arguments.size()); + + if (!isDateOrDate32OrDateTimeOrDateTime64(*arguments[0].type) || !isNumber(*arguments[1].type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be Date(32) or DateTime(64), second - numeric", getName()); + if (isTimeChange(Traits::EnumName)) { if (isDate(arguments[0].type)) @@ -86,10 +93,10 @@ public: if (isDate32(arguments[0].type)) return std::make_shared(DataTypeDateTime64::default_scale); } - + return arguments[0].type; } - + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const auto & input_type = arguments[0].type; @@ -109,8 +116,7 @@ public: return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } - - + template ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & input_type, const DataTypePtr & result_type, size_t input_rows_count) const { @@ -127,16 +133,16 @@ public: } else result_column = ResultDataType::ColumnType::create(result_rows_count); - + auto & result_data = result_column->getData(); - + auto input_column = arguments[0].column->convertToFullIfNeeded(); const auto & input_column_data = typeid_cast(*input_column).getData(); - + auto new_value_column = castColumn(arguments[1], std::make_shared()); new_value_column = new_value_column->convertToFullIfNeeded(); - const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); - + const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + for (size_t i = 0; i < result_rows_count; ++i) { if constexpr (isDateTime64()) @@ -146,17 +152,17 @@ public: Int64 deg = 1; for (size_t j = 0; j < scale; ++j) deg *= 10; - + Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); Int64 fraction = input_column_data[i] % deg; - + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, deg, fraction); } else if constexpr (isDate32() && isDateTime64()) { const auto & date_lut = DateLUT::instance(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; - + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1'000, 0); } else @@ -167,7 +173,7 @@ public: time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; else time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - + if (isDateOrDate32(result_type)) result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else @@ -177,10 +183,10 @@ public: if (is_const) return ColumnConst::create(std::move(result_column), input_rows_count); - + return result_column; } - + Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 deg = 0, Int64 fraction = 0) const { auto year = time / 10'000'000'000; @@ -189,7 +195,7 @@ public: auto hours = (time % 1'000'000) / 10'000; auto minutes = (time % 10'000) / 100; auto seconds = time % 100; - + Int64 min_date, max_date; Int16 min_year, max_year; if (isDate(result_type)) @@ -220,7 +226,7 @@ public: min_year = 1900; max_year = 2299; } - + switch (Traits::EnumName) { case ChangeDateFunctionsNames::CHANGE_YEAR: @@ -256,60 +262,60 @@ public: seconds = static_cast(new_value); break; } - + Int64 result; if (isDateOrDate32(result_type)) result = date_lut.makeDayNum(year, month, day); else result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; - + if (result > max_date) return max_date; - + return result; } }; - - + + struct ChangeYearTraits { static constexpr auto Name = "changeYear"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_YEAR; }; - + struct ChangeMonthTraits { static constexpr auto Name = "changeMonth"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MONTH; }; - + struct ChangeDayTraits { static constexpr auto Name = "changeDay"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_DAY; }; - + struct ChangeHourTraits { static constexpr auto Name = "changeHour"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_HOUR; }; - + struct ChangeMinuteTraits { static constexpr auto Name = "changeMinute"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MINUTE; }; - + struct ChangeSecondTraits { static constexpr auto Name = "changeSecond"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_SECOND; }; - - + + } - + REGISTER_FUNCTION(ChangeDate) { factory.registerFunction>(); @@ -319,5 +325,5 @@ REGISTER_FUNCTION(ChangeDate) factory.registerFunction>(); factory.registerFunction>(); } - + } From f4990f26d914b078d92674061ec5427b3daa4ab7 Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Wed, 8 May 2024 16:38:53 +0000 Subject: [PATCH 06/72] fixes --- src/Functions/changeDate.cpp | 39 +++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index ed6b3255cfc..dfa93c9d218 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -156,28 +156,33 @@ public: Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); Int64 fraction = input_column_data[i] % deg; - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, deg, fraction); + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, scale, fraction); } else if constexpr (isDate32() && isDateTime64()) { const auto & date_lut = DateLUT::instance(); - Int64 time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; + Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1'000, 0); + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); } else { const auto & date_lut = DateLUT::instance(); Int64 time; - if (isDateOrDate32(input_type)) + if (isDate(input_type)) time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; + else if (isDate32(input_type)) + time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; else time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - if (isDateOrDate32(result_type)) + + if (isDate(result_type)) + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + else if (isDate32(result_type)) result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1, 0)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); } } @@ -187,7 +192,7 @@ public: return result_column; } - Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 deg = 0, Int64 fraction = 0) const + Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const { auto year = time / 10'000'000'000; auto month = (time % 10'000'000'000) / 100'000'000; @@ -221,8 +226,17 @@ public: } else { - min_date = date_lut.makeDateTime(1900, 1, 1, 0, 0, 0) * deg; - max_date = date_lut.makeDateTime(2299, 12, 31, 23, 59, 59) * deg + (deg - 1); + min_date = DecimalUtils::decimalFromComponents( + date_lut.makeDateTime(1900, 1, 1, 0, 0, 0), + static_cast(0), + static_cast(scale)); + Int64 deg = 1; + for (Int64 j = 0; j < scale; ++j) + deg *= 10; + max_date = DecimalUtils::decimalFromComponents( + date_lut.makeDateTime(2299, 12, 31, 23, 59, 59), + static_cast(deg - 1), + static_cast(scale)); min_year = 1900; max_year = 2299; } @@ -266,8 +280,13 @@ public: Int64 result; if (isDateOrDate32(result_type)) result = date_lut.makeDayNum(year, month, day); + else if (isDateTime(result_type)) + result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); else - result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; + result = DecimalUtils::decimalFromComponents( + date_lut.makeDateTime(year, month, day, hours, minutes, seconds), + static_cast(fraction), + static_cast(scale)); if (result > max_date) return max_date; From 2f8341d8c1686b9b73bfad36d12651d63b8ffd08 Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Wed, 8 May 2024 16:52:36 +0000 Subject: [PATCH 07/72] style --- src/Functions/changeDate.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index dfa93c9d218..2265ee1b26b 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -176,7 +176,6 @@ public: else time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - if (isDate(result_type)) result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else if (isDate32(result_type)) From c82687c9a61bd6f53949ac5310d26c9b410fabbf Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Wed, 8 May 2024 22:17:59 +0000 Subject: [PATCH 08/72] fixes + functions docs --- src/Functions/changeDate.cpp | 82 ++++++++++++++++--- .../queries/0_stateless/02982_changeDate.sql | 24 +++--- 2 files changed, 82 insertions(+), 24 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 2265ee1b26b..73ac9eff867 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -56,6 +56,12 @@ constexpr bool isDate32() return DataType::type_id == TypeIndex::Date32; } +template +constexpr bool isDateTime() +{ + return DataType::type_id == TypeIndex::DateTime; +} + template constexpr bool isDateTime64() { @@ -86,7 +92,7 @@ public: if (!isDateOrDate32OrDateTimeOrDateTime64(*arguments[0].type) || !isNumber(*arguments[1].type)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be Date(32) or DateTime(64), second - numeric", getName()); - if (isTimeChange(Traits::EnumName)) + if constexpr (isTimeChange(Traits::EnumName)) { if (isDate(arguments[0].type)) return std::make_shared(); @@ -102,13 +108,13 @@ public: const auto & input_type = arguments[0].type; if (isDate(input_type)) { - if (isTimeChange(Traits::EnumName)) + if constexpr (isTimeChange(Traits::EnumName)) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDate32(input_type)) { - if (isTimeChange(Traits::EnumName)) + if constexpr (isTimeChange(Traits::EnumName)) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } @@ -145,6 +151,7 @@ public: for (size_t i = 0; i < result_rows_count; ++i) { + if constexpr (isDateTime64()) { const auto scale = typeid_cast(*result_type).getScale(); @@ -165,16 +172,21 @@ public: result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); } + else if constexpr (isDateTime()) + { + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); + + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + } else { const auto & date_lut = DateLUT::instance(); Int64 time; if (isDate(input_type)) time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; - else if (isDate32(input_type)) - time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; else - time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); + time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; if (isDate(result_type)) result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); @@ -336,12 +348,58 @@ struct ChangeSecondTraits REGISTER_FUNCTION(ChangeDate) { - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Changes the year of the given Date(32) or DateTime(64). +Returns the same type as the input data. +)", + .categories{"Dates and Times"} + } + ); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Same as changeYear function, but changes month of the date. +)", + .categories{"Dates and Times"} + } + ); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Same as changeYear function, but changes day_of_month of the date. +)", + .categories{"Dates and Times"} + } + ); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Changes the hour of the given Date(32) or DateTime(64). +If the input data is Date, return DateTime; +if the input data is Date32, return DateTime64; +In other cases returns the same type as the input data. +)", + .categories{"Dates and Times"} + } + ); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Same as changeHour function, but changes minute of the date. +)", + .categories{"Dates and Times"} + } + ); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Same as changeHour function, but changes seconds of the date. +)", + .categories{"Dates and Times"} + } + ); } } diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index fbfe3771b33..9f7f5eacfc5 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -13,18 +13,18 @@ SELECT changeDay(makeDate32(1970, 01, 01), 03); SELECT changeDay(makeDateTime(1970, 01, 01, 11, 22, 33), 04); SELECT changeDay(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 05); -SELECT changeHour(makeDate(1970, 01, 01), 12); -SELECT changeHour(makeDate32(1970, 01, 01), 13); +SELECT toTimeZone(changeHour(makeDate(1970, 01, 01), 12), 'UTC'); +SELECT toTimeZone(changeHour(makeDate32(1970, 01, 01), 13), 'UTC'); SELECT changeHour(makeDateTime(1970, 01, 01, 11, 22, 33), 14); SELECT changeHour(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 15); -SELECT changeMinute(makeDate(1970, 01, 01), 23); -SELECT changeMinute(makeDate32(1970, 01, 01), 24); +SELECT toTimeZone(changeMinute(makeDate(1970, 01, 01), 23), 'UTC'); +SELECT toTimeZone(changeMinute(makeDate32(1970, 01, 01), 24), 'UTC'); SELECT changeMinute(makeDateTime(1970, 01, 01, 11, 22, 33), 25); SELECT changeMinute(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 26); -SELECT changeSecond(makeDate(1970, 01, 01), 34); -SELECT changeSecond(makeDate32(1970, 01, 01), 35); +SELECT toTimeZone(changeSecond(makeDate(1970, 01, 01), 34), 'UTC'); +SELECT toTimeZone(changeSecond(makeDate32(1970, 01, 01), 35), 'UTC'); SELECT changeSecond(makeDateTime(1970, 01, 01, 11, 22, 33), 36); SELECT changeSecond(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 37); @@ -34,9 +34,9 @@ SELECT changeMonth(makeDate(2149, 01, 01), 07); SELECT changeMonth(makeDate(2000, 06, 07), 13); SELECT changeDay(makeDate(2000, 01, 01), 0); SELECT changeDay(makeDate(2000, 06, 07), 32); -SELECT changeHour(makeDate(2000, 01, 01), -1); -SELECT changeHour(makeDate(2000, 06, 07), 24); -SELECT changeMinute(makeDate(2000, 01, 01), -1); -SELECT changeMinute(makeDate(2000, 06, 07), 60); -SELECT changeSecond(makeDate(2000, 01, 01), -1); -SELECT changeSecond(makeDate(2000, 06, 07), 60); \ No newline at end of file +SELECT toTimeZone(changeHour(makeDate(2000, 01, 01), -1), 'UTC'); +SELECT toTimeZone(changeHour(makeDate(2000, 06, 07), 24), 'UTC'); +SELECT toTimeZone(changeMinute(makeDate(2000, 01, 01), -1), 'UTC'); +SELECT toTimeZone(changeMinute(makeDate(2000, 06, 07), 60), 'UTC'); +SELECT toTimeZone(changeSecond(makeDate(2000, 01, 01), -1), 'UTC'); +SELECT toTimeZone(changeSecond(makeDate(2000, 06, 07), 60), 'UTC'); \ No newline at end of file From f52c24f753b35240d6d4925aaf582c9ebac829b2 Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Thu, 9 May 2024 08:18:02 +0000 Subject: [PATCH 09/72] tests fixes --- src/Functions/changeDate.cpp | 1 - .../queries/0_stateless/02982_changeDate.sql | 72 +++++++++---------- 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 73ac9eff867..a1827e1d94a 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -151,7 +151,6 @@ public: for (size_t i = 0; i < result_rows_count; ++i) { - if constexpr (isDateTime64()) { const auto scale = typeid_cast(*result_type).getScale(); diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index 9f7f5eacfc5..f438212f9fa 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -1,42 +1,42 @@ -SELECT changeYear(makeDate(1970, 01, 01), 2000); -SELECT changeYear(makeDate32(1970, 01, 01), 2001); -SELECT changeYear(makeDateTime(1970, 01, 01, 11, 22, 33), 2002); -SELECT changeYear(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 2003); +SELECT changeYear(toDate('1970-01-01', 'UTC'), 2000); +SELECT changeYear(toDate32('1900-01-01', 'UTC'), 2001); +SELECT changeYear(toDateTime('1970-01-01 11:22:33', 'UTC'), 2002); +SELECT changeYear(toDateTime64('1900-01-01 11:22:33.4444', 4, 'UTC'), 2003); -SELECT changeMonth(makeDate(1970, 01, 01), 02); -SELECT changeMonth(makeDate32(1970, 01, 01), 03); -SELECT changeMonth(makeDateTime(1970, 01, 01, 11, 22, 33), 04); -SELECT changeMonth(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 05); +SELECT changeMonth(toDate('1970-01-01', 'UTC'), 02); +SELECT changeMonth(toDate32('1970-01-01', 'UTC'), 03); +SELECT changeMonth(toDateTime('1970-01-01 11:22:33', 'UTC'), 04); +SELECT changeMonth(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 05); -SELECT changeDay(makeDate(1970, 01, 01), 02); -SELECT changeDay(makeDate32(1970, 01, 01), 03); -SELECT changeDay(makeDateTime(1970, 01, 01, 11, 22, 33), 04); -SELECT changeDay(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 05); +SELECT changeDay(toDate('1970-01-01', 'UTC'), 02); +SELECT changeDay(toDate32('1970-01-01', 'UTC'), 03); +SELECT changeDay(toDateTime('1970-01-01 11:22:33', 'UTC'), 04); +SELECT changeDay(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 05); -SELECT toTimeZone(changeHour(makeDate(1970, 01, 01), 12), 'UTC'); -SELECT toTimeZone(changeHour(makeDate32(1970, 01, 01), 13), 'UTC'); -SELECT changeHour(makeDateTime(1970, 01, 01, 11, 22, 33), 14); -SELECT changeHour(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 15); +SELECT toTimeZone(changeHour(toDate('1970-01-01', 'UTC'), 12), 'UTC'); +SELECT toTimeZone(changeHour(toDate32('1970-01-01', 'UTC'), 13), 'UTC'); +SELECT changeHour(toDateTime('1970-01-01 11:22:33', 'UTC'), 14); +SELECT changeHour(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 15); -SELECT toTimeZone(changeMinute(makeDate(1970, 01, 01), 23), 'UTC'); -SELECT toTimeZone(changeMinute(makeDate32(1970, 01, 01), 24), 'UTC'); -SELECT changeMinute(makeDateTime(1970, 01, 01, 11, 22, 33), 25); -SELECT changeMinute(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 26); +SELECT toTimeZone(changeMinute(toDate('1970-01-01', 'UTC'), 23), 'UTC'); +SELECT toTimeZone(changeMinute(toDate32('1970-01-01', 'UTC'), 24), 'UTC'); +SELECT changeMinute(toDateTime('1970-01-01 11:22:33', 'UTC'), 25); +SELECT changeMinute(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 26); -SELECT toTimeZone(changeSecond(makeDate(1970, 01, 01), 34), 'UTC'); -SELECT toTimeZone(changeSecond(makeDate32(1970, 01, 01), 35), 'UTC'); -SELECT changeSecond(makeDateTime(1970, 01, 01, 11, 22, 33), 36); -SELECT changeSecond(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 37); +SELECT toTimeZone(changeSecond(toDate('1970-01-01', 'UTC'), 34), 'UTC'); +SELECT toTimeZone(changeSecond(toDate32('1970-01-01', 'UTC'), 35), 'UTC'); +SELECT changeSecond(toDateTime('1970-01-01 11:22:33', 'UTC'), 36); +SELECT changeSecond(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 37); -SELECT changeYear(makeDate(2000, 01, 01), 1969.0); -SELECT changeYear(makeDate(2000, 06, 07), 2149.0); -SELECT changeMonth(makeDate(2149, 01, 01), 07); -SELECT changeMonth(makeDate(2000, 06, 07), 13); -SELECT changeDay(makeDate(2000, 01, 01), 0); -SELECT changeDay(makeDate(2000, 06, 07), 32); -SELECT toTimeZone(changeHour(makeDate(2000, 01, 01), -1), 'UTC'); -SELECT toTimeZone(changeHour(makeDate(2000, 06, 07), 24), 'UTC'); -SELECT toTimeZone(changeMinute(makeDate(2000, 01, 01), -1), 'UTC'); -SELECT toTimeZone(changeMinute(makeDate(2000, 06, 07), 60), 'UTC'); -SELECT toTimeZone(changeSecond(makeDate(2000, 01, 01), -1), 'UTC'); -SELECT toTimeZone(changeSecond(makeDate(2000, 06, 07), 60), 'UTC'); \ No newline at end of file +SELECT changeYear(toDate('2000-01-01', 'UTC'), 1969.0); +SELECT changeYear(toDate('2000-06-07', 'UTC'), 2149.0); +SELECT changeMonth(toDate('2149-01-01', 'UTC'), 07); +SELECT changeMonth(toDate('2000-01-01', 'UTC'), 13); +SELECT changeDay(toDate('2000-01-01', 'UTC'), 0); +SELECT changeDay(toDate('2000-01-01', 'UTC'), 32); +SELECT toTimeZone(changeHour(toDate('2000-01-01', 'UTC'), -1), 'UTC'); +SELECT toTimeZone(changeHour(toDate('2000-01-01', 'UTC'), 24), 'UTC'); +SELECT toTimeZone(changeMinute(toDate('2000-01-01', 'UTC'), -1), 'UTC'); +SELECT toTimeZone(changeMinute(toDate('2000-01-01', 'UTC'), 60), 'UTC'); +SELECT toTimeZone(changeSecond(toDate('2000-01-01', 'UTC'), -1), 'UTC'); +SELECT toTimeZone(changeSecond(toDate('2000-01-01', 'UTC'), 60), 'UTC'); \ No newline at end of file From ff9c0934cecb5ee8b938adc92498cec035d2e26f Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Thu, 9 May 2024 17:52:50 +0000 Subject: [PATCH 10/72] changes in tests and functions because of time zone --- src/Functions/changeDate.cpp | 30 +++++--- .../0_stateless/02982_changeDate.reference | 12 ++-- .../queries/0_stateless/02982_changeDate.sql | 72 +++++++++---------- 3 files changed, 64 insertions(+), 50 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index a1827e1d94a..c7815263323 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -50,6 +50,12 @@ constexpr bool isTimeChange(const ChangeDateFunctionsNames & type) type == ChangeDateFunctionsNames::CHANGE_SECOND; } +template +constexpr bool isDate() +{ + return DataType::type_id == TypeIndex::Date; +} + template constexpr bool isDate32() { @@ -68,6 +74,7 @@ constexpr bool isDateTime64() return DataType::type_id == TypeIndex::DateTime64; } + template class FunctionChangeDate : public IFunction { @@ -162,21 +169,21 @@ public: Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); Int64 fraction = input_column_data[i] % deg; - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, scale, fraction); + result_data[i] = getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut, scale, fraction); } else if constexpr (isDate32() && isDateTime64()) { const auto & date_lut = DateLUT::instance(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); + result_data[i] = getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut, 3, 0); } else if constexpr (isDateTime()) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); } else { @@ -188,11 +195,11 @@ public: time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; if (isDate(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); else if (isDate32(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); } } @@ -202,7 +209,7 @@ public: return result_column; } - Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const + Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & input_type, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const { auto year = time / 10'000'000'000; auto month = (time % 10'000'000'000) / 100'000'000; @@ -291,12 +298,20 @@ public: if (isDateOrDate32(result_type)) result = date_lut.makeDayNum(year, month, day); else if (isDateTime(result_type)) + { result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); + if (isDate(input_type)) + result += date_lut.timezoneOffset(result); + } else + { result = DecimalUtils::decimalFromComponents( date_lut.makeDateTime(year, month, day, hours, minutes, seconds), static_cast(fraction), static_cast(scale)); + if (isDate32(input_type)) + result += date_lut.timezoneOffset(result); + } if (result > max_date) return max_date; @@ -342,7 +357,6 @@ struct ChangeSecondTraits static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_SECOND; }; - } REGISTER_FUNCTION(ChangeDate) diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference index d7d4edf4b43..67747922ae5 100644 --- a/tests/queries/0_stateless/02982_changeDate.reference +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -10,16 +10,16 @@ 1970-01-03 1970-01-04 11:22:33 1970-01-05 11:22:33.4444 -1970-01-01 12:00:00 -1970-01-01 13:00:00.000 +12 +13 1970-01-01 14:22:33 1970-01-01 15:22:33.4444 -1970-01-01 00:23:00 -1970-01-01 00:24:00.000 +23 +24 1970-01-01 11:25:33 1970-01-01 11:26:33.4444 -1970-01-01 00:00:34 -1970-01-01 00:00:35.000 +34 +35 1970-01-01 11:22:36 1970-01-01 11:22:37.4444 1970-01-01 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index f438212f9fa..786e27808e8 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -1,42 +1,42 @@ -SELECT changeYear(toDate('1970-01-01', 'UTC'), 2000); -SELECT changeYear(toDate32('1900-01-01', 'UTC'), 2001); -SELECT changeYear(toDateTime('1970-01-01 11:22:33', 'UTC'), 2002); -SELECT changeYear(toDateTime64('1900-01-01 11:22:33.4444', 4, 'UTC'), 2003); +SELECT changeYear(toDate('1970-01-01'), 2000); +SELECT changeYear(toDate32('1900-01-01'), 2001); +SELECT changeYear(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 2002); +SELECT changeYear(toDateTime64('1900-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 2003); -SELECT changeMonth(toDate('1970-01-01', 'UTC'), 02); -SELECT changeMonth(toDate32('1970-01-01', 'UTC'), 03); -SELECT changeMonth(toDateTime('1970-01-01 11:22:33', 'UTC'), 04); -SELECT changeMonth(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 05); +SELECT changeMonth(toDate('1970-01-01'), 02); +SELECT changeMonth(toDate32('1970-01-01'), 03); +SELECT changeMonth(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 04); +SELECT changeMonth(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 05); -SELECT changeDay(toDate('1970-01-01', 'UTC'), 02); -SELECT changeDay(toDate32('1970-01-01', 'UTC'), 03); -SELECT changeDay(toDateTime('1970-01-01 11:22:33', 'UTC'), 04); -SELECT changeDay(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 05); +SELECT changeDay(toDate('1970-01-01'), 02); +SELECT changeDay(toDate32('1970-01-01'), 03); +SELECT changeDay(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 04); +SELECT changeDay(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 05); -SELECT toTimeZone(changeHour(toDate('1970-01-01', 'UTC'), 12), 'UTC'); -SELECT toTimeZone(changeHour(toDate32('1970-01-01', 'UTC'), 13), 'UTC'); -SELECT changeHour(toDateTime('1970-01-01 11:22:33', 'UTC'), 14); -SELECT changeHour(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 15); +SELECT toHour(changeHour(toDate('1970-01-01'), 12)); +SELECT toHour(changeHour(toDate32('1970-01-01'), 13)); +SELECT changeHour(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 14); +SELECT changeHour(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 15); -SELECT toTimeZone(changeMinute(toDate('1970-01-01', 'UTC'), 23), 'UTC'); -SELECT toTimeZone(changeMinute(toDate32('1970-01-01', 'UTC'), 24), 'UTC'); -SELECT changeMinute(toDateTime('1970-01-01 11:22:33', 'UTC'), 25); -SELECT changeMinute(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 26); +SELECT toMinute(changeMinute(toDate('1970-01-01'), 23)); +SELECT toMinute(changeMinute(toDate32('1970-01-01'), 24)); +SELECT changeMinute(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 25); +SELECT changeMinute(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 26); -SELECT toTimeZone(changeSecond(toDate('1970-01-01', 'UTC'), 34), 'UTC'); -SELECT toTimeZone(changeSecond(toDate32('1970-01-01', 'UTC'), 35), 'UTC'); -SELECT changeSecond(toDateTime('1970-01-01 11:22:33', 'UTC'), 36); -SELECT changeSecond(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 37); +SELECT toSecond(changeSecond(toDate('1970-01-01'), 34)); +SELECT toSecond(changeSecond(toDate32('1970-01-01'), 35)); +SELECT changeSecond(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 36); +SELECT changeSecond(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 37); -SELECT changeYear(toDate('2000-01-01', 'UTC'), 1969.0); -SELECT changeYear(toDate('2000-06-07', 'UTC'), 2149.0); -SELECT changeMonth(toDate('2149-01-01', 'UTC'), 07); -SELECT changeMonth(toDate('2000-01-01', 'UTC'), 13); -SELECT changeDay(toDate('2000-01-01', 'UTC'), 0); -SELECT changeDay(toDate('2000-01-01', 'UTC'), 32); -SELECT toTimeZone(changeHour(toDate('2000-01-01', 'UTC'), -1), 'UTC'); -SELECT toTimeZone(changeHour(toDate('2000-01-01', 'UTC'), 24), 'UTC'); -SELECT toTimeZone(changeMinute(toDate('2000-01-01', 'UTC'), -1), 'UTC'); -SELECT toTimeZone(changeMinute(toDate('2000-01-01', 'UTC'), 60), 'UTC'); -SELECT toTimeZone(changeSecond(toDate('2000-01-01', 'UTC'), -1), 'UTC'); -SELECT toTimeZone(changeSecond(toDate('2000-01-01', 'UTC'), 60), 'UTC'); \ No newline at end of file +SELECT changeYear(toDate('2000-01-01'), 1969.0); +SELECT changeYear(toDate('2000-06-07'), 2149.0); +SELECT changeMonth(toDate('2149-01-01'), 07); +SELECT changeMonth(toDate('2000-01-01'), 13); +SELECT changeDay(toDate('2000-01-01'), 0); +SELECT changeDay(toDate('2000-01-01'), 32); +SELECT changeHour(toDate('2000-01-01'), -1); +SELECT changeHour(toDate('2000-01-01'), 24); +SELECT changeMinute(toDate('2000-01-01'), -1); +SELECT changeMinute(toDate('2000-01-01'), 60); +SELECT changeSecond(toDate('2000-01-01'), -1); +SELECT changeSecond(toDate('2000-01-01'), 60); \ No newline at end of file From 3e7041c1f4106359ab6792ddbab86db3a4792555 Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Thu, 9 May 2024 18:33:46 +0000 Subject: [PATCH 11/72] . --- src/Functions/changeDate.cpp | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index c7815263323..e77aaec30fe 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -169,21 +169,21 @@ public: Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); Int64 fraction = input_column_data[i] % deg; - result_data[i] = getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut, scale, fraction); + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, scale, fraction); } else if constexpr (isDate32() && isDateTime64()) { const auto & date_lut = DateLUT::instance(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; - result_data[i] = getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut, 3, 0); + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); } else if constexpr (isDateTime()) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); } else { @@ -195,11 +195,11 @@ public: time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; if (isDate(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else if (isDate32(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); } } @@ -209,7 +209,7 @@ public: return result_column; } - Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & input_type, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const + Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const { auto year = time / 10'000'000'000; auto month = (time % 10'000'000'000) / 100'000'000; @@ -298,20 +298,12 @@ public: if (isDateOrDate32(result_type)) result = date_lut.makeDayNum(year, month, day); else if (isDateTime(result_type)) - { result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); - if (isDate(input_type)) - result += date_lut.timezoneOffset(result); - } else - { result = DecimalUtils::decimalFromComponents( date_lut.makeDateTime(year, month, day, hours, minutes, seconds), static_cast(fraction), static_cast(scale)); - if (isDate32(input_type)) - result += date_lut.timezoneOffset(result); - } if (result > max_date) return max_date; From afb47e418303b4ee125f8c9700985372f827da8b Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Thu, 9 May 2024 19:32:43 +0000 Subject: [PATCH 12/72] . --- src/Functions/changeDate.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index e77aaec30fe..fd279a028b7 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -173,11 +173,18 @@ public: } else if constexpr (isDate32() && isDateTime64()) { - const auto & date_lut = DateLUT::instance(); + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); } + else if constexpr (isDate() && isDateTime()) + { + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; + + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + } else if constexpr (isDateTime()) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); @@ -196,10 +203,8 @@ public: if (isDate(result_type)) result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); - else if (isDate32(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); } } From 50377d2450701ea31ea3fcde8c6ee51b96c971cf Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Sun, 19 May 2024 08:41:30 +0000 Subject: [PATCH 13/72] fix date_min of DateTime --- src/Functions/changeDate.cpp | 8 ++- .../0_stateless/02982_changeDate.reference | 58 ++++++++--------- .../queries/0_stateless/02982_changeDate.sql | 62 +++++++++---------- 3 files changed, 66 insertions(+), 62 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index fd279a028b7..237d9082566 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -162,6 +162,7 @@ public: { const auto scale = typeid_cast(*result_type).getScale(); const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 deg = 1; for (size_t j = 0; j < scale; ++j) deg *= 10; @@ -223,7 +224,7 @@ public: auto minutes = (time % 10'000) / 100; auto seconds = time % 100; - Int64 min_date, max_date; + Int64 min_date = 0, max_date = 0; Int16 min_year, max_year; if (isDate(result_type)) { @@ -242,7 +243,7 @@ public: else if (isDateTime(result_type)) { min_date = 0; - max_date = 0x0ffffffffll; + max_date = 0x0FFFFFFFFLL; min_year = 1970; max_year = 2106; } @@ -310,6 +311,9 @@ public: static_cast(fraction), static_cast(scale)); + if (result < min_date) + return min_date; + if (result > max_date) return max_date; diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference index 67747922ae5..c64abc89ed2 100644 --- a/tests/queries/0_stateless/02982_changeDate.reference +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -1,36 +1,36 @@ -2000-01-01 2001-01-01 -2002-01-01 11:22:33 -2003-01-01 11:22:33.4444 -1970-02-01 -1970-03-01 -1970-04-01 11:22:33 -1970-05-01 11:22:33.4444 -1970-01-02 -1970-01-03 -1970-01-04 11:22:33 -1970-01-05 11:22:33.4444 -12 -13 -1970-01-01 14:22:33 -1970-01-01 15:22:33.4444 -23 -24 -1970-01-01 11:25:33 -1970-01-01 11:26:33.4444 -34 -35 -1970-01-01 11:22:36 -1970-01-01 11:22:37.4444 +2002-01-01 +2003-01-01 11:22:33 +2004-01-01 11:22:33.4444 +2000-02-01 +2000-03-01 +2000-04-01 11:22:33 +2000-05-01 11:22:33.4444 +2000-01-02 +2000-01-03 +2000-01-04 11:22:33 +2000-01-05 11:22:33.4444 +2000-01-01 12:00:00 +2000-01-01 13:00:00.000 +2000-01-01 14:22:33 +2000-01-01 15:22:33.4444 +2000-01-01 00:23:00 +2000-01-01 00:24:00.000 +2000-01-01 11:25:33 +2000-01-01 11:26:33.4444 +2000-01-01 00:00:34 +2000-01-01 00:00:35.000 +2000-01-01 11:22:36 +2000-01-01 11:22:37.4444 1970-01-01 2149-06-06 2149-06-06 1970-01-01 1970-01-01 1970-01-01 -1970-01-01 00:00:00 -1970-01-01 00:00:00 -1970-01-01 00:00:00 -1970-01-01 00:00:00 -1970-01-01 00:00:00 -1970-01-01 00:00:00 +1970-01-01 07:00:00 +1970-01-01 07:00:00 +1970-01-01 07:00:00 +1970-01-01 07:00:00 +1970-01-01 07:00:00 +1970-01-01 07:00:00 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index 786e27808e8..0d1bd75e130 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -1,42 +1,42 @@ -SELECT changeYear(toDate('1970-01-01'), 2000); -SELECT changeYear(toDate32('1900-01-01'), 2001); -SELECT changeYear(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 2002); -SELECT changeYear(toDateTime64('1900-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 2003); +SELECT changeYear(toDate('2000-01-01'), 2001); +SELECT changeYear(toDate32('2000-01-01'), 2002); +SELECT changeYear(toDateTime('2000-01-01 11:22:33'), 2003); +SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), 2004); -SELECT changeMonth(toDate('1970-01-01'), 02); -SELECT changeMonth(toDate32('1970-01-01'), 03); -SELECT changeMonth(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 04); -SELECT changeMonth(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 05); +SELECT changeMonth(toDate('2000-01-01'), 02); +SELECT changeMonth(toDate32('2000-01-01'), 03); +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 04); +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 05); -SELECT changeDay(toDate('1970-01-01'), 02); -SELECT changeDay(toDate32('1970-01-01'), 03); -SELECT changeDay(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 04); -SELECT changeDay(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 05); +SELECT changeDay(toDate('2000-01-01'), 02); +SELECT changeDay(toDate32('2000-01-01'), 03); +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 04); +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 05); -SELECT toHour(changeHour(toDate('1970-01-01'), 12)); -SELECT toHour(changeHour(toDate32('1970-01-01'), 13)); -SELECT changeHour(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 14); -SELECT changeHour(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 15); +SELECT changeHour(toDate('2000-01-01'), 12); +SELECT changeHour(toDate32('2000-01-01'), 13); +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 14); +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 15); -SELECT toMinute(changeMinute(toDate('1970-01-01'), 23)); -SELECT toMinute(changeMinute(toDate32('1970-01-01'), 24)); -SELECT changeMinute(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 25); -SELECT changeMinute(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 26); +SELECT changeMinute(toDate('2000-01-01'), 23); +SELECT changeMinute(toDate32('2000-01-01'), 24); +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 25); +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 26); -SELECT toSecond(changeSecond(toDate('1970-01-01'), 34)); -SELECT toSecond(changeSecond(toDate32('1970-01-01'), 35)); -SELECT changeSecond(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 36); -SELECT changeSecond(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 37); +SELECT changeSecond(toDate('2000-01-01'), 34); +SELECT changeSecond(toDate32('2000-01-01'), 35); +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 36); +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 37); -SELECT changeYear(toDate('2000-01-01'), 1969.0); +SELECT changeYear(toDate('2000-01-01'), 1969.0); SELECT changeYear(toDate('2000-06-07'), 2149.0); SELECT changeMonth(toDate('2149-01-01'), 07); SELECT changeMonth(toDate('2000-01-01'), 13); SELECT changeDay(toDate('2000-01-01'), 0); SELECT changeDay(toDate('2000-01-01'), 32); -SELECT changeHour(toDate('2000-01-01'), -1); -SELECT changeHour(toDate('2000-01-01'), 24); -SELECT changeMinute(toDate('2000-01-01'), -1); -SELECT changeMinute(toDate('2000-01-01'), 60); -SELECT changeSecond(toDate('2000-01-01'), -1); -SELECT changeSecond(toDate('2000-01-01'), 60); \ No newline at end of file +SELECT changeHour(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeHour(toDate('2000-01-01'), 24) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeMinute(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeMinute(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeSecond(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; \ No newline at end of file From eecbd44ce5e6673072c728097f50b41e2935ae90 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 19 May 2024 13:23:30 +0000 Subject: [PATCH 14/72] Some fixups --- src/Functions/changeDate.cpp | 172 ++++++++---------- .../queries/0_stateless/02982_changeDate.sql | 2 +- 2 files changed, 75 insertions(+), 99 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 237d9082566..6725afe3356 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -1,24 +1,23 @@ -#include -#include -#include +#include "Common/DateLUTImpl.h" +#include "Common/Exception.h" +#include +#include +#include +#include +#include +#include +#include #include #include #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include -#include "Common/DateLUTImpl.h" -#include "Common/Exception.h" -#include -#include -#include "Columns/IColumn.h" -#include "DataTypes/IDataType.h" -#include #include namespace DB @@ -26,88 +25,62 @@ namespace DB namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } namespace { -enum class ChangeDateFunctionsNames +enum class Component { - CHANGE_YEAR = 0, - CHANGE_MONTH = 1, - CHANGE_DAY = 2, - CHANGE_HOUR = 3, - CHANGE_MINUTE = 4, - CHANGE_SECOND = 5 + Year, + Month, + Day, + Hour, + Minute, + Second }; -constexpr bool isTimeChange(const ChangeDateFunctionsNames & type) +bool isTimeComponentChange(Component type) { - return type == ChangeDateFunctionsNames::CHANGE_HOUR || - type == ChangeDateFunctionsNames::CHANGE_MINUTE || - type == ChangeDateFunctionsNames::CHANGE_SECOND; + return type == Component::Hour || + type == Component::Minute || + type == Component::Second; } -template -constexpr bool isDate() -{ - return DataType::type_id == TypeIndex::Date; -} - -template -constexpr bool isDate32() -{ - return DataType::type_id == TypeIndex::Date32; -} - -template -constexpr bool isDateTime() -{ - return DataType::type_id == TypeIndex::DateTime; -} - -template -constexpr bool isDateTime64() -{ - return DataType::type_id == TypeIndex::DateTime64; -} - - template class FunctionChangeDate : public IFunction { public: - static constexpr auto name = Traits::Name; - - static constexpr std::array mandatory_argument_names = {"date", "new_value"}; - - String getName() const override { return name; } - - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - - size_t getNumberOfArguments() const override { return mandatory_argument_names.size(); } + static constexpr auto name = Traits::name; static FunctionPtr create(ContextPtr) { return std::make_shared(); } + String getName() const override { return Traits::name; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + size_t getNumberOfArguments() const override { return 2; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { if (arguments.size() != 2) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires 2 parameters: date, new_value. Passed {}.", getName(), arguments.size()); - if (!isDateOrDate32OrDateTimeOrDateTime64(*arguments[0].type) || !isNumber(*arguments[1].type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be Date(32) or DateTime(64), second - numeric", getName()); + if (!isDateOrDate32OrDateTimeOrDateTime64(*arguments[0].type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be Date, Date32, DateTime or DateTime64", getName()); + if (!isNumber(*arguments[1].type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Second argument for function {} must be numeric", getName()); - if constexpr (isTimeChange(Traits::EnumName)) + const auto & input_type = arguments[0].type; + + if (isTimeComponentChange(Traits::component)) { - if (isDate(arguments[0].type)) + if (isDate(input_type)) return std::make_shared(); - if (isDate32(arguments[0].type)) + if (isDate32(input_type)) return std::make_shared(DataTypeDateTime64::default_scale); } - return arguments[0].type; + return input_type; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override @@ -115,50 +88,53 @@ public: const auto & input_type = arguments[0].type; if (isDate(input_type)) { - if constexpr (isTimeChange(Traits::EnumName)) + if (isTimeComponentChange(Traits::component)) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDate32(input_type)) { - if constexpr (isTimeChange(Traits::EnumName)) + if (isTimeComponentChange(Traits::component)) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDateTime(input_type)) return execute(arguments, input_type, result_type, input_rows_count); - return execute(arguments, input_type, result_type, input_rows_count); + if (isDateTime64(input_type)) + return execute(arguments, input_type, result_type, input_rows_count); + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid input type"); } - template + template ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & input_type, const DataTypePtr & result_type, size_t input_rows_count) const { bool is_const = (isColumnConst(*arguments[0].column) && isColumnConst(*arguments[1].column)); size_t result_rows_count = (is_const ? 1 : input_rows_count); typename ResultDataType::ColumnType::MutablePtr result_column; - if constexpr (isDateTime64()) + if constexpr (std::is_same_v) { auto scale = DataTypeDateTime64::default_scale; - if constexpr (isDateTime64()) + if constexpr (std::is_same_v) scale = typeid_cast(*result_type).getScale(); result_column = ResultDataType::ColumnType::create(result_rows_count, scale); } else result_column = ResultDataType::ColumnType::create(result_rows_count); - auto & result_data = result_column->getData(); - auto input_column = arguments[0].column->convertToFullIfNeeded(); - const auto & input_column_data = typeid_cast(*input_column).getData(); + const auto & input_column_data = typeid_cast(*input_column).getData(); auto new_value_column = castColumn(arguments[1], std::make_shared()); new_value_column = new_value_column->convertToFullIfNeeded(); const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + auto & result_data = result_column->getData(); + for (size_t i = 0; i < result_rows_count; ++i) { - if constexpr (isDateTime64()) + if constexpr (std::is_same_v) { const auto scale = typeid_cast(*result_type).getScale(); const auto & date_lut = typeid_cast(*result_type).getTimeZone(); @@ -172,21 +148,21 @@ public: result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, scale, fraction); } - else if constexpr (isDate32() && isDateTime64()) + else if constexpr (std::is_same_v && std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); } - else if constexpr (isDate() && isDateTime()) + else if constexpr (std::is_same_v && std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); } - else if constexpr (isDateTime()) + else if constexpr (std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); @@ -264,36 +240,36 @@ public: max_year = 2299; } - switch (Traits::EnumName) + switch (Traits::component) { - case ChangeDateFunctionsNames::CHANGE_YEAR: + case Component::Year: if (new_value < min_year) return min_date; else if (new_value > max_year) return max_date; year = static_cast(new_value); break; - case ChangeDateFunctionsNames::CHANGE_MONTH: + case Component::Month: if (new_value < 1 || new_value > 12) return min_date; month = static_cast(new_value); break; - case ChangeDateFunctionsNames::CHANGE_DAY: + case Component::Day: if (new_value < 1 || new_value > 31) return min_date; day = static_cast(new_value); break; - case ChangeDateFunctionsNames::CHANGE_HOUR: + case Component::Hour: if (new_value < 0 || new_value > 23) return min_date; hours = static_cast(new_value); break; - case ChangeDateFunctionsNames::CHANGE_MINUTE: + case Component::Minute: if (new_value < 0 || new_value > 59) return min_date; minutes = static_cast(new_value); break; - case ChangeDateFunctionsNames::CHANGE_SECOND: + case Component::Second: if (new_value < 0 || new_value > 59) return min_date; seconds = static_cast(new_value); @@ -324,38 +300,38 @@ public: struct ChangeYearTraits { - static constexpr auto Name = "changeYear"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_YEAR; + static constexpr auto name = "changeYear"; + static constexpr auto component = Component::Year; }; struct ChangeMonthTraits { - static constexpr auto Name = "changeMonth"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MONTH; + static constexpr auto name = "changeMonth"; + static constexpr auto component = Component::Month; }; struct ChangeDayTraits { - static constexpr auto Name = "changeDay"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_DAY; + static constexpr auto name = "changeDay"; + static constexpr auto component = Component::Day; }; struct ChangeHourTraits { - static constexpr auto Name = "changeHour"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_HOUR; + static constexpr auto name = "changeHour"; + static constexpr auto component = Component::Hour; }; struct ChangeMinuteTraits { - static constexpr auto Name = "changeMinute"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MINUTE; + static constexpr auto name = "changeMinute"; + static constexpr auto component = Component::Minute; }; struct ChangeSecondTraits { - static constexpr auto Name = "changeSecond"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_SECOND; + static constexpr auto name = "changeSecond"; + static constexpr auto component = Component::Second; }; } diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index 0d1bd75e130..26d53b2f7f4 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -39,4 +39,4 @@ SELECT changeHour(toDate('2000-01-01'), 24) SETTINGS session_timezone = 'Asia/No SELECT changeMinute(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeMinute(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeSecond(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; -SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; \ No newline at end of file +SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; From b1d3eb4c4d67e2e95d09dabf027ea9065751634d Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 19 May 2024 13:27:17 +0000 Subject: [PATCH 15/72] Some fixups, pt. II --- src/Functions/changeDate.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 6725afe3356..d17787f4f55 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -26,6 +26,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } From 4fdcdc284d6c41c3a1f97f6d72c7b9c73816a9df Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 19 May 2024 13:28:38 +0000 Subject: [PATCH 16/72] Some fixups, pt. III --- src/DataTypes/DataTypeDate.h | 1 - src/DataTypes/DataTypeDate32.h | 1 - src/DataTypes/DataTypeDateTime.h | 1 - 3 files changed, 3 deletions(-) diff --git a/src/DataTypes/DataTypeDate.h b/src/DataTypes/DataTypeDate.h index 72b7ef2509f..0e08b9ba2ca 100644 --- a/src/DataTypes/DataTypeDate.h +++ b/src/DataTypes/DataTypeDate.h @@ -10,7 +10,6 @@ class DataTypeDate final : public DataTypeNumberBase { public: static constexpr auto family_name = "Date"; - static constexpr auto type_id = TypeIndex::Date; TypeIndex getTypeId() const override { return TypeIndex::Date; } TypeIndex getColumnType() const override { return TypeIndex::UInt16; } diff --git a/src/DataTypes/DataTypeDate32.h b/src/DataTypes/DataTypeDate32.h index 414c8301558..65633e7a228 100644 --- a/src/DataTypes/DataTypeDate32.h +++ b/src/DataTypes/DataTypeDate32.h @@ -9,7 +9,6 @@ class DataTypeDate32 final : public DataTypeNumberBase { public: static constexpr auto family_name = "Date32"; - static constexpr auto type_id = TypeIndex::Date32; TypeIndex getTypeId() const override { return TypeIndex::Date32; } TypeIndex getColumnType() const override { return TypeIndex::Int32; } diff --git a/src/DataTypes/DataTypeDateTime.h b/src/DataTypes/DataTypeDateTime.h index 3b1212d910d..5519240dee1 100644 --- a/src/DataTypes/DataTypeDateTime.h +++ b/src/DataTypes/DataTypeDateTime.h @@ -36,7 +36,6 @@ public: explicit DataTypeDateTime(const TimezoneMixin & time_zone); static constexpr auto family_name = "DateTime"; - static constexpr auto type_id = TypeIndex::DateTime; const char * getFamilyName() const override { return family_name; } String doGetName() const override; From ba1e0e0317e9b408bfeafaac3fdda87eb16296fb Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Mon, 20 May 2024 11:43:31 +0000 Subject: [PATCH 17/72] added docks + updated tests --- .../functions/date-time-functions.md | 85 +++++++++++++++++++ .../0_stateless/02982_changeDate.reference | 17 ++++ .../queries/0_stateless/02982_changeDate.sql | 26 +++++- 3 files changed, 127 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 5622097537e..1ce6fce6667 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -3039,3 +3039,88 @@ Result: - Blog: [Working with time series data in ClickHouse](https://clickhouse.com/blog/working-with-time-series-data-and-functions-ClickHouse) +## changeYear + +Changes a year of the passed date argument. + +**Syntax** + +``` sql +changeYear(date, new_value) +``` + +**Arguments** + +- `date` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) + +- `new_value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). + +**Return value** + +- A date or date with time. Same data type as input date argument. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeYear(toDate('1999-01-01'), 2000), hangeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); +``` + +Result: +``` +┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ +│ 2000-01-01 │ 2000-01-01 00:00:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + +## changeMonth + +Like [changeYear](#changeYear) but changes a month of the passed date argument. + +## changeMonth + +Like [changeYear](#changeYear) but changes a day of year of the passed date argument. + +## changeHour + +Changes an hour of the passed date argument. + +**Syntax** + +``` sql +changeHour(date, new_value) +``` + +**Arguments** + +- `date` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) + +- `new_value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). + +**Return value** + +- A date with time. If date argument is Date - returns DateTime, if Date32 - returns DateTime64, otherwise returns same data type as input date argument. + +Type: [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Exapmle** + +``` sql +SELECT changeHour(toDate('1999-01-01'), 12), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 12); +``` + +Result: +``` +┌─changeHour(toDate('1999-01-01'), 12)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 12)─┐ +│ 1999-01-01 12:00:00 │ 1999-01-01 12:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeMinute + +Like [changeHour](#changeHour) but changes a minute of the passed date argument. + +## changeSecond + +Like [changeHour](#changeHour) but changes a seconds of the passed date argument. \ No newline at end of file diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference index c64abc89ed2..3e8ad45d9a8 100644 --- a/tests/queries/0_stateless/02982_changeDate.reference +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -34,3 +34,20 @@ 1970-01-01 07:00:00 1970-01-01 07:00:00 1970-01-01 07:00:00 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2299-12-31 +1900-01-01 +2106-02-07 13:28:15 +1970-01-01 07:00:00 +2299-12-31 23:59:59.999 +1900-01-01 00:00:00.000 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index 0d1bd75e130..1a9d062bafe 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -39,4 +39,28 @@ SELECT changeHour(toDate('2000-01-01'), 24) SETTINGS session_timezone = 'Asia/No SELECT changeMinute(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeMinute(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeSecond(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; -SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; \ No newline at end of file +SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; + +SELECT changeYear(toDate('2000-01-01')); -- { serverError 42 } +SELECT changeYear(toDate('2000-01-01'), 2001, 2002); -- { serverError 42 } +SELECT changeYear(toDate('2000-01-01'), '2001'); -- { serverError 43 } + +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int8)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int16)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int32)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int64)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt8)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt16)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt32)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt64)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Float32)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Float64)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Decimal(10, 5))); + +SELECT changeYear(toDate32('2000-01-01'), 2300); +SELECT changeYear(toDate32('2000-01-01'), 1899); +SELECT changeSecond(toDateTime('2106-02-07 13:28:15'), 16) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeHour(toDateTime('1970-01-01 23:59:59'), 6) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeYear(toDateTime64('2000-01-01 00:00:00.000', 3), 2300) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeYear(toDateTime64('2000-01-01 00:00:00.000', 3), 1899) SETTINGS session_timezone = 'Asia/Novosibirsk'; + From d115adf462859fd82a5bb5a83a6dedaeae799a08 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 20 May 2024 15:02:51 +0000 Subject: [PATCH 18/72] Some fixups, pt. IV --- .../functions/date-time-functions.md | 180 ++++++++++++--- src/Functions/changeDate.cpp | 126 ++++++----- .../0_stateless/02982_changeDate.reference | 181 +++++++++++++-- .../queries/0_stateless/02982_changeDate.sql | 207 +++++++++++++++--- 4 files changed, 546 insertions(+), 148 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 43bb5b6f6cb..8bf301d76c2 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -4104,33 +4104,33 @@ timeDiff(toDateTime64('1927-01-01 00:00:00', 3), toDate32('1927-01-02')); ## changeYear -Changes a year of the passed date argument. +Changes the year component of a date or date time. **Syntax** ``` sql -changeYear(date, new_value) +changeYear(date_or_datetime, value) ``` **Arguments** -- `date` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) - -- `new_value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md). **Return value** -- A date or date with time. Same data type as input date argument. +- The same type as `date_or_datetime`. Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). **Example** ``` sql - SELECT changeYear(toDate('1999-01-01'), 2000), hangeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); + SELECT changeYear(toDate('1999-01-01'), 2000), changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); ``` Result: + ``` ┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ │ 2000-01-01 │ 2000-01-01 00:00:00.000 │ @@ -4139,51 +4139,175 @@ Result: ## changeMonth -Like [changeYear](#changeYear) but changes a month of the passed date argument. - -## changeMonth - -Like [changeYear](#changeYear) but changes a day of year of the passed date argument. - -## changeHour - -Changes an hour of the passed date argument. +Changes the month component of a date or date time. **Syntax** ``` sql -changeHour(date, new_value) +changeMonth(date_or_datetime, value) ``` **Arguments** -- `date` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) - -- `new_value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the month. [Integer](../../sql-reference/data-types/int-uint.md). **Return value** -- A date with time. If date argument is Date - returns DateTime, if Date32 - returns DateTime64, otherwise returns same data type as input date argument. +- The same type as `date_or_datetime`. -Type: [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). -**Exapmle** +**Example** ``` sql -SELECT changeHour(toDate('1999-01-01'), 12), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 12); + SELECT changeMonth(toDate('1999-01-01'), 2), changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2); ``` Result: + ``` -┌─changeHour(toDate('1999-01-01'), 12)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 12)─┐ -│ 1999-01-01 12:00:00 │ 1999-01-01 12:00:00.000 │ +┌─changeMonth(toDate('1999-01-01'), 2)─┬─changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2)─┐ +│ 1999-02-01 │ 1999-02-01 00:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeDay + +Changes the day component of a date or date time. + +**Syntax** + +``` sql +changeDay(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the day. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeDay(toDate('1999-01-01'), 5), changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5); +``` + +Result: + +``` +┌─changeDay(toDate('1999-01-01'), 5)─┬─changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5)─┐ +│ 1999-01-05 │ 1999-01-05 00:00:00.000 │ +└────────────────────────────────────┴──────────────────────────────────────────────────────────┘ +``` + +## changeHour + +Changes the hour component of a date or date time. + +**Syntax** + +``` sql +changeHour(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeHour(toDate('1999-01-01'), 14), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14); +``` + +Result: + +``` +┌─changeHour(toDate('1999-01-01'), 14)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14)─┐ +│ 1999-01-01 14:00:00 │ 1999-01-01 14:00:00.000 │ └──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ ``` ## changeMinute -Like [changeHour](#changeHour) but changes a minute of the passed date argument. +Changes the minute component of a date or date time. + +**Syntax** + +``` sql +changeMinute(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the minute. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeMinute(toDate('1999-01-01'), 15), changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeMinute(toDate('1999-01-01'), 15)─┬─changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:15:00 │ 1999-01-01 00:15:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` ## changeSecond -Like [changeHour](#changeHour) but changes a seconds of the passed date argument. \ No newline at end of file +Changes the second component of a date or date time. + +**Syntax** + +``` sql +changeSecond(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the second. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeSecond(toDate('1999-01-01'), 15), changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeSecond(toDate('1999-01-01'), 15)─┬─changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:00:15 │ 1999-01-01 00:00:15.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index d17787f4f55..bd73154a6f1 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -45,9 +45,9 @@ enum class Component bool isTimeComponentChange(Component type) { - return type == Component::Hour || - type == Component::Minute || - type == Component::Second; + return type == Component::Hour || type == Component::Minute || type == Component::Second; +} + } template @@ -63,13 +63,11 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() != 2) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires 2 parameters: date, new_value. Passed {}.", getName(), arguments.size()); - - if (!isDateOrDate32OrDateTimeOrDateTime64(*arguments[0].type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be Date, Date32, DateTime or DateTime64", getName()); - if (!isNumber(*arguments[1].type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Second argument for function {} must be numeric", getName()); + FunctionArgumentDescriptors args{ + {"date_or_datetime", static_cast(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date or date with time"}, + {"value", static_cast(&isNativeInteger), nullptr, "Integer"} + }; + validateFunctionArgumentTypes(*this, arguments, args); const auto & input_type = arguments[0].type; @@ -335,62 +333,62 @@ struct ChangeSecondTraits static constexpr auto component = Component::Second; }; -} - REGISTER_FUNCTION(ChangeDate) { - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Changes the year of the given Date(32) or DateTime(64). -Returns the same type as the input data. -)", - .categories{"Dates and Times"} - } - ); - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Same as changeYear function, but changes month of the date. -)", - .categories{"Dates and Times"} - } - ); - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Same as changeYear function, but changes day_of_month of the date. -)", - .categories{"Dates and Times"} - } - ); - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Changes the hour of the given Date(32) or DateTime(64). -If the input data is Date, return DateTime; -if the input data is Date32, return DateTime64; -In other cases returns the same type as the input data. -)", - .categories{"Dates and Times"} - } - ); - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Same as changeHour function, but changes minute of the date. -)", - .categories{"Dates and Times"} - } - ); - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Same as changeHour function, but changes seconds of the date. -)", - .categories{"Dates and Times"} - } - ); + { + FunctionDocumentation::Description description = "Changes the year component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeYear(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } + { + FunctionDocumentation::Description description = "Changes the month component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeMonth(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } + { + FunctionDocumentation::Description description = "Changes the day component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeDay(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } + { + FunctionDocumentation::Description description = "Changes the hour component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeHour(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } + { + FunctionDocumentation::Description description = "Changes the minute component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeMinute(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } + { + FunctionDocumentation::Description description = "Changes the second component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeSecond(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } } } diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference index 3e8ad45d9a8..8ce647481bb 100644 --- a/tests/queries/0_stateless/02982_changeDate.reference +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -1,37 +1,170 @@ +Negative tests +changeYear +-- Date 2001-01-01 -2002-01-01 -2003-01-01 11:22:33 -2004-01-01 11:22:33.4444 +1970-01-01 +1970-01-01 +2149-06-06 +-- Date32 +2001-01-01 +1900-01-01 +1900-01-01 +2299-12-31 +-- DateTime +2001-01-01 11:22:33 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +2106-02-07 07:28:15 +-- DateTime64 +2001-01-01 11:22:33.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +2299-12-31 23:59:59.9999 +changeMonth +-- Date +2000-01-01 2000-02-01 -2000-03-01 -2000-04-01 11:22:33 -2000-05-01 11:22:33.4444 +2000-12-01 +1970-01-01 +1970-01-01 +1970-01-01 +-- Date32 +2000-01-01 +2000-02-01 +2000-12-01 +1900-01-01 +1900-01-01 +1900-01-01 +-- DateTime +2000-01-01 11:22:33 +2000-02-01 11:22:33 +2000-12-01 11:22:33 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- DateTime64 +2000-01-01 11:22:33.4444 +2000-02-01 11:22:33.4444 +2000-12-01 11:22:33.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +changeDay +-- Date +2000-01-01 2000-01-02 -2000-01-03 -2000-01-04 11:22:33 -2000-01-05 11:22:33.4444 -2000-01-01 12:00:00 -2000-01-01 13:00:00.000 -2000-01-01 14:22:33 -2000-01-01 15:22:33.4444 -2000-01-01 00:23:00 -2000-01-01 00:24:00.000 -2000-01-01 11:25:33 -2000-01-01 11:26:33.4444 -2000-01-01 00:00:34 -2000-01-01 00:00:35.000 -2000-01-01 11:22:36 -2000-01-01 11:22:37.4444 -1970-01-01 -2149-06-06 -2149-06-06 +2000-01-31 1970-01-01 1970-01-01 1970-01-01 +-- Date32 +2000-01-01 +2000-01-02 +2000-01-31 +1900-01-01 +1900-01-01 +1900-01-01 +-- DateTime +2000-01-01 11:22:33 +2000-01-02 11:22:33 +2000-01-31 11:22:33 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- DateTime64 +2000-01-01 11:22:33.4444 +2000-01-02 11:22:33.4444 +2000-01-31 11:22:33.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +-- Special case: change to 29 Feb in a leap year +2000-02-29 +2000-02-29 +2000-02-29 11:22:33 +2000-02-29 11:22:33.4444 +changeHour +-- Date +2000-01-01 00:00:00 +2000-01-01 02:00:00 +2000-01-01 23:00:00 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- Date32 +2000-01-01 00:00:00.000 +2000-01-01 02:00:00.000 +2000-01-01 23:00:00.000 +1900-01-01 00:00:00.000 +1900-01-01 00:00:00.000 +-- DateTime +2000-01-01 00:22:33 +2000-01-01 02:22:33 +2000-01-01 23:22:33 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- DateTime64 +2000-01-01 00:22:33.4444 +2000-01-01 02:22:33.4444 +2000-01-01 23:22:33.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +-- With different timezone 1970-01-01 07:00:00 1970-01-01 07:00:00 +changeMinute +-- Date +2000-01-01 00:00:00 +2000-01-01 00:02:00 +2000-01-01 00:59:00 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- Date32 +2000-01-01 00:00:00.000 +2000-01-01 00:02:00.000 +2000-01-01 00:59:00.000 +1900-01-01 00:00:00.000 +1900-01-01 00:00:00.000 +-- DateTime +2000-01-01 11:00:33 +2000-01-01 11:02:33 +2000-01-01 11:59:33 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- DateTime64 +2000-01-01 11:00:33.4444 +2000-01-01 11:02:33.4444 +2000-01-01 11:59:33.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +-- With different timezone 1970-01-01 07:00:00 1970-01-01 07:00:00 +changeSecond +-- Date +2000-01-01 00:00:00 +2000-01-01 00:00:02 +2000-01-01 00:00:59 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- Date32 +2000-01-01 00:00:00.000 +2000-01-01 00:00:02.000 +2000-01-01 00:00:59.000 +1900-01-01 00:00:00.000 +1900-01-01 00:00:00.000 +-- DateTime +2000-01-01 11:22:00 +2000-01-01 11:22:02 +2000-01-01 11:22:59 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- DateTime64 +2000-01-01 11:22:00.4444 +2000-01-01 11:22:02.4444 +2000-01-01 11:22:59.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +-- With different timezone 1970-01-01 07:00:00 1970-01-01 07:00:00 2000-02-01 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index dec95c2d79d..62232079d61 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -1,43 +1,186 @@ +SELECT 'Negative tests'; +-- as changeYear, changeMonth, changeDay, changeMinute, changeSecond share the same implementation, just testing one of them +SELECT changeYear(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT changeYear(toDate('2000-01-01')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT changeYear(toDate('2000-01-01'), 2000, 1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT changeYear(1999, 2000); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT changeYear(toDate('2000-01-01'), 'abc'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT changeYear(toDate('2000-01-01'), 1.5); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +-- Disable timezone randomization +SET session_timezone='CET'; + +SELECT 'changeYear'; +SELECT '-- Date'; SELECT changeYear(toDate('2000-01-01'), 2001); -SELECT changeYear(toDate32('2000-01-01'), 2002); -SELECT changeYear(toDateTime('2000-01-01 11:22:33'), 2003); -SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), 2004); +SELECT changeYear(toDate('2000-01-01'), 1800); -- out-of-bounds +SELECT changeYear(toDate('2000-01-01'), -5000); -- out-of-bounds +SELECT changeYear(toDate('2000-01-01'), 2500); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeYear(toDate32('2000-01-01'), 2001); +SELECT changeYear(toDate32('2000-01-01'), 1800); -- out-of-bounds +SELECT changeYear(toDate32('2000-01-01'), -5000); -- out-of-bounds +SELECT changeYear(toDate32('2000-01-01'), 2500); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeYear(toDateTime('2000-01-01 11:22:33'), 2001); +SELECT changeYear(toDateTime('2000-01-01 11:22:33'), 1800); -- out-of-bounds +SELECT changeYear(toDateTime('2000-01-01 11:22:33'), -5000); -- out-of-bounds +SELECT changeYear(toDateTime('2000-01-01 11:22:33'), 2500); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), 2001); +SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), 1800); -- out-of-bounds +SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), -5000); -- out-of-bounds +SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), 2500); -- out-of-bounds -SELECT changeMonth(toDate('2000-01-01'), 02); -SELECT changeMonth(toDate32('2000-01-01'), 03); -SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 04); -SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 05); +SELECT 'changeMonth'; +SELECT '-- Date'; +SELECT changeMonth(toDate('2000-01-01'), 1); +SELECT changeMonth(toDate('2000-01-01'), 2); +SELECT changeMonth(toDate('2000-01-01'), 12); +SELECT changeMonth(toDate('2000-01-01'), 0); -- out-of-bounds +SELECT changeMonth(toDate('2000-01-01'), -1); -- out-of-bounds +SELECT changeMonth(toDate('2000-01-01'), 13); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeMonth(toDate32('2000-01-01'), 1); +SELECT changeMonth(toDate32('2000-01-01'), 2); +SELECT changeMonth(toDate32('2000-01-01'), 12); +SELECT changeMonth(toDate32('2000-01-01'), 0); -- out-of-bounds +SELECT changeMonth(toDate32('2000-01-01'), -1); -- out-of-bounds +SELECT changeMonth(toDate32('2000-01-01'), 13); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 1); +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 2); +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 12); +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 0); -- out-of-bounds +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), -1); -- out-of-bounds +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 13); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 1); +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 2); +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 12); +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 0); -- out-of-bounds +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), -1); -- out-of-bounds +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 13); -- out-of-bounds -SELECT changeDay(toDate('2000-01-01'), 02); -SELECT changeDay(toDate32('2000-01-01'), 03); -SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 04); -SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 05); +SELECT 'changeDay'; +SELECT '-- Date'; +SELECT changeDay(toDate('2000-01-01'), 1); +SELECT changeDay(toDate('2000-01-01'), 2); +SELECT changeDay(toDate('2000-01-01'), 31); +SELECT changeDay(toDate('2000-01-01'), 0); -- out-of-bounds +SELECT changeDay(toDate('2000-01-01'), -1); -- out-of-bounds +SELECT changeDay(toDate('2000-01-01'), 32); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeDay(toDate32('2000-01-01'), 1); +SELECT changeDay(toDate32('2000-01-01'), 2); +SELECT changeDay(toDate32('2000-01-01'), 31); +SELECT changeDay(toDate32('2000-01-01'), 0); -- out-of-bounds +SELECT changeDay(toDate32('2000-01-01'), -1); -- out-of-bounds +SELECT changeDay(toDate32('2000-01-01'), 32); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 1); +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 2); +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 31); +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 0); -- out-of-bounds +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), -1); -- out-of-bounds +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 32); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 1); +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 2); +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 31); +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 0); -- out-of-bounds +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), -1); -- out-of-bounds +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 32); -- out-of-bounds +SELECT '-- Special case: change to 29 Feb in a leap year'; +SELECT changeDay(toDate('2000-02-28'), 29); +SELECT changeDay(toDate32('2000-02-01'), 29); +SELECT changeDay(toDateTime('2000-02-01 11:22:33'), 29); +SELECT changeDay(toDateTime64('2000-02-01 11:22:33.4444', 4), 29); -SELECT changeHour(toDate('2000-01-01'), 12); -SELECT changeHour(toDate32('2000-01-01'), 13); -SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 14); -SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 15); - -SELECT changeMinute(toDate('2000-01-01'), 23); -SELECT changeMinute(toDate32('2000-01-01'), 24); -SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 25); -SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 26); - -SELECT changeSecond(toDate('2000-01-01'), 34); -SELECT changeSecond(toDate32('2000-01-01'), 35); -SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 36); -SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 37); - -SELECT changeYear(toDate('2000-01-01'), 1969.0); -SELECT changeYear(toDate('2000-06-07'), 2149.0); -SELECT changeMonth(toDate('2149-01-01'), 07); -SELECT changeMonth(toDate('2000-01-01'), 13); -SELECT changeDay(toDate('2000-01-01'), 0); -SELECT changeDay(toDate('2000-01-01'), 32); +SELECT 'changeHour'; +SELECT '-- Date'; +SELECT changeHour(toDate('2000-01-01'), 0); +SELECT changeHour(toDate('2000-01-01'), 2); +SELECT changeHour(toDate('2000-01-01'), 23); +SELECT changeHour(toDate('2000-01-01'), -1); -- out-of-bounds +SELECT changeHour(toDate('2000-01-01'), 24); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeHour(toDate32('2000-01-01'), 0); +SELECT changeHour(toDate32('2000-01-01'), 2); +SELECT changeHour(toDate32('2000-01-01'), 23); +SELECT changeHour(toDate32('2000-01-01'), -1); -- out-of-bounds +SELECT changeHour(toDate32('2000-01-01'), 24); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 0); +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 2); +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 23); +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), -1); -- out-of-bounds +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 24); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 0); +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 2); +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 23); +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), -1); -- out-of-bounds +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 24); -- out-of-bounds +SELECT '-- With different timezone'; SELECT changeHour(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeHour(toDate('2000-01-01'), 24) SETTINGS session_timezone = 'Asia/Novosibirsk'; + +SELECT 'changeMinute'; +SELECT '-- Date'; +SELECT changeMinute(toDate('2000-01-01'), 0); +SELECT changeMinute(toDate('2000-01-01'), 2); +SELECT changeMinute(toDate('2000-01-01'), 59); +SELECT changeMinute(toDate('2000-01-01'), -1); -- out-of-bounds +SELECT changeMinute(toDate('2000-01-01'), 60); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeMinute(toDate32('2000-01-01'), 0); +SELECT changeMinute(toDate32('2000-01-01'), 2); +SELECT changeMinute(toDate32('2000-01-01'), 59); +SELECT changeMinute(toDate32('2000-01-01'), -1); -- out-of-bounds +SELECT changeMinute(toDate32('2000-01-01'), 60); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 0); +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 2); +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 59); +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), -1); -- out-of-bounds +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 60); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 0); +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 2); +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 59); +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), -1); -- out-of-bounds +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 60); -- out-of-bounds +SELECT '-- With different timezone'; SELECT changeMinute(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeMinute(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; + +SELECT 'changeSecond'; +SELECT '-- Date'; +SELECT changeSecond(toDate('2000-01-01'), 0); +SELECT changeSecond(toDate('2000-01-01'), 2); +SELECT changeSecond(toDate('2000-01-01'), 59); +SELECT changeSecond(toDate('2000-01-01'), -1); -- out-of-bounds +SELECT changeSecond(toDate('2000-01-01'), 60); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeSecond(toDate32('2000-01-01'), 0); +SELECT changeSecond(toDate32('2000-01-01'), 2); +SELECT changeSecond(toDate32('2000-01-01'), 59); +SELECT changeSecond(toDate32('2000-01-01'), -1); -- out-of-bounds +SELECT changeSecond(toDate32('2000-01-01'), 60); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 0); +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 2); +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 59); +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), -1); -- out-of-bounds +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 60); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 0); +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 2); +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 59); +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), -1); -- out-of-bounds +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 60); -- out-of-bounds +SELECT '-- With different timezone'; SELECT changeSecond(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; From 6ff3cf82819e61020f3483687934a70ade67be67 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 20 May 2024 15:36:25 +0000 Subject: [PATCH 19/72] Some fixups, pt. V --- .../functions/date-time-functions.md | 419 +++++++++--------- 1 file changed, 210 insertions(+), 209 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 8bf301d76c2..7a39cc74372 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -3520,6 +3520,216 @@ Result: └───────────────────────────────────────────────────────────────────────┘ ``` +## changeYear + +Changes the year component of a date or date time. + +**Syntax** + +``` sql +changeYear(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeYear(toDate('1999-01-01'), 2000), changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); +``` + +Result: + +``` +┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ +│ 2000-01-01 │ 2000-01-01 00:00:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + +## changeMonth + +Changes the month component of a date or date time. + +**Syntax** + +``` sql +changeMonth(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the month. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeMonth(toDate('1999-01-01'), 2), changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2); +``` + +Result: + +``` +┌─changeMonth(toDate('1999-01-01'), 2)─┬─changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2)─┐ +│ 1999-02-01 │ 1999-02-01 00:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeDay + +Changes the day component of a date or date time. + +**Syntax** + +``` sql +changeDay(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the day. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeDay(toDate('1999-01-01'), 5), changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5); +``` + +Result: + +``` +┌─changeDay(toDate('1999-01-01'), 5)─┬─changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5)─┐ +│ 1999-01-05 │ 1999-01-05 00:00:00.000 │ +└────────────────────────────────────┴──────────────────────────────────────────────────────────┘ +``` + +## changeHour + +Changes the hour component of a date or date time. + +**Syntax** + +``` sql +changeHour(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeHour(toDate('1999-01-01'), 14), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14); +``` + +Result: + +``` +┌─changeHour(toDate('1999-01-01'), 14)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14)─┐ +│ 1999-01-01 14:00:00 │ 1999-01-01 14:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeMinute + +Changes the minute component of a date or date time. + +**Syntax** + +``` sql +changeMinute(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the minute. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeMinute(toDate('1999-01-01'), 15), changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeMinute(toDate('1999-01-01'), 15)─┬─changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:15:00 │ 1999-01-01 00:15:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + +## changeSecond + +Changes the second component of a date or date time. + +**Syntax** + +``` sql +changeSecond(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the second. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeSecond(toDate('1999-01-01'), 15), changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeSecond(toDate('1999-01-01'), 15)─┬─changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:00:15 │ 1999-01-01 00:00:15.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + ## timeSlots(StartTime, Duration,\[, Size\]) For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. @@ -4102,212 +4312,3 @@ timeDiff(toDateTime64('1927-01-01 00:00:00', 3), toDate32('1927-01-02')); - Blog: [Working with time series data in ClickHouse](https://clickhouse.com/blog/working-with-time-series-data-and-functions-ClickHouse) -## changeYear - -Changes the year component of a date or date time. - -**Syntax** - -``` sql -changeYear(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeYear(toDate('1999-01-01'), 2000), changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); -``` - -Result: - -``` -┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ -│ 2000-01-01 │ 2000-01-01 00:00:00.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` - -## changeMonth - -Changes the month component of a date or date time. - -**Syntax** - -``` sql -changeMonth(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the month. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeMonth(toDate('1999-01-01'), 2), changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2); -``` - -Result: - -``` -┌─changeMonth(toDate('1999-01-01'), 2)─┬─changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2)─┐ -│ 1999-02-01 │ 1999-02-01 00:00:00.000 │ -└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ -``` - -## changeDay - -Changes the day component of a date or date time. - -**Syntax** - -``` sql -changeDay(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the day. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeDay(toDate('1999-01-01'), 5), changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5); -``` - -Result: - -``` -┌─changeDay(toDate('1999-01-01'), 5)─┬─changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5)─┐ -│ 1999-01-05 │ 1999-01-05 00:00:00.000 │ -└────────────────────────────────────┴──────────────────────────────────────────────────────────┘ -``` - -## changeHour - -Changes the hour component of a date or date time. - -**Syntax** - -``` sql -changeHour(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeHour(toDate('1999-01-01'), 14), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14); -``` - -Result: - -``` -┌─changeHour(toDate('1999-01-01'), 14)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14)─┐ -│ 1999-01-01 14:00:00 │ 1999-01-01 14:00:00.000 │ -└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ -``` - -## changeMinute - -Changes the minute component of a date or date time. - -**Syntax** - -``` sql -changeMinute(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the minute. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeMinute(toDate('1999-01-01'), 15), changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15); -``` - -Result: - -``` -┌─changeMinute(toDate('1999-01-01'), 15)─┬─changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ -│ 1999-01-01 00:15:00 │ 1999-01-01 00:15:00.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` - -## changeSecond - -Changes the second component of a date or date time. - -**Syntax** - -``` sql -changeSecond(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the second. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeSecond(toDate('1999-01-01'), 15), changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15); -``` - -Result: - -``` -┌─changeSecond(toDate('1999-01-01'), 15)─┬─changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ -│ 1999-01-01 00:00:15 │ 1999-01-01 00:00:15.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` From b47524a9a5d5c379227da1ac290938f42d2b0586 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 21 May 2024 05:19:49 +0000 Subject: [PATCH 20/72] Fix spelling --- .../aspell-ignore/en/aspell-dict.txt | 116 +++++++++--------- 1 file changed, 61 insertions(+), 55 deletions(-) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 2a9aa259fdd..ade822f508a 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -29,13 +29,6 @@ Alexey AnyEvent AppleClang Approximative -arrayDotProduct -arrayEnumerateDenseRanked -arrayEnumerateUniqRanked -arrayFirstOrNull -arrayLastOrNull -arrayPartialShuffle -arrayShuffle ArrayJoin ArrowStream AsyncInsertCacheSize @@ -184,7 +177,6 @@ ComplexKeyCache ComplexKeyDirect ComplexKeyHashed Composable -composable Config ConnectionDetails Const @@ -394,8 +386,6 @@ InterserverThreads IsPentagon IsResClassIII IsValid -isNotDistinctFrom -isNullable JBOD JOINed JOINs @@ -464,8 +454,6 @@ KittenHouse Klickhouse Kolmogorov Konstantin -kostik -kostikConsistentHash Korzeniewski Kubernetes LDAP @@ -475,9 +463,8 @@ LLDB LLVM's LOCALTIME LOCALTIMESTAMP -LOONGARCH LONGLONG -LoongArch +LOONGARCH Levenshtein Liao LibFuzzer @@ -495,6 +482,7 @@ LocalThreadActive LogQL Logstash LookML +LoongArch LowCardinality LpDistance LpNorm @@ -569,17 +557,6 @@ MindsDB Mongodb Monotonicity MsgPack -multiSearchAllPositionsCaseInsensitive -multiSearchAllPositionsCaseInsensitiveUTF -multiSearchAnyCaseInsensitive -multiSearchAnyCaseInsensitiveUTF -multiSearchAnyUTF -multiSearchFirstIndexCaseInsensitive -multiSearchFirstIndexCaseInsensitiveUTF -multiSearchFirstIndexUTF -multiSearchFirstPositionCaseInsensitive -multiSearchFirstPositionCaseInsensitiveUTF -multiSearchFirstPositionUTF MultiPolygon Multiline Multiqueries @@ -681,8 +658,8 @@ OSUserTimeNormalized OTLP OUTFILE ObjectId -Observability Oblakov +Observability Octonica Ok OnTime @@ -883,7 +860,6 @@ Simhash SimpleAggregateFunction SimpleState SipHash -sigmoid Smirnov's Smirnov'test Soundex @@ -929,7 +905,6 @@ TAVG TCPConnection TCPThreads TDigest -ThreadMonotonic TINYINT TLSv TMAX @@ -955,7 +930,6 @@ TablesLoaderForegroundThreads TablesLoaderForegroundThreadsActive TablesToDropQueueSize TargetSpecific -tanh Telegraf TemplateIgnoreSpaces TemporaryFilesForAggregation @@ -965,6 +939,7 @@ TemporaryFilesUnknown Testflows Tgz Theil's +ThreadMonotonic ThreadPoolFSReaderThreads ThreadPoolFSReaderThreadsActive ThreadPoolRemoteFSReaderThreads @@ -1025,7 +1000,6 @@ UncompressedCacheBytes UncompressedCacheCells UnidirectionalEdgeIsValid UniqThetaSketch -unshuffled Updatable Uppercased Uptime @@ -1092,6 +1066,7 @@ activerecord addDate addDays addHours +addInterval addMicroseconds addMilliseconds addMinutes @@ -1099,10 +1074,9 @@ addMonths addNanoseconds addQuarters addSeconds +addTupleOfIntervals addWeeks addYears -addInterval -addTupleOfIntervals addr addressToLine addressToLineWithInlines @@ -1144,15 +1118,19 @@ arrayCumSum arrayCumSumNonNegative arrayDifference arrayDistinct +arrayDotProduct arrayElement arrayEnumerate arrayEnumerateDense +arrayEnumerateDenseRanked arrayEnumerateUniq +arrayEnumerateUniqRanked arrayExists arrayFill arrayFilter arrayFirst arrayFirstIndex +arrayFirstOrNull arrayFlatten arrayFold arrayIntersect @@ -1160,10 +1138,12 @@ arrayJaccardIndex arrayJoin arrayLast arrayLastIndex +arrayLastOrNull arrayMap arrayMax arrayMin arrayPartialReverseSort +arrayPartialShuffle arrayPartialSort arrayPopBack arrayPopFront @@ -1183,6 +1163,7 @@ arrayRotateRight arrayShiftLeft arrayShiftRight arrayShingles +arrayShuffle arraySlice arraySort arraySplit @@ -1323,6 +1304,12 @@ cfg cgroup cgroups chadmin +changeDay +changeHour +changeMinute +changeMonth +changeSecond +changeYear changelog changelogs charset @@ -1364,6 +1351,7 @@ collapsingmergetree combinator combinators comparising +composable compressability concat concatAssumeInjective @@ -1725,8 +1713,8 @@ hasSubsequenceCaseInsensitive hasSubsequenceCaseInsensitiveUTF hasSubsequenceUTF hasSubstr -hasToken hasThreadFuzzer +hasToken hasTokenCaseInsensitive hasTokenCaseInsensitiveOrNull hasTokenOrNull @@ -1799,8 +1787,10 @@ isIPAddressInRange isIPv isInfinite isNaN +isNotDistinctFrom isNotNull isNull +isNullable isValidJSON isValidUTF isZeroOrNull @@ -1852,6 +1842,8 @@ kolmogorovSmirnovTest kolmogorovsmirnovtest kolya konsole +kostik +kostikConsistentHash kurtPop kurtSamp kurtosis @@ -1863,9 +1855,9 @@ laravel largestTriangleThreeBuckets latencies ldap -leftUTF leftPad leftPadUTF +leftUTF lemmatization lemmatize lemmatized @@ -1912,8 +1904,8 @@ logTrace logagent loghouse london -loongarch lookups +loongarch lowcardinality lowerUTF lowercased @@ -1984,8 +1976,8 @@ mispredictions mmap mmapped modularization -moduloOrZero moduli +moduloOrZero mongodb monotonicity monthName @@ -2002,10 +1994,21 @@ multiMatchAllIndices multiMatchAny multiMatchAnyIndex multiSearchAllPositions +multiSearchAllPositionsCaseInsensitive +multiSearchAllPositionsCaseInsensitiveUTF multiSearchAllPositionsUTF multiSearchAny +multiSearchAnyCaseInsensitive +multiSearchAnyCaseInsensitiveUTF +multiSearchAnyUTF multiSearchFirstIndex +multiSearchFirstIndexCaseInsensitive +multiSearchFirstIndexCaseInsensitiveUTF +multiSearchFirstIndexUTF multiSearchFirstPosition +multiSearchFirstPositionCaseInsensitive +multiSearchFirstPositionCaseInsensitiveUTF +multiSearchFirstPositionUTF multibyte multidirectory multiline @@ -2340,8 +2343,8 @@ retentions rethrow retransmit retriable -rewritable reverseUTF +rewritable rightPad rightPadUTF rightUTF @@ -2401,8 +2404,9 @@ sharded sharding shortcircuit shortkeys -showCertificate shoutout +showCertificate +sigmoid simdjson simpleJSON simpleJSONExtractBool @@ -2416,8 +2420,8 @@ simpleLinearRegression simpleaggregatefunction simplelinearregression simpod -singlepart singleValueOrNull +singlepart singlevalueornull sinh sipHash @@ -2462,13 +2466,13 @@ statbox stateful stddev stddevPop -stddevSamp -stddevpop -stddevsamp -stddevpopstable stddevPopStable -stddevsampstable +stddevSamp stddevSampStable +stddevpop +stddevpopstable +stddevsamp +stddevsampstable stderr stdin stdout @@ -2529,6 +2533,7 @@ substrings subtitiles subtractDays subtractHours +subtractInterval subtractMicroseconds subtractMilliseconds subtractMinutes @@ -2536,10 +2541,9 @@ subtractMonths subtractNanoseconds subtractQuarters subtractSeconds +subtractTupleOfIntervals subtractWeeks subtractYears -subtractInterval -subtractTupleOfIntervals subtree subtrees subtype @@ -2548,13 +2552,13 @@ sumCount sumKahan sumMap sumMapFiltered +sumMapFilteredWithOverflow +sumMapWithOverflow sumWithOverflow sumcount sumkahan summap summapwithoverflow -sumMapWithOverflow -sumMapFilteredWithOverflow summingmergetree sumwithoverflow superaggregates @@ -2577,6 +2581,7 @@ tabseparatedrawwithnames tabseparatedrawwithnamesandtypes tabseparatedwithnames tabseparatedwithnamesandtypes +tanh tcp tcpPort tcpnodelay @@ -2711,18 +2716,18 @@ tupleDivide tupleDivideByNumber tupleElement tupleHammingDistance +tupleIntDiv +tupleIntDivByNumber +tupleIntDivOrZero +tupleIntDivOrZeroByNumber tupleMinus +tupleModulo +tupleModuloByNumber tupleMultiply tupleMultiplyByNumber tupleNegate tuplePlus tupleToNameValuePairs -tupleIntDiv -tupleIntDivByNumber -tupleIntDivOrZero -tupleIntDivOrZeroByNumber -tupleModulo -tupleModuloByNumber turbostat txt typename @@ -2765,6 +2770,7 @@ unrealiable unreplicated unresolvable unrounded +unshuffled untracked untrusted untuple @@ -2775,8 +2781,8 @@ uptime uptrace uring url -urlencoded urlCluster +urlencoded urls usearch userspace From c026a5b7e938b797d3f8cb016457dab38c002aab Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 21 May 2024 05:20:46 +0000 Subject: [PATCH 21/72] Fix style check --- src/Functions/changeDate.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index bd73154a6f1..9868ac5b914 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -23,13 +23,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int LOGICAL_ERROR; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - namespace { From 336e791ea87c8298ff584ae2de8a140f0947f02a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 21 May 2024 20:50:54 +0000 Subject: [PATCH 22/72] Fix style check, pt. II --- src/Functions/changeDate.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 9868ac5b914..b400680d272 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -23,6 +23,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace { From ab2baf5e9664133097025c149a93a09e84cca152 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 22 May 2024 12:23:14 +0000 Subject: [PATCH 23/72] Fix expected results --- .../0_stateless/02982_changeDate.reference | 17 -------------- .../queries/0_stateless/02982_changeDate.sql | 23 ------------------- 2 files changed, 40 deletions(-) diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference index 8ce647481bb..4a7f093ca2b 100644 --- a/tests/queries/0_stateless/02982_changeDate.reference +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -167,20 +167,3 @@ changeSecond -- With different timezone 1970-01-01 07:00:00 1970-01-01 07:00:00 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2299-12-31 -1900-01-01 -2106-02-07 13:28:15 -1970-01-01 07:00:00 -2299-12-31 23:59:59.999 -1900-01-01 00:00:00.000 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index 62232079d61..2bc9aa95569 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -183,26 +183,3 @@ SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 60); -- out-of- SELECT '-- With different timezone'; SELECT changeSecond(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; - -SELECT changeYear(toDate('2000-01-01')); -- { serverError 42 } -SELECT changeYear(toDate('2000-01-01'), 2001, 2002); -- { serverError 42 } -SELECT changeYear(toDate('2000-01-01'), '2001'); -- { serverError 43 } - -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int8)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int16)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int32)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int64)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt8)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt16)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt32)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt64)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Float32)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Float64)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Decimal(10, 5))); - -SELECT changeYear(toDate32('2000-01-01'), 2300); -SELECT changeYear(toDate32('2000-01-01'), 1899); -SELECT changeSecond(toDateTime('2106-02-07 13:28:15'), 16) SETTINGS session_timezone = 'Asia/Novosibirsk'; -SELECT changeHour(toDateTime('1970-01-01 23:59:59'), 6) SETTINGS session_timezone = 'Asia/Novosibirsk'; -SELECT changeYear(toDateTime64('2000-01-01 00:00:00.000', 3), 2300) SETTINGS session_timezone = 'Asia/Novosibirsk'; -SELECT changeYear(toDateTime64('2000-01-01 00:00:00.000', 3), 1899) SETTINGS session_timezone = 'Asia/Novosibirsk'; \ No newline at end of file From cdd99a73a0e46801bb01f47df49c47e67fd9bb6f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 23 May 2024 10:12:32 +0000 Subject: [PATCH 24/72] Fix clang-tidy --- src/Functions/changeDate.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index b400680d272..e24391afe12 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -274,15 +274,25 @@ public: } Int64 result; - if (isDateOrDate32(result_type)) + if (isDate(result_type) || isDate32(result_type)) result = date_lut.makeDayNum(year, month, day); else if (isDateTime(result_type)) result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); else +#ifndef __clang_analyzer__ + /// ^^ This looks funny. It is the least terrible suppression of a false positive reported by clang-analyzer (a sub-class + /// of clang-tidy checks) deep down in 'decimalFromComponents'. Usual suppressions of the form NOLINT* don't work here (they + /// would only affect code in _this_ file), and suppressing the issue in 'decimalFromComponents' may suppress true positives. result = DecimalUtils::decimalFromComponents( date_lut.makeDateTime(year, month, day, hours, minutes, seconds), - static_cast(fraction), + fraction, static_cast(scale)); +#else + { + UNUSED(fraction); + result = 0; + } +#endif if (result < min_date) return min_date; From 4f61f530bd02fec686b273f4c2519cde77a689f2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 2 Jul 2024 11:22:05 +0200 Subject: [PATCH 25/72] Named collections in clickhouse-local --- programs/local/LocalServer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index b33e1595056..74906d8797c 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -376,6 +376,7 @@ void LocalServer::setupUsers() " " " default" " default" + " 1 " " " " " " From e45a905904b5afd94e960bf48de015017a351527 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 2 Jul 2024 11:52:51 +0200 Subject: [PATCH 26/72] Update LocalServer.cpp --- programs/local/LocalServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 74906d8797c..46b543e49e9 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -376,7 +376,7 @@ void LocalServer::setupUsers() " " " default" " default" - " 1 + " 1" " " " " " " From e2885b1cfa976e9fe072e6453d8142364c013aa9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Jul 2024 02:16:51 +0200 Subject: [PATCH 27/72] Add a test --- .../03201_local_named_collections.reference | 1 + .../03201_local_named_collections.sh | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/queries/0_stateless/03201_local_named_collections.reference create mode 100755 tests/queries/0_stateless/03201_local_named_collections.sh diff --git a/tests/queries/0_stateless/03201_local_named_collections.reference b/tests/queries/0_stateless/03201_local_named_collections.reference new file mode 100644 index 00000000000..af5626b4a11 --- /dev/null +++ b/tests/queries/0_stateless/03201_local_named_collections.reference @@ -0,0 +1 @@ +Hello, world! diff --git a/tests/queries/0_stateless/03201_local_named_collections.sh b/tests/queries/0_stateless/03201_local_named_collections.sh new file mode 100755 index 00000000000..54ca76a52d9 --- /dev/null +++ b/tests/queries/0_stateless/03201_local_named_collections.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --multiquery " +DROP TABLE IF EXISTS test; +CREATE TABLE test (s String) ORDER BY (); +INSERT INTO test VALUES ('Hello, world!'); +" + +${CLICKHOUSE_LOCAL} --multiquery " +CREATE NAMED COLLECTION mydb AS host = '${CLICKHOUSE_HOST}', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', db = '${CLICKHOUSE_DATABASE}'; +SELECT * FROM remote(mydb, table = 'test'); +" + +${CLICKHOUSE_CLIENT} --multiquery " +DROP TABLE test; +" From 5f28c025ce39b1f9e9ea2e48b3cee78ad9a432be Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Jul 2024 12:51:29 +0200 Subject: [PATCH 28/72] Fix test (connections use coroutines) --- tests/queries/0_stateless/03201_local_named_collections.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03201_local_named_collections.sh b/tests/queries/0_stateless/03201_local_named_collections.sh index 54ca76a52d9..2054a09df06 100755 --- a/tests/queries/0_stateless/03201_local_named_collections.sh +++ b/tests/queries/0_stateless/03201_local_named_collections.sh @@ -13,7 +13,7 @@ INSERT INTO test VALUES ('Hello, world!'); ${CLICKHOUSE_LOCAL} --multiquery " CREATE NAMED COLLECTION mydb AS host = '${CLICKHOUSE_HOST}', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', db = '${CLICKHOUSE_DATABASE}'; SELECT * FROM remote(mydb, table = 'test'); -" +" | grep --text -F -v "ASan doesn't fully support makecontext/swapcontext functions" ${CLICKHOUSE_CLIENT} --multiquery " DROP TABLE test; From 6098bc20d90fc3ab45128f62db70624efa0c05dc Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 8 Jul 2024 10:49:46 -0300 Subject: [PATCH 29/72] remove test --- .../__init__.py | 0 .../configs/config.xml | 4 - .../configs/host_regexp.xml | 11 --- .../configs/listen_host.xml | 5 -- .../coredns_config/Corefile | 8 -- .../coredns_config/example.com | 1 - .../scripts/stress_test.py | 62 ------------- .../test.py | 88 ------------------- 8 files changed, 179 deletions(-) delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/__init__.py delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/__init__.py b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml deleted file mode 100644 index 42a1f962705..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml +++ /dev/null @@ -1,4 +0,0 @@ - - 1 - 250 - diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml deleted file mode 100644 index 9329c8dbde2..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - test1\.example\.com$ - - default - - - diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml deleted file mode 100644 index 9c27c612f63..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml +++ /dev/null @@ -1,5 +0,0 @@ - - :: - 0.0.0.0 - 1 - diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile deleted file mode 100644 index 3edf37dafa5..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile +++ /dev/null @@ -1,8 +0,0 @@ -. { - hosts /example.com { - reload "20ms" - fallthrough - } - forward . 127.0.0.11 - log -} diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com deleted file mode 100644 index 9beb415c290..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com +++ /dev/null @@ -1 +0,0 @@ -filled in runtime, but needs to exist in order to be volume mapped in docker \ No newline at end of file diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py deleted file mode 100644 index 70419f95dd3..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py +++ /dev/null @@ -1,62 +0,0 @@ -import pycurl -import threading -from io import BytesIO -import sys - -client_ip = sys.argv[1] -server_ip = sys.argv[2] - -mutex = threading.Lock() -success_counter = 0 -number_of_threads = 100 -number_of_iterations = 50 - - -def perform_request(): - buffer = BytesIO() - crl = pycurl.Curl() - crl.setopt(pycurl.INTERFACE, client_ip) - crl.setopt(crl.WRITEDATA, buffer) - crl.setopt(crl.URL, f"http://{server_ip}:8123/?query=select+1&user=test_dns") - - crl.perform() - - # End curl session - crl.close() - - str_response = buffer.getvalue().decode("iso-8859-1") - expected_response = "1\n" - - mutex.acquire() - - global success_counter - - if str_response == expected_response: - success_counter += 1 - - mutex.release() - - -def perform_multiple_requests(n): - for request_number in range(n): - perform_request() - - -threads = [] - - -for i in range(number_of_threads): - thread = threading.Thread( - target=perform_multiple_requests, args=(number_of_iterations,) - ) - thread.start() - threads.append(thread) - -for thread in threads: - thread.join() - - -if success_counter == number_of_threads * number_of_iterations: - exit(0) - -exit(1) diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py deleted file mode 100644 index d73e8813e79..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py +++ /dev/null @@ -1,88 +0,0 @@ -import pytest -import socket -from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check -from time import sleep -import os - -DOCKER_COMPOSE_PATH = get_docker_compose_path() -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -cluster = ClickHouseCluster(__file__) - -ch_server = cluster.add_instance( - "clickhouse-server", - with_coredns=True, - main_configs=["configs/config.xml", "configs/listen_host.xml"], - user_configs=["configs/host_regexp.xml"], -) - -client = cluster.add_instance( - "clickhouse-client", -) - - -@pytest.fixture(scope="module") -def started_cluster(): - global cluster - try: - cluster.start() - yield cluster - - finally: - cluster.shutdown() - - -def check_ptr_record(ip, hostname): - try: - host, aliaslist, ipaddrlist = socket.gethostbyaddr(ip) - if hostname.lower() == host.lower(): - return True - except socket.herror: - pass - return False - - -def setup_dns_server(ip): - domains_string = "test3.example.com test2.example.com test1.example.com" - example_file_path = f'{ch_server.env_variables["COREDNS_CONFIG_DIR"]}/example.com' - run_and_check(f"echo '{ip} {domains_string}' > {example_file_path}", shell=True) - - # DNS server takes time to reload the configuration. - for try_num in range(10): - if all(check_ptr_record(ip, host) for host in domains_string.split()): - break - sleep(1) - - -def setup_ch_server(dns_server_ip): - ch_server.exec_in_container( - (["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"]) - ) - ch_server.exec_in_container( - (["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"]) - ) - ch_server.query("SYSTEM DROP DNS CACHE") - - -def build_endpoint_v4(ip): - return f"'http://{ip}:8123/?query=SELECT+1&user=test_dns'" - - -def build_endpoint_v6(ip): - return build_endpoint_v4(f"[{ip}]") - - -def test_host_regexp_multiple_ptr_v4(started_cluster): - server_ip = cluster.get_instance_ip("clickhouse-server") - client_ip = cluster.get_instance_ip("clickhouse-client") - dns_server_ip = cluster.get_instance_ip(cluster.coredns_host) - - setup_dns_server(client_ip) - setup_ch_server(dns_server_ip) - - current_dir = os.path.dirname(__file__) - client.copy_file_to_container( - os.path.join(current_dir, "scripts", "stress_test.py"), "stress_test.py" - ) - - client.exec_in_container(["python3", f"stress_test.py", client_ip, server_ip]) From db1817a633d1fc1dca13eefa75303f0cb78145b7 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 8 Jul 2024 19:33:52 +0000 Subject: [PATCH 30/72] Some minor fixups --- .../settings/merge-tree-settings.md | 2 +- .../functions/date-time-functions.md | 434 +++++++++--------- src/Functions/changeDate.cpp | 110 ++--- .../aspell-ignore/en/aspell-dict.txt | 3 + 4 files changed, 285 insertions(+), 264 deletions(-) diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 22c8c704ba2..7278b91f90d 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -1030,7 +1030,7 @@ A table with no primary key represents the extreme case of a single equivalence The fewer and the larger the equivalence classes are, the higher the degree of freedom when re-shuffling rows. -The heuristics applied to find the best row order within each equivalence class is suggested by D. Lemir, O. Kaser in [Reordering columns for smaller indexes](https://doi.org/10.1016/j.ins.2011.02.002) and based on sorting the rows within each equivalence class by ascending cardinality of the non-primary key columns. +The heuristics applied to find the best row order within each equivalence class is suggested by D. Lemire, O. Kaser in [Reordering columns for smaller indexes](https://doi.org/10.1016/j.ins.2011.02.002) and based on sorting the rows within each equivalence class by ascending cardinality of the non-primary key columns. It performs three steps: 1. Find all equivalence classes based on the row values in primary key columns. 2. For each equivalence class, calculate (usually estimate) the cardinalities of the non-primary-key columns. diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 1edd8d407eb..4f5e5a5d716 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -2698,6 +2698,204 @@ Like function `YYYYMMDDhhmmssToDate()` but produces a [DateTime64](../data-types Accepts an additional, optional `precision` parameter after the `timezone` parameter. +## changeYear + +Changes the year component of a date or date time. + +**Syntax** +``` sql + +changeYear(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- The same type as `date_or_datetime`. + +**Example** + +``` sql +SELECT changeYear(toDate('1999-01-01'), 2000), changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); +``` + +Result: + +``` +┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ +│ 2000-01-01 │ 2000-01-01 00:00:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + +## changeMonth + +Changes the month component of a date or date time. + +**Syntax** + +``` sql +changeMonth(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the month. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Returns a value of same type as `date_or_datetime`. + +**Example** + +``` sql +SELECT changeMonth(toDate('1999-01-01'), 2), changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2); +``` + +Result: + +``` +┌─changeMonth(toDate('1999-01-01'), 2)─┬─changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2)─┐ +│ 1999-02-01 │ 1999-02-01 00:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeDay + +Changes the day component of a date or date time. + +**Syntax** + +``` sql +changeDay(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the day. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Returns a value of same type as `date_or_datetime`. + +**Example** + +``` sql +SELECT changeDay(toDate('1999-01-01'), 5), changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5); +``` + +Result: + +``` +┌─changeDay(toDate('1999-01-01'), 5)─┬─changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5)─┐ +│ 1999-01-05 │ 1999-01-05 00:00:00.000 │ +└────────────────────────────────────┴──────────────────────────────────────────────────────────┘ +``` + +## changeHour + +Changes the hour component of a date or date time. + +**Syntax** + +``` sql +changeHour(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Returns a value of same type as `date_or_datetime`. If the input is a [Date](../data-types/date.md), return [DateTime](../data-types/datetime.md). If the input is a [Date32](../data-types/date32.md), return [DateTime64](../data-types/datetime64.md). + +**Example** + +``` sql +SELECT changeHour(toDate('1999-01-01'), 14), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14); +``` + +Result: + +``` +┌─changeHour(toDate('1999-01-01'), 14)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14)─┐ +│ 1999-01-01 14:00:00 │ 1999-01-01 14:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeMinute + +Changes the minute component of a date or date time. + +**Syntax** + +``` sql +changeMinute(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the minute. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Returns a value of same type as `date_or_datetime`. If the input is a [Date](../data-types/date.md), return [DateTime](../data-types/datetime.md). If the input is a [Date32](../data-types/date32.md), return [DateTime64](../data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeMinute(toDate('1999-01-01'), 15), changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeMinute(toDate('1999-01-01'), 15)─┬─changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:15:00 │ 1999-01-01 00:15:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + +## changeSecond + +Changes the second component of a date or date time. + +**Syntax** + +``` sql +changeSecond(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the second. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Returns a value of same type as `date_or_datetime`. If the input is a [Date](../data-types/date.md), return [DateTime](../data-types/datetime.md). If the input is a [Date32](../data-types/date32.md), return [DateTime64](../data-types/datetime64.md). + +**Example** + +``` sql +SELECT changeSecond(toDate('1999-01-01'), 15), changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeSecond(toDate('1999-01-01'), 15)─┬─changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:00:15 │ 1999-01-01 00:00:15.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + ## addYears Adds a specified number of years to a date, a date with time or a string-encoded date / date with time. @@ -2714,6 +2912,7 @@ addYears(date, num) - `num`: Number of years to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` years. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2751,6 +2950,7 @@ addQuarters(date, num) - `num`: Number of quarters to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` quarters. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2788,6 +2988,7 @@ addMonths(date, num) - `num`: Number of months to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` months. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2825,6 +3026,7 @@ addWeeks(date, num) - `num`: Number of weeks to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` weeks. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2862,6 +3064,7 @@ addDays(date, num) - `num`: Number of days to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` days. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2899,6 +3102,7 @@ addHours(date, num) - `num`: Number of hours to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** +o - Returns `date` plus `num` hours. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2936,6 +3140,7 @@ addMinutes(date, num) - `num`: Number of minutes to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` minutes. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2973,6 +3178,7 @@ addSeconds(date, num) - `num`: Number of seconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` seconds. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3010,6 +3216,7 @@ addMilliseconds(date_time, num) - `num`: Number of milliseconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` plus `num` milliseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3045,6 +3252,7 @@ addMicroseconds(date_time, num) - `num`: Number of microseconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` plus `num` microseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3080,6 +3288,7 @@ addNanoseconds(date_time, num) - `num`: Number of nanoseconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` plus `num` nanoseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3115,6 +3324,7 @@ addInterval(interval_1, interval_2) - `interval_2`: Second interval to be added. [interval](../data-types/special-data-types/interval.md). **Returned value** + - Returns a tuple of intervals. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). :::note @@ -3161,6 +3371,7 @@ addTupleOfIntervals(interval_1, interval_2) - `intervals`: Tuple of intervals to add to `date`. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). **Returned value** + - Returns `date` with added `intervals`. [date](../data-types/date.md)/[date32](../data-types/date32.md)/[datetime](../data-types/datetime.md)/[datetime64](../data-types/datetime64.md). **Example** @@ -3195,6 +3406,7 @@ subtractYears(date, num) - `num`: Number of years to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` years. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3232,6 +3444,7 @@ subtractQuarters(date, num) - `num`: Number of quarters to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` quarters. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3269,6 +3482,7 @@ subtractMonths(date, num) - `num`: Number of months to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` months. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3306,6 +3520,7 @@ subtractWeeks(date, num) - `num`: Number of weeks to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` weeks. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3343,6 +3558,7 @@ subtractDays(date, num) - `num`: Number of days to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` days. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3380,6 +3596,7 @@ subtractHours(date, num) - `num`: Number of hours to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` hours. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[Datetime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3417,6 +3634,7 @@ subtractMinutes(date, num) - `num`: Number of minutes to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` minutes. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3454,6 +3672,7 @@ subtractSeconds(date, num) - `num`: Number of seconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` seconds. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3491,6 +3710,7 @@ subtractMilliseconds(date_time, num) - `num`: Number of milliseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` minus `num` milliseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3526,6 +3746,7 @@ subtractMicroseconds(date_time, num) - `num`: Number of microseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` minus `num` microseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3561,6 +3782,7 @@ subtractNanoseconds(date_time, num) - `num`: Number of nanoseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` minus `num` nanoseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3596,6 +3818,7 @@ subtractInterval(interval_1, interval_2) - `interval_2`: Second interval to be negated. [interval](../data-types/special-data-types/interval.md). **Returned value** + - Returns a tuple of intervals. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). :::note @@ -3642,6 +3865,7 @@ subtractTupleOfIntervals(interval_1, interval_2) - `intervals`: Tuple of intervals to subtract from `date`. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). **Returned value** + - Returns `date` with subtracted `intervals`. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3660,216 +3884,6 @@ Result: └───────────────────────────────────────────────────────────────────────┘ ``` -## changeYear - -Changes the year component of a date or date time. - -**Syntax** - -``` sql -changeYear(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeYear(toDate('1999-01-01'), 2000), changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); -``` - -Result: - -``` -┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ -│ 2000-01-01 │ 2000-01-01 00:00:00.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` - -## changeMonth - -Changes the month component of a date or date time. - -**Syntax** - -``` sql -changeMonth(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the month. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeMonth(toDate('1999-01-01'), 2), changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2); -``` - -Result: - -``` -┌─changeMonth(toDate('1999-01-01'), 2)─┬─changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2)─┐ -│ 1999-02-01 │ 1999-02-01 00:00:00.000 │ -└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ -``` - -## changeDay - -Changes the day component of a date or date time. - -**Syntax** - -``` sql -changeDay(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the day. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeDay(toDate('1999-01-01'), 5), changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5); -``` - -Result: - -``` -┌─changeDay(toDate('1999-01-01'), 5)─┬─changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5)─┐ -│ 1999-01-05 │ 1999-01-05 00:00:00.000 │ -└────────────────────────────────────┴──────────────────────────────────────────────────────────┘ -``` - -## changeHour - -Changes the hour component of a date or date time. - -**Syntax** - -``` sql -changeHour(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeHour(toDate('1999-01-01'), 14), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14); -``` - -Result: - -``` -┌─changeHour(toDate('1999-01-01'), 14)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14)─┐ -│ 1999-01-01 14:00:00 │ 1999-01-01 14:00:00.000 │ -└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ -``` - -## changeMinute - -Changes the minute component of a date or date time. - -**Syntax** - -``` sql -changeMinute(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the minute. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeMinute(toDate('1999-01-01'), 15), changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15); -``` - -Result: - -``` -┌─changeMinute(toDate('1999-01-01'), 15)─┬─changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ -│ 1999-01-01 00:15:00 │ 1999-01-01 00:15:00.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` - -## changeSecond - -Changes the second component of a date or date time. - -**Syntax** - -``` sql -changeSecond(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the second. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeSecond(toDate('1999-01-01'), 15), changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15); -``` - -Result: - -``` -┌─changeSecond(toDate('1999-01-01'), 15)─┬─changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ -│ 1999-01-01 00:00:15 │ 1999-01-01 00:00:15.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` - ## timeSlots(StartTime, Duration,\[, Size\]) For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index e24391afe12..5965f3d1d00 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -41,11 +41,6 @@ enum class Component Second }; -bool isTimeComponentChange(Component type) -{ - return type == Component::Hour || type == Component::Minute || type == Component::Second; -} - } template @@ -65,11 +60,11 @@ public: {"date_or_datetime", static_cast(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date or date with time"}, {"value", static_cast(&isNativeInteger), nullptr, "Integer"} }; - validateFunctionArgumentTypes(*this, arguments, args); + validateFunctionArguments(*this, arguments, args); const auto & input_type = arguments[0].type; - if (isTimeComponentChange(Traits::component)) + if constexpr (Traits::component == Component::Hour || Traits::component == Component::Minute || Traits::component == Component::Second) { if (isDate(input_type)) return std::make_shared(); @@ -85,13 +80,13 @@ public: const auto & input_type = arguments[0].type; if (isDate(input_type)) { - if (isTimeComponentChange(Traits::component)) + if constexpr (Traits::component == Component::Hour || Traits::component == Component::Minute || Traits::component == Component::Second) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDate32(input_type)) { - if (isTimeComponentChange(Traits::component)) + if constexpr (Traits::component == Component::Hour || Traits::component == Component::Minute || Traits::component == Component::Second) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } @@ -109,83 +104,92 @@ public: bool is_const = (isColumnConst(*arguments[0].column) && isColumnConst(*arguments[1].column)); size_t result_rows_count = (is_const ? 1 : input_rows_count); - typename ResultDataType::ColumnType::MutablePtr result_column; + typename ResultDataType::ColumnType::MutablePtr result_col; if constexpr (std::is_same_v) { auto scale = DataTypeDateTime64::default_scale; if constexpr (std::is_same_v) scale = typeid_cast(*result_type).getScale(); - result_column = ResultDataType::ColumnType::create(result_rows_count, scale); + result_col = ResultDataType::ColumnType::create(result_rows_count, scale); } else - result_column = ResultDataType::ColumnType::create(result_rows_count); + result_col = ResultDataType::ColumnType::create(result_rows_count); - auto input_column = arguments[0].column->convertToFullIfNeeded(); - const auto & input_column_data = typeid_cast(*input_column).getData(); + auto date_time_col = arguments[0].column->convertToFullIfNeeded(); + const auto & date_time_col_data = typeid_cast(*date_time_col).getData(); - auto new_value_column = castColumn(arguments[1], std::make_shared()); - new_value_column = new_value_column->convertToFullIfNeeded(); - const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + auto value_col = castColumn(arguments[1], std::make_shared()); + value_col = value_col->convertToFullIfNeeded(); + const auto & value_col_data = typeid_cast(*value_col).getData(); - auto & result_data = result_column->getData(); + auto & result_col_data = result_col->getData(); - for (size_t i = 0; i < result_rows_count; ++i) + if constexpr (std::is_same_v) { - if constexpr (std::is_same_v) + const auto scale = typeid_cast(*result_type).getScale(); + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + + Int64 deg = 1; + for (size_t j = 0; j < scale; ++j) + deg *= 10; + + for (size_t i = 0; i < result_rows_count; ++i) { - const auto scale = typeid_cast(*result_type).getScale(); - const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 time = date_lut.toNumYYYYMMDDhhmmss(date_time_col_data[i] / deg); + Int64 fraction = date_time_col_data[i] % deg; - Int64 deg = 1; - for (size_t j = 0; j < scale; ++j) - deg *= 10; - - Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); - Int64 fraction = input_column_data[i] % deg; - - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, scale, fraction); + result_col_data[i] = getChangedDate(time, value_col_data[i], result_type, date_lut, scale, fraction); } - else if constexpr (std::is_same_v && std::is_same_v) + } + else if constexpr (std::is_same_v && std::is_same_v) + { + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + for (size_t i = 0; i < result_rows_count; ++i) { - const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; - - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); + Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(date_time_col_data[i]))) * 1'000'000; + result_col_data[i] = getChangedDate(time, value_col_data[i], result_type, date_lut, 3, 0); } - else if constexpr (std::is_same_v && std::is_same_v) + } + else if constexpr (std::is_same_v && std::is_same_v) + { + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + for (size_t i = 0; i < result_rows_count; ++i) { - const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; - - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(date_time_col_data[i]))) * 1'000'000; + result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); } - else if constexpr (std::is_same_v) + } + else if constexpr (std::is_same_v) + { + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + for (size_t i = 0; i < result_rows_count; ++i) { - const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + Int64 time = date_lut.toNumYYYYMMDDhhmmss(date_time_col_data[i]); + result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); } - else + } + else + { + const auto & date_lut = DateLUT::instance(); + for (size_t i = 0; i < result_rows_count; ++i) { - const auto & date_lut = DateLUT::instance(); Int64 time; if (isDate(input_type)) - time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; + time = static_cast(date_lut.toNumYYYYMMDD(DayNum(date_time_col_data[i]))) * 1'000'000; else - time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; + time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(date_time_col_data[i]))) * 1'000'000; if (isDate(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); } } if (is_const) - return ColumnConst::create(std::move(result_column), input_rows_count); + return ColumnConst::create(std::move(result_col), input_rows_count); - return result_column; + return result_col; } Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 5689f94d2ae..5e31a09effb 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -467,6 +467,7 @@ LOCALTIME LOCALTIMESTAMP LONGLONG LOONGARCH +Lemire Levenshtein Liao LibFuzzer @@ -1962,6 +1963,8 @@ loghouse london lookups loongarch +lowCardinalityIndices +lowCardinalityKeys lowcardinality lowerUTF lowercased From 595a12ee884840da76e2b50ae723511e00915b85 Mon Sep 17 00:00:00 2001 From: nauu Date: Tue, 9 Jul 2024 09:48:51 +0800 Subject: [PATCH 31/72] to avoid ambiguity, replace FilesystemCacheFailToReserveSpaceBecauseOfLockContention with FilesystemCacheFailToReserveSpaceBecauseOfCacheResize. to avoid ambiguity, replace FilesystemCacheFailToReserveSpaceBecauseOfLockContention with FilesystemCacheResize. --- src/Common/ProfileEvents.cpp | 1 + src/Interpreters/Cache/FileCache.cpp | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 439965a92fb..e80afc95e8d 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -508,6 +508,7 @@ The server successfully detected this situation and will download merged part fr M(FileSegmentHolderCompleteMicroseconds, "File segments holder complete() time") \ M(FileSegmentFailToIncreasePriority, "Number of times the priority was not increased due to a high contention on the cache lock") \ M(FilesystemCacheFailToReserveSpaceBecauseOfLockContention, "Number of times space reservation was skipped due to a high contention on the cache lock") \ + M(FilesystemCacheFailToReserveSpaceBecauseOfCacheResize, "Number of times space reservation was skipped due to the cache is being resized") \ M(FilesystemCacheHoldFileSegments, "Filesystem cache file segments count, which were hold") \ M(FilesystemCacheUnusedHoldFileSegments, "Filesystem cache file segments count, which were hold, but not used (because of seek or LIMIT n, etc)") \ M(FilesystemCacheFreeSpaceKeepingThreadRun, "Number of times background thread executed free space keeping job") \ diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 0d33e39ffa3..a3848fa3a75 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -30,6 +30,7 @@ namespace ProfileEvents extern const Event FilesystemCacheFailToReserveSpaceBecauseOfLockContention; extern const Event FilesystemCacheFreeSpaceKeepingThreadRun; extern const Event FilesystemCacheFreeSpaceKeepingThreadWorkMilliseconds; + extern const Event FilesystemCacheFailToReserveSpaceBecauseOfCacheResize; } namespace DB @@ -813,7 +814,7 @@ bool FileCache::tryReserve( /// ok compared to the number of cases this check will help. if (cache_is_being_resized.load(std::memory_order_relaxed)) { - ProfileEvents::increment(ProfileEvents::FilesystemCacheFailToReserveSpaceBecauseOfLockContention); + ProfileEvents::increment(ProfileEvents::FilesystemCacheFailToReserveSpaceBecauseOfCacheResize); return false; } From 4729161f4187e887e10757731b2381673234b43a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 10 Jul 2024 18:41:48 +0000 Subject: [PATCH 32/72] Collect logs from minio --- docker/test/stateful/run.sh | 5 +++++ docker/test/stateless/run.sh | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 2215ac2b37c..ce637b9a146 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -20,7 +20,10 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test /usr/share/clickhouse-test/config/install.sh azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence & + ./setup_minio.sh stateful +mc admin trace clickminio > /test_output/rubbish.log & +MC_ADMIN_PID=$! config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml @@ -251,6 +254,8 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||: fi +# Kill minio admin client to stop collecting logs +kill $MC_ADMIN_PID rg -Fa "" /var/log/clickhouse-server/clickhouse-server.log ||: zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst ||: diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 029c5a03151..a01abe5929f 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -46,6 +46,9 @@ source /utils.lib /usr/share/clickhouse-test/config/install.sh ./setup_minio.sh stateless +mc admin trace clickminio > /test_output/rubbish.log & +MC_ADMIN_PID=$! + ./setup_hdfs_minicluster.sh config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml @@ -325,6 +328,9 @@ if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||: fi +# Kill minio admin client to stop collecting logs +kill $MC_ADMIN_PID + rg -Fa "" /var/log/clickhouse-server/clickhouse-server.log ||: rg -A50 -Fa "============" /var/log/clickhouse-server/stderr.log ||: zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst & From a9a227ddc614834eb2943a20bf0f2fa722549fec Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 11 Jul 2024 07:53:54 +0200 Subject: [PATCH 33/72] Fix "Sending a batch of X files to Y (0.00 rows, 0.00 B bytes)." in case of batch restoring Previously it was always zeros. Signed-off-by: Azat Khuzhin --- .../Distributed/DistributedAsyncInsertBatch.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp index 06d4c185840..e1facec5b40 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp @@ -196,6 +196,16 @@ void DistributedAsyncInsertBatch::readText(ReadBuffer & in) UInt64 idx; in >> idx >> "\n"; files.push_back(std::filesystem::absolute(fmt::format("{}/{}.bin", parent.path, idx)).string()); + + ReadBufferFromFile header_buffer(files.back()); + const DistributedAsyncInsertHeader & header = DistributedAsyncInsertHeader::read(header_buffer, parent.log); + total_bytes += total_bytes; + + if (header.rows) + { + total_rows += header.rows; + total_bytes += header.bytes; + } } recovered = true; From 37c66c8976da45e1984e22e55a3932e02e7937cc Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 11 Jul 2024 08:04:44 +0200 Subject: [PATCH 34/72] Fix 03030_system_flush_distributed_settings flakiness Signed-off-by: Azat Khuzhin --- .../03030_system_flush_distributed_settings.reference | 2 +- .../03030_system_flush_distributed_settings.sql | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/03030_system_flush_distributed_settings.reference b/tests/queries/0_stateless/03030_system_flush_distributed_settings.reference index 5caff40c4a0..3a05c8b3ee8 100644 --- a/tests/queries/0_stateless/03030_system_flush_distributed_settings.reference +++ b/tests/queries/0_stateless/03030_system_flush_distributed_settings.reference @@ -1 +1 @@ -10000 +30000 diff --git a/tests/queries/0_stateless/03030_system_flush_distributed_settings.sql b/tests/queries/0_stateless/03030_system_flush_distributed_settings.sql index ac64135b593..fac673a4fe4 100644 --- a/tests/queries/0_stateless/03030_system_flush_distributed_settings.sql +++ b/tests/queries/0_stateless/03030_system_flush_distributed_settings.sql @@ -6,15 +6,17 @@ drop table if exists dist_out; create table ephemeral (key Int, value Int) engine=Null(); create table dist_in as ephemeral engine=Distributed(test_shard_localhost, currentDatabase(), ephemeral, key) settings background_insert_batch=1; -create table data (key Int, uniq_values Int) engine=Memory(); -create materialized view mv to data as select key, uniqExact(value) uniq_values from ephemeral group by key; +create table data (key Int, uniq_values Int) engine=TinyLog(); +create materialized view mv to data as select key, uniqExact(value::String) uniq_values from ephemeral group by key; system stop distributed sends dist_in; create table dist_out as data engine=Distributed(test_shard_localhost, currentDatabase(), data); set prefer_localhost_replica=0; SET optimize_trivial_insert_select = 1; -insert into dist_in select number/100, number from system.numbers limit 1e6 settings max_memory_usage='20Mi'; +-- due to pushing to MV with aggregation the query needs ~300MiB +-- but it will be done in background via "system flush distributed" +insert into dist_in select number/100, number from system.numbers limit 3e6 settings max_block_size=3e6, max_memory_usage='100Mi'; system flush distributed dist_in; -- { serverError MEMORY_LIMIT_EXCEEDED } system flush distributed dist_in settings max_memory_usage=0; select count() from dist_out; From 321a58c51931e8d39b8ba111108fed20ba0541cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 11 Jul 2024 08:33:10 +0000 Subject: [PATCH 35/72] Fix relative path to `mc` --- docker/test/stateful/run.sh | 2 +- docker/test/stateless/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index ce637b9a146..2efbed6aa4f 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -22,7 +22,7 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence & ./setup_minio.sh stateful -mc admin trace clickminio > /test_output/rubbish.log & +./mc admin trace clickminio > /test_output/rubbish.log & MC_ADMIN_PID=$! config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index a01abe5929f..d47728cddb9 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -46,7 +46,7 @@ source /utils.lib /usr/share/clickhouse-test/config/install.sh ./setup_minio.sh stateless -mc admin trace clickminio > /test_output/rubbish.log & +m./c admin trace clickminio > /test_output/rubbish.log & MC_ADMIN_PID=$! ./setup_hdfs_minicluster.sh From cdb3b3f2aaec64b8d5b3ec1ee6e38545e3a2b186 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 11 Jul 2024 16:42:29 +0200 Subject: [PATCH 36/72] Add query elapsed time for non-default format in play UI Signed-off-by: Azat Khuzhin --- programs/server/play.html | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/programs/server/play.html b/programs/server/play.html index 507a96382a7..b5bcc687c27 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -516,6 +516,9 @@ /// Save query in history only if it is different. let previous_query = ''; + /// Start of the last query + let last_query_start = 0; + const current_url = new URL(window.location); const opened_locally = location.protocol == 'file:'; @@ -567,6 +570,8 @@ '&password=' + encodeURIComponent(password) } + last_query_start = performance.now(); + const xhr = new XMLHttpRequest; xhr.open('POST', url, true); @@ -579,7 +584,8 @@ if (posted_request_num != request_num) { return; } else if (this.readyState === XMLHttpRequest.DONE) { - renderResponse(this.status, this.response); + const elapsed_msec = performance.now() - last_query_start; + renderResponse(this.status, this.response, elapsed_msec); /// The query is saved in browser history (in state JSON object) /// as well as in URL fragment identifier. @@ -587,7 +593,8 @@ const state = { query: query, status: this.status, - response: this.response.length > 100000 ? null : this.response /// Lower than the browser's limit. + response: this.response.length > 100000 ? null : this.response, /// Lower than the browser's limit. + elapsed_msec: elapsed_msec, }; const title = "ClickHouse Query: " + query; @@ -617,7 +624,7 @@ xhr.send(query); } - function renderResponse(status, response) { + function renderResponse(status, response, elapsed_msec) { document.getElementById('hourglass').style.display = 'none'; if (status === 200) { @@ -632,6 +639,7 @@ renderChart(json); } else { renderUnparsedResult(response); + stats.innerText = `Elapsed: ${(elapsed_msec/1000).toFixed(3)} sec.`; } document.getElementById('check-mark').style.display = 'inline'; } else { @@ -651,7 +659,7 @@ clear(); return; } - renderResponse(event.state.status, event.state.response); + renderResponse(event.state.status, event.state.response, event.state.elapsed_msec); }; if (window.location.hash) { From 966e4d2d6327987b05668ce625e9f145c28e264a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 11 Jul 2024 21:46:08 +0200 Subject: [PATCH 37/72] Remove noisy message --- src/Interpreters/DatabaseCatalog.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 964baea1891..bb2dd158710 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -1281,10 +1281,6 @@ void DatabaseCatalog::rescheduleDropTableTask() auto min_drop_time = getMinDropTime(); time_t schedule_after_ms = min_drop_time > current_time ? (min_drop_time - current_time) * 1000 : 0; - LOG_TRACE( - log, - "Have {} tables in queue to drop. Schedule background task in {} seconds", - tables_marked_dropped.size(), schedule_after_ms / 1000); (*drop_task)->scheduleAfter(schedule_after_ms); } From 364686731f1437e708f12d6e0f44d371b977dc9c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 11 Jul 2024 21:51:09 +0200 Subject: [PATCH 38/72] Fix test --- tests/queries/0_stateless/03201_local_named_collections.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03201_local_named_collections.sh b/tests/queries/0_stateless/03201_local_named_collections.sh index 2054a09df06..809b4b52f41 100755 --- a/tests/queries/0_stateless/03201_local_named_collections.sh +++ b/tests/queries/0_stateless/03201_local_named_collections.sh @@ -13,7 +13,7 @@ INSERT INTO test VALUES ('Hello, world!'); ${CLICKHOUSE_LOCAL} --multiquery " CREATE NAMED COLLECTION mydb AS host = '${CLICKHOUSE_HOST}', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', db = '${CLICKHOUSE_DATABASE}'; SELECT * FROM remote(mydb, table = 'test'); -" | grep --text -F -v "ASan doesn't fully support makecontext/swapcontext functions" +" 2>&1 | grep --text -F -v "ASan doesn't fully support makecontext/swapcontext functions" ${CLICKHOUSE_CLIENT} --multiquery " DROP TABLE test; From 713546e5102829f71b2c2f27021478c63da0ea3e Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 12 Jul 2024 11:44:34 +0200 Subject: [PATCH 39/72] Update src/Common/ProfileEvents.cpp --- src/Common/ProfileEvents.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index e80afc95e8d..ba9c4cbfdb2 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -508,7 +508,7 @@ The server successfully detected this situation and will download merged part fr M(FileSegmentHolderCompleteMicroseconds, "File segments holder complete() time") \ M(FileSegmentFailToIncreasePriority, "Number of times the priority was not increased due to a high contention on the cache lock") \ M(FilesystemCacheFailToReserveSpaceBecauseOfLockContention, "Number of times space reservation was skipped due to a high contention on the cache lock") \ - M(FilesystemCacheFailToReserveSpaceBecauseOfCacheResize, "Number of times space reservation was skipped due to the cache is being resized") \ + M(FilesystemCacheFailToReserveSpaceBecauseOfCacheResize, "Number of times space reservation was skipped due to the cache is being resized" ) \ M(FilesystemCacheHoldFileSegments, "Filesystem cache file segments count, which were hold") \ M(FilesystemCacheUnusedHoldFileSegments, "Filesystem cache file segments count, which were hold, but not used (because of seek or LIMIT n, etc)") \ M(FilesystemCacheFreeSpaceKeepingThreadRun, "Number of times background thread executed free space keeping job") \ From b4959b25dfc1a0cf590734bf2b9bc16bc5728188 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 12 Jul 2024 11:44:44 +0200 Subject: [PATCH 40/72] Update src/Common/ProfileEvents.cpp --- src/Common/ProfileEvents.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index ba9c4cbfdb2..e80afc95e8d 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -508,7 +508,7 @@ The server successfully detected this situation and will download merged part fr M(FileSegmentHolderCompleteMicroseconds, "File segments holder complete() time") \ M(FileSegmentFailToIncreasePriority, "Number of times the priority was not increased due to a high contention on the cache lock") \ M(FilesystemCacheFailToReserveSpaceBecauseOfLockContention, "Number of times space reservation was skipped due to a high contention on the cache lock") \ - M(FilesystemCacheFailToReserveSpaceBecauseOfCacheResize, "Number of times space reservation was skipped due to the cache is being resized" ) \ + M(FilesystemCacheFailToReserveSpaceBecauseOfCacheResize, "Number of times space reservation was skipped due to the cache is being resized") \ M(FilesystemCacheHoldFileSegments, "Filesystem cache file segments count, which were hold") \ M(FilesystemCacheUnusedHoldFileSegments, "Filesystem cache file segments count, which were hold, but not used (because of seek or LIMIT n, etc)") \ M(FilesystemCacheFreeSpaceKeepingThreadRun, "Number of times background thread executed free space keeping job") \ From c7180e67bde8fa7924c77b999a0dafdc2b4b283d Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 12 Jul 2024 13:08:53 +0100 Subject: [PATCH 41/72] impl --- cmake/limit_jobs.cmake | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 8e48fc9b9d8..3a759b90fe3 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -42,9 +42,14 @@ endif () # But use 2 parallel jobs, since: # - this is what llvm does # - and I've verfied that lld-11 does not use all available CPU time (in peak) while linking one binary -if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 2) - message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.") - set (PARALLEL_LINK_JOBS 2) +if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO) + if (ARCH_AARCH64) + message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 1.") + set (PARALLEL_LINK_JOBS 1) + elseif (PARALLEL_LINK_JOBS GREATER 2) + message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.") + set (PARALLEL_LINK_JOBS 2) + endif () endif() message(STATUS "Building sub-tree with ${PARALLEL_COMPILE_JOBS} compile jobs and ${PARALLEL_LINK_JOBS} linker jobs (system: ${NUMBER_OF_LOGICAL_CORES} cores, ${TOTAL_PHYSICAL_MEMORY} MB RAM, 'OFF' means the native core count).") From a3ab1ab5ca707c0e40b4ad738413dd868f2db606 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Fri, 12 Jul 2024 13:10:13 +0000 Subject: [PATCH 42/72] CI: Do not block on few number of test failures --- tests/ci/ci_config.py | 3 ++ tests/ci/ci_utils.py | 15 ++++++++- tests/ci/merge_pr.py | 78 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 80 insertions(+), 16 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 8eda6e6b96f..9a9aa553e1b 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -13,6 +13,9 @@ class CI: each config item in the below dicts should be an instance of JobConfig class or inherited from it """ + MAX_TOTAL_FAILURES_BEFORE_BLOCKING_CI = 2 + MAX_TOTAL_FAILURES_PER_JOB_BEFORE_BLOCKING_CI = 1 + # reimport types to CI class so that they visible as CI.* and mypy is happy # pylint:disable=useless-import-alias,reimported,import-outside-toplevel from ci_definitions import BuildConfig as BuildConfig diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py index 629f37289a9..abc4a88989d 100644 --- a/tests/ci/ci_utils.py +++ b/tests/ci/ci_utils.py @@ -1,8 +1,9 @@ import os +import re import subprocess from contextlib import contextmanager from pathlib import Path -from typing import Any, Iterator, List, Union +from typing import Any, Iterator, List, Union, Optional class WithIter(type): @@ -83,3 +84,15 @@ class Shell: check=False, ) return result.returncode == 0 + + +class Utils: + @staticmethod + def get_failed_tests_number(description: str) -> Optional[int]: + description = description.lower() + + pattern = r"fail:\s*(\d+)\s*(?=,|$)" + match = re.search(pattern, description) + if match: + return int(match.group(1)) + return None diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py index 37c08fc4efe..6b437731561 100644 --- a/tests/ci/merge_pr.py +++ b/tests/ci/merge_pr.py @@ -26,6 +26,8 @@ from pr_info import PRInfo from report import SUCCESS, FAILURE from env_helper import GITHUB_UPSTREAM_REPOSITORY, GITHUB_REPOSITORY from synchronizer_utils import SYNC_BRANCH_PREFIX +from ci_config import CI +from ci_utils import Utils # The team name for accepted approvals TEAM_NAME = getenv("GITHUB_TEAM_NAME", "core") @@ -251,23 +253,69 @@ def main(): # set mergeable check status and exit commit = get_commit(gh, args.pr_info.sha) statuses = get_commit_filtered_statuses(commit) - state = trigger_mergeable_check( - commit, - statuses, - workflow_failed=(args.wf_status != "success"), - ) - # Process upstream StatusNames.SYNC - pr_info = PRInfo() - if ( - pr_info.head_ref.startswith(f"{SYNC_BRANCH_PREFIX}/pr/") - and GITHUB_REPOSITORY != GITHUB_UPSTREAM_REPOSITORY - ): - print("Updating upstream statuses") - update_upstream_sync_status(pr_info, state) + max_failed_tests_per_job = 0 + job_name_with_max_failures = None + total_failed_tests = 0 + failed_to_get_info = False + has_failed_statuses = False + for status in statuses: + if not CI.is_required(status.context): + continue + if status.state == FAILURE: + has_failed_statuses = True + failed_cnt = Utils.get_failed_tests_number(status.description) + if failed_cnt is None: + failed_to_get_info = True + else: + if failed_cnt > max_failed_tests_per_job: + job_name_with_max_failures = status.context + max_failed_tests_per_job = failed_cnt + total_failed_tests += failed_cnt + elif status.state != SUCCESS: + has_failed_statuses = True + print( + f"Unexpected status for [{status.context}]: [{status.state}] - block further testing" + ) + failed_to_get_info = True - if args.wf_status != "success": - # exit with 1 to rerun on workflow failed job restart + can_continue = True + if total_failed_tests > CI.MAX_TOTAL_FAILURES_BEFORE_BLOCKING_CI: + print( + f"Required check has [{total_failed_tests}] failed - block further testing" + ) + can_continue = False + if max_failed_tests_per_job > CI.MAX_TOTAL_FAILURES_PER_JOB_BEFORE_BLOCKING_CI: + print( + f"Job [{job_name_with_max_failures}] has [{max_failed_tests_per_job}] failures - block further testing" + ) + can_continue = False + if failed_to_get_info: + print(f"Unexpected commit status state - block further testing") + can_continue = False + if args.wf_status != SUCCESS: + can_continue = False + print("Workflow has failures - block further testing") + + if args.wf_status == "success" or has_failed_statuses: + state = trigger_mergeable_check( + commit, + statuses, + ) + # Process upstream StatusNames.SYNC + pr_info = PRInfo() + if ( + pr_info.head_ref.startswith(f"{SYNC_BRANCH_PREFIX}/pr/") + and GITHUB_REPOSITORY != GITHUB_UPSTREAM_REPOSITORY + ): + print("Updating upstream statuses") + update_upstream_sync_status(pr_info, state) + else: + print( + "Workflow failed but no failed statuses found (died runner?) - cannot set Mergeable Check status" + ) + + if not can_continue: sys.exit(1) sys.exit(0) From c06589392b866bf4c799b1f5053197f7027f3db3 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Fri, 12 Jul 2024 17:21:11 +0200 Subject: [PATCH 43/72] Stateless tests: fix flaky tests --- .../01037_polygon_dicts_correctness_all.sh | 14 +- .../01037_polygon_dicts_correctness_fast.sh | 14 +- .../01037_polygon_dicts_simple_functions.ans | 208 +++++++++--------- .../01037_polygon_dicts_simple_functions.sh | 127 +++++------ 4 files changed, 181 insertions(+), 182 deletions(-) diff --git a/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh b/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh index fff786d6c06..39f235d9966 100755 --- a/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh +++ b/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh @@ -5,20 +5,22 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -TMP_DIR="/tmp" +TMP_DIR=${CLICKHOUSE_TMP}${CLICKHOUSE_DATABASE} +mkdir -p $TMP_DIR declare -a SearchTypes=("POLYGON" "POLYGON_SIMPLE" "POLYGON_INDEX_EACH" "POLYGON_INDEX_CELL") -tar -xf "${CURDIR}"/01037_test_data_search.tar.gz -C "${CURDIR}" +DATA_DIR=${CURDIR}/${CLICKHOUSE_DATABASE} +tar -xf "${CURDIR}"/01037_test_data_search.tar.gz -C "${DATA_DIR}" $CLICKHOUSE_CLIENT -n --query=" DROP TABLE IF EXISTS points; CREATE TABLE points (x Float64, y Float64) ENGINE = Memory; " -$CLICKHOUSE_CLIENT --query="INSERT INTO points FORMAT TSV" --max_insert_block_size=100000 < "${CURDIR}/01037_point_data" +$CLICKHOUSE_CLIENT --query="INSERT INTO points FORMAT TSV" --max_insert_block_size=100000 < "${DATA_DIR}/01037_point_data" -rm "${CURDIR}"/01037_point_data +rm "${DATA_DIR}"/01037_point_data $CLICKHOUSE_CLIENT -n --query=" DROP TABLE IF EXISTS polygons_array; @@ -32,9 +34,9 @@ CREATE TABLE polygons_array ENGINE = Memory; " -$CLICKHOUSE_CLIENT --query="INSERT INTO polygons_array FORMAT JSONEachRow" --min_chunk_bytes_for_parallel_parsing=10485760 --max_insert_block_size=100000 < "${CURDIR}/01037_polygon_data" +$CLICKHOUSE_CLIENT --query="INSERT INTO polygons_array FORMAT JSONEachRow" --min_chunk_bytes_for_parallel_parsing=10485760 --max_insert_block_size=100000 < "${DATA_DIR}/01037_polygon_data" -rm "${CURDIR}"/01037_polygon_data +rm "${DATA_DIR}"/01037_polygon_data for type in "${SearchTypes[@]}"; do diff --git a/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh b/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh index c9cd151a2d9..3e461abcefe 100755 --- a/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh +++ b/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh @@ -5,19 +5,21 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -TMP_DIR="/tmp" +TMP_DIR=${CLICKHOUSE_TMP}${CLICKHOUSE_DATABASE} +mkdir -p $TMP_DIR declare -a SearchTypes=("POLYGON_INDEX_EACH" "POLYGON_INDEX_CELL") -tar -xf "${CURDIR}"/01037_test_data_perf.tar.gz -C "${CURDIR}" +DATA_DIR=${CURDIR}/${CLICKHOUSE_DATABASE} +tar -xf "${CURDIR}"/01037_test_data_perf.tar.gz -C "${DATA_DIR}" $CLICKHOUSE_CLIENT -n --query=" CREATE TABLE points (x Float64, y Float64) ENGINE = Memory; " -$CLICKHOUSE_CLIENT --query="INSERT INTO points FORMAT TSV" --min_chunk_bytes_for_parallel_parsing=10485760 --max_insert_block_size=100000 < "${CURDIR}/01037_point_data" +$CLICKHOUSE_CLIENT --query="INSERT INTO points FORMAT TSV" --min_chunk_bytes_for_parallel_parsing=10485760 --max_insert_block_size=100000 < "${DATA_DIR}/01037_point_data" -rm "${CURDIR}"/01037_point_data +rm "${DATA_DIR}"/01037_point_data $CLICKHOUSE_CLIENT -n --query=" DROP TABLE IF EXISTS polygons_array; @@ -31,9 +33,9 @@ CREATE TABLE polygons_array ENGINE = Memory; " -$CLICKHOUSE_CLIENT --query="INSERT INTO polygons_array FORMAT JSONEachRow" --min_chunk_bytes_for_parallel_parsing=10485760 --max_insert_block_size=100000 < "${CURDIR}/01037_polygon_data" +$CLICKHOUSE_CLIENT --query="INSERT INTO polygons_array FORMAT JSONEachRow" --min_chunk_bytes_for_parallel_parsing=10485760 --max_insert_block_size=100000 < "${DATA_DIR}/01037_polygon_data" -rm "${CURDIR}"/01037_polygon_data +rm "${DATA_DIR}"/01037_polygon_data for type in "${SearchTypes[@]}"; do diff --git a/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.ans b/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.ans index dfad14fb113..937539643ec 100644 --- a/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.ans +++ b/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.ans @@ -1,104 +1,104 @@ -dictGet test_01037.dict_array (-100,-42) qqq 101 -dictGet test_01037.dict_array (-1,0) Click South 423 -dictGet test_01037.dict_array (-0.1,0) Click South 423 -dictGet test_01037.dict_array (0,-2) Click West 424 -dictGet test_01037.dict_array (0,-1.1) Click West 424 -dictGet test_01037.dict_array (0,1.1) Click North 422 -dictGet test_01037.dict_array (0,2) Click North 422 -dictGet test_01037.dict_array (0.1,0) Click East 421 -dictGet test_01037.dict_array (0.99,2.99) Click North 422 -dictGet test_01037.dict_array (1,0) Click East 421 -dictGet test_01037.dict_array (3,3) House 314159 -dictGet test_01037.dict_array (5,6) Click 42 -dictGet test_01037.dict_array (7.01,7.01) qqq 101 -dictGetOrDefault test_01037.dict_array (-100,-42) www 1234 -dictGetOrDefault test_01037.dict_array (-1,0) Click South 423 -dictGetOrDefault test_01037.dict_array (-0.1,0) Click South 423 -dictGetOrDefault test_01037.dict_array (0,-2) Click West 424 -dictGetOrDefault test_01037.dict_array (0,-1.1) Click West 424 -dictGetOrDefault test_01037.dict_array (0,1.1) Click North 422 -dictGetOrDefault test_01037.dict_array (0,2) Click North 422 -dictGetOrDefault test_01037.dict_array (0.1,0) Click East 421 -dictGetOrDefault test_01037.dict_array (0.99,2.99) Click North 422 -dictGetOrDefault test_01037.dict_array (1,0) Click East 421 -dictGetOrDefault test_01037.dict_array (3,3) House 314159 -dictGetOrDefault test_01037.dict_array (5,6) Click 42 -dictGetOrDefault test_01037.dict_array (7.01,7.01) www 1234 -dictGetOrDefault test_01037.dict_array (-100,-42) dd 44 -dictGetOrDefault test_01037.dict_array (-1,0) Click South 423 -dictGetOrDefault test_01037.dict_array (-0.1,0) Click South 423 -dictGetOrDefault test_01037.dict_array (0,-2) Click West 424 -dictGetOrDefault test_01037.dict_array (0,-1.1) Click West 424 -dictGetOrDefault test_01037.dict_array (0,1.1) Click North 422 -dictGetOrDefault test_01037.dict_array (0,2) Click North 422 -dictGetOrDefault test_01037.dict_array (0.1,0) Click East 421 -dictGetOrDefault test_01037.dict_array (0.99,2.99) Click North 422 -dictGetOrDefault test_01037.dict_array (1,0) Click East 421 -dictGetOrDefault test_01037.dict_array (3,3) House 314159 -dictGetOrDefault test_01037.dict_array (5,6) Click 42 -dictGetOrDefault test_01037.dict_array (7.01,7.01) ee 55 -dictGet test_01037.dict_tuple (-100,-42) qqq 101 -dictGet test_01037.dict_tuple (-1,0) Click South 423 -dictGet test_01037.dict_tuple (-0.1,0) Click South 423 -dictGet test_01037.dict_tuple (0,-2) Click West 424 -dictGet test_01037.dict_tuple (0,-1.1) Click West 424 -dictGet test_01037.dict_tuple (0,1.1) Click North 422 -dictGet test_01037.dict_tuple (0,2) Click North 422 -dictGet test_01037.dict_tuple (0.1,0) Click East 421 -dictGet test_01037.dict_tuple (0.99,2.99) Click North 422 -dictGet test_01037.dict_tuple (1,0) Click East 421 -dictGet test_01037.dict_tuple (3,3) House 314159 -dictGet test_01037.dict_tuple (5,6) Click 42 -dictGet test_01037.dict_tuple (7.01,7.01) qqq 101 -dictGetOrDefault test_01037.dict_tuple (-100,-42) www 1234 -dictGetOrDefault test_01037.dict_tuple (-1,0) Click South 423 -dictGetOrDefault test_01037.dict_tuple (-0.1,0) Click South 423 -dictGetOrDefault test_01037.dict_tuple (0,-2) Click West 424 -dictGetOrDefault test_01037.dict_tuple (0,-1.1) Click West 424 -dictGetOrDefault test_01037.dict_tuple (0,1.1) Click North 422 -dictGetOrDefault test_01037.dict_tuple (0,2) Click North 422 -dictGetOrDefault test_01037.dict_tuple (0.1,0) Click East 421 -dictGetOrDefault test_01037.dict_tuple (0.99,2.99) Click North 422 -dictGetOrDefault test_01037.dict_tuple (1,0) Click East 421 -dictGetOrDefault test_01037.dict_tuple (3,3) House 314159 -dictGetOrDefault test_01037.dict_tuple (5,6) Click 42 -dictGetOrDefault test_01037.dict_tuple (7.01,7.01) www 1234 -dictGetOrDefault test_01037.dict_tuple (-100,-42) dd 44 -dictGetOrDefault test_01037.dict_tuple (-1,0) Click South 423 -dictGetOrDefault test_01037.dict_tuple (-0.1,0) Click South 423 -dictGetOrDefault test_01037.dict_tuple (0,-2) Click West 424 -dictGetOrDefault test_01037.dict_tuple (0,-1.1) Click West 424 -dictGetOrDefault test_01037.dict_tuple (0,1.1) Click North 422 -dictGetOrDefault test_01037.dict_tuple (0,2) Click North 422 -dictGetOrDefault test_01037.dict_tuple (0.1,0) Click East 421 -dictGetOrDefault test_01037.dict_tuple (0.99,2.99) Click North 422 -dictGetOrDefault test_01037.dict_tuple (1,0) Click East 421 -dictGetOrDefault test_01037.dict_tuple (3,3) House 314159 -dictGetOrDefault test_01037.dict_tuple (5,6) Click 42 -dictGetOrDefault test_01037.dict_tuple (7.01,7.01) ee 55 -dictHas test_01037.dict_array (-100,-42) 0 -dictHas test_01037.dict_array (-1,0) 1 -dictHas test_01037.dict_array (-0.1,0) 1 -dictHas test_01037.dict_array (0,-2) 1 -dictHas test_01037.dict_array (0,-1.1) 1 -dictHas test_01037.dict_array (0,1.1) 1 -dictHas test_01037.dict_array (0,2) 1 -dictHas test_01037.dict_array (0.1,0) 1 -dictHas test_01037.dict_array (0.99,2.99) 1 -dictHas test_01037.dict_array (1,0) 1 -dictHas test_01037.dict_array (3,3) 1 -dictHas test_01037.dict_array (5,6) 1 -dictHas test_01037.dict_array (7.01,7.01) 0 -dictHas test_01037.dict_tuple (-100,-42) 0 -dictHas test_01037.dict_tuple (-1,0) 1 -dictHas test_01037.dict_tuple (-0.1,0) 1 -dictHas test_01037.dict_tuple (0,-2) 1 -dictHas test_01037.dict_tuple (0,-1.1) 1 -dictHas test_01037.dict_tuple (0,1.1) 1 -dictHas test_01037.dict_tuple (0,2) 1 -dictHas test_01037.dict_tuple (0.1,0) 1 -dictHas test_01037.dict_tuple (0.99,2.99) 1 -dictHas test_01037.dict_tuple (1,0) 1 -dictHas test_01037.dict_tuple (3,3) 1 -dictHas test_01037.dict_tuple (5,6) 1 -dictHas test_01037.dict_tuple (7.01,7.01) 0 +dictGet dict_array (-100,-42) qqq 101 +dictGet dict_array (-1,0) Click South 423 +dictGet dict_array (-0.1,0) Click South 423 +dictGet dict_array (0,-2) Click West 424 +dictGet dict_array (0,-1.1) Click West 424 +dictGet dict_array (0,1.1) Click North 422 +dictGet dict_array (0,2) Click North 422 +dictGet dict_array (0.1,0) Click East 421 +dictGet dict_array (0.99,2.99) Click North 422 +dictGet dict_array (1,0) Click East 421 +dictGet dict_array (3,3) House 314159 +dictGet dict_array (5,6) Click 42 +dictGet dict_array (7.01,7.01) qqq 101 +dictGetOrDefault dict_array (-100,-42) www 1234 +dictGetOrDefault dict_array (-1,0) Click South 423 +dictGetOrDefault dict_array (-0.1,0) Click South 423 +dictGetOrDefault dict_array (0,-2) Click West 424 +dictGetOrDefault dict_array (0,-1.1) Click West 424 +dictGetOrDefault dict_array (0,1.1) Click North 422 +dictGetOrDefault dict_array (0,2) Click North 422 +dictGetOrDefault dict_array (0.1,0) Click East 421 +dictGetOrDefault dict_array (0.99,2.99) Click North 422 +dictGetOrDefault dict_array (1,0) Click East 421 +dictGetOrDefault dict_array (3,3) House 314159 +dictGetOrDefault dict_array (5,6) Click 42 +dictGetOrDefault dict_array (7.01,7.01) www 1234 +dictGetOrDefault dict_array (-100,-42) dd 44 +dictGetOrDefault dict_array (-1,0) Click South 423 +dictGetOrDefault dict_array (-0.1,0) Click South 423 +dictGetOrDefault dict_array (0,-2) Click West 424 +dictGetOrDefault dict_array (0,-1.1) Click West 424 +dictGetOrDefault dict_array (0,1.1) Click North 422 +dictGetOrDefault dict_array (0,2) Click North 422 +dictGetOrDefault dict_array (0.1,0) Click East 421 +dictGetOrDefault dict_array (0.99,2.99) Click North 422 +dictGetOrDefault dict_array (1,0) Click East 421 +dictGetOrDefault dict_array (3,3) House 314159 +dictGetOrDefault dict_array (5,6) Click 42 +dictGetOrDefault dict_array (7.01,7.01) ee 55 +dictGet dict_tuple (-100,-42) qqq 101 +dictGet dict_tuple (-1,0) Click South 423 +dictGet dict_tuple (-0.1,0) Click South 423 +dictGet dict_tuple (0,-2) Click West 424 +dictGet dict_tuple (0,-1.1) Click West 424 +dictGet dict_tuple (0,1.1) Click North 422 +dictGet dict_tuple (0,2) Click North 422 +dictGet dict_tuple (0.1,0) Click East 421 +dictGet dict_tuple (0.99,2.99) Click North 422 +dictGet dict_tuple (1,0) Click East 421 +dictGet dict_tuple (3,3) House 314159 +dictGet dict_tuple (5,6) Click 42 +dictGet dict_tuple (7.01,7.01) qqq 101 +dictGetOrDefault dict_tuple (-100,-42) www 1234 +dictGetOrDefault dict_tuple (-1,0) Click South 423 +dictGetOrDefault dict_tuple (-0.1,0) Click South 423 +dictGetOrDefault dict_tuple (0,-2) Click West 424 +dictGetOrDefault dict_tuple (0,-1.1) Click West 424 +dictGetOrDefault dict_tuple (0,1.1) Click North 422 +dictGetOrDefault dict_tuple (0,2) Click North 422 +dictGetOrDefault dict_tuple (0.1,0) Click East 421 +dictGetOrDefault dict_tuple (0.99,2.99) Click North 422 +dictGetOrDefault dict_tuple (1,0) Click East 421 +dictGetOrDefault dict_tuple (3,3) House 314159 +dictGetOrDefault dict_tuple (5,6) Click 42 +dictGetOrDefault dict_tuple (7.01,7.01) www 1234 +dictGetOrDefault dict_tuple (-100,-42) dd 44 +dictGetOrDefault dict_tuple (-1,0) Click South 423 +dictGetOrDefault dict_tuple (-0.1,0) Click South 423 +dictGetOrDefault dict_tuple (0,-2) Click West 424 +dictGetOrDefault dict_tuple (0,-1.1) Click West 424 +dictGetOrDefault dict_tuple (0,1.1) Click North 422 +dictGetOrDefault dict_tuple (0,2) Click North 422 +dictGetOrDefault dict_tuple (0.1,0) Click East 421 +dictGetOrDefault dict_tuple (0.99,2.99) Click North 422 +dictGetOrDefault dict_tuple (1,0) Click East 421 +dictGetOrDefault dict_tuple (3,3) House 314159 +dictGetOrDefault dict_tuple (5,6) Click 42 +dictGetOrDefault dict_tuple (7.01,7.01) ee 55 +dictHas dict_array (-100,-42) 0 +dictHas dict_array (-1,0) 1 +dictHas dict_array (-0.1,0) 1 +dictHas dict_array (0,-2) 1 +dictHas dict_array (0,-1.1) 1 +dictHas dict_array (0,1.1) 1 +dictHas dict_array (0,2) 1 +dictHas dict_array (0.1,0) 1 +dictHas dict_array (0.99,2.99) 1 +dictHas dict_array (1,0) 1 +dictHas dict_array (3,3) 1 +dictHas dict_array (5,6) 1 +dictHas dict_array (7.01,7.01) 0 +dictHas dict_tuple (-100,-42) 0 +dictHas dict_tuple (-1,0) 1 +dictHas dict_tuple (-0.1,0) 1 +dictHas dict_tuple (0,-2) 1 +dictHas dict_tuple (0,-1.1) 1 +dictHas dict_tuple (0,1.1) 1 +dictHas dict_tuple (0,2) 1 +dictHas dict_tuple (0.1,0) 1 +dictHas dict_tuple (0.99,2.99) 1 +dictHas dict_tuple (1,0) 1 +dictHas dict_tuple (3,3) 1 +dictHas dict_tuple (5,6) 1 +dictHas dict_tuple (7.01,7.01) 0 diff --git a/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh b/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh index be983ec1be4..efc66783d62 100755 --- a/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh +++ b/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh @@ -1,56 +1,52 @@ #!/usr/bin/env bash -# Tags: no-debug, no-parallel +# Tags: no-debug CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -TMP_DIR="/tmp" +TMP_DIR=${CLICKHOUSE_TMP}${CLICKHOUSE_DATABASE} +mkdir -p $TMP_DIR $CLICKHOUSE_CLIENT -n --query=" -DROP DATABASE IF EXISTS test_01037; +DROP TABLE IF EXISTS polygons_array; -CREATE DATABASE test_01037; +CREATE TABLE polygons_array (key Array(Array(Array(Array(Float64)))), name String, value UInt64) ENGINE = Memory; +INSERT INTO polygons_array VALUES ([[[[1, 3], [1, 1], [3, 1], [3, -1], [1, -1], [1, -3], [-1, -3], [-1, -1], [-3, -1], [-3, 1], [-1, 1], [-1, 3]]], [[[5, 5], [5, 1], [7, 1], [7, 7], [1, 7], [1, 5]]]], 'Click', 42); +INSERT INTO polygons_array VALUES ([[[[5, 5], [5, -5], [-5, -5], [-5, 5]], [[1, 3], [1, 1], [3, 1], [3, -1], [1, -1], [1, -3], [-1, -3], [-1, -1], [-3, -1], [-3, 1], [-1, 1], [-1, 3]]]], 'House', 314159); +INSERT INTO polygons_array VALUES ([[[[3, 1], [0, 1], [0, -1], [3, -1]]]], 'Click East', 421); +INSERT INTO polygons_array VALUES ([[[[-1, 1], [1, 1], [1, 3], [-1, 3]]]], 'Click North', 422); +INSERT INTO polygons_array VALUES ([[[[-3, 1], [-3, -1], [0, -1], [0, 1]]]], 'Click South', 423); +INSERT INTO polygons_array VALUES ([[[[-1, -1], [1, -1], [1, -3], [-1, -3]]]], 'Click West', 424); -DROP TABLE IF EXISTS test_01037.polygons_array; +DROP TABLE IF EXISTS polygons_tuple; -CREATE TABLE test_01037.polygons_array (key Array(Array(Array(Array(Float64)))), name String, value UInt64) ENGINE = Memory; -INSERT INTO test_01037.polygons_array VALUES ([[[[1, 3], [1, 1], [3, 1], [3, -1], [1, -1], [1, -3], [-1, -3], [-1, -1], [-3, -1], [-3, 1], [-1, 1], [-1, 3]]], [[[5, 5], [5, 1], [7, 1], [7, 7], [1, 7], [1, 5]]]], 'Click', 42); -INSERT INTO test_01037.polygons_array VALUES ([[[[5, 5], [5, -5], [-5, -5], [-5, 5]], [[1, 3], [1, 1], [3, 1], [3, -1], [1, -1], [1, -3], [-1, -3], [-1, -1], [-3, -1], [-3, 1], [-1, 1], [-1, 3]]]], 'House', 314159); -INSERT INTO test_01037.polygons_array VALUES ([[[[3, 1], [0, 1], [0, -1], [3, -1]]]], 'Click East', 421); -INSERT INTO test_01037.polygons_array VALUES ([[[[-1, 1], [1, 1], [1, 3], [-1, 3]]]], 'Click North', 422); -INSERT INTO test_01037.polygons_array VALUES ([[[[-3, 1], [-3, -1], [0, -1], [0, 1]]]], 'Click South', 423); -INSERT INTO test_01037.polygons_array VALUES ([[[[-1, -1], [1, -1], [1, -3], [-1, -3]]]], 'Click West', 424); +CREATE TABLE polygons_tuple (key Array(Array(Array(Tuple(Float64, Float64)))), name String, value UInt64) ENGINE = Memory; +INSERT INTO polygons_tuple VALUES ([[[(1, 3), (1, 1), (3, 1), (3, -1), (1, -1), (1, -3), (-1, -3), (-1, -1), (-3, -1), (-3, 1), (-1, 1), (-1, 3)]], [[(5, 5), (5, 1), (7, 1), (7, 7), (1, 7), (1, 5)]]], 'Click', 42); +INSERT INTO polygons_tuple VALUES ([[[(5, 5), (5, -5), (-5, -5), (-5, 5)], [(1, 3), (1, 1), (3, 1), (3, -1), (1, -1), (1, -3), (-1, -3), (-1, -1), (-3, -1), (-3, 1), (-1, 1), (-1, 3)]]], 'House', 314159); +INSERT INTO polygons_tuple VALUES ([[[(3, 1), (0, 1), (0, -1), (3, -1)]]], 'Click East', 421); +INSERT INTO polygons_tuple VALUES ([[[(-1, 1), (1, 1), (1, 3), (-1, 3)]]], 'Click North', 422); +INSERT INTO polygons_tuple VALUES ([[[(-3, 1), (-3, -1), (0, -1), (0, 1)]]], 'Click South', 423); +INSERT INTO polygons_tuple VALUES ([[[(-1, -1), (1, -1), (1, -3), (-1, -3)]]], 'Click West', 424); -DROP TABLE IF EXISTS test_01037.polygons_tuple; +DROP TABLE IF EXISTS points; -CREATE TABLE test_01037.polygons_tuple (key Array(Array(Array(Tuple(Float64, Float64)))), name String, value UInt64) ENGINE = Memory; -INSERT INTO test_01037.polygons_tuple VALUES ([[[(1, 3), (1, 1), (3, 1), (3, -1), (1, -1), (1, -3), (-1, -3), (-1, -1), (-3, -1), (-3, 1), (-1, 1), (-1, 3)]], [[(5, 5), (5, 1), (7, 1), (7, 7), (1, 7), (1, 5)]]], 'Click', 42); -INSERT INTO test_01037.polygons_tuple VALUES ([[[(5, 5), (5, -5), (-5, -5), (-5, 5)], [(1, 3), (1, 1), (3, 1), (3, -1), (1, -1), (1, -3), (-1, -3), (-1, -1), (-3, -1), (-3, 1), (-1, 1), (-1, 3)]]], 'House', 314159); -INSERT INTO test_01037.polygons_tuple VALUES ([[[(3, 1), (0, 1), (0, -1), (3, -1)]]], 'Click East', 421); -INSERT INTO test_01037.polygons_tuple VALUES ([[[(-1, 1), (1, 1), (1, 3), (-1, 3)]]], 'Click North', 422); -INSERT INTO test_01037.polygons_tuple VALUES ([[[(-3, 1), (-3, -1), (0, -1), (0, 1)]]], 'Click South', 423); -INSERT INTO test_01037.polygons_tuple VALUES ([[[(-1, -1), (1, -1), (1, -3), (-1, -3)]]], 'Click West', 424); - -DROP TABLE IF EXISTS test_01037.points; - -CREATE TABLE test_01037.points (x Float64, y Float64, def_i UInt64, def_s String) ENGINE = Memory; -INSERT INTO test_01037.points VALUES (0.1, 0.0, 112, 'aax'); -INSERT INTO test_01037.points VALUES (-0.1, 0.0, 113, 'aay'); -INSERT INTO test_01037.points VALUES (0.0, 1.1, 114, 'aaz'); -INSERT INTO test_01037.points VALUES (0.0, -1.1, 115, 'aat'); -INSERT INTO test_01037.points VALUES (3.0, 3.0, 22, 'bb'); -INSERT INTO test_01037.points VALUES (5.0, 6.0, 33, 'cc'); -INSERT INTO test_01037.points VALUES (-100.0, -42.0, 44, 'dd'); -INSERT INTO test_01037.points VALUES (7.01, 7.01, 55, 'ee'); -INSERT INTO test_01037.points VALUES (0.99, 2.99, 66, 'ee'); -INSERT INTO test_01037.points VALUES (1.0, 0.0, 771, 'ffa'); -INSERT INTO test_01037.points VALUES (-1.0, 0.0, 772, 'ffb'); -INSERT INTO test_01037.points VALUES (0.0, 2.0, 773, 'ffc'); -INSERT INTO test_01037.points VALUES (0.0, -2.0, 774, 'ffd'); +CREATE TABLE points (x Float64, y Float64, def_i UInt64, def_s String) ENGINE = Memory; +INSERT INTO points VALUES (0.1, 0.0, 112, 'aax'); +INSERT INTO points VALUES (-0.1, 0.0, 113, 'aay'); +INSERT INTO points VALUES (0.0, 1.1, 114, 'aaz'); +INSERT INTO points VALUES (0.0, -1.1, 115, 'aat'); +INSERT INTO points VALUES (3.0, 3.0, 22, 'bb'); +INSERT INTO points VALUES (5.0, 6.0, 33, 'cc'); +INSERT INTO points VALUES (-100.0, -42.0, 44, 'dd'); +INSERT INTO points VALUES (7.01, 7.01, 55, 'ee'); +INSERT INTO points VALUES (0.99, 2.99, 66, 'ee'); +INSERT INTO points VALUES (1.0, 0.0, 771, 'ffa'); +INSERT INTO points VALUES (-1.0, 0.0, 772, 'ffb'); +INSERT INTO points VALUES (0.0, 2.0, 773, 'ffc'); +INSERT INTO points VALUES (0.0, -2.0, 774, 'ffd'); " - declare -a SearchTypes=("POLYGON" "POLYGON_SIMPLE" "POLYGON_INDEX_EACH" "POLYGON_INDEX_CELL") for type in "${SearchTypes[@]}"; @@ -58,63 +54,62 @@ do outputFile="${TMP_DIR}/results${type}.out" $CLICKHOUSE_CLIENT -n --query=" - DROP DICTIONARY IF EXISTS test_01037.dict_array; - CREATE DICTIONARY test_01037.dict_array + DROP DICTIONARY IF EXISTS dict_array; + CREATE DICTIONARY dict_array ( key Array(Array(Array(Array(Float64)))), name String DEFAULT 'qqq', value UInt64 DEFAULT 101 ) PRIMARY KEY key - SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'polygons_array' PASSWORD '' DB 'test_01037')) + SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'polygons_array' PASSWORD '' DB currentDatabase())) LIFETIME(0) LAYOUT($type()); - DROP DICTIONARY IF EXISTS test_01037.dict_tuple; + DROP DICTIONARY IF EXISTS dict_tuple; - CREATE DICTIONARY test_01037.dict_tuple + CREATE DICTIONARY dict_tuple ( key Array(Array(Array(Tuple(Float64, Float64)))), name String DEFAULT 'qqq', value UInt64 DEFAULT 101 ) PRIMARY KEY key - SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'polygons_tuple' PASSWORD '' DB 'test_01037')) + SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'polygons_tuple' PASSWORD '' DB currentDatabase())) LIFETIME(0) LAYOUT($type()); - select 'dictGet', 'test_01037.dict_array' as dict_name, tuple(x, y) as key, + select 'dictGet', 'dict_array' as dict_name, tuple(x, y) as key, dictGet(dict_name, 'name', key), - dictGet(dict_name, 'value', key) from test_01037.points order by x, y; - select 'dictGetOrDefault', 'test_01037.dict_array' as dict_name, tuple(x, y) as key, + dictGet(dict_name, 'value', key) from points order by x, y; + select 'dictGetOrDefault', 'dict_array' as dict_name, tuple(x, y) as key, dictGetOrDefault(dict_name, 'name', key, 'www'), - dictGetOrDefault(dict_name, 'value', key, toUInt64(1234)) from test_01037.points order by x, y; - select 'dictGetOrDefault', 'test_01037.dict_array' as dict_name, tuple(x, y) as key, + dictGetOrDefault(dict_name, 'value', key, toUInt64(1234)) from points order by x, y; + select 'dictGetOrDefault', 'dict_array' as dict_name, tuple(x, y) as key, dictGetOrDefault(dict_name, 'name', key, def_s), - dictGetOrDefault(dict_name, 'value', key, def_i) from test_01037.points order by x, y; - select 'dictGet', 'test_01037.dict_tuple' as dict_name, tuple(x, y) as key, + dictGetOrDefault(dict_name, 'value', key, def_i) from points order by x, y; + select 'dictGet', 'dict_tuple' as dict_name, tuple(x, y) as key, dictGet(dict_name, 'name', key), - dictGet(dict_name, 'value', key) from test_01037.points order by x, y; - select 'dictGetOrDefault', 'test_01037.dict_tuple' as dict_name, tuple(x, y) as key, + dictGet(dict_name, 'value', key) from points order by x, y; + select 'dictGetOrDefault', 'dict_tuple' as dict_name, tuple(x, y) as key, dictGetOrDefault(dict_name, 'name', key, 'www'), - dictGetOrDefault(dict_name, 'value', key, toUInt64(1234)) from test_01037.points order by x, y; - select 'dictGetOrDefault', 'test_01037.dict_tuple' as dict_name, tuple(x, y) as key, + dictGetOrDefault(dict_name, 'value', key, toUInt64(1234)) from points order by x, y; + select 'dictGetOrDefault', 'dict_tuple' as dict_name, tuple(x, y) as key, dictGetOrDefault(dict_name, 'name', key, def_s), - dictGetOrDefault(dict_name, 'value', key, def_i) from test_01037.points order by x, y; - select 'dictHas', 'test_01037.dict_array' as dict_name, tuple(x, y) as key, - dictHas(dict_name, key) from test_01037.points order by x, y; - select 'dictHas', 'test_01037.dict_tuple' as dict_name, tuple(x, y) as key, - dictHas(dict_name, key) from test_01037.points order by x, y; + dictGetOrDefault(dict_name, 'value', key, def_i) from points order by x, y; + select 'dictHas', 'dict_array' as dict_name, tuple(x, y) as key, + dictHas(dict_name, key) from points order by x, y; + select 'dictHas', 'dict_tuple' as dict_name, tuple(x, y) as key, + dictHas(dict_name, key) from points order by x, y; " > "$outputFile" diff -q "${CURDIR}/01037_polygon_dicts_simple_functions.ans" "$outputFile" done $CLICKHOUSE_CLIENT -n --query=" -DROP DICTIONARY test_01037.dict_array; -DROP DICTIONARY test_01037.dict_tuple; -DROP TABLE test_01037.polygons_array; -DROP TABLE test_01037.polygons_tuple; -DROP TABLE test_01037.points; -DROP DATABASE test_01037; +DROP DICTIONARY dict_array; +DROP DICTIONARY dict_tuple; +DROP TABLE polygons_array; +DROP TABLE polygons_tuple; +DROP TABLE points; " From b38928b09f75bc88c722c10ba401623a880a7b70 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 12 Jul 2024 19:25:32 +0200 Subject: [PATCH 44/72] Update 00992_system_parts_race_condition_zookeeper_long.sh --- .../00992_system_parts_race_condition_zookeeper_long.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh index d45cc3a6871..4887c409844 100755 --- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh +++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh @@ -41,7 +41,7 @@ function thread3() function thread4() { - while true; do $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table0 FINAL"; done + while true; do $CLICKHOUSE_CLIENT --receive_timeout=3 -q "OPTIMIZE TABLE alter_table0 FINAL" | grep -Fv "Timeout exceeded while receiving data from server"; done } function thread5() From 8d07c522f155b3b6ec9760b9ecef28c90598dd1f Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 12 Jul 2024 17:13:33 +0100 Subject: [PATCH 45/72] reduce amount of parallel linker jobs further --- cmake/limit_jobs.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 3a759b90fe3..17d8dd42a2c 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -44,8 +44,13 @@ endif () # - and I've verfied that lld-11 does not use all available CPU time (in peak) while linking one binary if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO) if (ARCH_AARCH64) + # aarch64 builds start to often fail with OOMs (reason not yet clear), for now let's limit the concurrency message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 1.") set (PARALLEL_LINK_JOBS 1) + if (LINKER_NAME MATCHES "lld") + math(EXPR LTO_JOBS ${NUMBER_OF_LOGICAL_CORES}/4) + set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--thinlto-jobs=${LTO_JOBS}") + endif() elseif (PARALLEL_LINK_JOBS GREATER 2) message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.") set (PARALLEL_LINK_JOBS 2) From 0bb3d07e8ed0f5e7005186e47446d57f8fea0e9f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 12 Jul 2024 20:12:39 +0200 Subject: [PATCH 46/72] fix --- src/Processors/Formats/Impl/NpyRowInputFormat.cpp | 3 +++ tests/queries/0_stateless/02895_npy_format.reference | 12 ++++++------ tests/queries/0_stateless/02895_npy_format.sh | 12 ++++++------ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/Processors/Formats/Impl/NpyRowInputFormat.cpp b/src/Processors/Formats/Impl/NpyRowInputFormat.cpp index 65e0f9dd192..773cbc9268e 100644 --- a/src/Processors/Formats/Impl/NpyRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/NpyRowInputFormat.cpp @@ -445,6 +445,9 @@ bool NpyRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & /* elements_in_current_column *= header.shape[i]; } + if (typeid_cast(current_column)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected nesting level of column '{}', expected {}", column->getName(), header.shape.size() - 1); + for (size_t i = 0; i != elements_in_current_column; ++i) readValue(current_column); diff --git a/tests/queries/0_stateless/02895_npy_format.reference b/tests/queries/0_stateless/02895_npy_format.reference index f9e77644a35..52972f0acbd 100644 --- a/tests/queries/0_stateless/02895_npy_format.reference +++ b/tests/queries/0_stateless/02895_npy_format.reference @@ -85,12 +85,12 @@ c [4,5,6] [[1,2],[3,4]] [[5,6],[7,8]] -0 -0 -0 -0 -0 -0 +1 +1 +1 +1 +1 +1 1 [2.199219,1.099609,3.300781] [4.25,3.34961,6.628906] diff --git a/tests/queries/0_stateless/02895_npy_format.sh b/tests/queries/0_stateless/02895_npy_format.sh index 9d05303a091..194b2bc1fe4 100755 --- a/tests/queries/0_stateless/02895_npy_format.sh +++ b/tests/queries/0_stateless/02895_npy_format.sh @@ -52,14 +52,14 @@ $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/two_dim.npy', Npy, 'v $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/three_dim.npy', Npy, 'value Array(Array(Int8))')" $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Array(Float32)')" 2>&1 | grep -c "BAD_ARGUMENTS" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value UUID')" 2>&1 | grep -c "BAD_ARGUMENTS" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Tuple(UInt8)')" 2>&1 | grep -c "BAD_ARGUMENTS" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value UUID')" 2>&1 | grep -c "UNKNOWN_TYPE" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Tuple(UInt8)')" 2>&1 | grep -c "UNKNOWN_TYPE" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Int8')" 2>&1 | grep -c "BAD_ARGUMENTS" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_str.npy', Npy, 'value Int8')" 2>&1 | grep -c "BAD_ARGUMENTS" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_unicode.npy', Npy, 'value Float32')" 2>&1 | grep -c "BAD_ARGUMENTS" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Int8')" 2>&1 | grep -c "ILLEGAL_COLUMN" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_str.npy', Npy, 'value Int8')" 2>&1 | grep -c "ILLEGAL_COLUMN" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_unicode.npy', Npy, 'value Float32')" 2>&1 | grep -c "ILLEGAL_COLUMN" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/complex.npy')" 2>&1 | grep -c "BAD_ARGUMENTS" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/complex.npy')" 2>&1 | grep -c "CANNOT_EXTRACT_TABLE_STRUCTURE" $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/float_16.npy')" From 5b1e9bebe47e6b6971e6432f788e9ad9ce1c5f2b Mon Sep 17 00:00:00 2001 From: Max K Date: Fri, 12 Jul 2024 18:19:30 +0200 Subject: [PATCH 47/72] change thresholds --- tests/ci/ci_config.py | 4 ++-- tests/ci/merge_pr.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 9a9aa553e1b..d9f8e7d3afd 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -13,8 +13,8 @@ class CI: each config item in the below dicts should be an instance of JobConfig class or inherited from it """ - MAX_TOTAL_FAILURES_BEFORE_BLOCKING_CI = 2 - MAX_TOTAL_FAILURES_PER_JOB_BEFORE_BLOCKING_CI = 1 + MAX_TOTAL_FAILURES_BEFORE_BLOCKING_CI = 5 + MAX_TOTAL_FAILURES_PER_JOB_BEFORE_BLOCKING_CI = 2 # reimport types to CI class so that they visible as CI.* and mypy is happy # pylint:disable=useless-import-alias,reimported,import-outside-toplevel diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py index 6b437731561..061376fc856 100644 --- a/tests/ci/merge_pr.py +++ b/tests/ci/merge_pr.py @@ -291,13 +291,14 @@ def main(): ) can_continue = False if failed_to_get_info: - print(f"Unexpected commit status state - block further testing") + print("Unexpected commit status state - block further testing") can_continue = False if args.wf_status != SUCCESS: can_continue = False print("Workflow has failures - block further testing") if args.wf_status == "success" or has_failed_statuses: + # do not set mergeable check status if args.wf_status == failure, apparently it has died runners and is to be restarted state = trigger_mergeable_check( commit, statuses, From 1495ef32180625b743f9f01cd86b4a257ae96ff0 Mon Sep 17 00:00:00 2001 From: Max K Date: Fri, 12 Jul 2024 20:32:01 +0200 Subject: [PATCH 48/72] CI: Set error status for job with OOM --- pyproject.toml | 1 + tests/ci/ci.py | 23 +++++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 90f089afa41..39511e1a0d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ disable = ''' bare-except, no-else-return, global-statement, + f-string-without-interpolation, ''' [tool.pylint.SIMILARITIES] diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 32b87698395..9f4a98114c5 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1125,6 +1125,7 @@ def main() -> int: ### POST action: start elif args.post: + has_oom_error = False if Shell.check( "sudo dmesg -T | grep -q -e 'Out of memory: Killed process' -e 'oom_reaper: reaped process' -e 'oom-kill:constraint=CONSTRAINT_NONE'" ): @@ -1132,6 +1133,7 @@ def main() -> int: CIBuddy(dry_run=not pr_info.is_release).post_error( "Out Of Memory", job_name=_get_ext_check_name(args.job_name) ) + has_oom_error = True job_report = JobReport.load() if JobReport.exist() else None if job_report: @@ -1235,8 +1237,25 @@ def main() -> int: ch_helper, ) else: - # no job report - print(f"No job report for {[args.job_name]} - do nothing") + if CI.is_test_job(args.job_name): + if has_oom_error: + description = "ERROR: Out Of Memory" + else: + description = "ERROR: Unknown job status" + gh = GitHub(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) + post_commit_status( + commit, + ERROR, + "", + description, + job_report.check_name or _get_ext_check_name(args.job_name), + pr_info, + dump_to_file=True, + ) + else: + # no job report + print(f"No job report for {[args.job_name]} - do nothing") ### POST action: end ### MARK SUCCESS action: start From 2dc7d1f510dfc7a4719835a31acf9a9a47a8c1dc Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Fri, 12 Jul 2024 21:09:10 +0200 Subject: [PATCH 49/72] Stateless tests: fix flaky tests 2 --- .../0_stateless/01037_polygon_dicts_correctness_all.sh | 5 +++-- .../0_stateless/01037_polygon_dicts_correctness_fast.sh | 5 +++-- .../0_stateless/01037_polygon_dicts_simple_functions.sh | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh b/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh index 39f235d9966..9a26f78a8ee 100755 --- a/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh +++ b/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh @@ -5,12 +5,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -TMP_DIR=${CLICKHOUSE_TMP}${CLICKHOUSE_DATABASE} +TMP_DIR=${CLICKHOUSE_TMP}/tmp +DATA_DIR=${CLICKHOUSE_TMP}/data mkdir -p $TMP_DIR +mkdir -p $DATA_DIR declare -a SearchTypes=("POLYGON" "POLYGON_SIMPLE" "POLYGON_INDEX_EACH" "POLYGON_INDEX_CELL") -DATA_DIR=${CURDIR}/${CLICKHOUSE_DATABASE} tar -xf "${CURDIR}"/01037_test_data_search.tar.gz -C "${DATA_DIR}" $CLICKHOUSE_CLIENT -n --query=" diff --git a/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh b/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh index 3e461abcefe..47f7a5c1c4f 100755 --- a/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh +++ b/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh @@ -5,12 +5,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -TMP_DIR=${CLICKHOUSE_TMP}${CLICKHOUSE_DATABASE} +TMP_DIR=${CLICKHOUSE_TMP}/tmp +DATA_DIR=${CLICKHOUSE_TMP}/data mkdir -p $TMP_DIR +mkdir -p $DATA_DIR declare -a SearchTypes=("POLYGON_INDEX_EACH" "POLYGON_INDEX_CELL") -DATA_DIR=${CURDIR}/${CLICKHOUSE_DATABASE} tar -xf "${CURDIR}"/01037_test_data_perf.tar.gz -C "${DATA_DIR}" $CLICKHOUSE_CLIENT -n --query=" diff --git a/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh b/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh index efc66783d62..d1ee3f283bc 100755 --- a/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh +++ b/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -TMP_DIR=${CLICKHOUSE_TMP}${CLICKHOUSE_DATABASE} +TMP_DIR=${CLICKHOUSE_TMP}/tmp mkdir -p $TMP_DIR $CLICKHOUSE_CLIENT -n --query=" From cc2cce97177552dcf82682f06418ffa3388760c1 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Sat, 13 Jul 2024 00:34:54 +0200 Subject: [PATCH 50/72] Stateless tests: fix flaky tests 3 --- tests/clickhouse-test | 2 +- tests/queries/0_stateless/02834_apache_arrow_abort.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 79f6b5d71d3..bc30b3c21b7 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -711,7 +711,7 @@ def get_localzone(): class SettingsRandomizer: settings = { - "max_insert_threads": lambda: 32 + "max_insert_threads": lambda: 12 if random.random() < 0.03 else random.randint(1, 3), "group_by_two_level_threshold": threshold_generator(0.2, 0.2, 1, 1000000), diff --git a/tests/queries/0_stateless/02834_apache_arrow_abort.sql b/tests/queries/0_stateless/02834_apache_arrow_abort.sql index bd29e95db9a..47e1c5d3951 100644 --- a/tests/queries/0_stateless/02834_apache_arrow_abort.sql +++ b/tests/queries/0_stateless/02834_apache_arrow_abort.sql @@ -1,4 +1,4 @@ -- Tags: no-fasttest -- This tests depends on internet access, but it does not matter, because it only has to check that there is no abort due to a bug in Apache Arrow library. - +SET optimize_trivial_insert_select=1; INSERT INTO TABLE FUNCTION url('https://clickhouse-public-datasets.s3.amazonaws.com/hits_compatible/athena_partitioned/hits_9.parquet') SELECT * FROM url('https://clickhouse-public-datasets.s3.amazonaws.com/hits_compatible/athena_partitioned/hits_9.parquet'); -- { serverError CANNOT_WRITE_TO_OSTREAM, RECEIVED_ERROR_FROM_REMOTE_IO_SERVER, POCO_EXCEPTION } From 8295a8e9b8360eec0128078fb05f6c3d50ce3b97 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Sat, 13 Jul 2024 00:39:53 +0200 Subject: [PATCH 51/72] Stateless tests: fix flaky tests 4 --- tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index bc30b3c21b7..0cf46732354 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -729,7 +729,7 @@ class SettingsRandomizer: "prefer_localhost_replica": lambda: random.randint(0, 1), "max_block_size": lambda: random.randint(8000, 100000), "max_joined_block_size_rows": lambda: random.randint(8000, 100000), - "max_threads": lambda: 64 if random.random() < 0.03 else random.randint(1, 3), + "max_threads": lambda: 32 if random.random() < 0.03 else random.randint(1, 3), "optimize_append_index": lambda: random.randint(0, 1), "optimize_if_chain_to_multiif": lambda: random.randint(0, 1), "optimize_if_transform_strings_to_enum": lambda: random.randint(0, 1), From 04525888f5db6f2c0e61e170cab5ad57626fbf17 Mon Sep 17 00:00:00 2001 From: Max K Date: Sat, 13 Jul 2024 11:55:25 +0200 Subject: [PATCH 52/72] fix for failed workflow status --- tests/ci/merge_pr.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py index 061376fc856..6fb6821ede4 100644 --- a/tests/ci/merge_pr.py +++ b/tests/ci/merge_pr.py @@ -293,11 +293,14 @@ def main(): if failed_to_get_info: print("Unexpected commit status state - block further testing") can_continue = False - if args.wf_status != SUCCESS: + if args.wf_status != SUCCESS and not has_failed_statuses: + # workflow failed but reason is unknown as no failed statuses present can_continue = False - print("Workflow has failures - block further testing") + print( + "WARNING: Either the runner is faulty or the operating status is unknown. The first is self-healing, the second requires investigation." + ) - if args.wf_status == "success" or has_failed_statuses: + if args.wf_status == SUCCESS or has_failed_statuses: # do not set mergeable check status if args.wf_status == failure, apparently it has died runners and is to be restarted state = trigger_mergeable_check( commit, From 8706145c467852e7d4b84e5a9823050b8de3e085 Mon Sep 17 00:00:00 2001 From: Max K Date: Sat, 13 Jul 2024 12:17:03 +0200 Subject: [PATCH 53/72] fix for not success status in Sync --- tests/ci/merge_pr.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py index 6fb6821ede4..59749abb4fa 100644 --- a/tests/ci/merge_pr.py +++ b/tests/ci/merge_pr.py @@ -272,7 +272,11 @@ def main(): job_name_with_max_failures = status.context max_failed_tests_per_job = failed_cnt total_failed_tests += failed_cnt - elif status.state != SUCCESS: + elif status.state != SUCCESS and status.context not in ( + CI.StatusNames.SYNC, + CI.StatusNames.PR_CHECK, + ): + # do not block CI on failures in (CI.StatusNames.SYNC, CI.StatusNames.PR_CHECK) has_failed_statuses = True print( f"Unexpected status for [{status.context}]: [{status.state}] - block further testing" From 11f3e406c6ab040cc42d209ac2471406367f577c Mon Sep 17 00:00:00 2001 From: Max K Date: Sat, 13 Jul 2024 12:48:48 +0200 Subject: [PATCH 54/72] CI: Cache AST fuzzers (run always) jobs in CI --- tests/ci/ci.py | 4 ++-- tests/ci/ci_cache.py | 4 ++-- tests/ci/ci_definitions.py | 9 ++++++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 32b87698395..57552985f62 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -325,8 +325,8 @@ def _mark_success_action( # do nothing, exit without failure print(f"ERROR: no status file for job [{job}]") - if job_config.run_always or job_config.run_by_label: - print(f"Job [{job}] runs always or by label in CI - do not cache") + if job_config.run_by_label or not job_config.has_digest(): + print(f"Job [{job}] has no digest or run by label in CI - do not cache") else: if pr_info.is_master: pass diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index 291ed56aeea..bc6761959b4 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -609,7 +609,7 @@ class CiCache: pushes pending records for all jobs that supposed to be run """ for job, job_config in self.jobs_to_do.items(): - if job_config.run_always: + if not job_config.has_digest(): continue pending_state = PendingState(time.time(), run_url=GITHUB_RUN_URL) assert job_config.batches @@ -680,7 +680,7 @@ class CiCache: It removes jobs from @jobs_to_do if it is a: 1. test job and it is in @jobs_to_wait (no need to wait not affected jobs in PRs) 2. test job and it has finished on release branch (even if failed) - 2. build job which is not required by any test job that is left in @jobs_to_do + 3. build job which is not required by any test job that is left in @jobs_to_do :return: """ diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index 4ae252560e9..a79097d8b55 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -327,6 +327,9 @@ class JobConfig: assert self.required_builds return self.required_builds[0] + def has_digest(self) -> bool: + return self.digest != DigestConfig() + class CommonJobConfigs: """ @@ -440,7 +443,11 @@ class CommonJobConfigs: ) ASTFUZZER_TEST = JobConfig( job_name_keyword="ast", - digest=DigestConfig(), + digest=DigestConfig( + include_paths=[ + "./tests/ci/ast_fuzzer_check.py", + ], + docker=["clickhouse/fuzzer"]), run_command="ast_fuzzer_check.py", run_always=True, runner_type=Runners.FUZZER_UNIT_TESTER, From fd9f91c796227a4b9d7273f812c626c2053b098a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 13 Jul 2024 11:07:52 +0000 Subject: [PATCH 55/72] Automatic style fix --- tests/ci/ci_definitions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index a79097d8b55..d2da73f4e46 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -447,7 +447,8 @@ class CommonJobConfigs: include_paths=[ "./tests/ci/ast_fuzzer_check.py", ], - docker=["clickhouse/fuzzer"]), + docker=["clickhouse/fuzzer"], + ), run_command="ast_fuzzer_check.py", run_always=True, runner_type=Runners.FUZZER_UNIT_TESTER, From 99cd83da1f6bf71545337fdd72ffd2ed7cf68bb2 Mon Sep 17 00:00:00 2001 From: Max K Date: Thu, 27 Jun 2024 19:43:10 +0200 Subject: [PATCH 56/72] New Create Release workflow --- .github/workflows/create_release.yml | 138 +++++- tests/ci/artifactory.py | 356 ++++++++++++++ tests/ci/create_release.py | 663 +++++++++++++++++++++++++++ tests/ci/docker_server.py | 24 +- tests/ci/version_helper.py | 6 + 5 files changed, 1175 insertions(+), 12 deletions(-) create mode 100644 tests/ci/artifactory.py create mode 100755 tests/ci/create_release.py diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 3988df3b2b1..e2ad16a05a4 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -6,8 +6,8 @@ concurrency: 'on': workflow_dispatch: inputs: - sha: - description: 'The SHA hash of the commit from which to create the release' + ref: + description: 'Git reference (branch or commit sha) from which to create the release' required: true type: string type: @@ -15,15 +15,139 @@ concurrency: required: true type: choice options: - - new + # TODO: + #- new - patch + dry-run: + description: 'Dry run' + required: false + default: true + type: boolean jobs: - Release: - runs-on: [self-hosted, style-checker-aarch64] + CreateRelease: + env: + GH_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }} + RELEASE_TYPE: + runs-on: [self-hosted, release-maker] steps: + - name: DebugInfo + uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 + - name: Set envs + # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings + run: | + cat >> "$GITHUB_ENV" << 'EOF' + ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" + echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV" + - name: Download All Release Artifacts + run: | + python3 ./tests/ci/create_release.py --infile "$RELEASE_INFO_FILE" --download-packages ${{ inputs.dry-run && '--dry-run' || '' }} + - name: Push Git Tag for the Release + run: | + python3 ./tests/ci/create_release.py --push-release-tag --infile "$RELEASE_INFO_FILE" ${{ inputs.dry-run && '--dry-run' || '' }} + - name: Bump CH Version and Update Contributors' List + run: | + python3 ./tests/ci/create_release.py --create-bump-version-pr --infile "$RELEASE_INFO_FILE" ${{ inputs.dry-run && '--dry-run' || '' }} + - name: Checkout master + run: | + git checkout master + - name: Bump Docker versions, Changelog, Security + run: | + [ "$(git branch --show-current)" != "master" ] && echo "not on the master" && exit 1 + echo "List versions" + ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv + echo "Update docker version" + ./utils/list-versions/update-docker-version.sh + echo "Generate ChangeLog" + export CI=1 + docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 -e CI=1 --network=host \ + --volume=".:/ClickHouse" clickhouse/style-test \ + /ClickHouse/tests/ci/changelog.py -v --debug-helpers \ + --gh-user-or-token="$GH_TOKEN" --jobs=5 \ + --output="/ClickHouse/docs/changelogs/${RELEASE_TAG}.md" "${RELEASE_TAG}" + git add ./docs/changelogs/${RELEASE_TAG}.md + echo "Generate Security" + python3 ./utils/security-generator/generate_security.py > SECURITY.md + git diff HEAD + - name: Create ChangeLog Pull Request + if: ${{ ! inputs.dry-run }} + uses: peter-evans/create-pull-request@v6 + with: + author: "robot-clickhouse " + token: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }} + committer: "robot-clickhouse " + commit-message: Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }} + branch: auto/${{ env.RELEASE_TAG }} + assignees: ${{ github.event.sender.login }} # assign the PR to the tag pusher + delete-branch: true + title: Update version_date.tsv and changelog after ${{ env.RELEASE_TAG }} + labels: do not test + body: | + Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }} + ### Changelog category (leave one): + - Not for changelog (changelog entry is not required) + - name: Reset changes if Dry-run + if: ${{ inputs.dry-run }} + run: | + git reset --hard HEAD + - name: Checkout back to GITHUB_REF + run: | + git checkout "$GITHUB_REF_NAME" + - name: Create GH Release + run: | + python3 ./tests/ci/create_release.py --create-gh-release \ + --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} + + - name: Export TGZ Packages + run: | + python3 ./tests/ci/artifactory.py --export-tgz --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} + - name: Test TGZ Packages + run: | + python3 ./tests/ci/artifactory.py --test-tgz --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} + - name: Export RPM Packages + run: | + python3 ./tests/ci/artifactory.py --export-rpm --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} + - name: Test RPM Packages + run: | + python3 ./tests/ci/artifactory.py --test-rpm --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} + - name: Export Debian Packages + run: | + python3 ./tests/ci/artifactory.py --export-debian --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} + - name: Test Debian Packages + run: | + python3 ./tests/ci/artifactory.py --test-debian --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} + - name: Docker clickhouse/clickhouse-server building + run: | + cd "./tests/ci" + export CHECK_NAME="Docker server image" + python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }} + - name: Docker clickhouse/clickhouse-keeper building + run: | + cd "./tests/ci" + export CHECK_NAME="Docker keeper image" + python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }} + - name: Post Slack Message + if: failure() + run: | + echo Slack Message \ No newline at end of file diff --git a/tests/ci/artifactory.py b/tests/ci/artifactory.py new file mode 100644 index 00000000000..2e18316cb78 --- /dev/null +++ b/tests/ci/artifactory.py @@ -0,0 +1,356 @@ +import argparse +import time +from pathlib import Path +from typing import Optional + +from create_release import PackageDownloader, ReleaseInfo, ShellRunner +from ci_utils import WithIter +from shutil import copy2 + + +class MountPointApp(metaclass=WithIter): + RCLONE = "rclone" + S3FS = "s3fs" + + +class R2MountPoint: + _TEST_BUCKET_NAME = "repo-test" + _PROD_BUCKET_NAME = "packages" + _CACHE_MAX_SIZE_GB = 20 + MOUNT_POINT = "/home/ubuntu/mountpoint" + API_ENDPOINT = "https://d4fd593eebab2e3a58a599400c4cd64d.r2.cloudflarestorage.com" + LOG_FILE = "/home/ubuntu/fuse_mount.log" + # mod time is not required by reprepro and createrepo - disable to simplify bucket's mount sync (applicable fro rclone) + NOMODTIME = True + # enable debug messages in mount log + DEBUG = True + # enable cache for mountpoint + CACHE_ENABLED = False + # TODO: which mode is better: minimal/writes/full/off + _RCLONE_CACHE_MODE = "minimal" + UMASK = "0000" + + def __init__(self, app: str, dry_run: bool) -> None: + assert app in MountPointApp + self.app = app + if dry_run: + self.bucket_name = self._TEST_BUCKET_NAME + else: + self.bucket_name = self._PROD_BUCKET_NAME + + self.aux_mount_options = "" + self.async_mount = False + if self.app == MountPointApp.S3FS: + self.cache_dir = "/home/ubuntu/s3fs_cache" + # self.aux_mount_options += "-o nomodtime " if self.NOMODTIME else "" not for s3fs + self.aux_mount_options += "--debug " if self.DEBUG else "" + self.aux_mount_options += ( + f"-o use_cache={self.cache_dir} -o cache_size_mb={self._CACHE_MAX_SIZE_GB * 1024} " + if self.CACHE_ENABLED + else "" + ) + # without -o nomultipart there are errors like "Error 5 writing to /home/ubuntu/***.deb: Input/output error" + self.mount_cmd = f"s3fs {self.bucket_name} {self.MOUNT_POINT} -o url={self.API_ENDPOINT} -o use_path_request_style -o umask=0000 -o nomultipart -o logfile={self.LOG_FILE} {self.aux_mount_options}" + elif self.app == MountPointApp.RCLONE: + # run rclone mount process asynchronously, otherwise subprocess.run(daemonized command) will not return + self.async_mount = True + self.cache_dir = "/home/ubuntu/rclone_cache" + self.aux_mount_options += "--no-modtime " if self.NOMODTIME else "" + self.aux_mount_options += "-v " if self.DEBUG else "" # -vv too verbose + self.aux_mount_options += ( + f"--vfs-cache-mode {self._RCLONE_CACHE_MODE} --vfs-cache-max-size {self._CACHE_MAX_SIZE_GB}G" + if self.CACHE_ENABLED + else "--vfs-cache-mode off" + ) + # Use --no-modtime to try to avoid: ERROR : rpm/lts/clickhouse-client-24.3.6.5.x86_64.rpm: Failed to apply pending mod time + self.mount_cmd = f"rclone mount remote:{self.bucket_name} {self.MOUNT_POINT} --daemon --cache-dir {self.cache_dir} --umask 0000 --log-file {self.LOG_FILE} {self.aux_mount_options}" + else: + assert False + + def init(self): + print(f"Mount bucket [{self.bucket_name}] to [{self.MOUNT_POINT}]") + _CLEAN_LOG_FILE_CMD = f"tail -n 1000 {self.LOG_FILE} > {self.LOG_FILE}_tmp && mv {self.LOG_FILE}_tmp {self.LOG_FILE} ||:" + _MKDIR_CMD = f"mkdir -p {self.MOUNT_POINT}" + _MKDIR_FOR_CACHE = f"mkdir -p {self.cache_dir}" + _UNMOUNT_CMD = ( + f"mount | grep -q {self.MOUNT_POINT} && umount {self.MOUNT_POINT} ||:" + ) + + _TEST_MOUNT_CMD = f"mount | grep -q {self.MOUNT_POINT}" + ShellRunner.run(_CLEAN_LOG_FILE_CMD) + ShellRunner.run(_UNMOUNT_CMD) + ShellRunner.run(_MKDIR_CMD) + ShellRunner.run(_MKDIR_FOR_CACHE) + ShellRunner.run(self.mount_cmd, async_=self.async_mount) + if self.async_mount: + time.sleep(3) + ShellRunner.run(_TEST_MOUNT_CMD) + + @classmethod + def teardown(cls): + print(f"Unmount [{cls.MOUNT_POINT}]") + ShellRunner.run(f"umount {cls.MOUNT_POINT}") + + +class RepoCodenames(metaclass=WithIter): + LTS = "lts" + STABLE = "stable" + + +class DebianArtifactory: + _TEST_REPO_URL = "https://pub-73dd1910f4284a81a02a67018967e028.r2.dev/deb" + _PROD_REPO_URL = "https://packages.clickhouse.com/deb" + + def __init__(self, release_info: ReleaseInfo, dry_run: bool): + self.codename = release_info.codename + self.version = release_info.version + if dry_run: + self.repo_url = self._TEST_REPO_URL + else: + self.repo_url = self._PROD_REPO_URL + assert self.codename in RepoCodenames + self.pd = PackageDownloader( + release=release_info.release_branch, + commit_sha=release_info.commit_sha, + version=release_info.version, + ) + + def export_packages(self): + assert self.pd.local_deb_packages_ready(), "BUG: Packages are not downloaded" + print("Start adding packages") + paths = [ + self.pd.LOCAL_DIR + "/" + file for file in self.pd.get_deb_packages_files() + ] + REPREPRO_CMD_PREFIX = f"reprepro --basedir {R2MountPoint.MOUNT_POINT}/configs/deb --outdir {R2MountPoint.MOUNT_POINT}/deb --verbose" + cmd = f"{REPREPRO_CMD_PREFIX} includedeb {self.codename} {' '.join(paths)}" + print("Running export command:") + print(f" {cmd}") + ShellRunner.run(cmd) + ShellRunner.run("sync") + + if self.codename == RepoCodenames.LTS: + packages_with_version = [ + package + "=" + self.version for package in self.pd.get_packages_names() + ] + print( + f"Copy packages from {RepoCodenames.LTS} to {RepoCodenames.STABLE} repository" + ) + cmd = f"{REPREPRO_CMD_PREFIX} copy {RepoCodenames.STABLE} {RepoCodenames.LTS} {' '.join(packages_with_version)}" + print("Running copy command:") + print(f" {cmd}") + ShellRunner.run(cmd) + ShellRunner.run("sync") + + def test_packages(self): + ShellRunner.run("docker pull ubuntu:latest") + print(f"Test packages installation, version [{self.version}]") + cmd = f"docker run --rm ubuntu:latest bash -c \"apt update -y; apt install -y sudo gnupg ca-certificates; apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754; echo 'deb {self.repo_url} stable main' | tee /etc/apt/sources.list.d/clickhouse.list; apt update -y; apt-get install -y clickhouse-client={self.version}\"" + print("Running test command:") + print(f" {cmd}") + ShellRunner.run(cmd) + + +def _copy_if_not_exists(src: Path, dst: Path) -> Path: + if dst.is_dir(): + dst = dst / src.name + if not dst.exists(): + return copy2(src, dst) # type: ignore + if src.stat().st_size == dst.stat().st_size: + return dst + return copy2(src, dst) # type: ignore + + +class RpmArtifactory: + _TEST_REPO_URL = ( + "https://pub-73dd1910f4284a81a02a67018967e028.r2.dev/rpm/clickhouse.repo" + ) + _PROD_REPO_URL = "https://packages.clickhouse.com/rpm/clickhouse.repo" + _SIGN_KEY = "885E2BDCF96B0B45ABF058453E4AD4719DDE9A38" + + def __init__(self, release_info: ReleaseInfo, dry_run: bool): + self.codename = release_info.codename + self.version = release_info.version + if dry_run: + self.repo_url = self._TEST_REPO_URL + else: + self.repo_url = self._PROD_REPO_URL + assert self.codename in RepoCodenames + self.pd = PackageDownloader( + release=release_info.release_branch, + commit_sha=release_info.commit_sha, + version=release_info.version, + ) + + def export_packages(self, codename: Optional[str] = None) -> None: + assert self.pd.local_rpm_packages_ready(), "BUG: Packages are not downloaded" + codename = codename or self.codename + print(f"Start adding packages to [{codename}]") + paths = [ + self.pd.LOCAL_DIR + "/" + file for file in self.pd.get_rpm_packages_files() + ] + + dest_dir = Path(R2MountPoint.MOUNT_POINT) / "rpm" / codename + + for package in paths: + _copy_if_not_exists(Path(package), dest_dir) + + commands = ( + f"createrepo_c --local-sqlite --workers=2 --update --verbose {dest_dir}", + f"gpg --sign-with {self._SIGN_KEY} --detach-sign --batch --yes --armor {dest_dir / 'repodata' / 'repomd.xml'}", + ) + print(f"Exporting RPM packages into [{codename}]") + + for command in commands: + print(f"Running command:") + print(f" {command}") + ShellRunner.run(command) + + update_public_key = f"gpg --armor --export {self._SIGN_KEY}" + pub_key_path = dest_dir / "repodata" / "repomd.xml.key" + print("Updating repomd.xml.key") + pub_key_path.write_text(ShellRunner.run(update_public_key)[1]) + if codename == RepoCodenames.LTS: + self.export_packages(RepoCodenames.STABLE) + ShellRunner.run("sync") + + def test_packages(self): + ShellRunner.run("docker pull fedora:latest") + print(f"Test package installation, version [{self.version}]") + cmd = f'docker run --rm fedora:latest /bin/bash -c "dnf -y install dnf-plugins-core && dnf config-manager --add-repo={self.repo_url} && dnf makecache && dnf -y install clickhouse-client-{self.version}-1"' + print("Running test command:") + print(f" {cmd}") + ShellRunner.run(cmd) + + +class TgzArtifactory: + _TEST_REPO_URL = "https://pub-73dd1910f4284a81a02a67018967e028.r2.dev/tgz" + _PROD_REPO_URL = "https://packages.clickhouse.com/tgz" + + def __init__(self, release_info: ReleaseInfo, dry_run: bool): + self.codename = release_info.codename + self.version = release_info.version + if dry_run: + self.repo_url = self._TEST_REPO_URL + else: + self.repo_url = self._PROD_REPO_URL + assert self.codename in RepoCodenames + self.pd = PackageDownloader( + release=release_info.release_branch, + commit_sha=release_info.commit_sha, + version=release_info.version, + ) + + def export_packages(self, codename: Optional[str] = None) -> None: + assert self.pd.local_tgz_packages_ready(), "BUG: Packages are not downloaded" + codename = codename or self.codename + + paths = [ + self.pd.LOCAL_DIR + "/" + file for file in self.pd.get_tgz_packages_files() + ] + + dest_dir = Path(R2MountPoint.MOUNT_POINT) / "tgz" / codename + + print(f"Exporting TGZ packages into [{codename}]") + + for package in paths: + _copy_if_not_exists(Path(package), dest_dir) + + if codename == RepoCodenames.LTS: + self.export_packages(RepoCodenames.STABLE) + ShellRunner.run("sync") + + def test_packages(self): + tgz_file = "/tmp/tmp.tgz" + tgz_sha_file = "/tmp/tmp.tgz.sha512" + ShellRunner.run( + f"curl -o {tgz_file} -f0 {self.repo_url}/stable/clickhouse-client-{self.version}-arm64.tgz" + ) + ShellRunner.run( + f"curl -o {tgz_sha_file} -f0 {self.repo_url}/stable/clickhouse-client-{self.version}-arm64.tgz.sha512" + ) + expected_checksum = ShellRunner.run(f"cut -d ' ' -f 1 {tgz_sha_file}") + actual_checksum = ShellRunner.run(f"sha512sum {tgz_file} | cut -d ' ' -f 1") + assert ( + expected_checksum == actual_checksum + ), f"[{actual_checksum} != {expected_checksum}]" + ShellRunner.run("rm /tmp/tmp.tgz*") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Adds release packages to the repository", + ) + parser.add_argument( + "--infile", + type=str, + required=True, + help="input file with release info", + ) + parser.add_argument( + "--export-debian", + action="store_true", + help="Export debian packages to repository", + ) + parser.add_argument( + "--export-rpm", + action="store_true", + help="Export rpm packages to repository", + ) + parser.add_argument( + "--export-tgz", + action="store_true", + help="Export tgz packages to repository", + ) + parser.add_argument( + "--test-debian", + action="store_true", + help="Test debian packages installation", + ) + parser.add_argument( + "--test-rpm", + action="store_true", + help="Test rpm packages installation", + ) + parser.add_argument( + "--test-tgz", + action="store_true", + help="Test tgz packages installation", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Dry run mode", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + assert args.dry_run + + release_info = ReleaseInfo.from_file(args.infile) + """ + Use S3FS. RCLONE has some errors with r2 remote which I didn't figure out how to resolve: + ERROR : IO error: NotImplemented: versionId not implemented + Failed to copy: NotImplemented: versionId not implemented + """ + mp = R2MountPoint(MountPointApp.S3FS, dry_run=args.dry_run) + if args.export_debian: + mp.init() + DebianArtifactory(release_info, dry_run=args.dry_run).export_packages() + mp.teardown() + if args.export_rpm: + mp.init() + RpmArtifactory(release_info, dry_run=args.dry_run).export_packages() + mp.teardown() + if args.export_tgz: + mp.init() + TgzArtifactory(release_info, dry_run=args.dry_run).export_packages() + mp.teardown() + if args.test_debian: + DebianArtifactory(release_info, dry_run=args.dry_run).test_packages() + if args.test_tgz: + TgzArtifactory(release_info, dry_run=args.dry_run).test_packages() + if args.test_rpm: + RpmArtifactory(release_info, dry_run=args.dry_run).test_packages() diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py new file mode 100755 index 00000000000..d749c85994b --- /dev/null +++ b/tests/ci/create_release.py @@ -0,0 +1,663 @@ +import argparse +import dataclasses +import json +import os +import subprocess + +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator, List + +from git_helper import Git, GIT_PREFIX, Runner +from ssh import SSHAgent +from env_helper import GITHUB_REPOSITORY, S3_BUILDS_BUCKET +from s3_helper import S3Helper +from version_helper import ( + FILE_WITH_VERSION_PATH, + GENERATED_CONTRIBUTORS, + get_abs_path, + get_version_from_repo, + update_cmake_version, + update_contributors, +) +from git_helper import git_runner as runner +from ci_config import CI + +CMAKE_PATH = get_abs_path(FILE_WITH_VERSION_PATH) +CONTRIBUTORS_PATH = get_abs_path(GENERATED_CONTRIBUTORS) + + +class ShellRunner: + + @classmethod + def run(cls, command, check_retcode=True, print_output=True, async_=False): + print(f"Running shell command: [{command}]") + if async_: + subprocess.Popen(command.split(" ")) + return 0, "" + result = subprocess.run( + command + " 2>&1", + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if print_output: + print(result.stdout) + if check_retcode: + assert result.returncode == 0, f"Return code [{result.returncode}]" + return result.returncode, result.stdout + + +@dataclasses.dataclass +class ReleaseInfo: + version: str + release_tag: str + release_branch: str + commit_sha: str + # lts or stable + codename: str + + @staticmethod + def from_file(file_path: str) -> "ReleaseInfo": + with open(file_path, "r", encoding="utf-8") as json_file: + res = json.load(json_file) + return ReleaseInfo(**res) + + @staticmethod + def prepare(commit_ref: str, release_type: str, outfile: str) -> None: + dir = Path(outfile).parent + dir.mkdir(parents=True, exist_ok=True) + Path(outfile).unlink(missing_ok=True) + version = None + release_branch = None + release_tag = None + codename = None + assert release_type in ("patch",) + # if release_type == "new": + # assert False, "TODO" + # git = Git() + # version = get_version_from_repo(git=git) + # assert runner.check_command( + # f"git merge-base --is-ancestor {commit_ref} origin/master" + # ) + # expected_tag = f"v{version.major}.{version.minor}-new" + # assert ( + # git.latest_tag == expected_tag + # ), f"BUG: latest tag [{git.latest_tag}], expected [{expected_tag}]" + # release_branch = "master" + if release_type == "patch": + with checkout(commit_ref): + # Git() must be inside "with checkout" contextmanager + commit_sha = Runner().run(f"git rev-parse {commit_ref}") + git = Git() + version = get_version_from_repo(git=git) + codename = version.get_stable_release_type() + version.with_description(codename) + release_branch = f"{version.major}.{version.minor}" + release_tag = version.describe + runner.run(f"{GIT_PREFIX} fetch origin {release_branch} --tags") + assert runner.check_command( + f"git merge-base --is-ancestor {commit_ref} origin/{release_branch}" + ) + if version.patch == 1: + expected_tag_prefix = f"v{version.major}.{version.minor+1}-" + expected_tag_suffix = "-new" + else: + expected_tag_prefix = ( + f"v{version.major}.{version.minor}.{version.patch-1}." + ) + expected_tag_suffix = f"-{version.get_stable_release_type()}" + if git.latest_tag.startswith( + expected_tag_prefix + ) and git.latest_tag.endswith(expected_tag_suffix): + pass + else: + assert ( + False + ), f"BUG: Unexpected latest tag [{git.latest_tag}] expected [{expected_tag_prefix}*{expected_tag_suffix}]" + + assert ( + release_branch + and commit_sha + and release_tag + and version.string + and codename in ("lts", "stable") + ) + res = ReleaseInfo( + release_branch=release_branch, + commit_sha=commit_sha, + release_tag=release_tag, + version=version.string, + codename=codename, + ) + with open(outfile, "w", encoding="utf-8") as f: + print(json.dumps(dataclasses.asdict(res), indent=2), file=f) + + def push_release_tag(self, dry_run: bool) -> None: + if dry_run: + # remove locally created tag from prev run + ShellRunner.run( + f"{GIT_PREFIX} tag -l | grep -q {self.release_tag} && git tag -d {self.release_tag} ||:" + ) + # Create release tag + print( + f"Create and push release tag [{self.release_tag}], commit [{self.commit_sha}]" + ) + tag_message = f"Release {self.release_tag}" + runner.run( + f"{GIT_PREFIX} tag -a -m '{tag_message}' {self.release_tag} {self.commit_sha}" + ) + cmd_push_tag = f"{GIT_PREFIX} push origin {self.release_tag}:{self.release_tag}" + if not dry_run: + # TODO: cannot push - workflow will start + # runner.run(cmd_commit_version_upd) + pass + else: + print("Dry run, would execute:") + print(f"* {cmd_push_tag}") + + def update_version_and_contributors_list(self, dry_run: bool) -> None: + # Bump version, update contributors list, create PR + branch_upd_version_contributors = f"bump_version_{self.version}" + with checkout(self.commit_sha): + git = Git() + version = get_version_from_repo(git=git) + version.with_description(version.get_stable_release_type()) + assert ( + version.string == self.version + ), "BUG: version in release info does not match version in git commit" + with checkout(self.release_branch): + with checkout_new(branch_upd_version_contributors): + update_cmake_version(version) + update_contributors(raise_error=True) + cmd_commit_version_upd = ( + f"{GIT_PREFIX} commit '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}' -m 'Update autogenerated version to {self.version} and contributors'", + ) + cmd_push_branch = f"{GIT_PREFIX} push --set-upstream origin {branch_upd_version_contributors}" + body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md") + actor = os.getenv("GITHUB_ACTOR", "") or "me" + cmd_create_pr = f"gh pr create --repo {GITHUB_REPOSITORY} --title 'Update version after release' --head {branch_upd_version_contributors} --base {self.release_branch} --body-file '{body_file} --label 'do not test' --assignee @{actor}" + if not dry_run: + runner.run(cmd_commit_version_upd) + runner.run(cmd_push_branch) + runner.run(cmd_create_pr) + else: + print("Dry run, would execute:") + print(f"* {cmd_commit_version_upd}") + print(f"* {cmd_push_branch}") + print(f"* {cmd_create_pr}") + print("Dry run, diff:") + print( + runner.run( + f"{GIT_PREFIX} diff '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'" + ) + ) + runner.run( + f"{GIT_PREFIX} checkout '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'" + ) + + def create_gh_release(self, packages_files: List[str], dry_run: bool) -> None: + repo = os.getenv("GITHUB_REPOSITORY") + assert repo + cmds = [] + cmds.append( + f"gh release create --repo {repo} --title 'Release {self.release_tag}' {self.release_tag}" + ) + for file in packages_files: + cmds.append(f"gh release upload {self.release_tag} {file}") + if not dry_run: + for cmd in cmds: + ShellRunner.run(cmd) + else: + print("Dry-run, would run commands:") + print("\n * ".join(cmds)) + + +class RepoTypes: + RPM = "rpm" + DEBIAN = "deb" + TGZ = "tgz" + + +class PackageDownloader: + PACKAGES = ( + "clickhouse-client", + "clickhouse-common-static", + "clickhouse-common-static-dbg", + "clickhouse-keeper", + "clickhouse-keeper-dbg", + "clickhouse-server", + ) + + EXTRA_PACKAGES = ( + "clickhouse-library-bridge", + "clickhouse-odbc-bridge", + ) + PACKAGE_TYPES = (CI.BuildNames.PACKAGE_RELEASE, CI.BuildNames.PACKAGE_AARCH64) + MACOS_PACKAGE_TO_BIN_SUFFIX = { + CI.BuildNames.BINARY_DARWIN: "macos", + CI.BuildNames.BINARY_DARWIN_AARCH64: "macos-aarch64", + } + LOCAL_DIR = "/tmp/packages" + + @classmethod + def _get_arch_suffix(cls, package_arch, repo_type): + if package_arch == CI.BuildNames.PACKAGE_RELEASE: + return ( + "amd64" if repo_type in (RepoTypes.DEBIAN, RepoTypes.TGZ) else "x86_64" + ) + elif package_arch == CI.BuildNames.PACKAGE_AARCH64: + return ( + "arm64" if repo_type in (RepoTypes.DEBIAN, RepoTypes.TGZ) else "aarch64" + ) + else: + assert False, "BUG" + + def __init__(self, release, commit_sha, version: str): + assert version.startswith(release), "Invalid release branch or version" + major, minor = map(int, release.split(".")) + self.package_names = self.PACKAGES + if major > 24 or (major == 24 and minor > 3): + self.package_names += self.EXTRA_PACKAGES + self.release = release + self.commit_sha = commit_sha + self.version = version + self.s3 = S3Helper() + self.deb_package_files = [] + self.rpm_package_files = [] + self.tgz_package_files = [] + # just binaries for macos + self.macos_package_files = ["clickhouse-macos", "clickhouse-macos-aarch64"] + self.file_to_type = {} + + ShellRunner.run(f"mkdir -p {self.LOCAL_DIR}") + + for package_type in self.PACKAGE_TYPES: + for package in self.package_names: + package_file_name = ( + package + + "_" + + self.version + + "_" + + self._get_arch_suffix(package_type, RepoTypes.DEBIAN) + + ".deb" + ) + self.deb_package_files.append(package_file_name) + self.file_to_type[package_file_name] = package_type + + package_file_name = ( + package + + "-" + + self.version + + "." + + self._get_arch_suffix(package_type, RepoTypes.RPM) + + ".rpm" + ) + self.rpm_package_files.append(package_file_name) + self.file_to_type[package_file_name] = package_type + + package_file_name = ( + package + + "-" + + self.version + + "-" + + self._get_arch_suffix(package_type, RepoTypes.TGZ) + + ".tgz" + ) + self.tgz_package_files.append(package_file_name) + self.file_to_type[package_file_name] = package_type + package_file_name += ".sha512" + self.tgz_package_files.append(package_file_name) + self.file_to_type[package_file_name] = package_type + + def get_deb_packages_files(self): + return self.deb_package_files + + def get_rpm_packages_files(self): + return self.rpm_package_files + + def get_tgz_packages_files(self): + return self.tgz_package_files + + def get_macos_packages_files(self): + return self.macos_package_files + + def get_packages_names(self): + return self.package_names + + def get_all_packages_files(self): + assert self.local_tgz_packages_ready() + assert self.local_deb_packages_ready() + assert self.local_rpm_packages_ready() + assert self.local_macos_packages_ready() + res = [] + for package_file in ( + self.deb_package_files + + self.rpm_package_files + + self.tgz_package_files + + self.macos_package_files + ): + res.append(self.LOCAL_DIR + "/" + package_file) + return res + + def run(self): + ShellRunner.run(f"rm -rf {self.LOCAL_DIR}/*") + for package_file in ( + self.deb_package_files + self.rpm_package_files + self.tgz_package_files + ): + print(f"Downloading: [{package_file}]") + s3_path = "/".join( + [ + self.release, + self.commit_sha, + self.file_to_type[package_file], + package_file, + ] + ) + self.s3.download_file( + bucket=S3_BUILDS_BUCKET, + s3_path=s3_path, + local_file_path="/".join([self.LOCAL_DIR, package_file]), + ) + + for macos_package, bin_suffix in self.MACOS_PACKAGE_TO_BIN_SUFFIX.items(): + binary_name = "clickhouse" + destination_binary_name = f"{binary_name}-{bin_suffix}" + assert destination_binary_name in self.macos_package_files + print( + f"Downloading: [{macos_package}] binary to [{destination_binary_name}]" + ) + s3_path = "/".join( + [ + self.release, + self.commit_sha, + macos_package, + binary_name, + ] + ) + self.s3.download_file( + bucket=S3_BUILDS_BUCKET, + s3_path=s3_path, + local_file_path="/".join([self.LOCAL_DIR, destination_binary_name]), + ) + + def local_deb_packages_ready(self) -> bool: + assert self.deb_package_files + for package_file in self.deb_package_files: + print(f"Check package is downloaded [{package_file}]") + if not Path(self.LOCAL_DIR + "/" + package_file).is_file(): + return False + return True + + def local_rpm_packages_ready(self) -> bool: + assert self.rpm_package_files + for package_file in self.rpm_package_files: + print(f"Check package is downloaded [{package_file}]") + if not Path(self.LOCAL_DIR + "/" + package_file).is_file(): + return False + return True + + def local_tgz_packages_ready(self) -> bool: + assert self.tgz_package_files + for package_file in self.tgz_package_files: + print(f"Check package is downloaded [{package_file}]") + if not Path(self.LOCAL_DIR + "/" + package_file).is_file(): + return False + return True + + def local_macos_packages_ready(self) -> bool: + assert self.macos_package_files + for package_file in self.macos_package_files: + print(f"Check package is downloaded [{package_file}]") + if not Path(self.LOCAL_DIR + "/" + package_file).is_file(): + return False + return True + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Creates release", + ) + parser.add_argument( + "--prepare-release-info", + action="store_true", + help="Initial step to prepare info like release branch, release tag, etc.", + ) + parser.add_argument( + "--push-release-tag", + action="store_true", + help="Creates and pushes git tag", + ) + parser.add_argument( + "--create-bump-version-pr", + action="store_true", + help="Updates version, contributors' list and creates PR", + ) + parser.add_argument( + "--download-packages", + action="store_true", + help="Downloads all required packages from s3", + ) + parser.add_argument( + "--create-gh-release", + action="store_true", + help="Create GH Release object and attach all packages", + ) + parser.add_argument( + "--ref", + type=str, + help="the commit hash or branch", + ) + parser.add_argument( + "--release-type", + choices=("new", "patch"), + # dest="release_type", + help="a release type to bump the major.minor.patch version part, " + "new branch is created only for the value 'new'", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="do not make any actual changes in the repo, just show what will be done", + ) + parser.add_argument( + "--outfile", + default="", + type=str, + help="output file to write json result to, if not set - stdout", + ) + parser.add_argument( + "--infile", + default="", + type=str, + help="input file with release info", + ) + + return parser.parse_args() + + +@contextmanager +def checkout(ref: str) -> Iterator[None]: + orig_ref = runner.run(f"{GIT_PREFIX} symbolic-ref --short HEAD") + rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}" + assert orig_ref + if ref not in (orig_ref,): + runner.run(f"{GIT_PREFIX} checkout {ref}") + try: + yield + except (Exception, KeyboardInterrupt) as e: + print(f"ERROR: Exception [{e}]") + runner.run(rollback_cmd) + raise + runner.run(rollback_cmd) + + +@contextmanager +def checkout_new(ref: str) -> Iterator[None]: + orig_ref = runner.run(f"{GIT_PREFIX} symbolic-ref --short HEAD") + rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}" + assert orig_ref + runner.run(f"{GIT_PREFIX} checkout -b {ref}") + try: + yield + except (Exception, KeyboardInterrupt) as e: + print(f"ERROR: Exception [{e}]") + runner.run(rollback_cmd) + raise + runner.run(rollback_cmd) + + +if __name__ == "__main__": + args = parse_args() + assert args.dry_run + + # prepare ssh for git if needed + _ssh_agent = None + _key_pub = None + if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""): + _key = os.getenv("ROBOT_CLICKHOUSE_SSH_KEY") + _ssh_agent = SSHAgent() + _key_pub = _ssh_agent.add(_key) + _ssh_agent.print_keys() + + if args.prepare_release_info: + assert ( + args.ref and args.release_type and args.outfile + ), "--ref, --release-type and --outfile must be provided with --prepare-release-info" + ReleaseInfo.prepare( + commit_ref=args.ref, release_type=args.release_type, outfile=args.outfile + ) + if args.push_release_tag: + assert args.infile, "--infile must be provided" + release_info = ReleaseInfo.from_file(args.infile) + release_info.push_release_tag(dry_run=args.dry_run) + if args.create_bump_version_pr: + # TODO: store link to PR in release info + assert args.infile, "--infile must be provided" + release_info = ReleaseInfo.from_file(args.infile) + release_info.update_version_and_contributors_list(dry_run=args.dry_run) + if args.download_packages: + assert args.infile, "--infile must be provided" + release_info = ReleaseInfo.from_file(args.infile) + p = PackageDownloader( + release=release_info.release_branch, + commit_sha=release_info.commit_sha, + version=release_info.version, + ) + p.run() + if args.create_gh_release: + assert args.infile, "--infile must be provided" + release_info = ReleaseInfo.from_file(args.infile) + p = PackageDownloader( + release=release_info.release_branch, + commit_sha=release_info.commit_sha, + version=release_info.version, + ) + release_info.create_gh_release(p.get_all_packages_files(), args.dry_run) + + # tear down ssh + if _ssh_agent and _key_pub: + _ssh_agent.remove(_key_pub) + + +""" +Prepare release machine (for arm machine): + +### INSTALL PACKAGES +sudo apt update +sudo apt install --yes --no-install-recommends python3-dev python3-pip gh unzip +sudo apt install --yes python3-boto3 +sudo apt install --yes python3-github +sudo apt install --yes python3-unidiff +sudo apt install --yes s3fs + +### INSTALL AWS CLI +cd /tmp +curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" +unzip awscliv2.zip +sudo ./aws/install +rm -rf aws* +cd - + +### INSTALL GH ACTIONS RUNNER: +# Create a folder +RUNNER_VERSION=2.317.0 +cd ~ +mkdir actions-runner && cd actions-runner +# Download the latest runner package +runner_arch() { + case $(uname -m) in + x86_64 ) + echo x64;; + aarch64 ) + echo arm64;; + esac +} +curl -O -L https://github.com/actions/runner/releases/download/v$RUNNER_VERSION/actions-runner-linux-$(runner_arch)-$RUNNER_VERSION.tar.gz +# Extract the installer +tar xzf ./actions-runner-linux-$(runner_arch)-$RUNNER_VERSION.tar.gz +rm ./actions-runner-linux-$(runner_arch)-$RUNNER_VERSION.tar.gz + +### Install reprepro: +cd ~ +sudo apt install dpkg-dev libgpgme-dev libdb-dev libbz2-dev liblzma-dev libarchive-dev shunit2 db-util debhelper +git clone https://salsa.debian.org/debian/reprepro.git +cd reprepro +dpkg-buildpackage -b --no-sign && sudo dpkg -i ../reprepro_$(dpkg-parsechangelog --show-field Version)_$(dpkg-architecture -q DEB_HOST_ARCH).deb + +### Install createrepo-c: +sudo apt install createrepo-c +createrepo_c --version +#Version: 0.17.3 (Features: DeltaRPM LegacyWeakdeps ) + +### Import gpg sign key +gpg --import key.pgp +gpg --list-secret-keys + +### Install docker +sudo su; cd ~ + +deb_arch() { + case $(uname -m) in + x86_64 ) + echo amd64;; + aarch64 ) + echo arm64;; + esac +} +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + +echo "deb [arch=$(deb_arch) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +sudo apt-get update +sudo apt-get install --yes --no-install-recommends docker-ce docker-buildx-plugin docker-ce-cli containerd.io + +sudo usermod -aG docker ubuntu + +# enable ipv6 in containers (fixed-cidr-v6 is some random network mask) +cat < /etc/docker/daemon.json +{ + "ipv6": true, + "fixed-cidr-v6": "2001:db8:1::/64", + "log-driver": "json-file", + "log-opts": { + "max-file": "5", + "max-size": "1000m" + }, + "insecure-registries" : ["dockerhub-proxy.dockerhub-proxy-zone:5000"], + "registry-mirrors" : ["http://dockerhub-proxy.dockerhub-proxy-zone:5000"] +} +EOT + +# if docker build does not work: + sudo systemctl restart docker + docker buildx rm mybuilder + docker buildx create --name mybuilder --driver docker-container --use + docker buildx inspect mybuilder --bootstrap + +### Install tailscale + +### Configure GH runner +""" diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 151cc5a4c02..2f556e3ed57 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -11,7 +11,6 @@ from os import path as p from pathlib import Path from typing import Dict, List -from build_check import get_release_or_pr from build_download_helper import read_build_urls from docker_images_helper import DockerImageData, docker_login from env_helper import ( @@ -22,7 +21,7 @@ from env_helper import ( TEMP_PATH, ) from git_helper import Git -from pr_info import PRInfo +from pr_info import PRInfo, EventType from report import FAILURE, SUCCESS, JobReport, TestResult, TestResults from stopwatch import Stopwatch from tee_popen import TeePopen @@ -63,6 +62,12 @@ def parse_args() -> argparse.Namespace: help="a version to build, automaticaly got from version_helper, accepts either " "tag ('refs/tags/' is removed automatically) or a normal 22.2.2.2 format", ) + parser.add_argument( + "--sha", + type=str, + default="", + help="sha of the commit to use packages from", + ) parser.add_argument( "--release-type", type=str, @@ -122,7 +127,7 @@ def parse_args() -> argparse.Namespace: def retry_popen(cmd: str, log_file: Path) -> int: - max_retries = 5 + max_retries = 2 for retry in range(max_retries): # From time to time docker build may failed. Curl issues, or even push # It will sleep progressively 5, 15, 30 and 50 seconds between retries @@ -370,13 +375,22 @@ def main(): tags = gen_tags(args.version, args.release_type) repo_urls = {} direct_urls: Dict[str, List[str]] = {} - release_or_pr, _ = get_release_or_pr(pr_info, args.version) + if pr_info.event_type == EventType.PULL_REQUEST: + release_or_pr = pr_info.number + sha = pr_info.sha + elif pr_info.event_type == EventType.PUSH and pr_info.is_master: + release_or_pr = 0 + sha = pr_info.sha + else: + release_or_pr = f"{args.version.major}.{args.version.minor}" + sha = args.sha + assert sha for arch, build_name in zip(ARCH, ("package_release", "package_aarch64")): if not args.bucket_prefix: repo_urls[arch] = ( f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/" - f"{release_or_pr}/{pr_info.sha}/{build_name}" + f"{release_or_pr}/{sha}/{build_name}" ) else: repo_urls[arch] = f"{args.bucket_prefix}/{build_name}" diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 50263f6ebb6..d48dd36b16a 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -148,6 +148,11 @@ class ClickHouseVersion: """our X.3 and X.8 are LTS""" return self.minor % 5 == 3 + def get_stable_release_type(self) -> str: + if self.is_lts: + return VersionType.LTS + return VersionType.STABLE + def as_dict(self) -> VERSIONS: return { "revision": self.revision, @@ -168,6 +173,7 @@ class ClickHouseVersion: raise ValueError(f"version type {version_type} not in {VersionType.VALID}") self._description = version_type self._describe = f"v{self.string}-{version_type}" + return self def copy(self) -> "ClickHouseVersion": copy = ClickHouseVersion( From f8e71707f234c10dd3567e9087a16e6cc2e21801 Mon Sep 17 00:00:00 2001 From: Max K Date: Wed, 10 Jul 2024 14:50:55 +0200 Subject: [PATCH 57/72] update black --- docker/test/style/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/style/requirements.txt b/docker/test/style/requirements.txt index bb0cd55dd1a..ed73d0d3636 100644 --- a/docker/test/style/requirements.txt +++ b/docker/test/style/requirements.txt @@ -3,7 +3,7 @@ aiosignal==1.3.1 astroid==3.1.0 async-timeout==4.0.3 attrs==23.2.0 -black==23.12.0 +black==24.4.2 boto3==1.34.131 botocore==1.34.131 certifi==2024.6.2 From 10502174452c7b1adf623b113e145f1bad24c1ea Mon Sep 17 00:00:00 2001 From: Max K Date: Wed, 10 Jul 2024 16:19:06 +0200 Subject: [PATCH 58/72] add support for new release branch Automatic style fix --- .github/workflows/create_release.yml | 27 +++- docker/test/libfuzzer/run_libfuzzer.py | 18 +-- tests/ci/create_release.py | 125 +++++++++++++++--- tests/ci/ssh.py | 6 +- tests/ci/test_ci_options.py | 14 +- tests/ci/version_helper.py | 13 ++ tests/integration/helpers/cluster.py | 18 +-- tests/integration/test_disk_types/test.py | 6 +- .../test_distributed_default_database/test.py | 1 + .../test_interserver_dns_retires/test.py | 1 + tests/integration/test_storage_s3/test.py | 4 +- .../test_case.py | 1 + .../test_case.py | 1 + ...411_long_accurate_number_comparison.python | 2 +- 14 files changed, 175 insertions(+), 62 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index e2ad16a05a4..26adc3adff9 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -15,9 +15,8 @@ concurrency: required: true type: choice options: - # TODO: - #- new - patch + - new dry-run: description: 'Dry run' required: false @@ -28,7 +27,6 @@ jobs: CreateRelease: env: GH_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }} - RELEASE_TYPE: runs-on: [self-hosted, release-maker] steps: - name: DebugInfo @@ -61,11 +59,16 @@ jobs: echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV" echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV" - name: Download All Release Artifacts + if: ${{ inputs.type == 'patch' }} run: | python3 ./tests/ci/create_release.py --infile "$RELEASE_INFO_FILE" --download-packages ${{ inputs.dry-run && '--dry-run' || '' }} - name: Push Git Tag for the Release run: | python3 ./tests/ci/create_release.py --push-release-tag --infile "$RELEASE_INFO_FILE" ${{ inputs.dry-run && '--dry-run' || '' }} + - name: Push New Release Branch + if: ${{ inputs.type == 'new' }} + run: | + python3 ./tests/ci/create_release.py --push-new-release-branch --infile "$RELEASE_INFO_FILE" ${{ inputs.dry-run && '--dry-run' || '' }} - name: Bump CH Version and Update Contributors' List run: | python3 ./tests/ci/create_release.py --create-bump-version-pr --infile "$RELEASE_INFO_FILE" ${{ inputs.dry-run && '--dry-run' || '' }} @@ -73,6 +76,7 @@ jobs: run: | git checkout master - name: Bump Docker versions, Changelog, Security + if: ${{ inputs.type == 'patch' }} run: | [ "$(git branch --show-current)" != "master" ] && echo "not on the master" && exit 1 echo "List versions" @@ -90,8 +94,8 @@ jobs: echo "Generate Security" python3 ./utils/security-generator/generate_security.py > SECURITY.md git diff HEAD - - name: Create ChangeLog Pull Request - if: ${{ ! inputs.dry-run }} + - name: Generate ChangeLog + if: ${{ inputs.type == 'patch' && ! inputs.dry-run }} uses: peter-evans/create-pull-request@v6 with: author: "robot-clickhouse " @@ -115,39 +119,48 @@ jobs: run: | git checkout "$GITHUB_REF_NAME" - name: Create GH Release + if: ${{ inputs.type == 'patch' }} run: | python3 ./tests/ci/create_release.py --create-gh-release \ --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} - name: Export TGZ Packages + if: ${{ inputs.type == 'patch' }} run: | python3 ./tests/ci/artifactory.py --export-tgz --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} - name: Test TGZ Packages + if: ${{ inputs.type == 'patch' }} run: | python3 ./tests/ci/artifactory.py --test-tgz --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} - name: Export RPM Packages + if: ${{ inputs.type == 'patch' }} run: | python3 ./tests/ci/artifactory.py --export-rpm --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} - name: Test RPM Packages + if: ${{ inputs.type == 'patch' }} run: | python3 ./tests/ci/artifactory.py --test-rpm --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} - name: Export Debian Packages + if: ${{ inputs.type == 'patch' }} run: | python3 ./tests/ci/artifactory.py --export-debian --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} - name: Test Debian Packages + if: ${{ inputs.type == 'patch' }} run: | python3 ./tests/ci/artifactory.py --test-debian --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} - name: Docker clickhouse/clickhouse-server building + if: ${{ inputs.type == 'patch' }} run: | cd "./tests/ci" export CHECK_NAME="Docker server image" python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }} - name: Docker clickhouse/clickhouse-keeper building + if: ${{ inputs.type == 'patch' }} run: | cd "./tests/ci" export CHECK_NAME="Docker keeper image" python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }} - name: Post Slack Message - if: failure() + if: always() run: | - echo Slack Message \ No newline at end of file + echo Slack Message diff --git a/docker/test/libfuzzer/run_libfuzzer.py b/docker/test/libfuzzer/run_libfuzzer.py index 5ed019490d5..fa67805dfa5 100755 --- a/docker/test/libfuzzer/run_libfuzzer.py +++ b/docker/test/libfuzzer/run_libfuzzer.py @@ -27,19 +27,19 @@ def run_fuzzer(fuzzer: str): parser.read(path) if parser.has_section("asan"): - os.environ[ - "ASAN_OPTIONS" - ] = f"{os.environ['ASAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['asan'].items())}" + os.environ["ASAN_OPTIONS"] = ( + f"{os.environ['ASAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['asan'].items())}" + ) if parser.has_section("msan"): - os.environ[ - "MSAN_OPTIONS" - ] = f"{os.environ['MSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['msan'].items())}" + os.environ["MSAN_OPTIONS"] = ( + f"{os.environ['MSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['msan'].items())}" + ) if parser.has_section("ubsan"): - os.environ[ - "UBSAN_OPTIONS" - ] = f"{os.environ['UBSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['ubsan'].items())}" + os.environ["UBSAN_OPTIONS"] = ( + f"{os.environ['UBSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['ubsan'].items())}" + ) if parser.has_section("libfuzzer"): custom_libfuzzer_options = " ".join( diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py index d749c85994b..65ab48865ef 100755 --- a/tests/ci/create_release.py +++ b/tests/ci/create_release.py @@ -5,6 +5,7 @@ import os import subprocess from contextlib import contextmanager +from copy import copy from pathlib import Path from typing import Iterator, List @@ -12,6 +13,7 @@ from git_helper import Git, GIT_PREFIX, Runner from ssh import SSHAgent from env_helper import GITHUB_REPOSITORY, S3_BUILDS_BUCKET from s3_helper import S3Helper +from autoscale_runners_lambda.lambda_shared.pr import Labels from version_helper import ( FILE_WITH_VERSION_PATH, GENERATED_CONTRIBUTORS, @@ -19,6 +21,7 @@ from version_helper import ( get_version_from_repo, update_cmake_version, update_contributors, + VersionType, ) from git_helper import git_runner as runner from ci_config import CI @@ -30,7 +33,12 @@ CONTRIBUTORS_PATH = get_abs_path(GENERATED_CONTRIBUTORS) class ShellRunner: @classmethod - def run(cls, command, check_retcode=True, print_output=True, async_=False): + def run( + cls, command, check_retcode=True, print_output=True, async_=False, dry_run=False + ): + if dry_run: + print(f"Dry-run: Would run shell command: [{command}]") + return 0, "" print(f"Running shell command: [{command}]") if async_: subprocess.Popen(command.split(" ")) @@ -73,23 +81,31 @@ class ReleaseInfo: release_branch = None release_tag = None codename = None - assert release_type in ("patch",) - # if release_type == "new": - # assert False, "TODO" - # git = Git() - # version = get_version_from_repo(git=git) - # assert runner.check_command( - # f"git merge-base --is-ancestor {commit_ref} origin/master" - # ) - # expected_tag = f"v{version.major}.{version.minor}-new" - # assert ( - # git.latest_tag == expected_tag - # ), f"BUG: latest tag [{git.latest_tag}], expected [{expected_tag}]" - # release_branch = "master" + assert release_type in ("patch", "new") + if release_type == "new": + # check commit_ref is right and on a right branch + ShellRunner.run( + f"git merge-base --is-ancestor origin/{commit_ref} origin/master" + ) + with checkout(commit_ref): + commit_sha = Runner().run(f"git rev-parse {commit_ref}") + # Git() must be inside "with checkout" contextmanager + git = Git() + version = get_version_from_repo(git=git) + release_branch = "master" + expected_prev_tag = f"v{version.major}.{version.minor}.1.1-new" + version.bump().with_description(VersionType.NEW) + assert ( + git.latest_tag == expected_prev_tag + ), f"BUG: latest tag [{git.latest_tag}], expected [{expected_prev_tag}]" + release_tag = version.describe + codename = ( + VersionType.STABLE + ) # dummy value (artifactory won't be updated for new release) if release_type == "patch": with checkout(commit_ref): - # Git() must be inside "with checkout" contextmanager commit_sha = Runner().run(f"git rev-parse {commit_ref}") + # Git() must be inside "with checkout" contextmanager git = Git() version = get_version_from_repo(git=git) codename = version.get_stable_release_type() @@ -97,11 +113,16 @@ class ReleaseInfo: release_branch = f"{version.major}.{version.minor}" release_tag = version.describe runner.run(f"{GIT_PREFIX} fetch origin {release_branch} --tags") - assert runner.check_command( + # check commit is right and on a right branch + ShellRunner.run( f"git merge-base --is-ancestor {commit_ref} origin/{release_branch}" ) if version.patch == 1: - expected_tag_prefix = f"v{version.major}.{version.minor+1}-" + expected_version = copy(version) + expected_version.bump() + expected_tag_prefix = ( + f"v{expected_version.major}.{expected_version.minor}-" + ) expected_tag_suffix = "-new" else: expected_tag_prefix = ( @@ -157,16 +178,71 @@ class ReleaseInfo: print("Dry run, would execute:") print(f"* {cmd_push_tag}") + @staticmethod + def _create_gh_label(label: str, color_hex: str, dry_run: bool): + cmd = f"gh api repos/{GITHUB_REPOSITORY}/labels -f name={label} -f color={color_hex}" + ShellRunner.run(cmd, dry_run=dry_run) + + def push_new_release_branch(self, dry_run: bool) -> None: + assert ( + self.release_branch == "master" + ), "New release branch can be created only for release type [new]" + git = Git() + version = get_version_from_repo(git=git) + new_release_branch = f"{version.major}.{version.minor}" + stable_release_type = version.get_stable_release_type() + version_after_release = copy(version) + version_after_release.bump() + assert ( + version_after_release.string == self.version + ), f"Unexpected current version in git, must precede [{self.version}] by one step, actual [{version.string}]" + if dry_run: + # remove locally created branch from prev run + ShellRunner.run( + f"{GIT_PREFIX} branch -l | grep -q {new_release_branch} && git branch -d {new_release_branch} ||:" + ) + print( + f"Create and push new release branch [{new_release_branch}], commit [{self.commit_sha}]" + ) + with checkout(self.release_branch): + with checkout_new(new_release_branch): + pr_labels = f"--label {Labels.RELEASE}" + if stable_release_type == VersionType.LTS: + pr_labels += f" --label {Labels.RELEASE_LTS}" + cmd_push_branch = ( + f"{GIT_PREFIX} push --set-upstream origin {new_release_branch}" + ) + ShellRunner.run(cmd_push_branch, dry_run=dry_run) + + print("Create and push backport tags for new release branch") + ReleaseInfo._create_gh_label( + f"v{new_release_branch}-must-backport", "10dbed", dry_run=dry_run + ) + ReleaseInfo._create_gh_label( + f"v{new_release_branch}-affected", "c2bfff", dry_run=dry_run + ) + ShellRunner.run( + f"""gh pr create --repo {GITHUB_REPOSITORY} --title 'Release pull request for branch {new_release_branch}' + --head {new_release_branch} {pr_labels} + --body 'This PullRequest is a part of ClickHouse release cycle. It is used by CI system only. Do not perform any changes with it.' + """, + dry_run=dry_run, + ) + def update_version_and_contributors_list(self, dry_run: bool) -> None: # Bump version, update contributors list, create PR branch_upd_version_contributors = f"bump_version_{self.version}" with checkout(self.commit_sha): git = Git() version = get_version_from_repo(git=git) - version.with_description(version.get_stable_release_type()) + if self.release_branch == "master": + version.bump() + version.with_description(VersionType.TESTING) + else: + version.with_description(version.get_stable_release_type()) assert ( version.string == self.version - ), "BUG: version in release info does not match version in git commit" + ), f"BUG: version in release info does not match version in git commit, expected [{self.version}], got [{version.string}]" with checkout(self.release_branch): with checkout_new(branch_upd_version_contributors): update_cmake_version(version) @@ -430,6 +506,11 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Creates and pushes git tag", ) + parser.add_argument( + "--push-new-release-branch", + action="store_true", + help="Creates and pushes new release branch and corresponding service gh tags for backports", + ) parser.add_argument( "--create-bump-version-pr", action="store_true", @@ -533,6 +614,10 @@ if __name__ == "__main__": assert args.infile, "--infile must be provided" release_info = ReleaseInfo.from_file(args.infile) release_info.push_release_tag(dry_run=args.dry_run) + if args.push_new_release_branch: + assert args.infile, "--infile must be provided" + release_info = ReleaseInfo.from_file(args.infile) + release_info.push_new_release_branch(dry_run=args.dry_run) if args.create_bump_version_pr: # TODO: store link to PR in release info assert args.infile, "--infile must be provided" @@ -563,7 +648,7 @@ if __name__ == "__main__": """ -Prepare release machine (for arm machine): +Prepare release machine: ### INSTALL PACKAGES sudo apt update diff --git a/tests/ci/ssh.py b/tests/ci/ssh.py index 321826fcf44..89d90d724d2 100644 --- a/tests/ci/ssh.py +++ b/tests/ci/ssh.py @@ -37,9 +37,9 @@ class SSHAgent: ssh_options = ( "," + os.environ["SSH_OPTIONS"] if os.environ.get("SSH_OPTIONS") else "" ) - os.environ[ - "SSH_OPTIONS" - ] = f"{ssh_options}UserKnownHostsFile=/dev/null,StrictHostKeyChecking=no" + os.environ["SSH_OPTIONS"] = ( + f"{ssh_options}UserKnownHostsFile=/dev/null,StrictHostKeyChecking=no" + ) def add(self, key): key_pub = self._key_pub(key) diff --git a/tests/ci/test_ci_options.py b/tests/ci/test_ci_options.py index 3f158e79f30..f4d14a17512 100644 --- a/tests/ci/test_ci_options.py +++ b/tests/ci/test_ci_options.py @@ -172,14 +172,10 @@ class TestCIOptions(unittest.TestCase): job: CI.JobConfig(runner_type=CI.Runners.STYLE_CHECKER) for job in _TEST_JOB_LIST } - jobs_configs[ - "fuzzers" - ].run_by_label = ( + jobs_configs["fuzzers"].run_by_label = ( "TEST_LABEL" # check "fuzzers" appears in the result due to the label ) - jobs_configs[ - "Integration tests (asan)" - ].release_only = ( + jobs_configs["Integration tests (asan)"].release_only = ( True # still must be included as it's set with include keywords ) filtered_jobs = list( @@ -311,9 +307,9 @@ class TestCIOptions(unittest.TestCase): job: CI.JobConfig(runner_type=CI.Runners.STYLE_CHECKER) for job in _TEST_JOB_LIST } - jobs_configs[ - "fuzzers" - ].run_by_label = "TEST_LABEL" # check "fuzzers" does not appears in the result + jobs_configs["fuzzers"].run_by_label = ( + "TEST_LABEL" # check "fuzzers" does not appears in the result + ) jobs_configs["Integration tests (asan)"].release_only = True filtered_jobs = list( ci_options.apply( diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index d48dd36b16a..07a7a9601c0 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -72,6 +72,19 @@ class ClickHouseVersion: return self.patch_update() raise KeyError(f"wrong part {part} is used") + def bump(self) -> "ClickHouseVersion": + if self.minor < 12: + self._minor += 1 + self._revision += 1 + self._patch = 1 + self._tweak = 1 + else: + self._major += 1 + self._revision += 1 + self._patch = 1 + self._tweak = 1 + return self + def major_update(self) -> "ClickHouseVersion": if self._git is not None: self._git.update() diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 34f5c28fef8..548b58a17e8 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1454,9 +1454,9 @@ class ClickHouseCluster: def setup_azurite_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_azurite = True env_variables["AZURITE_PORT"] = str(self.azurite_port) - env_variables[ - "AZURITE_STORAGE_ACCOUNT_URL" - ] = f"http://azurite1:{env_variables['AZURITE_PORT']}/devstoreaccount1" + env_variables["AZURITE_STORAGE_ACCOUNT_URL"] = ( + f"http://azurite1:{env_variables['AZURITE_PORT']}/devstoreaccount1" + ) env_variables["AZURITE_CONNECTION_STRING"] = ( f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" f"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" @@ -1653,9 +1653,9 @@ class ClickHouseCluster: # Code coverage files will be placed in database directory # (affect only WITH_COVERAGE=1 build) - env_variables[ - "LLVM_PROFILE_FILE" - ] = "/var/lib/clickhouse/server_%h_%p_%m.profraw" + env_variables["LLVM_PROFILE_FILE"] = ( + "/var/lib/clickhouse/server_%h_%p_%m.profraw" + ) clickhouse_start_command = CLICKHOUSE_START_COMMAND if clickhouse_log_file: @@ -1668,9 +1668,9 @@ class ClickHouseCluster: cluster=self, base_path=self.base_dir, name=name, - base_config_dir=base_config_dir - if base_config_dir - else self.base_config_dir, + base_config_dir=( + base_config_dir if base_config_dir else self.base_config_dir + ), custom_main_configs=main_configs or [], custom_user_configs=user_configs or [], custom_dictionaries=dictionaries or [], diff --git a/tests/integration/test_disk_types/test.py b/tests/integration/test_disk_types/test.py index 1cc5048eb69..a5e2456ef4f 100644 --- a/tests/integration/test_disk_types/test.py +++ b/tests/integration/test_disk_types/test.py @@ -19,9 +19,9 @@ def cluster(): cluster = ClickHouseCluster(__file__) cluster.add_instance( "node", - main_configs=["configs/storage_arm.xml"] - if is_arm() - else ["configs/storage_amd.xml"], + main_configs=( + ["configs/storage_arm.xml"] if is_arm() else ["configs/storage_amd.xml"] + ), with_minio=True, with_hdfs=not is_arm(), ) diff --git a/tests/integration/test_distributed_default_database/test.py b/tests/integration/test_distributed_default_database/test.py index ef69533416b..7da9a368997 100644 --- a/tests/integration/test_distributed_default_database/test.py +++ b/tests/integration/test_distributed_default_database/test.py @@ -5,6 +5,7 @@ in this test we write into per-node tables and read from the distributed table. The default database in the distributed table definition is left empty on purpose to test default database deduction. """ + import pytest from helpers.client import QueryRuntimeException diff --git a/tests/integration/test_interserver_dns_retires/test.py b/tests/integration/test_interserver_dns_retires/test.py index f0c581e6450..7c81f278737 100644 --- a/tests/integration/test_interserver_dns_retires/test.py +++ b/tests/integration/test_interserver_dns_retires/test.py @@ -2,6 +2,7 @@ This test makes sure interserver cluster queries handle invalid DNS records for replicas. """ + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster, ClickHouseInstance diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 9a0cb352088..40cbf4b44a6 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -197,7 +197,9 @@ def test_partition_by_string_column(started_cluster): started_cluster, bucket, "test_foo/bar.csv" ) assert '3,"йцук"\n' == get_s3_file_content(started_cluster, bucket, "test_йцук.csv") - assert '78,"你好"\n' == get_s3_file_content(started_cluster, bucket, "test_你好.csv") + assert '78,"你好"\n' == get_s3_file_content( + started_cluster, bucket, "test_你好.csv" + ) def test_partition_by_const_column(started_cluster): diff --git a/tests/integration/test_tcp_handler_http_responses/test_case.py b/tests/integration/test_tcp_handler_http_responses/test_case.py index 2fc53674ca4..98f4b74223e 100644 --- a/tests/integration/test_tcp_handler_http_responses/test_case.py +++ b/tests/integration/test_tcp_handler_http_responses/test_case.py @@ -1,4 +1,5 @@ """Test HTTP responses given by the TCP Handler.""" + from pathlib import Path import pytest from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py b/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py index 62581996f3b..b20ab48795b 100644 --- a/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py +++ b/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py @@ -1,4 +1,5 @@ """Test Interserver responses on configured IP.""" + from pathlib import Path import pytest from helpers.cluster import ClickHouseCluster diff --git a/tests/queries/0_stateless/00411_long_accurate_number_comparison.python b/tests/queries/0_stateless/00411_long_accurate_number_comparison.python index 183a2637d36..045de9ee7ee 100644 --- a/tests/queries/0_stateless/00411_long_accurate_number_comparison.python +++ b/tests/queries/0_stateless/00411_long_accurate_number_comparison.python @@ -50,7 +50,7 @@ TYPES = { "UInt32": {"bits": 32, "sign": False, "float": False}, "Int32": {"bits": 32, "sign": True, "float": False}, "UInt64": {"bits": 64, "sign": False, "float": False}, - "Int64": {"bits": 64, "sign": True, "float": False} + "Int64": {"bits": 64, "sign": True, "float": False}, # "Float32" : { "bits" : 32, "sign" : True, "float" : True }, # "Float64" : { "bits" : 64, "sign" : True, "float" : True } } From 2cd1a39ff4f283122208bc5b2be5244a60e0bfcf Mon Sep 17 00:00:00 2001 From: Max K Date: Wed, 10 Jul 2024 21:01:49 +0200 Subject: [PATCH 59/72] style fixes Automatic style fix --- .github/workflows/create_release.yml | 2 +- tests/ci/artifactory.py | 5 +- tests/ci/create_release.py | 116 +++++++++------------------ 3 files changed, 42 insertions(+), 81 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 26adc3adff9..7879c28ab8d 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -49,7 +49,7 @@ jobs: run: | python3 ./tests/ci/create_release.py --prepare-release-info \ --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} \ - --outfile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} + --outfile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }} echo "::group::Release Info" python3 -m json.tool "$RELEASE_INFO_FILE" echo "::endgroup::" diff --git a/tests/ci/artifactory.py b/tests/ci/artifactory.py index 2e18316cb78..1a062d05a23 100644 --- a/tests/ci/artifactory.py +++ b/tests/ci/artifactory.py @@ -2,10 +2,9 @@ import argparse import time from pathlib import Path from typing import Optional - +from shutil import copy2 from create_release import PackageDownloader, ReleaseInfo, ShellRunner from ci_utils import WithIter -from shutil import copy2 class MountPointApp(metaclass=WithIter): @@ -201,7 +200,7 @@ class RpmArtifactory: print(f"Exporting RPM packages into [{codename}]") for command in commands: - print(f"Running command:") + print("Running command:") print(f" {command}") ShellRunner.run(command) diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py index 65ab48865ef..83a51c66bce 100755 --- a/tests/ci/create_release.py +++ b/tests/ci/create_release.py @@ -9,7 +9,7 @@ from copy import copy from pathlib import Path from typing import Iterator, List -from git_helper import Git, GIT_PREFIX, Runner +from git_helper import Git, GIT_PREFIX from ssh import SSHAgent from env_helper import GITHUB_REPOSITORY, S3_BUILDS_BUCKET from s3_helper import S3Helper @@ -23,7 +23,6 @@ from version_helper import ( update_contributors, VersionType, ) -from git_helper import git_runner as runner from ci_config import CI CMAKE_PATH = get_abs_path(FILE_WITH_VERSION_PATH) @@ -49,6 +48,7 @@ class ShellRunner: stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + check=True, ) if print_output: print(result.stdout) @@ -74,8 +74,7 @@ class ReleaseInfo: @staticmethod def prepare(commit_ref: str, release_type: str, outfile: str) -> None: - dir = Path(outfile).parent - dir.mkdir(parents=True, exist_ok=True) + Path(outfile).parent.mkdir(parents=True, exist_ok=True) Path(outfile).unlink(missing_ok=True) version = None release_branch = None @@ -88,7 +87,7 @@ class ReleaseInfo: f"git merge-base --is-ancestor origin/{commit_ref} origin/master" ) with checkout(commit_ref): - commit_sha = Runner().run(f"git rev-parse {commit_ref}") + _, commit_sha = ShellRunner.run(f"git rev-parse {commit_ref}") # Git() must be inside "with checkout" contextmanager git = Git() version = get_version_from_repo(git=git) @@ -104,7 +103,7 @@ class ReleaseInfo: ) # dummy value (artifactory won't be updated for new release) if release_type == "patch": with checkout(commit_ref): - commit_sha = Runner().run(f"git rev-parse {commit_ref}") + _, commit_sha = ShellRunner.run(f"git rev-parse {commit_ref}") # Git() must be inside "with checkout" contextmanager git = Git() version = get_version_from_repo(git=git) @@ -112,7 +111,7 @@ class ReleaseInfo: version.with_description(codename) release_branch = f"{version.major}.{version.minor}" release_tag = version.describe - runner.run(f"{GIT_PREFIX} fetch origin {release_branch} --tags") + ShellRunner.run(f"{GIT_PREFIX} fetch origin {release_branch} --tags") # check commit is right and on a right branch ShellRunner.run( f"git merge-base --is-ancestor {commit_ref} origin/{release_branch}" @@ -142,7 +141,7 @@ class ReleaseInfo: release_branch and commit_sha and release_tag - and version.string + and version and codename in ("lts", "stable") ) res = ReleaseInfo( @@ -166,20 +165,14 @@ class ReleaseInfo: f"Create and push release tag [{self.release_tag}], commit [{self.commit_sha}]" ) tag_message = f"Release {self.release_tag}" - runner.run( + ShellRunner.run( f"{GIT_PREFIX} tag -a -m '{tag_message}' {self.release_tag} {self.commit_sha}" ) cmd_push_tag = f"{GIT_PREFIX} push origin {self.release_tag}:{self.release_tag}" - if not dry_run: - # TODO: cannot push - workflow will start - # runner.run(cmd_commit_version_upd) - pass - else: - print("Dry run, would execute:") - print(f"* {cmd_push_tag}") + ShellRunner.run(cmd_push_tag, dry_run=dry_run) @staticmethod - def _create_gh_label(label: str, color_hex: str, dry_run: bool): + def _create_gh_label(label: str, color_hex: str, dry_run: bool) -> None: cmd = f"gh api repos/{GITHUB_REPOSITORY}/labels -f name={label} -f color={color_hex}" ShellRunner.run(cmd, dry_run=dry_run) @@ -247,29 +240,19 @@ class ReleaseInfo: with checkout_new(branch_upd_version_contributors): update_cmake_version(version) update_contributors(raise_error=True) - cmd_commit_version_upd = ( - f"{GIT_PREFIX} commit '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}' -m 'Update autogenerated version to {self.version} and contributors'", - ) + cmd_commit_version_upd = f"{GIT_PREFIX} commit '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}' -m 'Update autogenerated version to {self.version} and contributors'" cmd_push_branch = f"{GIT_PREFIX} push --set-upstream origin {branch_upd_version_contributors}" body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md") actor = os.getenv("GITHUB_ACTOR", "") or "me" cmd_create_pr = f"gh pr create --repo {GITHUB_REPOSITORY} --title 'Update version after release' --head {branch_upd_version_contributors} --base {self.release_branch} --body-file '{body_file} --label 'do not test' --assignee @{actor}" - if not dry_run: - runner.run(cmd_commit_version_upd) - runner.run(cmd_push_branch) - runner.run(cmd_create_pr) - else: - print("Dry run, would execute:") - print(f"* {cmd_commit_version_upd}") - print(f"* {cmd_push_branch}") - print(f"* {cmd_create_pr}") - print("Dry run, diff:") - print( - runner.run( - f"{GIT_PREFIX} diff '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'" - ) + ShellRunner.run(cmd_commit_version_upd, dry_run=dry_run) + ShellRunner.run(cmd_push_branch, dry_run=dry_run) + ShellRunner.run(cmd_create_pr, dry_run=dry_run) + if dry_run: + ShellRunner.run( + f"{GIT_PREFIX} diff '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'" ) - runner.run( + ShellRunner.run( f"{GIT_PREFIX} checkout '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'" ) @@ -330,7 +313,7 @@ class PackageDownloader: else: assert False, "BUG" - def __init__(self, release, commit_sha, version: str): + def __init__(self, release, commit_sha, version): assert version.startswith(release), "Invalid release branch or version" major, minor = map(int, release.split(".")) self.package_names = self.PACKAGES @@ -351,41 +334,20 @@ class PackageDownloader: for package_type in self.PACKAGE_TYPES: for package in self.package_names: - package_file_name = ( - package - + "_" - + self.version - + "_" - + self._get_arch_suffix(package_type, RepoTypes.DEBIAN) - + ".deb" - ) - self.deb_package_files.append(package_file_name) - self.file_to_type[package_file_name] = package_type + deb_package_file_name = f"{package}_{self.version}_{self._get_arch_suffix(package_type, RepoTypes.DEBIAN)}.deb" + self.deb_package_files.append(deb_package_file_name) + self.file_to_type[deb_package_file_name] = package_type - package_file_name = ( - package - + "-" - + self.version - + "." - + self._get_arch_suffix(package_type, RepoTypes.RPM) - + ".rpm" - ) - self.rpm_package_files.append(package_file_name) - self.file_to_type[package_file_name] = package_type + rpm_package_file_name = f"{package}-{self.version}.{self._get_arch_suffix(package_type, RepoTypes.RPM)}.rpm" + self.rpm_package_files.append(rpm_package_file_name) + self.file_to_type[rpm_package_file_name] = package_type - package_file_name = ( - package - + "-" - + self.version - + "-" - + self._get_arch_suffix(package_type, RepoTypes.TGZ) - + ".tgz" - ) - self.tgz_package_files.append(package_file_name) - self.file_to_type[package_file_name] = package_type - package_file_name += ".sha512" - self.tgz_package_files.append(package_file_name) - self.file_to_type[package_file_name] = package_type + tgz_package_file_name = f"{package}-{self.version}-{self._get_arch_suffix(package_type, RepoTypes.TGZ)}.tgz" + self.tgz_package_files.append(tgz_package_file_name) + self.file_to_type[tgz_package_file_name] = package_type + tgz_package_file_name += ".sha512" + self.tgz_package_files.append(tgz_package_file_name) + self.file_to_type[tgz_package_file_name] = package_type def get_deb_packages_files(self): return self.deb_package_files @@ -561,33 +523,33 @@ def parse_args() -> argparse.Namespace: @contextmanager def checkout(ref: str) -> Iterator[None]: - orig_ref = runner.run(f"{GIT_PREFIX} symbolic-ref --short HEAD") + _, orig_ref = ShellRunner.run(f"{GIT_PREFIX} symbolic-ref --short HEAD") rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}" assert orig_ref if ref not in (orig_ref,): - runner.run(f"{GIT_PREFIX} checkout {ref}") + ShellRunner.run(f"{GIT_PREFIX} checkout {ref}") try: yield except (Exception, KeyboardInterrupt) as e: print(f"ERROR: Exception [{e}]") - runner.run(rollback_cmd) + ShellRunner.run(rollback_cmd) raise - runner.run(rollback_cmd) + ShellRunner.run(rollback_cmd) @contextmanager def checkout_new(ref: str) -> Iterator[None]: - orig_ref = runner.run(f"{GIT_PREFIX} symbolic-ref --short HEAD") + _, orig_ref = ShellRunner.run(f"{GIT_PREFIX} symbolic-ref --short HEAD") rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}" assert orig_ref - runner.run(f"{GIT_PREFIX} checkout -b {ref}") + ShellRunner.run(f"{GIT_PREFIX} checkout -b {ref}") try: yield except (Exception, KeyboardInterrupt) as e: print(f"ERROR: Exception [{e}]") - runner.run(rollback_cmd) + ShellRunner.run(rollback_cmd) raise - runner.run(rollback_cmd) + ShellRunner.run(rollback_cmd) if __name__ == "__main__": From 9f1520ae219e39f445b0948f48a2b34a99abf568 Mon Sep 17 00:00:00 2001 From: Max K Date: Wed, 10 Jul 2024 21:28:08 +0200 Subject: [PATCH 60/72] do not autofix if not only black failed --- tests/ci/style_check.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 9deae06d9f4..6b58ecece8d 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -192,15 +192,6 @@ def main(): future = executor.submit(subprocess.run, cmd_shell, shell=True) _ = future.result() - autofix_description = "" - if args.push: - try: - commit_push_staged(pr_info) - except subprocess.SubprocessError: - # do not fail the whole script if the autofix didn't work out - logging.error("Unable to push the autofix. Continue.") - autofix_description = "Failed to push autofix to the PR. " - subprocess.check_call( f"python3 ../../utils/check-style/process_style_check_result.py --in-results-dir {temp_path} " f"--out-results-file {temp_path}/test_results.tsv --out-status-file {temp_path}/check_status.tsv || " @@ -210,6 +201,21 @@ def main(): state, description, test_results, additional_files = process_result(temp_path) + autofix_description = "" + fail_cnt = 0 + for result in test_results: + if result.status == FAILURE: + # do not autofix if not only back failed + fail_cnt += 1 + + if args.push and fail_cnt == 1: + try: + commit_push_staged(pr_info) + except subprocess.SubprocessError: + # do not fail the whole script if the autofix didn't work out + logging.error("Unable to push the autofix. Continue.") + autofix_description = "Failed to push autofix to the PR. " + JobReport( description=f"{autofix_description}{description}", test_results=test_results, From 578d22ae94ea1704819e41ca1f0f7c8e58d92e32 Mon Sep 17 00:00:00 2001 From: Max K Date: Thu, 11 Jul 2024 09:24:25 +0200 Subject: [PATCH 61/72] style Automatic style fix Automatic style fix --- .github/actionlint.yml | 1 + .github/workflows/create_release.yml | 4 ++-- pyproject.toml | 2 ++ tests/ci/create_release.py | 6 +++--- tests/ci/docker_server.py | 4 ++-- tests/ci/style_check.py | 14 +++++++++++--- tests/clickhouse-test | 6 +++--- 7 files changed, 24 insertions(+), 13 deletions(-) diff --git a/.github/actionlint.yml b/.github/actionlint.yml index 0f88f30d42c..4357bd3eb6b 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -7,3 +7,4 @@ self-hosted-runner: - stress-tester - style-checker - style-checker-aarch64 + - release-maker diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 7879c28ab8d..972aff90195 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -89,8 +89,8 @@ jobs: --volume=".:/ClickHouse" clickhouse/style-test \ /ClickHouse/tests/ci/changelog.py -v --debug-helpers \ --gh-user-or-token="$GH_TOKEN" --jobs=5 \ - --output="/ClickHouse/docs/changelogs/${RELEASE_TAG}.md" "${RELEASE_TAG}" - git add ./docs/changelogs/${RELEASE_TAG}.md + --output="/ClickHouse/docs/changelogs/${{ env.RELEASE_TAG }}.md" ${{ env.RELEASE_TAG }} + git add ./docs/changelogs/${{ env.RELEASE_TAG }}.md echo "Generate Security" python3 ./utils/security-generator/generate_security.py > SECURITY.md git diff HEAD diff --git a/pyproject.toml b/pyproject.toml index 39511e1a0d3..c89d46c0929 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,8 @@ src_paths = ["src", "tests/ci", "tests/sqllogic"] [tool.pylint.'MESSAGES CONTROL'] # pytest.mark.parametrize is not callable (not-callable) disable = ''' + pointless-string-statement, + line-too-long, missing-docstring, too-few-public-methods, invalid-name, diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py index 83a51c66bce..7f4cf8c787a 100755 --- a/tests/ci/create_release.py +++ b/tests/ci/create_release.py @@ -40,7 +40,7 @@ class ShellRunner: return 0, "" print(f"Running shell command: [{command}]") if async_: - subprocess.Popen(command.split(" ")) + subprocess.Popen(command.split(" ")) # pylint:disable=consider-using-with return 0, "" result = subprocess.run( command + " 2>&1", @@ -316,9 +316,9 @@ class PackageDownloader: def __init__(self, release, commit_sha, version): assert version.startswith(release), "Invalid release branch or version" major, minor = map(int, release.split(".")) - self.package_names = self.PACKAGES + self.package_names = list(self.PACKAGES) if major > 24 or (major == 24 and minor > 3): - self.package_names += self.EXTRA_PACKAGES + self.package_names += list(self.EXTRA_PACKAGES) self.release = release self.commit_sha = commit_sha self.version = version diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 2f556e3ed57..21fc02ce02a 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -376,10 +376,10 @@ def main(): repo_urls = {} direct_urls: Dict[str, List[str]] = {} if pr_info.event_type == EventType.PULL_REQUEST: - release_or_pr = pr_info.number + release_or_pr = str(pr_info.number) sha = pr_info.sha elif pr_info.event_type == EventType.PUSH and pr_info.is_master: - release_or_pr = 0 + release_or_pr = str(0) sha = pr_info.sha else: release_or_pr = f"{args.version.major}.{args.version.minor}" diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 6b58ecece8d..36620d44a2d 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -16,7 +16,15 @@ from docker_images_helper import get_docker_image, pull_image from env_helper import IS_CI, REPO_COPY, TEMP_PATH, GITHUB_EVENT_PATH from git_helper import GIT_PREFIX, git_runner from pr_info import PRInfo -from report import ERROR, FAILURE, SUCCESS, JobReport, TestResults, read_test_results +from report import ( + ERROR, + FAILURE, + SUCCESS, + JobReport, + TestResults, + read_test_results, + FAIL, +) from ssh import SSHKey from stopwatch import Stopwatch @@ -204,8 +212,8 @@ def main(): autofix_description = "" fail_cnt = 0 for result in test_results: - if result.status == FAILURE: - # do not autofix if not only back failed + if result.status in (FAILURE, FAIL): + # do not autofix if not only black failed fail_cnt += 1 if args.push and fail_cnt == 1: diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 0cf46732354..90fb9611151 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -711,9 +711,9 @@ def get_localzone(): class SettingsRandomizer: settings = { - "max_insert_threads": lambda: 12 - if random.random() < 0.03 - else random.randint(1, 3), + "max_insert_threads": lambda: ( + 12 if random.random() < 0.03 else random.randint(1, 3) + ), "group_by_two_level_threshold": threshold_generator(0.2, 0.2, 1, 1000000), "group_by_two_level_threshold_bytes": threshold_generator( 0.2, 0.2, 1, 50000000 From d99c56ae8cf2ddcc8e404b20eee4885c71a43fb1 Mon Sep 17 00:00:00 2001 From: Max K Date: Sat, 13 Jul 2024 16:13:56 +0200 Subject: [PATCH 62/72] CI: Autoskip all non-affected jobs in PRs --- .github/workflows/pull_request.yml | 2 +- tests/ci/ci.py | 19 ++++- tests/ci/ci_cache.py | 107 +++++++++++++------------- tests/ci/ci_config.py | 11 ++- tests/ci/ci_definitions.py | 6 +- tests/ci/report.py | 1 + tests/ci/test_ci_config.py | 119 +++++++++++++++-------------- 7 files changed, 146 insertions(+), 119 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 259e6d41110..c9f4f858825 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -172,7 +172,7 @@ jobs: ################################# Stage Final ################################# # FinishCheck: - if: ${{ !failure() }} + if: ${{ !failure() && !cancelled() }} needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3] runs-on: [self-hosted, style-checker-aarch64] steps: diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 53bbf588e1b..9065e2798c8 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -996,7 +996,7 @@ def main() -> int: args.skip_jobs, ) - if IS_CI and pr_info.is_pr: + if IS_CI and pr_info.is_pr and not ci_settings.no_ci_cache: ci_cache.filter_out_not_affected_jobs() ci_cache.print_status() @@ -1086,6 +1086,16 @@ def main() -> int: print(status) print("::endgroup::") previous_status = status.state + print("Create dummy job report with job_skipped flag") + JobReport( + status=status.state, + description="", + test_results=[], + start_time="", + duration=0.0, + additional_files=[], + job_skipped=True, + ).dump() # ci cache check if not previous_status and not ci_settings.no_ci_cache: @@ -1136,7 +1146,7 @@ def main() -> int: has_oom_error = True job_report = JobReport.load() if JobReport.exist() else None - if job_report: + if job_report and not job_report.job_skipped: ch_helper = ClickHouseHelper() check_url = "" @@ -1242,6 +1252,9 @@ def main() -> int: description = "ERROR: Out Of Memory" else: description = "ERROR: Unknown job status" + print( + f"No job report for {[args.job_name]} - post status [{description}]" + ) gh = GitHub(get_best_robot_token(), per_page=100) commit = get_commit(gh, pr_info.sha) post_commit_status( @@ -1249,7 +1262,7 @@ def main() -> int: ERROR, "", description, - job_report.check_name or _get_ext_check_name(args.job_name), + _get_ext_check_name(args.job_name), pr_info, dump_to_file=True, ) diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index bc6761959b4..ae3b8f9e9a4 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -520,6 +520,35 @@ class CiCache: self.RecordType.SUCCESSFUL, job, batch, num_batches, release_branch ) + def has_evidence(self, job: str, job_config: CI.JobConfig) -> bool: + """ + checks if the job has been seen in master/release CI + function is to be used to check if change did not affect the job + :param job_config: + :param job: + :return: + """ + return ( + self.is_successful( + job=job, + batch=0, + num_batches=job_config.num_batches, + release_branch=not job_config.pr_only, + ) + or self.is_pending( + job=job, + batch=0, + num_batches=job_config.num_batches, + release_branch=not job_config.pr_only, + ) + or self.is_failed( + job=job, + batch=0, + num_batches=job_config.num_batches, + release_branch=not job_config.pr_only, + ) + ) + def is_failed( self, job: str, batch: int, num_batches: int, release_branch: bool ) -> bool: @@ -677,74 +706,46 @@ class CiCache: def filter_out_not_affected_jobs(self): """ Filter is to be applied in PRs to remove jobs that are not affected by the change - It removes jobs from @jobs_to_do if it is a: - 1. test job and it is in @jobs_to_wait (no need to wait not affected jobs in PRs) - 2. test job and it has finished on release branch (even if failed) - 3. build job which is not required by any test job that is left in @jobs_to_do - :return: """ - # 1. - remove_from_await_list = [] - for job_name, job_config in self.jobs_to_wait.items(): - if CI.is_test_job(job_name) and job_name != CI.JobNames.BUILD_CHECK: - remove_from_await_list.append(job_name) - for job in remove_from_await_list: - print(f"Filter job [{job}] - test job and not affected by the change") - del self.jobs_to_wait[job] - del self.jobs_to_do[job] - - # 2. remove_from_to_do = [] + required_builds = [] for job_name, job_config in self.jobs_to_do.items(): if CI.is_test_job(job_name) and job_name != CI.JobNames.BUILD_CHECK: - batches_to_remove = [] - assert job_config.batches is not None - for batch in job_config.batches: - if self.is_failed( - job_name, batch, job_config.num_batches, release_branch=True - ): - print( - f"Filter [{job_name}/{batch}] - not affected by the change (failed on release branch)" - ) - batches_to_remove.append(batch) - for batch in batches_to_remove: - job_config.batches.remove(batch) - if not job_config.batches: - print( - f"Filter [{job_name}] - not affected by the change (failed on release branch)" - ) + if job_config.reference_job_name: + reference_name = job_config.reference_job_name + reference_config = self.jobs_to_do[reference_name] + else: + reference_name = job_name + reference_config = job_config + if self.has_evidence( + job=reference_name, + job_config=reference_config, + ): remove_from_to_do.append(job_name) - for job in remove_from_to_do: - del self.jobs_to_do[job] + else: + required_builds += ( + job_config.required_builds if job_config.required_builds else [] + ) - # 3. - required_builds = [] # type: List[str] - for job_name, job_config in self.jobs_to_do.items(): - if CI.is_test_job(job_name) and job_config.required_builds: - required_builds += job_config.required_builds - required_builds = list(set(required_builds)) - - remove_builds = [] # type: List[str] has_builds_to_do = False for job_name, job_config in self.jobs_to_do.items(): if CI.is_build_job(job_name): if job_name not in required_builds: - remove_builds.append(job_name) + remove_from_to_do.append(job_name) else: has_builds_to_do = True - for build_job in remove_builds: - print( - f"Filter build job [{build_job}] - not affected and not required by test jobs" - ) - del self.jobs_to_do[build_job] - if build_job in self.jobs_to_wait: - del self.jobs_to_wait[build_job] + if not has_builds_to_do: + remove_from_to_do.append(CI.JobNames.BUILD_CHECK) - if not has_builds_to_do and CI.JobNames.BUILD_CHECK in self.jobs_to_do: - print(f"Filter job [{CI.JobNames.BUILD_CHECK}] - no builds to do") - del self.jobs_to_do[CI.JobNames.BUILD_CHECK] + for job in remove_from_to_do: + print(f"Filter job [{job}] - not affected by the change") + if job in self.jobs_to_do: + del self.jobs_to_do[job] + if job in self.jobs_to_wait: + del self.jobs_to_wait[job] + self.jobs_to_skip.append(job) def await_pending_jobs(self, is_release: bool, dry_run: bool = False) -> None: """ diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index d9f8e7d3afd..9b9ddee5326 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -413,7 +413,9 @@ class CI: release_only=True, ), JobNames.INTEGRATION_TEST_FLAKY: CommonJobConfigs.INTEGRATION_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_ASAN], pr_only=True + required_builds=[BuildNames.PACKAGE_ASAN], + pr_only=True, + reference_job_name=JobNames.INTEGRATION_TEST_TSAN, ), JobNames.COMPATIBILITY_TEST: CommonJobConfigs.COMPATIBILITY_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], @@ -455,7 +457,10 @@ class CI: required_builds=[BuildNames.PACKAGE_UBSAN], ), JobNames.STATELESS_TEST_FLAKY_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_ASAN], pr_only=True, timeout=3600 + required_builds=[BuildNames.PACKAGE_ASAN], + pr_only=True, + timeout=3600, + reference_job_name=JobNames.STATELESS_TEST_RELEASE, ), JobNames.JEPSEN_KEEPER: JobConfig( required_builds=[BuildNames.BINARY_RELEASE], @@ -640,7 +645,7 @@ class CI: @classmethod def is_test_job(cls, job: str) -> bool: - return not cls.is_build_job(job) and job != cls.JobNames.STYLE_CHECK + return not cls.is_build_job(job) @classmethod def is_docs_job(cls, job: str) -> bool: diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index d2da73f4e46..acd9b7fa904 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -284,8 +284,12 @@ class JobConfig: # GH Runner type (tag from @Runners) runner_type: str - # used for config validation in ci unittests + # used in ci unittests for config validation job_name_keyword: str = "" + # name of another job that (if provided) should be used to check if job was affected by the change or not (in CiCache.has_evidence(job=@reference_job_name) call) + # for example: "Stateless flaky check" can use reference_job_name="Stateless tests (release)". "Stateless flaky check" does not run on master + # and there cannot be an evidence for it, so instead "Stateless tests (release)" job name can be used to check the evidence + reference_job_name: str = "" # builds required for the job (applicable for test jobs) required_builds: Optional[List[str]] = None # build config for the build job (applicable for builds) diff --git a/tests/ci/report.py b/tests/ci/report.py index bdaa2e15130..039c21e9c9b 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -296,6 +296,7 @@ class JobReport: build_dir_for_upload: Union[Path, str] = "" # if False no GH commit status will be created by CI need_commit_status: bool = True + job_skipped: bool = False def __post_init__(self): assert self.status in (SUCCESS, ERROR, FAILURE, PENDING) diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py index 558faca915e..e12a67bfc92 100644 --- a/tests/ci/test_ci_config.py +++ b/tests/ci/test_ci_config.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import unittest +import random + from ci_config import CI import ci as CIPY from ci_settings import CiSettings @@ -57,6 +59,18 @@ class TestCIConfig(unittest.TestCase): f"Job [{job}] apparently uses wrong common config with job keyword [{CI.JOB_CONFIGS[job].job_name_keyword}]", ) + def test_job_config_has_proper_values(self): + for job in CI.JobNames: + if CI.JOB_CONFIGS[job].reference_job_name: + reference_job_config = CI.JOB_CONFIGS[ + CI.JOB_CONFIGS[job].reference_job_name + ] + # reference job must run in all workflows and has digest + self.assertTrue(reference_job_config.pr_only == False) + self.assertTrue(reference_job_config.release_only == False) + self.assertTrue(reference_job_config.run_always == False) + self.assertTrue(reference_job_config.digest != CI.DigestConfig()) + def test_required_checks(self): for job in CI.REQUIRED_CHECKS: if job in (CI.StatusNames.PR_CHECK, CI.StatusNames.SYNC): @@ -497,79 +511,68 @@ class TestCIConfig(unittest.TestCase): settings = CiSettings() settings.no_ci_cache = True pr_info = PRInfo(github_event=_TEST_EVENT_JSON) - pr_info.event_type = EventType.PUSH - pr_info.number = 0 - assert pr_info.is_release and not pr_info.is_merge_queue + pr_info.event_type = EventType.PULL_REQUEST + pr_info.number = 123 + assert pr_info.is_pr ci_cache = CIPY._configure_jobs( S3Helper(), pr_info, settings, skip_jobs=False, dry_run=True ) self.assertTrue(not ci_cache.jobs_to_skip, "Must be no jobs in skip list") - all_jobs_in_wf = list(ci_cache.jobs_to_do) assert not ci_cache.jobs_to_wait assert not ci_cache.jobs_to_skip + MOCK_AFFECTED_JOBS = [ + CI.JobNames.STATELESS_TEST_S3_DEBUG, + CI.JobNames.STRESS_TEST_TSAN, + ] + MOCK_REQUIRED_BUILDS = [] + # pretend there are pending jobs that we need to wait for job, job_config in ci_cache.jobs_to_do.items(): - ci_cache.jobs_to_wait[job] = job_config + if job in MOCK_AFFECTED_JOBS: + MOCK_REQUIRED_BUILDS += job_config.required_builds + elif job not in MOCK_AFFECTED_JOBS: + ci_cache.jobs_to_wait[job] = job_config - # remove couple tests from to_wait and - # expect they are preserved in @jobs_to_to along with required package_asan - del ci_cache.jobs_to_wait[CI.JobNames.STATELESS_TEST_ASAN] - del ci_cache.jobs_to_wait[CI.JobNames.INTEGRATION_TEST_TSAN] - del ci_cache.jobs_to_wait[CI.JobNames.STATELESS_TEST_MSAN] - - # pretend we have some batches failed for one of the job from the to_do list - failed_job = CI.JobNames.INTEGRATION_TEST_TSAN - failed_job_config = ci_cache.jobs_to_do[failed_job] - FAILED_BATCHES = [0, 3] - for batch in FAILED_BATCHES: - assert batch < failed_job_config.num_batches - record = CiCache.Record( - record_type=CiCache.RecordType.FAILED, - job_name=failed_job, - job_digest=ci_cache.job_digests[failed_job], - batch=batch, - num_batches=failed_job_config.num_batches, - release_branch=True, - ) - for record_t_, records_ in ci_cache.records.items(): - if record_t_.value == CiCache.RecordType.FAILED.value: - records_[record.to_str_key()] = record - - # pretend we have all batches failed for one of the job from the to_do list - failed_job = CI.JobNames.STATELESS_TEST_MSAN - failed_job_config = ci_cache.jobs_to_do[failed_job] - assert failed_job_config.num_batches > 1 - for batch in range(failed_job_config.num_batches): - record = CiCache.Record( - record_type=CiCache.RecordType.FAILED, - job_name=failed_job, - job_digest=ci_cache.job_digests[failed_job], - batch=batch, - num_batches=failed_job_config.num_batches, - release_branch=True, - ) - for record_t_, records_ in ci_cache.records.items(): - if record_t_.value == CiCache.RecordType.FAILED.value: - records_[record.to_str_key()] = record + for job, job_config in ci_cache.jobs_to_do.items(): + if job_config.reference_job_name: + # jobs with reference_job_name in config are not supposed to have records in the cache - continue + continue + if job in MOCK_AFFECTED_JOBS: + continue + for batch in range(job_config.num_batches): + # add any record into cache + record = CiCache.Record( + record_type=random.choice( + [ + CiCache.RecordType.FAILED, + CiCache.RecordType.PENDING, + CiCache.RecordType.SUCCESSFUL, + ] + ), + job_name=job, + job_digest=ci_cache.job_digests[job], + batch=batch, + num_batches=job_config.num_batches, + release_branch=True, + ) + for record_t_, records_ in ci_cache.records.items(): + if record_t_.value == CiCache.RecordType.FAILED.value: + records_[record.to_str_key()] = record ci_cache.filter_out_not_affected_jobs() - expected_to_do = [ - CI.JobNames.STATELESS_TEST_ASAN, - CI.BuildNames.PACKAGE_ASAN, - CI.JobNames.INTEGRATION_TEST_TSAN, - CI.BuildNames.PACKAGE_TSAN, - CI.JobNames.BUILD_CHECK, - ] + expected_to_do = ( + [ + CI.JobNames.BUILD_CHECK, + ] + + MOCK_AFFECTED_JOBS + + MOCK_REQUIRED_BUILDS + ) self.assertCountEqual( list(ci_cache.jobs_to_wait), [ - CI.BuildNames.PACKAGE_ASAN, - CI.BuildNames.PACKAGE_TSAN, CI.JobNames.BUILD_CHECK, - ], + ] + + MOCK_REQUIRED_BUILDS, ) self.assertCountEqual(list(ci_cache.jobs_to_do), expected_to_do) - self.assertTrue(ci_cache.jobs_to_do[CI.JobNames.INTEGRATION_TEST_TSAN].batches) - for batch in ci_cache.jobs_to_do[CI.JobNames.INTEGRATION_TEST_TSAN].batches: - self.assertTrue(batch not in FAILED_BATCHES) From 162d875aee7ee8f9b2c055b8506c1e6467ad2d77 Mon Sep 17 00:00:00 2001 From: Max K Date: Sat, 13 Jul 2024 18:28:13 +0200 Subject: [PATCH 63/72] persistent job report path --- tests/ci/report.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ci/report.py b/tests/ci/report.py index 039c21e9c9b..5ca911f638f 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -23,7 +23,7 @@ from typing import ( from build_download_helper import get_gh_api from ci_config import CI from ci_utils import normalize_string -from env_helper import REPORT_PATH, TEMP_PATH +from env_helper import REPORT_PATH, GITHUB_WORKSPACE logger = logging.getLogger(__name__) @@ -244,7 +244,8 @@ HTML_TEST_PART = """ """ BASE_HEADERS = ["Test name", "Test status"] -JOB_REPORT_FILE = Path(TEMP_PATH) / "job_report.json" +# should not be in TEMP directory or any directory that may be cleaned during the job execution +JOB_REPORT_FILE = Path(GITHUB_WORKSPACE) / "job_report.json" @dataclass From f9b1f2498a2c5044f6b8cc0859abaf66a87034a0 Mon Sep 17 00:00:00 2001 From: Max K Date: Sat, 13 Jul 2024 18:49:23 +0200 Subject: [PATCH 64/72] fix --- tests/ci/ci.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 9065e2798c8..5e13def95f1 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1246,6 +1246,8 @@ def main() -> int: indata["build"], ch_helper, ) + elif job_report.job_skipped: + print(f"Skipped after rerun check {[args.job_name]} - do nothing") else: if CI.is_test_job(args.job_name): if has_oom_error: From 96d3c311900fcb089cfccef965847634bd23de4a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jul 2024 18:51:00 +0200 Subject: [PATCH 65/72] Update play.html --- programs/server/play.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/play.html b/programs/server/play.html index b5bcc687c27..9590a65524c 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -639,7 +639,7 @@ renderChart(json); } else { renderUnparsedResult(response); - stats.innerText = `Elapsed: ${(elapsed_msec/1000).toFixed(3)} sec.`; + stats.innerText = `Elapsed (client-side): ${(elapsed_msec / 1000).toFixed(3)} sec.`; } document.getElementById('check-mark').style.display = 'inline'; } else { From 09b753ecc59e4e8bec78f63faac97e89df4c6c80 Mon Sep 17 00:00:00 2001 From: Max K Date: Sat, 13 Jul 2024 20:15:47 +0200 Subject: [PATCH 66/72] CI: Check job's exit status and report if killed --- tests/ci/ci.py | 52 +++++++++++++++++++++++--------------------- tests/ci/ci_utils.py | 12 ++++++++++ tests/ci/report.py | 26 ++++++++++++++++++++++ 3 files changed, 65 insertions(+), 25 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 5e13def95f1..f99a5dad92f 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -15,7 +15,7 @@ import upload_result_helper from build_check import get_release_or_pr from ci_config import CI from ci_metadata import CiMetadata -from ci_utils import GHActions, normalize_string, Shell +from ci_utils import GHActions, normalize_string, Utils from clickhouse_helper import ( CiLogsCredentials, ClickHouseHelper, @@ -264,7 +264,7 @@ def check_missing_images_on_dockerhub( def _pre_action(s3, indata, pr_info): print("Clear dmesg") - Shell.run("sudo dmesg --clear ||:") + Utils.clear_dmesg() CommitStatusData.cleanup() JobReport.cleanup() BuildResult.cleanup() @@ -1035,6 +1035,7 @@ def main() -> int: elif args.pre: assert indata, "Run config must be provided via --infile" _pre_action(s3, indata, pr_info) + JobReport.create_pre_report().dump() ### RUN action: start elif args.run: @@ -1131,22 +1132,22 @@ def main() -> int: exit_code = 1 else: exit_code = _run_test(check_name, args.run_command) + job_report = JobReport.load() if JobReport.exist() else None + assert ( + job_report + ), "BUG. There must be job report either real report, or pre-report if job was killed" + job_report.exit_code = exit_code + job_report.dump() ### RUN action: end ### POST action: start elif args.post: - has_oom_error = False - if Shell.check( - "sudo dmesg -T | grep -q -e 'Out of memory: Killed process' -e 'oom_reaper: reaped process' -e 'oom-kill:constraint=CONSTRAINT_NONE'" - ): - print("WARNING: OOM while job execution") - CIBuddy(dry_run=not pr_info.is_release).post_error( - "Out Of Memory", job_name=_get_ext_check_name(args.job_name) - ) - has_oom_error = True - job_report = JobReport.load() if JobReport.exist() else None - if job_report and not job_report.job_skipped: + assert ( + job_report + ), "BUG. There must be job report either real report, or pre-report if job was killed" + if not job_report.job_skipped and not job_report.pre_report: + # it's a real job report ch_helper = ClickHouseHelper() check_url = "" @@ -1248,29 +1249,30 @@ def main() -> int: ) elif job_report.job_skipped: print(f"Skipped after rerun check {[args.job_name]} - do nothing") - else: + elif job_report.job_skipped: + print(f"Job was skipped {[args.job_name]} - do nothing") + elif job_report.pre_report: + print(f"ERROR: Job was killed - generate evidence") + job_report.update_duration() + # Job was killed! + if Utils.is_killed_with_oom(): + print("WARNING: OOM while job execution") + error = f"Out Of Memory, exit_code {job_report.exit_code}, after {job_report.duration}s" + else: + error = f"Unknown, exit_code {job_report.exit_code}, after {job_report.duration}s" + CIBuddy().post_error(error, job_name=_get_ext_check_name(args.job_name)) if CI.is_test_job(args.job_name): - if has_oom_error: - description = "ERROR: Out Of Memory" - else: - description = "ERROR: Unknown job status" - print( - f"No job report for {[args.job_name]} - post status [{description}]" - ) gh = GitHub(get_best_robot_token(), per_page=100) commit = get_commit(gh, pr_info.sha) post_commit_status( commit, ERROR, "", - description, + "Error: " + error, _get_ext_check_name(args.job_name), pr_info, dump_to_file=True, ) - else: - # no job report - print(f"No job report for {[args.job_name]} - do nothing") ### POST action: end ### MARK SUCCESS action: start diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py index abc4a88989d..44bd37fe260 100644 --- a/tests/ci/ci_utils.py +++ b/tests/ci/ci_utils.py @@ -96,3 +96,15 @@ class Utils: if match: return int(match.group(1)) return None + + @staticmethod + def is_killed_with_oom(): + if Shell.check( + "sudo dmesg -T | grep -q -e 'Out of memory: Killed process' -e 'oom_reaper: reaped process' -e 'oom-kill:constraint=CONSTRAINT_NONE'" + ): + return True + return False + + @staticmethod + def clear_dmesg(): + Shell.run("sudo dmesg --clear ||:") diff --git a/tests/ci/report.py b/tests/ci/report.py index 5ca911f638f..4be7b438f4f 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -297,7 +297,33 @@ class JobReport: build_dir_for_upload: Union[Path, str] = "" # if False no GH commit status will be created by CI need_commit_status: bool = True + # indicates that this is not real job report but report for the job that was skipped by rerun check job_skipped: bool = False + # indicates that report generated by CI script in order to check later if job was killed before real report is generated + pre_report: bool = False + exit_code: int = -1 + + @staticmethod + def create_pre_report() -> "JobReport": + return JobReport( + status=ERROR, + description="", + test_results=[], + start_time=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), + duration=0.0, + additional_files=[], + pre_report=True, + ) + + def update_duration(self): + if not self.start_time: + self.duration = 0.0 + else: + start_time = datetime.datetime.strptime( + self.start_time, "%Y-%m-%d %H:%M:%S" + ) + current_time = datetime.datetime.utcnow() + self.duration = (current_time - start_time).total_seconds() def __post_init__(self): assert self.status in (SUCCESS, ERROR, FAILURE, PENDING) From 707994b876145b93bcd57e204f1b2d449496e998 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jul 2024 21:48:53 +0200 Subject: [PATCH 67/72] Add a test for #37557 --- .../03204_distributed_with_scalar_subquery.reference | 0 .../03204_distributed_with_scalar_subquery.sql | 10 ++++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/03204_distributed_with_scalar_subquery.reference create mode 100644 tests/queries/0_stateless/03204_distributed_with_scalar_subquery.sql diff --git a/tests/queries/0_stateless/03204_distributed_with_scalar_subquery.reference b/tests/queries/0_stateless/03204_distributed_with_scalar_subquery.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03204_distributed_with_scalar_subquery.sql b/tests/queries/0_stateless/03204_distributed_with_scalar_subquery.sql new file mode 100644 index 00000000000..0a07ce48268 --- /dev/null +++ b/tests/queries/0_stateless/03204_distributed_with_scalar_subquery.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS t_c3oollc8r; +CREATE TABLE t_c3oollc8r (c_k37 Int32, c_y String, c_bou Int32, c_g1 Int32, c_lfntfzg Int32, c_kntw50q Int32) ENGINE = MergeTree ORDER BY (); + +SELECT ( + SELECT c_k37 + FROM t_c3oollc8r + ) > c_lfntfzg +FROM remote('127.0.0.{1,2}', currentDatabase(), t_c3oollc8r); + +DROP TABLE t_c3oollc8r; From dfb831f86119588f63a48f12807c44d568995ab4 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 13 Jul 2024 22:07:46 +0000 Subject: [PATCH 68/72] Try to fix an msan issue --- src/Functions/changeDate.cpp | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 5965f3d1d00..19e4c165ee3 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -1,6 +1,5 @@ #include "Common/DateLUTImpl.h" #include "Common/Exception.h" -#include #include #include #include @@ -101,19 +100,16 @@ public: template ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & input_type, const DataTypePtr & result_type, size_t input_rows_count) const { - bool is_const = (isColumnConst(*arguments[0].column) && isColumnConst(*arguments[1].column)); - size_t result_rows_count = (is_const ? 1 : input_rows_count); - typename ResultDataType::ColumnType::MutablePtr result_col; if constexpr (std::is_same_v) { auto scale = DataTypeDateTime64::default_scale; if constexpr (std::is_same_v) scale = typeid_cast(*result_type).getScale(); - result_col = ResultDataType::ColumnType::create(result_rows_count, scale); + result_col = ResultDataType::ColumnType::create(input_rows_count, scale); } else - result_col = ResultDataType::ColumnType::create(result_rows_count); + result_col = ResultDataType::ColumnType::create(input_rows_count); auto date_time_col = arguments[0].column->convertToFullIfNeeded(); const auto & date_time_col_data = typeid_cast(*date_time_col).getData(); @@ -133,7 +129,7 @@ public: for (size_t j = 0; j < scale; ++j) deg *= 10; - for (size_t i = 0; i < result_rows_count; ++i) + for (size_t i = 0; i < input_rows_count; ++i) { Int64 time = date_lut.toNumYYYYMMDDhhmmss(date_time_col_data[i] / deg); Int64 fraction = date_time_col_data[i] % deg; @@ -144,7 +140,7 @@ public: else if constexpr (std::is_same_v && std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - for (size_t i = 0; i < result_rows_count; ++i) + for (size_t i = 0; i < input_rows_count; ++i) { Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(date_time_col_data[i]))) * 1'000'000; result_col_data[i] = getChangedDate(time, value_col_data[i], result_type, date_lut, 3, 0); @@ -153,7 +149,7 @@ public: else if constexpr (std::is_same_v && std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - for (size_t i = 0; i < result_rows_count; ++i) + for (size_t i = 0; i < input_rows_count; ++i) { Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(date_time_col_data[i]))) * 1'000'000; result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); @@ -162,7 +158,7 @@ public: else if constexpr (std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - for (size_t i = 0; i < result_rows_count; ++i) + for (size_t i = 0; i < input_rows_count; ++i) { Int64 time = date_lut.toNumYYYYMMDDhhmmss(date_time_col_data[i]); result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); @@ -171,7 +167,7 @@ public: else { const auto & date_lut = DateLUT::instance(); - for (size_t i = 0; i < result_rows_count; ++i) + for (size_t i = 0; i < input_rows_count; ++i) { Int64 time; if (isDate(input_type)) @@ -186,9 +182,6 @@ public: } } - if (is_const) - return ColumnConst::create(std::move(result_col), input_rows_count); - return result_col; } From 80c8511004008e5caaec08ad7f869baa578cad6c Mon Sep 17 00:00:00 2001 From: Max K Date: Sun, 14 Jul 2024 11:39:31 +0200 Subject: [PATCH 69/72] CI: Add retry for GH set_comment_status call --- tests/ci/ci.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index f99a5dad92f..979d108378d 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -6,6 +6,7 @@ import os import re import subprocess import sys +import time from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, List, Optional @@ -550,7 +551,17 @@ def _update_gh_statuses_action(indata: Dict, s3: S3Helper) -> None: except Exception as e: raise e print("Going to update overall CI report") - set_status_comment(commit, pr_info) + for retry in range(2): + try: + set_status_comment(commit, pr_info) + break + except Exception as e: + print( + f"WARNING: Failed to update CI Running status, attempt [{retry + 1}], exception [{e}]" + ) + time.sleep(1) + else: + print("ERROR: All retry attempts failed.") print("... CI report update - done") From 379706d2d5118dcd53c87f661b7a06add00bee18 Mon Sep 17 00:00:00 2001 From: Max K Date: Sun, 14 Jul 2024 11:47:18 +0200 Subject: [PATCH 70/72] fix for job filtering --- tests/ci/ci.py | 4 ++-- tests/ci/ci_cache.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 979d108378d..8dcf3fc4c69 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1007,10 +1007,10 @@ def main() -> int: args.skip_jobs, ) + ci_cache.print_status() if IS_CI and pr_info.is_pr and not ci_settings.no_ci_cache: ci_cache.filter_out_not_affected_jobs() - - ci_cache.print_status() + ci_cache.print_status() if IS_CI and not pr_info.is_merge_queue: # wait for pending jobs to be finished, await_jobs is a long blocking call diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index ae3b8f9e9a4..7552b10b873 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -714,7 +714,7 @@ class CiCache: if CI.is_test_job(job_name) and job_name != CI.JobNames.BUILD_CHECK: if job_config.reference_job_name: reference_name = job_config.reference_job_name - reference_config = self.jobs_to_do[reference_name] + reference_config = CI.JOB_CONFIGS[reference_name] else: reference_name = job_name reference_config = job_config @@ -745,7 +745,8 @@ class CiCache: del self.jobs_to_do[job] if job in self.jobs_to_wait: del self.jobs_to_wait[job] - self.jobs_to_skip.append(job) + if job in self.jobs_to_skip: + self.jobs_to_skip.remove(job) def await_pending_jobs(self, is_release: bool, dry_run: bool = False) -> None: """ From f374cdfe69a8d2bc1f7360ead578a5323331650d Mon Sep 17 00:00:00 2001 From: Max K Date: Sun, 14 Jul 2024 13:53:20 +0200 Subject: [PATCH 71/72] CI: Fix for job filtering in PRs --- tests/ci/ci_cache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index 7552b10b873..5c6d3b05021 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -533,19 +533,19 @@ class CiCache: job=job, batch=0, num_batches=job_config.num_batches, - release_branch=not job_config.pr_only, + release_branch=True, ) or self.is_pending( job=job, batch=0, num_batches=job_config.num_batches, - release_branch=not job_config.pr_only, + release_branch=True, ) or self.is_failed( job=job, batch=0, num_batches=job_config.num_batches, - release_branch=not job_config.pr_only, + release_branch=True, ) ) From 9dabf205e623680807bffe022d8b8b6b856779d8 Mon Sep 17 00:00:00 2001 From: Max K Date: Sun, 14 Jul 2024 14:01:10 +0200 Subject: [PATCH 72/72] CI: Fix for await to always await for builds --- tests/ci/ci_cache.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index 5c6d3b05021..5229c754d70 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -765,14 +765,19 @@ class CiCache: MAX_JOB_NUM_TO_WAIT = 3 round_cnt = 0 - # FIXME: temporary experiment: lets enable await for PR' workflows but for a shorter time + def _has_build_job(): + for job in self.jobs_to_wait: + if CI.is_build_job(job): + return True + return False + if not is_release: - MAX_ROUNDS_TO_WAIT = 3 + # in PRs we can wait only for builds, TIMEOUT*MAX_ROUNDS_TO_WAIT=100min is enough + MAX_ROUNDS_TO_WAIT = 2 while ( - len(self.jobs_to_wait) > MAX_JOB_NUM_TO_WAIT - and round_cnt < MAX_ROUNDS_TO_WAIT - ): + len(self.jobs_to_wait) > MAX_JOB_NUM_TO_WAIT or _has_build_job() + ) and round_cnt < MAX_ROUNDS_TO_WAIT: round_cnt += 1 GHActions.print_in_group( f"Wait pending jobs, round [{round_cnt}/{MAX_ROUNDS_TO_WAIT}]:",