From d84e272ed8687f8d13f0286ac37b93db3f874715 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 30 Apr 2024 13:15:23 +0000 Subject: [PATCH 001/161] Add changeDate functions --- src/Functions/changeDate.cpp | 342 ++++++++++++++++++ .../0_stateless/02982_changeDate.reference | 36 ++ .../queries/0_stateless/02982_changeDate.sql | 42 +++ 3 files changed, 420 insertions(+) create mode 100644 src/Functions/changeDate.cpp create mode 100644 tests/queries/0_stateless/02982_changeDate.reference create mode 100644 tests/queries/0_stateless/02982_changeDate.sql diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp new file mode 100644 index 00000000000..14f6887af47 --- /dev/null +++ b/src/Functions/changeDate.cpp @@ -0,0 +1,342 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Common/DateLUTImpl.h" +#include +#include +#include "Columns/IColumn.h" +#include "DataTypes/IDataType.h" +#include "base/DayNum.h" + +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; +} + +namespace +{ + +enum ChangeDateFunctionsNames +{ + CHANGE_YEAR = 0, + CHANGE_MONTH = 1, + CHANGE_DAY = 2, + CHANGE_HOUR = 3, + CHANGE_MINUTE = 4, + CHANGE_SECOND = 5 +}; + +template +class FunctionChangeDate : public IFunction +{ +public: + static constexpr auto name = Traits::Name; + + static constexpr std::array mandatory_argument_names = {"date", "new_value"}; + + String getName() const override { return name; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + size_t getNumberOfArguments() const override { return mandatory_argument_names.size(); } + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + FunctionArgumentDescriptors args{ + {mandatory_argument_names[0], &isDateOrDate32OrDateTimeOrDateTime64, nullptr, "Date"}, + {mandatory_argument_names[1], &isNumber, nullptr, "Number"} + }; + validateFunctionArgumentTypes(*this, arguments, args); + + if (Traits::EnumName >= 3) + { + if (isDate(arguments[0].type)) + return std::make_shared(); + if (isDate32(arguments[0].type)) + return std::make_shared(DataTypeDateTime64::default_scale); + } + + return arguments[0].type; + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + const auto & input_type = arguments[0].type; + if (isDate(input_type)) + { + if (Traits::EnumName >= 3) + return execute(arguments, input_type, result_type, input_rows_count); + return execute(arguments, input_type, result_type, input_rows_count); + } + if (isDate32(input_type)) + { + if (Traits::EnumName >= 3) + return executeDate32ToDateTime64(arguments, result_type, input_rows_count); + return execute(arguments, input_type, result_type, input_rows_count); + } + if (isDateTime(input_type)) + return execute(arguments, input_type, result_type, input_rows_count); + return executeDateTime64(arguments, result_type, input_rows_count); + } + + + template + ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & input_type, const DataTypePtr & result_type, size_t input_rows_count) const + { + const auto & date_lut = DateLUT::instance(); + + auto result_column = ResultDataType::ColumnType::create(input_rows_count); + auto & result_data = result_column->getData(); + + auto input_column = castColumn(arguments[0], std::make_shared()); + input_column = input_column->convertToFullColumnIfConst(); + const auto & input_column_data = typeid_cast(*input_column).getData(); + + auto new_value_column = castColumn(arguments[1], std::make_shared()); + new_value_column = new_value_column->convertToFullColumnIfConst(); + const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + + for (size_t i = 0; i < input_rows_count; ++i) + { + Int64 time; + if (isDateOrDate32(input_type)) + time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1000'000; + else + time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); + + if (isDateOrDate32(result_type)) + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + else + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + } + + return result_column; + } + + ColumnPtr executeDate32ToDateTime64(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const + { + const auto & date_lut = DateLUT::instance(); + + auto result_column = ColumnDateTime64::create(input_rows_count, DataTypeDateTime64::default_scale); + auto & result_data = result_column->getData(); + + auto input_column = arguments[0].column->convertToFullColumnIfConst(); + const auto & input_column_data = typeid_cast(*input_column).getData(); + + auto new_value_column = castColumn(arguments[1], std::make_shared()); + new_value_column = new_value_column->convertToFullColumnIfConst(); + const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + + for (size_t i = 0; i < input_rows_count; ++i) + { + Int64 time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1000'000; + + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1000, 0); + } + + return result_column; + } + + ColumnPtr executeDateTime64(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const + { + auto result_column = ColumnDateTime64::create(input_rows_count, DataTypeDateTime64::default_scale); + auto & result_data = result_column->getData(); + + auto input_column = arguments[0].column->convertToFullColumnIfConst(); + const auto & input_column_data = typeid_cast(*input_column).getData(); + + auto new_value_column = castColumn(arguments[1], std::make_shared()); + new_value_column = new_value_column->convertToFullColumnIfConst(); + const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + + const auto scale = typeid_cast(*result_type).getScale(); + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 deg = 1; + for (size_t i = 0; i < scale; ++i) { + deg *= 10; + } + + for (size_t i = 0; i < input_rows_count; ++i) + { + Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); + Int64 fraction = input_column_data[i] % deg; + + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, deg, fraction); + } + + return result_column; + } + + Int64 getChangedDate(Int64 time, Float32 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 deg = 0, Int64 fraction = 0) const + { + auto year = time / 10'000'000'000; + auto month = (time % 10'000'000'000) / 100'000'000; + auto day = (time % 100'000'000) / 1000'000; + auto hours = (time % 1000'000) / 10'000; + auto minutes = (time % 10'000) / 100; + auto seconds = time % 100; + + Int64 min_date, max_date; + Int16 min_year, max_year; + if (isDate(result_type)) + { + min_date = date_lut.makeDayNum(1970, 1, 1); + max_date = date_lut.makeDayNum(2149, 6, 6); + min_year = 1970; + max_year = 2149; + } + else if (isDate32(result_type)) + { + min_date = date_lut.makeDayNum(1900, 1, 1); + max_date = date_lut.makeDayNum(2299, 12, 31); + min_year = 1900; + max_year = 2299; + } + else if (isDateTime(result_type)) + { + min_date = 0; + max_date = 0x0ffffffffll; + min_year = 1970; + max_year = 2106; + } + else + { + min_date = date_lut.makeDateTime(1900, 1, 1, 0,0 , 0) * deg; + max_date = date_lut.makeDateTime(2299, 12, 31, 23, 59, 59) * deg + (deg - 1); + min_year = 1900; + max_year = 2299; + } + + Int8 fl = 0; + + switch (Traits::EnumName) + { + case CHANGE_YEAR: + if (new_value < min_year) + fl = 1; + else if (new_value > max_year) + fl = 2; + year = static_cast(new_value); + break; + case CHANGE_MONTH: + if (new_value < 1 || new_value > 12) + fl = 1; + month = static_cast(new_value); + break; + case CHANGE_DAY: + if (new_value < 1 || new_value > 31) + fl = 1; + day = static_cast(new_value); + break; + case CHANGE_HOUR: + if (new_value < 0 || new_value > 23) + fl = 1; + hours = static_cast(new_value); + break; + case CHANGE_MINUTE: + if (new_value < 0 || new_value > 59) + fl = 1; + minutes = static_cast(new_value); + break; + case CHANGE_SECOND: + if (new_value < 0 || new_value > 59) + fl = 1; + seconds = static_cast(new_value); + break; + } + + if (fl == 1) + return min_date; + + if (fl == 2) + return max_date; + + Int64 result; + if (isDateOrDate32(result_type)) + result = date_lut.makeDayNum(year, month, day); + else + { + if (isDateTime(result_type)) + result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); + else + result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; + } + + + if (result > max_date) + return max_date; + + return result; + } +}; + + +struct ChangeYearTraits +{ + static constexpr auto Name = "changeYear"; + static constexpr auto EnumName = CHANGE_YEAR; +}; + +struct ChangeMonthTraits +{ + static constexpr auto Name = "changeMonth"; + static constexpr auto EnumName = CHANGE_MONTH; +}; + +struct ChangeDayTraits +{ + static constexpr auto Name = "changeDay"; + static constexpr auto EnumName = CHANGE_DAY; +}; + +struct ChangeHourTraits +{ + static constexpr auto Name = "changeHour"; + static constexpr auto EnumName = CHANGE_HOUR; +}; + +struct ChangeMinuteTraits +{ + static constexpr auto Name = "changeMinute"; + static constexpr auto EnumName = CHANGE_MINUTE; +}; + +struct ChangeSecondTraits +{ + static constexpr auto Name = "changeSecond"; + static constexpr auto EnumName = CHANGE_SECOND; +}; + + +} + +REGISTER_FUNCTION(ChangeDate) +{ + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); +} + +} diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference new file mode 100644 index 00000000000..d7d4edf4b43 --- /dev/null +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -0,0 +1,36 @@ +2000-01-01 +2001-01-01 +2002-01-01 11:22:33 +2003-01-01 11:22:33.4444 +1970-02-01 +1970-03-01 +1970-04-01 11:22:33 +1970-05-01 11:22:33.4444 +1970-01-02 +1970-01-03 +1970-01-04 11:22:33 +1970-01-05 11:22:33.4444 +1970-01-01 12:00:00 +1970-01-01 13:00:00.000 +1970-01-01 14:22:33 +1970-01-01 15:22:33.4444 +1970-01-01 00:23:00 +1970-01-01 00:24:00.000 +1970-01-01 11:25:33 +1970-01-01 11:26:33.4444 +1970-01-01 00:00:34 +1970-01-01 00:00:35.000 +1970-01-01 11:22:36 +1970-01-01 11:22:37.4444 +1970-01-01 +2149-06-06 +2149-06-06 +1970-01-01 +1970-01-01 +1970-01-01 +1970-01-01 00:00:00 +1970-01-01 00:00:00 +1970-01-01 00:00:00 +1970-01-01 00:00:00 +1970-01-01 00:00:00 +1970-01-01 00:00:00 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql new file mode 100644 index 00000000000..fbfe3771b33 --- /dev/null +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -0,0 +1,42 @@ +SELECT changeYear(makeDate(1970, 01, 01), 2000); +SELECT changeYear(makeDate32(1970, 01, 01), 2001); +SELECT changeYear(makeDateTime(1970, 01, 01, 11, 22, 33), 2002); +SELECT changeYear(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 2003); + +SELECT changeMonth(makeDate(1970, 01, 01), 02); +SELECT changeMonth(makeDate32(1970, 01, 01), 03); +SELECT changeMonth(makeDateTime(1970, 01, 01, 11, 22, 33), 04); +SELECT changeMonth(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 05); + +SELECT changeDay(makeDate(1970, 01, 01), 02); +SELECT changeDay(makeDate32(1970, 01, 01), 03); +SELECT changeDay(makeDateTime(1970, 01, 01, 11, 22, 33), 04); +SELECT changeDay(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 05); + +SELECT changeHour(makeDate(1970, 01, 01), 12); +SELECT changeHour(makeDate32(1970, 01, 01), 13); +SELECT changeHour(makeDateTime(1970, 01, 01, 11, 22, 33), 14); +SELECT changeHour(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 15); + +SELECT changeMinute(makeDate(1970, 01, 01), 23); +SELECT changeMinute(makeDate32(1970, 01, 01), 24); +SELECT changeMinute(makeDateTime(1970, 01, 01, 11, 22, 33), 25); +SELECT changeMinute(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 26); + +SELECT changeSecond(makeDate(1970, 01, 01), 34); +SELECT changeSecond(makeDate32(1970, 01, 01), 35); +SELECT changeSecond(makeDateTime(1970, 01, 01, 11, 22, 33), 36); +SELECT changeSecond(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 37); + +SELECT changeYear(makeDate(2000, 01, 01), 1969.0); +SELECT changeYear(makeDate(2000, 06, 07), 2149.0); +SELECT changeMonth(makeDate(2149, 01, 01), 07); +SELECT changeMonth(makeDate(2000, 06, 07), 13); +SELECT changeDay(makeDate(2000, 01, 01), 0); +SELECT changeDay(makeDate(2000, 06, 07), 32); +SELECT changeHour(makeDate(2000, 01, 01), -1); +SELECT changeHour(makeDate(2000, 06, 07), 24); +SELECT changeMinute(makeDate(2000, 01, 01), -1); +SELECT changeMinute(makeDate(2000, 06, 07), 60); +SELECT changeSecond(makeDate(2000, 01, 01), -1); +SELECT changeSecond(makeDate(2000, 06, 07), 60); \ No newline at end of file From 8f11262666132a9a940c2a10e67c5be8f0151c3a Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Tue, 30 Apr 2024 13:36:36 +0000 Subject: [PATCH 002/161] ome style changes --- src/Functions/changeDate.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 14f6887af47..4edc31384c3 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -25,10 +25,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int ARGUMENT_OUT_OF_BOUND; -} namespace { @@ -171,7 +167,8 @@ public: const auto scale = typeid_cast(*result_type).getScale(); const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 deg = 1; - for (size_t i = 0; i < scale; ++i) { + for (size_t i = 0; i < scale; ++i) + { deg *= 10; } @@ -266,7 +263,7 @@ public: if (fl == 1) return min_date; - + if (fl == 2) return max_date; @@ -280,7 +277,7 @@ public: else result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; } - + if (result > max_date) return max_date; From 32f267999da8e9edd68c843431d7a1ae26c7250e Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Sat, 4 May 2024 21:28:32 +0000 Subject: [PATCH 003/161] changeDate implementation fixes --- src/DataTypes/DataTypeDate.h | 1 + src/DataTypes/DataTypeDate32.h | 1 + src/DataTypes/DataTypeDateTime.h | 1 + src/Functions/changeDate.cpp | 296 +++++++++++++++---------------- 4 files changed, 143 insertions(+), 156 deletions(-) diff --git a/src/DataTypes/DataTypeDate.h b/src/DataTypes/DataTypeDate.h index 0e08b9ba2ca..72b7ef2509f 100644 --- a/src/DataTypes/DataTypeDate.h +++ b/src/DataTypes/DataTypeDate.h @@ -10,6 +10,7 @@ class DataTypeDate final : public DataTypeNumberBase { public: static constexpr auto family_name = "Date"; + static constexpr auto type_id = TypeIndex::Date; TypeIndex getTypeId() const override { return TypeIndex::Date; } TypeIndex getColumnType() const override { return TypeIndex::UInt16; } diff --git a/src/DataTypes/DataTypeDate32.h b/src/DataTypes/DataTypeDate32.h index 02e818f10df..052bb39fc31 100644 --- a/src/DataTypes/DataTypeDate32.h +++ b/src/DataTypes/DataTypeDate32.h @@ -10,6 +10,7 @@ class DataTypeDate32 final : public DataTypeNumberBase { public: static constexpr auto family_name = "Date32"; + static constexpr auto type_id = TypeIndex::Date32; TypeIndex getTypeId() const override { return TypeIndex::Date32; } TypeIndex getColumnType() const override { return TypeIndex::Int32; } diff --git a/src/DataTypes/DataTypeDateTime.h b/src/DataTypes/DataTypeDateTime.h index 5519240dee1..3b1212d910d 100644 --- a/src/DataTypes/DataTypeDateTime.h +++ b/src/DataTypes/DataTypeDateTime.h @@ -36,6 +36,7 @@ public: explicit DataTypeDateTime(const TimezoneMixin & time_zone); static constexpr auto family_name = "DateTime"; + static constexpr auto type_id = TypeIndex::DateTime; const char * getFamilyName() const override { return family_name; } String doGetName() const override; diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 4edc31384c3..8a5d0a87ca2 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -10,26 +10,23 @@ #include #include #include -#include #include - #include "Common/DateLUTImpl.h" #include #include #include "Columns/IColumn.h" #include "DataTypes/IDataType.h" -#include "base/DayNum.h" - + #include #include - + namespace DB { - + namespace { - -enum ChangeDateFunctionsNames + +enum class ChangeDateFunctionsNames { CHANGE_YEAR = 0, CHANGE_MONTH = 1, @@ -38,160 +35,161 @@ enum ChangeDateFunctionsNames CHANGE_MINUTE = 4, CHANGE_SECOND = 5 }; - + +constexpr bool isTimeChange(const ChangeDateFunctionsNames & type) +{ + return type == ChangeDateFunctionsNames::CHANGE_HOUR || + type == ChangeDateFunctionsNames::CHANGE_MINUTE || + type == ChangeDateFunctionsNames::CHANGE_SECOND; +} + +template +constexpr bool isDate32() +{ + return DataType::type_id == TypeIndex::Date32; +} + +template +constexpr bool isDateTime64() +{ + return DataType::type_id == TypeIndex::DateTime64; +} + template class FunctionChangeDate : public IFunction { public: static constexpr auto name = Traits::Name; - + static constexpr std::array mandatory_argument_names = {"date", "new_value"}; - + String getName() const override { return name; } - + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - + size_t getNumberOfArguments() const override { return mandatory_argument_names.size(); } - + static FunctionPtr create(ContextPtr) { return std::make_shared(); } - + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { FunctionArgumentDescriptors args{ - {mandatory_argument_names[0], &isDateOrDate32OrDateTimeOrDateTime64, nullptr, "Date"}, + {mandatory_argument_names[0], &isDateOrDate32OrDateTimeOrDateTime64, nullptr, "Date(32) or DateTime(64)"}, {mandatory_argument_names[1], &isNumber, nullptr, "Number"} }; validateFunctionArgumentTypes(*this, arguments, args); - - if (Traits::EnumName >= 3) + + if (isTimeChange(Traits::EnumName)) { if (isDate(arguments[0].type)) return std::make_shared(); if (isDate32(arguments[0].type)) return std::make_shared(DataTypeDateTime64::default_scale); } - + return arguments[0].type; } - + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const auto & input_type = arguments[0].type; if (isDate(input_type)) { - if (Traits::EnumName >= 3) + if (isTimeChange(Traits::EnumName)) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDate32(input_type)) { - if (Traits::EnumName >= 3) - return executeDate32ToDateTime64(arguments, result_type, input_rows_count); + if (isTimeChange(Traits::EnumName)) + return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDateTime(input_type)) return execute(arguments, input_type, result_type, input_rows_count); - return executeDateTime64(arguments, result_type, input_rows_count); + return execute(arguments, input_type, result_type, input_rows_count); } - - + + template ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & input_type, const DataTypePtr & result_type, size_t input_rows_count) const { - const auto & date_lut = DateLUT::instance(); + bool is_const = (isColumnConst(*arguments[0].column) && isColumnConst(*arguments[1].column)); + size_t result_rows_count = (is_const ? 1 : input_rows_count); - auto result_column = ResultDataType::ColumnType::create(input_rows_count); + typename ResultDataType::ColumnType::MutablePtr result_column; + if constexpr (isDateTime64()) + { + auto scale = DataTypeDateTime64::default_scale; + if constexpr (isDateTime64()) + scale = typeid_cast(*result_type).getScale(); + result_column = ResultDataType::ColumnType::create(result_rows_count, scale); + } + else + result_column = ResultDataType::ColumnType::create(result_rows_count); + auto & result_data = result_column->getData(); - - auto input_column = castColumn(arguments[0], std::make_shared()); - input_column = input_column->convertToFullColumnIfConst(); + + auto input_column = arguments[0].column->convertToFullIfNeeded(); const auto & input_column_data = typeid_cast(*input_column).getData(); - - auto new_value_column = castColumn(arguments[1], std::make_shared()); - new_value_column = new_value_column->convertToFullColumnIfConst(); - const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); - - for (size_t i = 0; i < input_rows_count; ++i) + + auto new_value_column = castColumn(arguments[1], std::make_shared()); + new_value_column = new_value_column->convertToFullIfNeeded(); + const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + + for (size_t i = 0; i < result_rows_count; ++i) { - Int64 time; - if (isDateOrDate32(input_type)) - time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1000'000; + if constexpr (isDateTime64()) + { + const auto scale = typeid_cast(*result_type).getScale(); + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 deg = 1; + for (size_t j = 0; j < scale; ++j) + deg *= 10; + + Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); + Int64 fraction = input_column_data[i] % deg; + + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, deg, fraction); + } + else if constexpr (isDate32() && isDateTime64()) + { + const auto & date_lut = DateLUT::instance(); + Int64 time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; + + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1'000, 0); + } else - time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - - if (isDateOrDate32(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); - else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + { + const auto & date_lut = DateLUT::instance(); + Int64 time; + if (isDateOrDate32(input_type)) + time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; + else + time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); + + if (isDateOrDate32(result_type)) + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + else + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1, 0)); + } } + if (is_const) + return ColumnConst::create(std::move(result_column), input_rows_count); + return result_column; } - - ColumnPtr executeDate32ToDateTime64(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const - { - const auto & date_lut = DateLUT::instance(); - - auto result_column = ColumnDateTime64::create(input_rows_count, DataTypeDateTime64::default_scale); - auto & result_data = result_column->getData(); - - auto input_column = arguments[0].column->convertToFullColumnIfConst(); - const auto & input_column_data = typeid_cast(*input_column).getData(); - - auto new_value_column = castColumn(arguments[1], std::make_shared()); - new_value_column = new_value_column->convertToFullColumnIfConst(); - const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); - - for (size_t i = 0; i < input_rows_count; ++i) - { - Int64 time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1000'000; - - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1000, 0); - } - - return result_column; - } - - ColumnPtr executeDateTime64(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const - { - auto result_column = ColumnDateTime64::create(input_rows_count, DataTypeDateTime64::default_scale); - auto & result_data = result_column->getData(); - - auto input_column = arguments[0].column->convertToFullColumnIfConst(); - const auto & input_column_data = typeid_cast(*input_column).getData(); - - auto new_value_column = castColumn(arguments[1], std::make_shared()); - new_value_column = new_value_column->convertToFullColumnIfConst(); - const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); - - const auto scale = typeid_cast(*result_type).getScale(); - const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - Int64 deg = 1; - for (size_t i = 0; i < scale; ++i) - { - deg *= 10; - } - - for (size_t i = 0; i < input_rows_count; ++i) - { - Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); - Int64 fraction = input_column_data[i] % deg; - - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, deg, fraction); - } - - return result_column; - } - - Int64 getChangedDate(Int64 time, Float32 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 deg = 0, Int64 fraction = 0) const + + Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 deg = 0, Int64 fraction = 0) const { auto year = time / 10'000'000'000; auto month = (time % 10'000'000'000) / 100'000'000; - auto day = (time % 100'000'000) / 1000'000; - auto hours = (time % 1000'000) / 10'000; + auto day = (time % 100'000'000) / 1'000'000; + auto hours = (time % 1'000'000) / 10'000; auto minutes = (time % 10'000) / 100; auto seconds = time % 100; - + Int64 min_date, max_date; Int16 min_year, max_year; if (isDate(result_type)) @@ -217,115 +215,101 @@ public: } else { - min_date = date_lut.makeDateTime(1900, 1, 1, 0,0 , 0) * deg; + min_date = date_lut.makeDateTime(1900, 1, 1, 0, 0, 0) * deg; max_date = date_lut.makeDateTime(2299, 12, 31, 23, 59, 59) * deg + (deg - 1); min_year = 1900; max_year = 2299; } - - Int8 fl = 0; - + switch (Traits::EnumName) { - case CHANGE_YEAR: + case ChangeDateFunctionsNames::CHANGE_YEAR: if (new_value < min_year) - fl = 1; + return min_date; else if (new_value > max_year) - fl = 2; + return max_date; year = static_cast(new_value); break; - case CHANGE_MONTH: + case ChangeDateFunctionsNames::CHANGE_MONTH: if (new_value < 1 || new_value > 12) - fl = 1; + return min_date; month = static_cast(new_value); break; - case CHANGE_DAY: + case ChangeDateFunctionsNames::CHANGE_DAY: if (new_value < 1 || new_value > 31) - fl = 1; + return min_date; day = static_cast(new_value); break; - case CHANGE_HOUR: + case ChangeDateFunctionsNames::CHANGE_HOUR: if (new_value < 0 || new_value > 23) - fl = 1; + return min_date; hours = static_cast(new_value); break; - case CHANGE_MINUTE: + case ChangeDateFunctionsNames::CHANGE_MINUTE: if (new_value < 0 || new_value > 59) - fl = 1; + return min_date; minutes = static_cast(new_value); break; - case CHANGE_SECOND: + case ChangeDateFunctionsNames::CHANGE_SECOND: if (new_value < 0 || new_value > 59) - fl = 1; + return min_date; seconds = static_cast(new_value); break; } - - if (fl == 1) - return min_date; - - if (fl == 2) - return max_date; - + Int64 result; if (isDateOrDate32(result_type)) result = date_lut.makeDayNum(year, month, day); else - { - if (isDateTime(result_type)) - result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); - else - result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; - } - - + result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; + if (result > max_date) return max_date; - + return result; } }; - - + + struct ChangeYearTraits { static constexpr auto Name = "changeYear"; - static constexpr auto EnumName = CHANGE_YEAR; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_YEAR; }; - + struct ChangeMonthTraits { static constexpr auto Name = "changeMonth"; - static constexpr auto EnumName = CHANGE_MONTH; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MONTH; }; - + struct ChangeDayTraits { static constexpr auto Name = "changeDay"; - static constexpr auto EnumName = CHANGE_DAY; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_DAY; }; - + struct ChangeHourTraits { static constexpr auto Name = "changeHour"; - static constexpr auto EnumName = CHANGE_HOUR; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_HOUR; }; - + struct ChangeMinuteTraits { static constexpr auto Name = "changeMinute"; - static constexpr auto EnumName = CHANGE_MINUTE; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MINUTE; }; - + struct ChangeSecondTraits { static constexpr auto Name = "changeSecond"; - static constexpr auto EnumName = CHANGE_SECOND; + static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_SECOND; }; - - + + } - + REGISTER_FUNCTION(ChangeDate) { factory.registerFunction>(); @@ -335,5 +319,5 @@ REGISTER_FUNCTION(ChangeDate) factory.registerFunction>(); factory.registerFunction>(); } - + } From fb6c931262c106bf3e076949fc47a28cf06181ba Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Sat, 4 May 2024 22:49:10 +0000 Subject: [PATCH 004/161] Style fixes --- src/Functions/changeDate.cpp | 108 ++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 8a5d0a87ca2..ed6b3255cfc 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -12,20 +12,27 @@ #include #include #include "Common/DateLUTImpl.h" +#include "Common/Exception.h" #include #include #include "Columns/IColumn.h" #include "DataTypes/IDataType.h" - + #include #include - + namespace DB { - + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + namespace { - + enum class ChangeDateFunctionsNames { CHANGE_YEAR = 0, @@ -35,50 +42,50 @@ enum class ChangeDateFunctionsNames CHANGE_MINUTE = 4, CHANGE_SECOND = 5 }; - + constexpr bool isTimeChange(const ChangeDateFunctionsNames & type) { return type == ChangeDateFunctionsNames::CHANGE_HOUR || type == ChangeDateFunctionsNames::CHANGE_MINUTE || type == ChangeDateFunctionsNames::CHANGE_SECOND; } - + template constexpr bool isDate32() { return DataType::type_id == TypeIndex::Date32; } - + template constexpr bool isDateTime64() { return DataType::type_id == TypeIndex::DateTime64; } - + template class FunctionChangeDate : public IFunction { public: static constexpr auto name = Traits::Name; - + static constexpr std::array mandatory_argument_names = {"date", "new_value"}; - + String getName() const override { return name; } - + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - + size_t getNumberOfArguments() const override { return mandatory_argument_names.size(); } - + static FunctionPtr create(ContextPtr) { return std::make_shared(); } - + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - FunctionArgumentDescriptors args{ - {mandatory_argument_names[0], &isDateOrDate32OrDateTimeOrDateTime64, nullptr, "Date(32) or DateTime(64)"}, - {mandatory_argument_names[1], &isNumber, nullptr, "Number"} - }; - validateFunctionArgumentTypes(*this, arguments, args); - + if (arguments.size() != 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires 2 parameters: date, new_value. Passed {}.", getName(), arguments.size()); + + if (!isDateOrDate32OrDateTimeOrDateTime64(*arguments[0].type) || !isNumber(*arguments[1].type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be Date(32) or DateTime(64), second - numeric", getName()); + if (isTimeChange(Traits::EnumName)) { if (isDate(arguments[0].type)) @@ -86,10 +93,10 @@ public: if (isDate32(arguments[0].type)) return std::make_shared(DataTypeDateTime64::default_scale); } - + return arguments[0].type; } - + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const auto & input_type = arguments[0].type; @@ -109,8 +116,7 @@ public: return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } - - + template ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & input_type, const DataTypePtr & result_type, size_t input_rows_count) const { @@ -127,16 +133,16 @@ public: } else result_column = ResultDataType::ColumnType::create(result_rows_count); - + auto & result_data = result_column->getData(); - + auto input_column = arguments[0].column->convertToFullIfNeeded(); const auto & input_column_data = typeid_cast(*input_column).getData(); - + auto new_value_column = castColumn(arguments[1], std::make_shared()); new_value_column = new_value_column->convertToFullIfNeeded(); - const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); - + const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + for (size_t i = 0; i < result_rows_count; ++i) { if constexpr (isDateTime64()) @@ -146,17 +152,17 @@ public: Int64 deg = 1; for (size_t j = 0; j < scale; ++j) deg *= 10; - + Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); Int64 fraction = input_column_data[i] % deg; - + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, deg, fraction); } else if constexpr (isDate32() && isDateTime64()) { const auto & date_lut = DateLUT::instance(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; - + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1'000, 0); } else @@ -167,7 +173,7 @@ public: time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; else time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - + if (isDateOrDate32(result_type)) result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else @@ -177,10 +183,10 @@ public: if (is_const) return ColumnConst::create(std::move(result_column), input_rows_count); - + return result_column; } - + Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 deg = 0, Int64 fraction = 0) const { auto year = time / 10'000'000'000; @@ -189,7 +195,7 @@ public: auto hours = (time % 1'000'000) / 10'000; auto minutes = (time % 10'000) / 100; auto seconds = time % 100; - + Int64 min_date, max_date; Int16 min_year, max_year; if (isDate(result_type)) @@ -220,7 +226,7 @@ public: min_year = 1900; max_year = 2299; } - + switch (Traits::EnumName) { case ChangeDateFunctionsNames::CHANGE_YEAR: @@ -256,60 +262,60 @@ public: seconds = static_cast(new_value); break; } - + Int64 result; if (isDateOrDate32(result_type)) result = date_lut.makeDayNum(year, month, day); else result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; - + if (result > max_date) return max_date; - + return result; } }; - - + + struct ChangeYearTraits { static constexpr auto Name = "changeYear"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_YEAR; }; - + struct ChangeMonthTraits { static constexpr auto Name = "changeMonth"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MONTH; }; - + struct ChangeDayTraits { static constexpr auto Name = "changeDay"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_DAY; }; - + struct ChangeHourTraits { static constexpr auto Name = "changeHour"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_HOUR; }; - + struct ChangeMinuteTraits { static constexpr auto Name = "changeMinute"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MINUTE; }; - + struct ChangeSecondTraits { static constexpr auto Name = "changeSecond"; static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_SECOND; }; - - + + } - + REGISTER_FUNCTION(ChangeDate) { factory.registerFunction>(); @@ -319,5 +325,5 @@ REGISTER_FUNCTION(ChangeDate) factory.registerFunction>(); factory.registerFunction>(); } - + } From f4990f26d914b078d92674061ec5427b3daa4ab7 Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Wed, 8 May 2024 16:38:53 +0000 Subject: [PATCH 005/161] fixes --- src/Functions/changeDate.cpp | 39 +++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index ed6b3255cfc..dfa93c9d218 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -156,28 +156,33 @@ public: Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); Int64 fraction = input_column_data[i] % deg; - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, deg, fraction); + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, scale, fraction); } else if constexpr (isDate32() && isDateTime64()) { const auto & date_lut = DateLUT::instance(); - Int64 time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; + Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1'000, 0); + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); } else { const auto & date_lut = DateLUT::instance(); Int64 time; - if (isDateOrDate32(input_type)) + if (isDate(input_type)) time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; + else if (isDate32(input_type)) + time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; else time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - if (isDateOrDate32(result_type)) + + if (isDate(result_type)) + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + else if (isDate32(result_type)) result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut, 1, 0)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); } } @@ -187,7 +192,7 @@ public: return result_column; } - Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 deg = 0, Int64 fraction = 0) const + Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const { auto year = time / 10'000'000'000; auto month = (time % 10'000'000'000) / 100'000'000; @@ -221,8 +226,17 @@ public: } else { - min_date = date_lut.makeDateTime(1900, 1, 1, 0, 0, 0) * deg; - max_date = date_lut.makeDateTime(2299, 12, 31, 23, 59, 59) * deg + (deg - 1); + min_date = DecimalUtils::decimalFromComponents( + date_lut.makeDateTime(1900, 1, 1, 0, 0, 0), + static_cast(0), + static_cast(scale)); + Int64 deg = 1; + for (Int64 j = 0; j < scale; ++j) + deg *= 10; + max_date = DecimalUtils::decimalFromComponents( + date_lut.makeDateTime(2299, 12, 31, 23, 59, 59), + static_cast(deg - 1), + static_cast(scale)); min_year = 1900; max_year = 2299; } @@ -266,8 +280,13 @@ public: Int64 result; if (isDateOrDate32(result_type)) result = date_lut.makeDayNum(year, month, day); + else if (isDateTime(result_type)) + result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); else - result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds) * deg + fraction; + result = DecimalUtils::decimalFromComponents( + date_lut.makeDateTime(year, month, day, hours, minutes, seconds), + static_cast(fraction), + static_cast(scale)); if (result > max_date) return max_date; From 2f8341d8c1686b9b73bfad36d12651d63b8ffd08 Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Wed, 8 May 2024 16:52:36 +0000 Subject: [PATCH 006/161] style --- src/Functions/changeDate.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index dfa93c9d218..2265ee1b26b 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -176,7 +176,6 @@ public: else time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - if (isDate(result_type)) result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else if (isDate32(result_type)) From c82687c9a61bd6f53949ac5310d26c9b410fabbf Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Wed, 8 May 2024 22:17:59 +0000 Subject: [PATCH 007/161] fixes + functions docs --- src/Functions/changeDate.cpp | 82 ++++++++++++++++--- .../queries/0_stateless/02982_changeDate.sql | 24 +++--- 2 files changed, 82 insertions(+), 24 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 2265ee1b26b..73ac9eff867 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -56,6 +56,12 @@ constexpr bool isDate32() return DataType::type_id == TypeIndex::Date32; } +template +constexpr bool isDateTime() +{ + return DataType::type_id == TypeIndex::DateTime; +} + template constexpr bool isDateTime64() { @@ -86,7 +92,7 @@ public: if (!isDateOrDate32OrDateTimeOrDateTime64(*arguments[0].type) || !isNumber(*arguments[1].type)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be Date(32) or DateTime(64), second - numeric", getName()); - if (isTimeChange(Traits::EnumName)) + if constexpr (isTimeChange(Traits::EnumName)) { if (isDate(arguments[0].type)) return std::make_shared(); @@ -102,13 +108,13 @@ public: const auto & input_type = arguments[0].type; if (isDate(input_type)) { - if (isTimeChange(Traits::EnumName)) + if constexpr (isTimeChange(Traits::EnumName)) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDate32(input_type)) { - if (isTimeChange(Traits::EnumName)) + if constexpr (isTimeChange(Traits::EnumName)) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } @@ -145,6 +151,7 @@ public: for (size_t i = 0; i < result_rows_count; ++i) { + if constexpr (isDateTime64()) { const auto scale = typeid_cast(*result_type).getScale(); @@ -165,16 +172,21 @@ public: result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); } + else if constexpr (isDateTime()) + { + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); + + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + } else { const auto & date_lut = DateLUT::instance(); Int64 time; if (isDate(input_type)) time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; - else if (isDate32(input_type)) - time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; else - time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); + time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; if (isDate(result_type)) result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); @@ -336,12 +348,58 @@ struct ChangeSecondTraits REGISTER_FUNCTION(ChangeDate) { - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Changes the year of the given Date(32) or DateTime(64). +Returns the same type as the input data. +)", + .categories{"Dates and Times"} + } + ); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Same as changeYear function, but changes month of the date. +)", + .categories{"Dates and Times"} + } + ); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Same as changeYear function, but changes day_of_month of the date. +)", + .categories{"Dates and Times"} + } + ); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Changes the hour of the given Date(32) or DateTime(64). +If the input data is Date, return DateTime; +if the input data is Date32, return DateTime64; +In other cases returns the same type as the input data. +)", + .categories{"Dates and Times"} + } + ); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Same as changeHour function, but changes minute of the date. +)", + .categories{"Dates and Times"} + } + ); + factory.registerFunction>( + FunctionDocumentation{ + .description = R"( +Same as changeHour function, but changes seconds of the date. +)", + .categories{"Dates and Times"} + } + ); } } diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index fbfe3771b33..9f7f5eacfc5 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -13,18 +13,18 @@ SELECT changeDay(makeDate32(1970, 01, 01), 03); SELECT changeDay(makeDateTime(1970, 01, 01, 11, 22, 33), 04); SELECT changeDay(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 05); -SELECT changeHour(makeDate(1970, 01, 01), 12); -SELECT changeHour(makeDate32(1970, 01, 01), 13); +SELECT toTimeZone(changeHour(makeDate(1970, 01, 01), 12), 'UTC'); +SELECT toTimeZone(changeHour(makeDate32(1970, 01, 01), 13), 'UTC'); SELECT changeHour(makeDateTime(1970, 01, 01, 11, 22, 33), 14); SELECT changeHour(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 15); -SELECT changeMinute(makeDate(1970, 01, 01), 23); -SELECT changeMinute(makeDate32(1970, 01, 01), 24); +SELECT toTimeZone(changeMinute(makeDate(1970, 01, 01), 23), 'UTC'); +SELECT toTimeZone(changeMinute(makeDate32(1970, 01, 01), 24), 'UTC'); SELECT changeMinute(makeDateTime(1970, 01, 01, 11, 22, 33), 25); SELECT changeMinute(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 26); -SELECT changeSecond(makeDate(1970, 01, 01), 34); -SELECT changeSecond(makeDate32(1970, 01, 01), 35); +SELECT toTimeZone(changeSecond(makeDate(1970, 01, 01), 34), 'UTC'); +SELECT toTimeZone(changeSecond(makeDate32(1970, 01, 01), 35), 'UTC'); SELECT changeSecond(makeDateTime(1970, 01, 01, 11, 22, 33), 36); SELECT changeSecond(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 37); @@ -34,9 +34,9 @@ SELECT changeMonth(makeDate(2149, 01, 01), 07); SELECT changeMonth(makeDate(2000, 06, 07), 13); SELECT changeDay(makeDate(2000, 01, 01), 0); SELECT changeDay(makeDate(2000, 06, 07), 32); -SELECT changeHour(makeDate(2000, 01, 01), -1); -SELECT changeHour(makeDate(2000, 06, 07), 24); -SELECT changeMinute(makeDate(2000, 01, 01), -1); -SELECT changeMinute(makeDate(2000, 06, 07), 60); -SELECT changeSecond(makeDate(2000, 01, 01), -1); -SELECT changeSecond(makeDate(2000, 06, 07), 60); \ No newline at end of file +SELECT toTimeZone(changeHour(makeDate(2000, 01, 01), -1), 'UTC'); +SELECT toTimeZone(changeHour(makeDate(2000, 06, 07), 24), 'UTC'); +SELECT toTimeZone(changeMinute(makeDate(2000, 01, 01), -1), 'UTC'); +SELECT toTimeZone(changeMinute(makeDate(2000, 06, 07), 60), 'UTC'); +SELECT toTimeZone(changeSecond(makeDate(2000, 01, 01), -1), 'UTC'); +SELECT toTimeZone(changeSecond(makeDate(2000, 06, 07), 60), 'UTC'); \ No newline at end of file From f52c24f753b35240d6d4925aaf582c9ebac829b2 Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Thu, 9 May 2024 08:18:02 +0000 Subject: [PATCH 008/161] tests fixes --- src/Functions/changeDate.cpp | 1 - .../queries/0_stateless/02982_changeDate.sql | 72 +++++++++---------- 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 73ac9eff867..a1827e1d94a 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -151,7 +151,6 @@ public: for (size_t i = 0; i < result_rows_count; ++i) { - if constexpr (isDateTime64()) { const auto scale = typeid_cast(*result_type).getScale(); diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index 9f7f5eacfc5..f438212f9fa 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -1,42 +1,42 @@ -SELECT changeYear(makeDate(1970, 01, 01), 2000); -SELECT changeYear(makeDate32(1970, 01, 01), 2001); -SELECT changeYear(makeDateTime(1970, 01, 01, 11, 22, 33), 2002); -SELECT changeYear(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 2003); +SELECT changeYear(toDate('1970-01-01', 'UTC'), 2000); +SELECT changeYear(toDate32('1900-01-01', 'UTC'), 2001); +SELECT changeYear(toDateTime('1970-01-01 11:22:33', 'UTC'), 2002); +SELECT changeYear(toDateTime64('1900-01-01 11:22:33.4444', 4, 'UTC'), 2003); -SELECT changeMonth(makeDate(1970, 01, 01), 02); -SELECT changeMonth(makeDate32(1970, 01, 01), 03); -SELECT changeMonth(makeDateTime(1970, 01, 01, 11, 22, 33), 04); -SELECT changeMonth(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 05); +SELECT changeMonth(toDate('1970-01-01', 'UTC'), 02); +SELECT changeMonth(toDate32('1970-01-01', 'UTC'), 03); +SELECT changeMonth(toDateTime('1970-01-01 11:22:33', 'UTC'), 04); +SELECT changeMonth(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 05); -SELECT changeDay(makeDate(1970, 01, 01), 02); -SELECT changeDay(makeDate32(1970, 01, 01), 03); -SELECT changeDay(makeDateTime(1970, 01, 01, 11, 22, 33), 04); -SELECT changeDay(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 05); +SELECT changeDay(toDate('1970-01-01', 'UTC'), 02); +SELECT changeDay(toDate32('1970-01-01', 'UTC'), 03); +SELECT changeDay(toDateTime('1970-01-01 11:22:33', 'UTC'), 04); +SELECT changeDay(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 05); -SELECT toTimeZone(changeHour(makeDate(1970, 01, 01), 12), 'UTC'); -SELECT toTimeZone(changeHour(makeDate32(1970, 01, 01), 13), 'UTC'); -SELECT changeHour(makeDateTime(1970, 01, 01, 11, 22, 33), 14); -SELECT changeHour(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 15); +SELECT toTimeZone(changeHour(toDate('1970-01-01', 'UTC'), 12), 'UTC'); +SELECT toTimeZone(changeHour(toDate32('1970-01-01', 'UTC'), 13), 'UTC'); +SELECT changeHour(toDateTime('1970-01-01 11:22:33', 'UTC'), 14); +SELECT changeHour(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 15); -SELECT toTimeZone(changeMinute(makeDate(1970, 01, 01), 23), 'UTC'); -SELECT toTimeZone(changeMinute(makeDate32(1970, 01, 01), 24), 'UTC'); -SELECT changeMinute(makeDateTime(1970, 01, 01, 11, 22, 33), 25); -SELECT changeMinute(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 26); +SELECT toTimeZone(changeMinute(toDate('1970-01-01', 'UTC'), 23), 'UTC'); +SELECT toTimeZone(changeMinute(toDate32('1970-01-01', 'UTC'), 24), 'UTC'); +SELECT changeMinute(toDateTime('1970-01-01 11:22:33', 'UTC'), 25); +SELECT changeMinute(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 26); -SELECT toTimeZone(changeSecond(makeDate(1970, 01, 01), 34), 'UTC'); -SELECT toTimeZone(changeSecond(makeDate32(1970, 01, 01), 35), 'UTC'); -SELECT changeSecond(makeDateTime(1970, 01, 01, 11, 22, 33), 36); -SELECT changeSecond(makeDateTime64(1970, 01, 01, 11, 22, 33, 4444, 4), 37); +SELECT toTimeZone(changeSecond(toDate('1970-01-01', 'UTC'), 34), 'UTC'); +SELECT toTimeZone(changeSecond(toDate32('1970-01-01', 'UTC'), 35), 'UTC'); +SELECT changeSecond(toDateTime('1970-01-01 11:22:33', 'UTC'), 36); +SELECT changeSecond(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 37); -SELECT changeYear(makeDate(2000, 01, 01), 1969.0); -SELECT changeYear(makeDate(2000, 06, 07), 2149.0); -SELECT changeMonth(makeDate(2149, 01, 01), 07); -SELECT changeMonth(makeDate(2000, 06, 07), 13); -SELECT changeDay(makeDate(2000, 01, 01), 0); -SELECT changeDay(makeDate(2000, 06, 07), 32); -SELECT toTimeZone(changeHour(makeDate(2000, 01, 01), -1), 'UTC'); -SELECT toTimeZone(changeHour(makeDate(2000, 06, 07), 24), 'UTC'); -SELECT toTimeZone(changeMinute(makeDate(2000, 01, 01), -1), 'UTC'); -SELECT toTimeZone(changeMinute(makeDate(2000, 06, 07), 60), 'UTC'); -SELECT toTimeZone(changeSecond(makeDate(2000, 01, 01), -1), 'UTC'); -SELECT toTimeZone(changeSecond(makeDate(2000, 06, 07), 60), 'UTC'); \ No newline at end of file +SELECT changeYear(toDate('2000-01-01', 'UTC'), 1969.0); +SELECT changeYear(toDate('2000-06-07', 'UTC'), 2149.0); +SELECT changeMonth(toDate('2149-01-01', 'UTC'), 07); +SELECT changeMonth(toDate('2000-01-01', 'UTC'), 13); +SELECT changeDay(toDate('2000-01-01', 'UTC'), 0); +SELECT changeDay(toDate('2000-01-01', 'UTC'), 32); +SELECT toTimeZone(changeHour(toDate('2000-01-01', 'UTC'), -1), 'UTC'); +SELECT toTimeZone(changeHour(toDate('2000-01-01', 'UTC'), 24), 'UTC'); +SELECT toTimeZone(changeMinute(toDate('2000-01-01', 'UTC'), -1), 'UTC'); +SELECT toTimeZone(changeMinute(toDate('2000-01-01', 'UTC'), 60), 'UTC'); +SELECT toTimeZone(changeSecond(toDate('2000-01-01', 'UTC'), -1), 'UTC'); +SELECT toTimeZone(changeSecond(toDate('2000-01-01', 'UTC'), 60), 'UTC'); \ No newline at end of file From ff9c0934cecb5ee8b938adc92498cec035d2e26f Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Thu, 9 May 2024 17:52:50 +0000 Subject: [PATCH 009/161] changes in tests and functions because of time zone --- src/Functions/changeDate.cpp | 30 +++++--- .../0_stateless/02982_changeDate.reference | 12 ++-- .../queries/0_stateless/02982_changeDate.sql | 72 +++++++++---------- 3 files changed, 64 insertions(+), 50 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index a1827e1d94a..c7815263323 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -50,6 +50,12 @@ constexpr bool isTimeChange(const ChangeDateFunctionsNames & type) type == ChangeDateFunctionsNames::CHANGE_SECOND; } +template +constexpr bool isDate() +{ + return DataType::type_id == TypeIndex::Date; +} + template constexpr bool isDate32() { @@ -68,6 +74,7 @@ constexpr bool isDateTime64() return DataType::type_id == TypeIndex::DateTime64; } + template class FunctionChangeDate : public IFunction { @@ -162,21 +169,21 @@ public: Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); Int64 fraction = input_column_data[i] % deg; - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, scale, fraction); + result_data[i] = getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut, scale, fraction); } else if constexpr (isDate32() && isDateTime64()) { const auto & date_lut = DateLUT::instance(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); + result_data[i] = getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut, 3, 0); } else if constexpr (isDateTime()) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); } else { @@ -188,11 +195,11 @@ public: time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; if (isDate(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); else if (isDate32(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); } } @@ -202,7 +209,7 @@ public: return result_column; } - Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const + Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & input_type, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const { auto year = time / 10'000'000'000; auto month = (time % 10'000'000'000) / 100'000'000; @@ -291,12 +298,20 @@ public: if (isDateOrDate32(result_type)) result = date_lut.makeDayNum(year, month, day); else if (isDateTime(result_type)) + { result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); + if (isDate(input_type)) + result += date_lut.timezoneOffset(result); + } else + { result = DecimalUtils::decimalFromComponents( date_lut.makeDateTime(year, month, day, hours, minutes, seconds), static_cast(fraction), static_cast(scale)); + if (isDate32(input_type)) + result += date_lut.timezoneOffset(result); + } if (result > max_date) return max_date; @@ -342,7 +357,6 @@ struct ChangeSecondTraits static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_SECOND; }; - } REGISTER_FUNCTION(ChangeDate) diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference index d7d4edf4b43..67747922ae5 100644 --- a/tests/queries/0_stateless/02982_changeDate.reference +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -10,16 +10,16 @@ 1970-01-03 1970-01-04 11:22:33 1970-01-05 11:22:33.4444 -1970-01-01 12:00:00 -1970-01-01 13:00:00.000 +12 +13 1970-01-01 14:22:33 1970-01-01 15:22:33.4444 -1970-01-01 00:23:00 -1970-01-01 00:24:00.000 +23 +24 1970-01-01 11:25:33 1970-01-01 11:26:33.4444 -1970-01-01 00:00:34 -1970-01-01 00:00:35.000 +34 +35 1970-01-01 11:22:36 1970-01-01 11:22:37.4444 1970-01-01 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index f438212f9fa..786e27808e8 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -1,42 +1,42 @@ -SELECT changeYear(toDate('1970-01-01', 'UTC'), 2000); -SELECT changeYear(toDate32('1900-01-01', 'UTC'), 2001); -SELECT changeYear(toDateTime('1970-01-01 11:22:33', 'UTC'), 2002); -SELECT changeYear(toDateTime64('1900-01-01 11:22:33.4444', 4, 'UTC'), 2003); +SELECT changeYear(toDate('1970-01-01'), 2000); +SELECT changeYear(toDate32('1900-01-01'), 2001); +SELECT changeYear(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 2002); +SELECT changeYear(toDateTime64('1900-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 2003); -SELECT changeMonth(toDate('1970-01-01', 'UTC'), 02); -SELECT changeMonth(toDate32('1970-01-01', 'UTC'), 03); -SELECT changeMonth(toDateTime('1970-01-01 11:22:33', 'UTC'), 04); -SELECT changeMonth(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 05); +SELECT changeMonth(toDate('1970-01-01'), 02); +SELECT changeMonth(toDate32('1970-01-01'), 03); +SELECT changeMonth(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 04); +SELECT changeMonth(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 05); -SELECT changeDay(toDate('1970-01-01', 'UTC'), 02); -SELECT changeDay(toDate32('1970-01-01', 'UTC'), 03); -SELECT changeDay(toDateTime('1970-01-01 11:22:33', 'UTC'), 04); -SELECT changeDay(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 05); +SELECT changeDay(toDate('1970-01-01'), 02); +SELECT changeDay(toDate32('1970-01-01'), 03); +SELECT changeDay(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 04); +SELECT changeDay(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 05); -SELECT toTimeZone(changeHour(toDate('1970-01-01', 'UTC'), 12), 'UTC'); -SELECT toTimeZone(changeHour(toDate32('1970-01-01', 'UTC'), 13), 'UTC'); -SELECT changeHour(toDateTime('1970-01-01 11:22:33', 'UTC'), 14); -SELECT changeHour(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 15); +SELECT toHour(changeHour(toDate('1970-01-01'), 12)); +SELECT toHour(changeHour(toDate32('1970-01-01'), 13)); +SELECT changeHour(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 14); +SELECT changeHour(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 15); -SELECT toTimeZone(changeMinute(toDate('1970-01-01', 'UTC'), 23), 'UTC'); -SELECT toTimeZone(changeMinute(toDate32('1970-01-01', 'UTC'), 24), 'UTC'); -SELECT changeMinute(toDateTime('1970-01-01 11:22:33', 'UTC'), 25); -SELECT changeMinute(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 26); +SELECT toMinute(changeMinute(toDate('1970-01-01'), 23)); +SELECT toMinute(changeMinute(toDate32('1970-01-01'), 24)); +SELECT changeMinute(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 25); +SELECT changeMinute(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 26); -SELECT toTimeZone(changeSecond(toDate('1970-01-01', 'UTC'), 34), 'UTC'); -SELECT toTimeZone(changeSecond(toDate32('1970-01-01', 'UTC'), 35), 'UTC'); -SELECT changeSecond(toDateTime('1970-01-01 11:22:33', 'UTC'), 36); -SELECT changeSecond(toDateTime64('1970-01-01 11:22:33.4444', 4, 'UTC'), 37); +SELECT toSecond(changeSecond(toDate('1970-01-01'), 34)); +SELECT toSecond(changeSecond(toDate32('1970-01-01'), 35)); +SELECT changeSecond(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 36); +SELECT changeSecond(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 37); -SELECT changeYear(toDate('2000-01-01', 'UTC'), 1969.0); -SELECT changeYear(toDate('2000-06-07', 'UTC'), 2149.0); -SELECT changeMonth(toDate('2149-01-01', 'UTC'), 07); -SELECT changeMonth(toDate('2000-01-01', 'UTC'), 13); -SELECT changeDay(toDate('2000-01-01', 'UTC'), 0); -SELECT changeDay(toDate('2000-01-01', 'UTC'), 32); -SELECT toTimeZone(changeHour(toDate('2000-01-01', 'UTC'), -1), 'UTC'); -SELECT toTimeZone(changeHour(toDate('2000-01-01', 'UTC'), 24), 'UTC'); -SELECT toTimeZone(changeMinute(toDate('2000-01-01', 'UTC'), -1), 'UTC'); -SELECT toTimeZone(changeMinute(toDate('2000-01-01', 'UTC'), 60), 'UTC'); -SELECT toTimeZone(changeSecond(toDate('2000-01-01', 'UTC'), -1), 'UTC'); -SELECT toTimeZone(changeSecond(toDate('2000-01-01', 'UTC'), 60), 'UTC'); \ No newline at end of file +SELECT changeYear(toDate('2000-01-01'), 1969.0); +SELECT changeYear(toDate('2000-06-07'), 2149.0); +SELECT changeMonth(toDate('2149-01-01'), 07); +SELECT changeMonth(toDate('2000-01-01'), 13); +SELECT changeDay(toDate('2000-01-01'), 0); +SELECT changeDay(toDate('2000-01-01'), 32); +SELECT changeHour(toDate('2000-01-01'), -1); +SELECT changeHour(toDate('2000-01-01'), 24); +SELECT changeMinute(toDate('2000-01-01'), -1); +SELECT changeMinute(toDate('2000-01-01'), 60); +SELECT changeSecond(toDate('2000-01-01'), -1); +SELECT changeSecond(toDate('2000-01-01'), 60); \ No newline at end of file From 3e7041c1f4106359ab6792ddbab86db3a4792555 Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Thu, 9 May 2024 18:33:46 +0000 Subject: [PATCH 010/161] . --- src/Functions/changeDate.cpp | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index c7815263323..e77aaec30fe 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -169,21 +169,21 @@ public: Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); Int64 fraction = input_column_data[i] % deg; - result_data[i] = getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut, scale, fraction); + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, scale, fraction); } else if constexpr (isDate32() && isDateTime64()) { const auto & date_lut = DateLUT::instance(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; - result_data[i] = getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut, 3, 0); + result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); } else if constexpr (isDateTime()) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); } else { @@ -195,11 +195,11 @@ public: time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; if (isDate(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else if (isDate32(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], input_type, result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); } } @@ -209,7 +209,7 @@ public: return result_column; } - Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & input_type, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const + Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const { auto year = time / 10'000'000'000; auto month = (time % 10'000'000'000) / 100'000'000; @@ -298,20 +298,12 @@ public: if (isDateOrDate32(result_type)) result = date_lut.makeDayNum(year, month, day); else if (isDateTime(result_type)) - { result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); - if (isDate(input_type)) - result += date_lut.timezoneOffset(result); - } else - { result = DecimalUtils::decimalFromComponents( date_lut.makeDateTime(year, month, day, hours, minutes, seconds), static_cast(fraction), static_cast(scale)); - if (isDate32(input_type)) - result += date_lut.timezoneOffset(result); - } if (result > max_date) return max_date; From afb47e418303b4ee125f8c9700985372f827da8b Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Thu, 9 May 2024 19:32:43 +0000 Subject: [PATCH 011/161] . --- src/Functions/changeDate.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index e77aaec30fe..fd279a028b7 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -173,11 +173,18 @@ public: } else if constexpr (isDate32() && isDateTime64()) { - const auto & date_lut = DateLUT::instance(); + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); } + else if constexpr (isDate() && isDateTime()) + { + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; + + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + } else if constexpr (isDateTime()) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); @@ -196,10 +203,8 @@ public: if (isDate(result_type)) result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); - else if (isDate32(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); } } From eb1cce7b07bae5dcb80868f449db1b4dd7dbeb52 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 11 May 2024 03:09:25 +0200 Subject: [PATCH 012/161] Everything should work with Analyzer --- .../0_stateless/01487_distributed_in_not_default_db.sql | 4 ---- tests/queries/0_stateless/02942_variant_cast.sql | 1 - tests/queries/0_stateless/02944_variant_as_common_type.sql | 3 --- 3 files changed, 8 deletions(-) diff --git a/tests/queries/0_stateless/01487_distributed_in_not_default_db.sql b/tests/queries/0_stateless/01487_distributed_in_not_default_db.sql index cd027530ac8..ccd2c571290 100644 --- a/tests/queries/0_stateless/01487_distributed_in_not_default_db.sql +++ b/tests/queries/0_stateless/01487_distributed_in_not_default_db.sql @@ -25,10 +25,6 @@ CREATE TABLE d AS t ENGINE = Distributed(test_cluster_two_shards_different_datab USE test_01487; DROP DATABASE test_01487; --- After the default database is dropped QueryAnalysisPass cannot process the following SELECT query. --- That query is invalid on the initiator node. -set allow_experimental_analyzer = 0; - SELECT * FROM main_01487.d WHERE value IN (SELECT l.value FROM l) ORDER BY value; USE main_01487; diff --git a/tests/queries/0_stateless/02942_variant_cast.sql b/tests/queries/0_stateless/02942_variant_cast.sql index fc2d1d63657..33587e3e438 100644 --- a/tests/queries/0_stateless/02942_variant_cast.sql +++ b/tests/queries/0_stateless/02942_variant_cast.sql @@ -1,5 +1,4 @@ set allow_experimental_variant_type=1; -set allow_experimental_analyzer=0; -- It's currently doesn't work with analyzer because of the way it works with constants, but it will be refactored and fixed in future select NULL::Variant(String, UInt64); select 42::UInt64::Variant(String, UInt64); diff --git a/tests/queries/0_stateless/02944_variant_as_common_type.sql b/tests/queries/0_stateless/02944_variant_as_common_type.sql index e985cf365dd..b3b86427b06 100644 --- a/tests/queries/0_stateless/02944_variant_as_common_type.sql +++ b/tests/queries/0_stateless/02944_variant_as_common_type.sql @@ -1,5 +1,3 @@ -set allow_experimental_analyzer=0; -- The result type for if function with constant is different with analyzer. It wil be fixed after refactoring around constants in analyzer. - set allow_experimental_variant_type=1; set use_variant_as_common_type=1; @@ -73,4 +71,3 @@ select toTypeName(res), array([1, 2, 3], [[1, 2, 3]]) as res; select toTypeName(res), map('a', 1, 'b', 'str_1') as res; select toTypeName(res), map('a', 1, 'b', map('c', 2, 'd', 'str_1')) as res; select toTypeName(res), map('a', 1, 'b', [1, 2, 3], 'c', [[4, 5, 6]]) as res; - From 50377d2450701ea31ea3fcde8c6ee51b96c971cf Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Sun, 19 May 2024 08:41:30 +0000 Subject: [PATCH 013/161] fix date_min of DateTime --- src/Functions/changeDate.cpp | 8 ++- .../0_stateless/02982_changeDate.reference | 58 ++++++++--------- .../queries/0_stateless/02982_changeDate.sql | 62 +++++++++---------- 3 files changed, 66 insertions(+), 62 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index fd279a028b7..237d9082566 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -162,6 +162,7 @@ public: { const auto scale = typeid_cast(*result_type).getScale(); const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 deg = 1; for (size_t j = 0; j < scale; ++j) deg *= 10; @@ -223,7 +224,7 @@ public: auto minutes = (time % 10'000) / 100; auto seconds = time % 100; - Int64 min_date, max_date; + Int64 min_date = 0, max_date = 0; Int16 min_year, max_year; if (isDate(result_type)) { @@ -242,7 +243,7 @@ public: else if (isDateTime(result_type)) { min_date = 0; - max_date = 0x0ffffffffll; + max_date = 0x0FFFFFFFFLL; min_year = 1970; max_year = 2106; } @@ -310,6 +311,9 @@ public: static_cast(fraction), static_cast(scale)); + if (result < min_date) + return min_date; + if (result > max_date) return max_date; diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference index 67747922ae5..c64abc89ed2 100644 --- a/tests/queries/0_stateless/02982_changeDate.reference +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -1,36 +1,36 @@ -2000-01-01 2001-01-01 -2002-01-01 11:22:33 -2003-01-01 11:22:33.4444 -1970-02-01 -1970-03-01 -1970-04-01 11:22:33 -1970-05-01 11:22:33.4444 -1970-01-02 -1970-01-03 -1970-01-04 11:22:33 -1970-01-05 11:22:33.4444 -12 -13 -1970-01-01 14:22:33 -1970-01-01 15:22:33.4444 -23 -24 -1970-01-01 11:25:33 -1970-01-01 11:26:33.4444 -34 -35 -1970-01-01 11:22:36 -1970-01-01 11:22:37.4444 +2002-01-01 +2003-01-01 11:22:33 +2004-01-01 11:22:33.4444 +2000-02-01 +2000-03-01 +2000-04-01 11:22:33 +2000-05-01 11:22:33.4444 +2000-01-02 +2000-01-03 +2000-01-04 11:22:33 +2000-01-05 11:22:33.4444 +2000-01-01 12:00:00 +2000-01-01 13:00:00.000 +2000-01-01 14:22:33 +2000-01-01 15:22:33.4444 +2000-01-01 00:23:00 +2000-01-01 00:24:00.000 +2000-01-01 11:25:33 +2000-01-01 11:26:33.4444 +2000-01-01 00:00:34 +2000-01-01 00:00:35.000 +2000-01-01 11:22:36 +2000-01-01 11:22:37.4444 1970-01-01 2149-06-06 2149-06-06 1970-01-01 1970-01-01 1970-01-01 -1970-01-01 00:00:00 -1970-01-01 00:00:00 -1970-01-01 00:00:00 -1970-01-01 00:00:00 -1970-01-01 00:00:00 -1970-01-01 00:00:00 +1970-01-01 07:00:00 +1970-01-01 07:00:00 +1970-01-01 07:00:00 +1970-01-01 07:00:00 +1970-01-01 07:00:00 +1970-01-01 07:00:00 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index 786e27808e8..0d1bd75e130 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -1,42 +1,42 @@ -SELECT changeYear(toDate('1970-01-01'), 2000); -SELECT changeYear(toDate32('1900-01-01'), 2001); -SELECT changeYear(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 2002); -SELECT changeYear(toDateTime64('1900-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 2003); +SELECT changeYear(toDate('2000-01-01'), 2001); +SELECT changeYear(toDate32('2000-01-01'), 2002); +SELECT changeYear(toDateTime('2000-01-01 11:22:33'), 2003); +SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), 2004); -SELECT changeMonth(toDate('1970-01-01'), 02); -SELECT changeMonth(toDate32('1970-01-01'), 03); -SELECT changeMonth(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 04); -SELECT changeMonth(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 05); +SELECT changeMonth(toDate('2000-01-01'), 02); +SELECT changeMonth(toDate32('2000-01-01'), 03); +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 04); +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 05); -SELECT changeDay(toDate('1970-01-01'), 02); -SELECT changeDay(toDate32('1970-01-01'), 03); -SELECT changeDay(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 04); -SELECT changeDay(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 05); +SELECT changeDay(toDate('2000-01-01'), 02); +SELECT changeDay(toDate32('2000-01-01'), 03); +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 04); +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 05); -SELECT toHour(changeHour(toDate('1970-01-01'), 12)); -SELECT toHour(changeHour(toDate32('1970-01-01'), 13)); -SELECT changeHour(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 14); -SELECT changeHour(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 15); +SELECT changeHour(toDate('2000-01-01'), 12); +SELECT changeHour(toDate32('2000-01-01'), 13); +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 14); +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 15); -SELECT toMinute(changeMinute(toDate('1970-01-01'), 23)); -SELECT toMinute(changeMinute(toDate32('1970-01-01'), 24)); -SELECT changeMinute(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 25); -SELECT changeMinute(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 26); +SELECT changeMinute(toDate('2000-01-01'), 23); +SELECT changeMinute(toDate32('2000-01-01'), 24); +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 25); +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 26); -SELECT toSecond(changeSecond(toDate('1970-01-01'), 34)); -SELECT toSecond(changeSecond(toDate32('1970-01-01'), 35)); -SELECT changeSecond(toDateTime('1970-01-01 11:22:33', 'Antarctica/Palmer'), 36); -SELECT changeSecond(toDateTime64('1970-01-01 11:22:33.4444', 4, 'Antarctica/Palmer'), 37); +SELECT changeSecond(toDate('2000-01-01'), 34); +SELECT changeSecond(toDate32('2000-01-01'), 35); +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 36); +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 37); -SELECT changeYear(toDate('2000-01-01'), 1969.0); +SELECT changeYear(toDate('2000-01-01'), 1969.0); SELECT changeYear(toDate('2000-06-07'), 2149.0); SELECT changeMonth(toDate('2149-01-01'), 07); SELECT changeMonth(toDate('2000-01-01'), 13); SELECT changeDay(toDate('2000-01-01'), 0); SELECT changeDay(toDate('2000-01-01'), 32); -SELECT changeHour(toDate('2000-01-01'), -1); -SELECT changeHour(toDate('2000-01-01'), 24); -SELECT changeMinute(toDate('2000-01-01'), -1); -SELECT changeMinute(toDate('2000-01-01'), 60); -SELECT changeSecond(toDate('2000-01-01'), -1); -SELECT changeSecond(toDate('2000-01-01'), 60); \ No newline at end of file +SELECT changeHour(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeHour(toDate('2000-01-01'), 24) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeMinute(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeMinute(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeSecond(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; \ No newline at end of file From eecbd44ce5e6673072c728097f50b41e2935ae90 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 19 May 2024 13:23:30 +0000 Subject: [PATCH 014/161] Some fixups --- src/Functions/changeDate.cpp | 172 ++++++++---------- .../queries/0_stateless/02982_changeDate.sql | 2 +- 2 files changed, 75 insertions(+), 99 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 237d9082566..6725afe3356 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -1,24 +1,23 @@ -#include -#include -#include +#include "Common/DateLUTImpl.h" +#include "Common/Exception.h" +#include +#include +#include +#include +#include +#include +#include #include #include #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include -#include "Common/DateLUTImpl.h" -#include "Common/Exception.h" -#include -#include -#include "Columns/IColumn.h" -#include "DataTypes/IDataType.h" -#include #include namespace DB @@ -26,88 +25,62 @@ namespace DB namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } namespace { -enum class ChangeDateFunctionsNames +enum class Component { - CHANGE_YEAR = 0, - CHANGE_MONTH = 1, - CHANGE_DAY = 2, - CHANGE_HOUR = 3, - CHANGE_MINUTE = 4, - CHANGE_SECOND = 5 + Year, + Month, + Day, + Hour, + Minute, + Second }; -constexpr bool isTimeChange(const ChangeDateFunctionsNames & type) +bool isTimeComponentChange(Component type) { - return type == ChangeDateFunctionsNames::CHANGE_HOUR || - type == ChangeDateFunctionsNames::CHANGE_MINUTE || - type == ChangeDateFunctionsNames::CHANGE_SECOND; + return type == Component::Hour || + type == Component::Minute || + type == Component::Second; } -template -constexpr bool isDate() -{ - return DataType::type_id == TypeIndex::Date; -} - -template -constexpr bool isDate32() -{ - return DataType::type_id == TypeIndex::Date32; -} - -template -constexpr bool isDateTime() -{ - return DataType::type_id == TypeIndex::DateTime; -} - -template -constexpr bool isDateTime64() -{ - return DataType::type_id == TypeIndex::DateTime64; -} - - template class FunctionChangeDate : public IFunction { public: - static constexpr auto name = Traits::Name; - - static constexpr std::array mandatory_argument_names = {"date", "new_value"}; - - String getName() const override { return name; } - - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - - size_t getNumberOfArguments() const override { return mandatory_argument_names.size(); } + static constexpr auto name = Traits::name; static FunctionPtr create(ContextPtr) { return std::make_shared(); } + String getName() const override { return Traits::name; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + size_t getNumberOfArguments() const override { return 2; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { if (arguments.size() != 2) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires 2 parameters: date, new_value. Passed {}.", getName(), arguments.size()); - if (!isDateOrDate32OrDateTimeOrDateTime64(*arguments[0].type) || !isNumber(*arguments[1].type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be Date(32) or DateTime(64), second - numeric", getName()); + if (!isDateOrDate32OrDateTimeOrDateTime64(*arguments[0].type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be Date, Date32, DateTime or DateTime64", getName()); + if (!isNumber(*arguments[1].type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Second argument for function {} must be numeric", getName()); - if constexpr (isTimeChange(Traits::EnumName)) + const auto & input_type = arguments[0].type; + + if (isTimeComponentChange(Traits::component)) { - if (isDate(arguments[0].type)) + if (isDate(input_type)) return std::make_shared(); - if (isDate32(arguments[0].type)) + if (isDate32(input_type)) return std::make_shared(DataTypeDateTime64::default_scale); } - return arguments[0].type; + return input_type; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override @@ -115,50 +88,53 @@ public: const auto & input_type = arguments[0].type; if (isDate(input_type)) { - if constexpr (isTimeChange(Traits::EnumName)) + if (isTimeComponentChange(Traits::component)) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDate32(input_type)) { - if constexpr (isTimeChange(Traits::EnumName)) + if (isTimeComponentChange(Traits::component)) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDateTime(input_type)) return execute(arguments, input_type, result_type, input_rows_count); - return execute(arguments, input_type, result_type, input_rows_count); + if (isDateTime64(input_type)) + return execute(arguments, input_type, result_type, input_rows_count); + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid input type"); } - template + template ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & input_type, const DataTypePtr & result_type, size_t input_rows_count) const { bool is_const = (isColumnConst(*arguments[0].column) && isColumnConst(*arguments[1].column)); size_t result_rows_count = (is_const ? 1 : input_rows_count); typename ResultDataType::ColumnType::MutablePtr result_column; - if constexpr (isDateTime64()) + if constexpr (std::is_same_v) { auto scale = DataTypeDateTime64::default_scale; - if constexpr (isDateTime64()) + if constexpr (std::is_same_v) scale = typeid_cast(*result_type).getScale(); result_column = ResultDataType::ColumnType::create(result_rows_count, scale); } else result_column = ResultDataType::ColumnType::create(result_rows_count); - auto & result_data = result_column->getData(); - auto input_column = arguments[0].column->convertToFullIfNeeded(); - const auto & input_column_data = typeid_cast(*input_column).getData(); + const auto & input_column_data = typeid_cast(*input_column).getData(); auto new_value_column = castColumn(arguments[1], std::make_shared()); new_value_column = new_value_column->convertToFullIfNeeded(); const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + auto & result_data = result_column->getData(); + for (size_t i = 0; i < result_rows_count; ++i) { - if constexpr (isDateTime64()) + if constexpr (std::is_same_v) { const auto scale = typeid_cast(*result_type).getScale(); const auto & date_lut = typeid_cast(*result_type).getTimeZone(); @@ -172,21 +148,21 @@ public: result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, scale, fraction); } - else if constexpr (isDate32() && isDateTime64()) + else if constexpr (std::is_same_v && std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); } - else if constexpr (isDate() && isDateTime()) + else if constexpr (std::is_same_v && std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); } - else if constexpr (isDateTime()) + else if constexpr (std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); @@ -264,36 +240,36 @@ public: max_year = 2299; } - switch (Traits::EnumName) + switch (Traits::component) { - case ChangeDateFunctionsNames::CHANGE_YEAR: + case Component::Year: if (new_value < min_year) return min_date; else if (new_value > max_year) return max_date; year = static_cast(new_value); break; - case ChangeDateFunctionsNames::CHANGE_MONTH: + case Component::Month: if (new_value < 1 || new_value > 12) return min_date; month = static_cast(new_value); break; - case ChangeDateFunctionsNames::CHANGE_DAY: + case Component::Day: if (new_value < 1 || new_value > 31) return min_date; day = static_cast(new_value); break; - case ChangeDateFunctionsNames::CHANGE_HOUR: + case Component::Hour: if (new_value < 0 || new_value > 23) return min_date; hours = static_cast(new_value); break; - case ChangeDateFunctionsNames::CHANGE_MINUTE: + case Component::Minute: if (new_value < 0 || new_value > 59) return min_date; minutes = static_cast(new_value); break; - case ChangeDateFunctionsNames::CHANGE_SECOND: + case Component::Second: if (new_value < 0 || new_value > 59) return min_date; seconds = static_cast(new_value); @@ -324,38 +300,38 @@ public: struct ChangeYearTraits { - static constexpr auto Name = "changeYear"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_YEAR; + static constexpr auto name = "changeYear"; + static constexpr auto component = Component::Year; }; struct ChangeMonthTraits { - static constexpr auto Name = "changeMonth"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MONTH; + static constexpr auto name = "changeMonth"; + static constexpr auto component = Component::Month; }; struct ChangeDayTraits { - static constexpr auto Name = "changeDay"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_DAY; + static constexpr auto name = "changeDay"; + static constexpr auto component = Component::Day; }; struct ChangeHourTraits { - static constexpr auto Name = "changeHour"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_HOUR; + static constexpr auto name = "changeHour"; + static constexpr auto component = Component::Hour; }; struct ChangeMinuteTraits { - static constexpr auto Name = "changeMinute"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_MINUTE; + static constexpr auto name = "changeMinute"; + static constexpr auto component = Component::Minute; }; struct ChangeSecondTraits { - static constexpr auto Name = "changeSecond"; - static constexpr auto EnumName = ChangeDateFunctionsNames::CHANGE_SECOND; + static constexpr auto name = "changeSecond"; + static constexpr auto component = Component::Second; }; } diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index 0d1bd75e130..26d53b2f7f4 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -39,4 +39,4 @@ SELECT changeHour(toDate('2000-01-01'), 24) SETTINGS session_timezone = 'Asia/No SELECT changeMinute(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeMinute(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeSecond(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; -SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; \ No newline at end of file +SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; From b1d3eb4c4d67e2e95d09dabf027ea9065751634d Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 19 May 2024 13:27:17 +0000 Subject: [PATCH 015/161] Some fixups, pt. II --- src/Functions/changeDate.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 6725afe3356..d17787f4f55 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -26,6 +26,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } From 4fdcdc284d6c41c3a1f97f6d72c7b9c73816a9df Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 19 May 2024 13:28:38 +0000 Subject: [PATCH 016/161] Some fixups, pt. III --- src/DataTypes/DataTypeDate.h | 1 - src/DataTypes/DataTypeDate32.h | 1 - src/DataTypes/DataTypeDateTime.h | 1 - 3 files changed, 3 deletions(-) diff --git a/src/DataTypes/DataTypeDate.h b/src/DataTypes/DataTypeDate.h index 72b7ef2509f..0e08b9ba2ca 100644 --- a/src/DataTypes/DataTypeDate.h +++ b/src/DataTypes/DataTypeDate.h @@ -10,7 +10,6 @@ class DataTypeDate final : public DataTypeNumberBase { public: static constexpr auto family_name = "Date"; - static constexpr auto type_id = TypeIndex::Date; TypeIndex getTypeId() const override { return TypeIndex::Date; } TypeIndex getColumnType() const override { return TypeIndex::UInt16; } diff --git a/src/DataTypes/DataTypeDate32.h b/src/DataTypes/DataTypeDate32.h index 414c8301558..65633e7a228 100644 --- a/src/DataTypes/DataTypeDate32.h +++ b/src/DataTypes/DataTypeDate32.h @@ -9,7 +9,6 @@ class DataTypeDate32 final : public DataTypeNumberBase { public: static constexpr auto family_name = "Date32"; - static constexpr auto type_id = TypeIndex::Date32; TypeIndex getTypeId() const override { return TypeIndex::Date32; } TypeIndex getColumnType() const override { return TypeIndex::Int32; } diff --git a/src/DataTypes/DataTypeDateTime.h b/src/DataTypes/DataTypeDateTime.h index 3b1212d910d..5519240dee1 100644 --- a/src/DataTypes/DataTypeDateTime.h +++ b/src/DataTypes/DataTypeDateTime.h @@ -36,7 +36,6 @@ public: explicit DataTypeDateTime(const TimezoneMixin & time_zone); static constexpr auto family_name = "DateTime"; - static constexpr auto type_id = TypeIndex::DateTime; const char * getFamilyName() const override { return family_name; } String doGetName() const override; From ba1e0e0317e9b408bfeafaac3fdda87eb16296fb Mon Sep 17 00:00:00 2001 From: Maksim Galkin Date: Mon, 20 May 2024 11:43:31 +0000 Subject: [PATCH 017/161] added docks + updated tests --- .../functions/date-time-functions.md | 85 +++++++++++++++++++ .../0_stateless/02982_changeDate.reference | 17 ++++ .../queries/0_stateless/02982_changeDate.sql | 26 +++++- 3 files changed, 127 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 5622097537e..1ce6fce6667 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -3039,3 +3039,88 @@ Result: - Blog: [Working with time series data in ClickHouse](https://clickhouse.com/blog/working-with-time-series-data-and-functions-ClickHouse) +## changeYear + +Changes a year of the passed date argument. + +**Syntax** + +``` sql +changeYear(date, new_value) +``` + +**Arguments** + +- `date` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) + +- `new_value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). + +**Return value** + +- A date or date with time. Same data type as input date argument. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeYear(toDate('1999-01-01'), 2000), hangeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); +``` + +Result: +``` +┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ +│ 2000-01-01 │ 2000-01-01 00:00:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + +## changeMonth + +Like [changeYear](#changeYear) but changes a month of the passed date argument. + +## changeMonth + +Like [changeYear](#changeYear) but changes a day of year of the passed date argument. + +## changeHour + +Changes an hour of the passed date argument. + +**Syntax** + +``` sql +changeHour(date, new_value) +``` + +**Arguments** + +- `date` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) + +- `new_value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). + +**Return value** + +- A date with time. If date argument is Date - returns DateTime, if Date32 - returns DateTime64, otherwise returns same data type as input date argument. + +Type: [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Exapmle** + +``` sql +SELECT changeHour(toDate('1999-01-01'), 12), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 12); +``` + +Result: +``` +┌─changeHour(toDate('1999-01-01'), 12)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 12)─┐ +│ 1999-01-01 12:00:00 │ 1999-01-01 12:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeMinute + +Like [changeHour](#changeHour) but changes a minute of the passed date argument. + +## changeSecond + +Like [changeHour](#changeHour) but changes a seconds of the passed date argument. \ No newline at end of file diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference index c64abc89ed2..3e8ad45d9a8 100644 --- a/tests/queries/0_stateless/02982_changeDate.reference +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -34,3 +34,20 @@ 1970-01-01 07:00:00 1970-01-01 07:00:00 1970-01-01 07:00:00 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2000-02-01 +2299-12-31 +1900-01-01 +2106-02-07 13:28:15 +1970-01-01 07:00:00 +2299-12-31 23:59:59.999 +1900-01-01 00:00:00.000 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index 0d1bd75e130..1a9d062bafe 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -39,4 +39,28 @@ SELECT changeHour(toDate('2000-01-01'), 24) SETTINGS session_timezone = 'Asia/No SELECT changeMinute(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeMinute(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeSecond(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; -SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; \ No newline at end of file +SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; + +SELECT changeYear(toDate('2000-01-01')); -- { serverError 42 } +SELECT changeYear(toDate('2000-01-01'), 2001, 2002); -- { serverError 42 } +SELECT changeYear(toDate('2000-01-01'), '2001'); -- { serverError 43 } + +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int8)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int16)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int32)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int64)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt8)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt16)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt32)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt64)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Float32)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Float64)); +SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Decimal(10, 5))); + +SELECT changeYear(toDate32('2000-01-01'), 2300); +SELECT changeYear(toDate32('2000-01-01'), 1899); +SELECT changeSecond(toDateTime('2106-02-07 13:28:15'), 16) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeHour(toDateTime('1970-01-01 23:59:59'), 6) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeYear(toDateTime64('2000-01-01 00:00:00.000', 3), 2300) SETTINGS session_timezone = 'Asia/Novosibirsk'; +SELECT changeYear(toDateTime64('2000-01-01 00:00:00.000', 3), 1899) SETTINGS session_timezone = 'Asia/Novosibirsk'; + From d115adf462859fd82a5bb5a83a6dedaeae799a08 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 20 May 2024 15:02:51 +0000 Subject: [PATCH 018/161] Some fixups, pt. IV --- .../functions/date-time-functions.md | 180 ++++++++++++--- src/Functions/changeDate.cpp | 126 ++++++----- .../0_stateless/02982_changeDate.reference | 181 +++++++++++++-- .../queries/0_stateless/02982_changeDate.sql | 207 +++++++++++++++--- 4 files changed, 546 insertions(+), 148 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 43bb5b6f6cb..8bf301d76c2 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -4104,33 +4104,33 @@ timeDiff(toDateTime64('1927-01-01 00:00:00', 3), toDate32('1927-01-02')); ## changeYear -Changes a year of the passed date argument. +Changes the year component of a date or date time. **Syntax** ``` sql -changeYear(date, new_value) +changeYear(date_or_datetime, value) ``` **Arguments** -- `date` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) - -- `new_value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md). **Return value** -- A date or date with time. Same data type as input date argument. +- The same type as `date_or_datetime`. Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). **Example** ``` sql - SELECT changeYear(toDate('1999-01-01'), 2000), hangeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); + SELECT changeYear(toDate('1999-01-01'), 2000), changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); ``` Result: + ``` ┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ │ 2000-01-01 │ 2000-01-01 00:00:00.000 │ @@ -4139,51 +4139,175 @@ Result: ## changeMonth -Like [changeYear](#changeYear) but changes a month of the passed date argument. - -## changeMonth - -Like [changeYear](#changeYear) but changes a day of year of the passed date argument. - -## changeHour - -Changes an hour of the passed date argument. +Changes the month component of a date or date time. **Syntax** ``` sql -changeHour(date, new_value) +changeMonth(date_or_datetime, value) ``` **Arguments** -- `date` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) - -- `new_value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the month. [Integer](../../sql-reference/data-types/int-uint.md). **Return value** -- A date with time. If date argument is Date - returns DateTime, if Date32 - returns DateTime64, otherwise returns same data type as input date argument. +- The same type as `date_or_datetime`. -Type: [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). -**Exapmle** +**Example** ``` sql -SELECT changeHour(toDate('1999-01-01'), 12), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 12); + SELECT changeMonth(toDate('1999-01-01'), 2), changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2); ``` Result: + ``` -┌─changeHour(toDate('1999-01-01'), 12)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 12)─┐ -│ 1999-01-01 12:00:00 │ 1999-01-01 12:00:00.000 │ +┌─changeMonth(toDate('1999-01-01'), 2)─┬─changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2)─┐ +│ 1999-02-01 │ 1999-02-01 00:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeDay + +Changes the day component of a date or date time. + +**Syntax** + +``` sql +changeDay(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the day. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeDay(toDate('1999-01-01'), 5), changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5); +``` + +Result: + +``` +┌─changeDay(toDate('1999-01-01'), 5)─┬─changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5)─┐ +│ 1999-01-05 │ 1999-01-05 00:00:00.000 │ +└────────────────────────────────────┴──────────────────────────────────────────────────────────┘ +``` + +## changeHour + +Changes the hour component of a date or date time. + +**Syntax** + +``` sql +changeHour(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeHour(toDate('1999-01-01'), 14), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14); +``` + +Result: + +``` +┌─changeHour(toDate('1999-01-01'), 14)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14)─┐ +│ 1999-01-01 14:00:00 │ 1999-01-01 14:00:00.000 │ └──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ ``` ## changeMinute -Like [changeHour](#changeHour) but changes a minute of the passed date argument. +Changes the minute component of a date or date time. + +**Syntax** + +``` sql +changeMinute(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the minute. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeMinute(toDate('1999-01-01'), 15), changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeMinute(toDate('1999-01-01'), 15)─┬─changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:15:00 │ 1999-01-01 00:15:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` ## changeSecond -Like [changeHour](#changeHour) but changes a seconds of the passed date argument. \ No newline at end of file +Changes the second component of a date or date time. + +**Syntax** + +``` sql +changeSecond(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the second. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeSecond(toDate('1999-01-01'), 15), changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeSecond(toDate('1999-01-01'), 15)─┬─changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:00:15 │ 1999-01-01 00:00:15.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index d17787f4f55..bd73154a6f1 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -45,9 +45,9 @@ enum class Component bool isTimeComponentChange(Component type) { - return type == Component::Hour || - type == Component::Minute || - type == Component::Second; + return type == Component::Hour || type == Component::Minute || type == Component::Second; +} + } template @@ -63,13 +63,11 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() != 2) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires 2 parameters: date, new_value. Passed {}.", getName(), arguments.size()); - - if (!isDateOrDate32OrDateTimeOrDateTime64(*arguments[0].type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be Date, Date32, DateTime or DateTime64", getName()); - if (!isNumber(*arguments[1].type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Second argument for function {} must be numeric", getName()); + FunctionArgumentDescriptors args{ + {"date_or_datetime", static_cast(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date or date with time"}, + {"value", static_cast(&isNativeInteger), nullptr, "Integer"} + }; + validateFunctionArgumentTypes(*this, arguments, args); const auto & input_type = arguments[0].type; @@ -335,62 +333,62 @@ struct ChangeSecondTraits static constexpr auto component = Component::Second; }; -} - REGISTER_FUNCTION(ChangeDate) { - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Changes the year of the given Date(32) or DateTime(64). -Returns the same type as the input data. -)", - .categories{"Dates and Times"} - } - ); - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Same as changeYear function, but changes month of the date. -)", - .categories{"Dates and Times"} - } - ); - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Same as changeYear function, but changes day_of_month of the date. -)", - .categories{"Dates and Times"} - } - ); - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Changes the hour of the given Date(32) or DateTime(64). -If the input data is Date, return DateTime; -if the input data is Date32, return DateTime64; -In other cases returns the same type as the input data. -)", - .categories{"Dates and Times"} - } - ); - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Same as changeHour function, but changes minute of the date. -)", - .categories{"Dates and Times"} - } - ); - factory.registerFunction>( - FunctionDocumentation{ - .description = R"( -Same as changeHour function, but changes seconds of the date. -)", - .categories{"Dates and Times"} - } - ); + { + FunctionDocumentation::Description description = "Changes the year component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeYear(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } + { + FunctionDocumentation::Description description = "Changes the month component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeMonth(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } + { + FunctionDocumentation::Description description = "Changes the day component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeDay(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } + { + FunctionDocumentation::Description description = "Changes the hour component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeHour(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } + { + FunctionDocumentation::Description description = "Changes the minute component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeMinute(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } + { + FunctionDocumentation::Description description = "Changes the second component of a date or date time."; + FunctionDocumentation::Syntax syntax = "changeSecond(date_or_datetime, value);"; + FunctionDocumentation::Arguments arguments = {{"date_or_datetime", "The value to change. Type: Date, Date32, DateTime, or DateTime64"}, {"value", "The new value. Type: [U]Int*"}}; + FunctionDocumentation::ReturnedValue returned_value = "The same type as date_or_datetime. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64."; + FunctionDocumentation::Categories categories = {"Dates and Times"}; + FunctionDocumentation function_documentation = {.description = description, .syntax = syntax, .arguments = arguments, .returned_value = returned_value, .categories = categories}; + factory.registerFunction>(function_documentation); + } } } diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference index 3e8ad45d9a8..8ce647481bb 100644 --- a/tests/queries/0_stateless/02982_changeDate.reference +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -1,37 +1,170 @@ +Negative tests +changeYear +-- Date 2001-01-01 -2002-01-01 -2003-01-01 11:22:33 -2004-01-01 11:22:33.4444 +1970-01-01 +1970-01-01 +2149-06-06 +-- Date32 +2001-01-01 +1900-01-01 +1900-01-01 +2299-12-31 +-- DateTime +2001-01-01 11:22:33 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +2106-02-07 07:28:15 +-- DateTime64 +2001-01-01 11:22:33.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +2299-12-31 23:59:59.9999 +changeMonth +-- Date +2000-01-01 2000-02-01 -2000-03-01 -2000-04-01 11:22:33 -2000-05-01 11:22:33.4444 +2000-12-01 +1970-01-01 +1970-01-01 +1970-01-01 +-- Date32 +2000-01-01 +2000-02-01 +2000-12-01 +1900-01-01 +1900-01-01 +1900-01-01 +-- DateTime +2000-01-01 11:22:33 +2000-02-01 11:22:33 +2000-12-01 11:22:33 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- DateTime64 +2000-01-01 11:22:33.4444 +2000-02-01 11:22:33.4444 +2000-12-01 11:22:33.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +changeDay +-- Date +2000-01-01 2000-01-02 -2000-01-03 -2000-01-04 11:22:33 -2000-01-05 11:22:33.4444 -2000-01-01 12:00:00 -2000-01-01 13:00:00.000 -2000-01-01 14:22:33 -2000-01-01 15:22:33.4444 -2000-01-01 00:23:00 -2000-01-01 00:24:00.000 -2000-01-01 11:25:33 -2000-01-01 11:26:33.4444 -2000-01-01 00:00:34 -2000-01-01 00:00:35.000 -2000-01-01 11:22:36 -2000-01-01 11:22:37.4444 -1970-01-01 -2149-06-06 -2149-06-06 +2000-01-31 1970-01-01 1970-01-01 1970-01-01 +-- Date32 +2000-01-01 +2000-01-02 +2000-01-31 +1900-01-01 +1900-01-01 +1900-01-01 +-- DateTime +2000-01-01 11:22:33 +2000-01-02 11:22:33 +2000-01-31 11:22:33 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- DateTime64 +2000-01-01 11:22:33.4444 +2000-01-02 11:22:33.4444 +2000-01-31 11:22:33.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +-- Special case: change to 29 Feb in a leap year +2000-02-29 +2000-02-29 +2000-02-29 11:22:33 +2000-02-29 11:22:33.4444 +changeHour +-- Date +2000-01-01 00:00:00 +2000-01-01 02:00:00 +2000-01-01 23:00:00 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- Date32 +2000-01-01 00:00:00.000 +2000-01-01 02:00:00.000 +2000-01-01 23:00:00.000 +1900-01-01 00:00:00.000 +1900-01-01 00:00:00.000 +-- DateTime +2000-01-01 00:22:33 +2000-01-01 02:22:33 +2000-01-01 23:22:33 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- DateTime64 +2000-01-01 00:22:33.4444 +2000-01-01 02:22:33.4444 +2000-01-01 23:22:33.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +-- With different timezone 1970-01-01 07:00:00 1970-01-01 07:00:00 +changeMinute +-- Date +2000-01-01 00:00:00 +2000-01-01 00:02:00 +2000-01-01 00:59:00 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- Date32 +2000-01-01 00:00:00.000 +2000-01-01 00:02:00.000 +2000-01-01 00:59:00.000 +1900-01-01 00:00:00.000 +1900-01-01 00:00:00.000 +-- DateTime +2000-01-01 11:00:33 +2000-01-01 11:02:33 +2000-01-01 11:59:33 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- DateTime64 +2000-01-01 11:00:33.4444 +2000-01-01 11:02:33.4444 +2000-01-01 11:59:33.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +-- With different timezone 1970-01-01 07:00:00 1970-01-01 07:00:00 +changeSecond +-- Date +2000-01-01 00:00:00 +2000-01-01 00:00:02 +2000-01-01 00:00:59 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- Date32 +2000-01-01 00:00:00.000 +2000-01-01 00:00:02.000 +2000-01-01 00:00:59.000 +1900-01-01 00:00:00.000 +1900-01-01 00:00:00.000 +-- DateTime +2000-01-01 11:22:00 +2000-01-01 11:22:02 +2000-01-01 11:22:59 +1970-01-01 01:00:00 +1970-01-01 01:00:00 +-- DateTime64 +2000-01-01 11:22:00.4444 +2000-01-01 11:22:02.4444 +2000-01-01 11:22:59.4444 +1900-01-01 00:00:00.0000 +1900-01-01 00:00:00.0000 +-- With different timezone 1970-01-01 07:00:00 1970-01-01 07:00:00 2000-02-01 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index dec95c2d79d..62232079d61 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -1,43 +1,186 @@ +SELECT 'Negative tests'; +-- as changeYear, changeMonth, changeDay, changeMinute, changeSecond share the same implementation, just testing one of them +SELECT changeYear(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT changeYear(toDate('2000-01-01')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT changeYear(toDate('2000-01-01'), 2000, 1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT changeYear(1999, 2000); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT changeYear(toDate('2000-01-01'), 'abc'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT changeYear(toDate('2000-01-01'), 1.5); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +-- Disable timezone randomization +SET session_timezone='CET'; + +SELECT 'changeYear'; +SELECT '-- Date'; SELECT changeYear(toDate('2000-01-01'), 2001); -SELECT changeYear(toDate32('2000-01-01'), 2002); -SELECT changeYear(toDateTime('2000-01-01 11:22:33'), 2003); -SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), 2004); +SELECT changeYear(toDate('2000-01-01'), 1800); -- out-of-bounds +SELECT changeYear(toDate('2000-01-01'), -5000); -- out-of-bounds +SELECT changeYear(toDate('2000-01-01'), 2500); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeYear(toDate32('2000-01-01'), 2001); +SELECT changeYear(toDate32('2000-01-01'), 1800); -- out-of-bounds +SELECT changeYear(toDate32('2000-01-01'), -5000); -- out-of-bounds +SELECT changeYear(toDate32('2000-01-01'), 2500); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeYear(toDateTime('2000-01-01 11:22:33'), 2001); +SELECT changeYear(toDateTime('2000-01-01 11:22:33'), 1800); -- out-of-bounds +SELECT changeYear(toDateTime('2000-01-01 11:22:33'), -5000); -- out-of-bounds +SELECT changeYear(toDateTime('2000-01-01 11:22:33'), 2500); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), 2001); +SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), 1800); -- out-of-bounds +SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), -5000); -- out-of-bounds +SELECT changeYear(toDateTime64('2000-01-01 11:22:33.4444', 4), 2500); -- out-of-bounds -SELECT changeMonth(toDate('2000-01-01'), 02); -SELECT changeMonth(toDate32('2000-01-01'), 03); -SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 04); -SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 05); +SELECT 'changeMonth'; +SELECT '-- Date'; +SELECT changeMonth(toDate('2000-01-01'), 1); +SELECT changeMonth(toDate('2000-01-01'), 2); +SELECT changeMonth(toDate('2000-01-01'), 12); +SELECT changeMonth(toDate('2000-01-01'), 0); -- out-of-bounds +SELECT changeMonth(toDate('2000-01-01'), -1); -- out-of-bounds +SELECT changeMonth(toDate('2000-01-01'), 13); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeMonth(toDate32('2000-01-01'), 1); +SELECT changeMonth(toDate32('2000-01-01'), 2); +SELECT changeMonth(toDate32('2000-01-01'), 12); +SELECT changeMonth(toDate32('2000-01-01'), 0); -- out-of-bounds +SELECT changeMonth(toDate32('2000-01-01'), -1); -- out-of-bounds +SELECT changeMonth(toDate32('2000-01-01'), 13); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 1); +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 2); +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 12); +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 0); -- out-of-bounds +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), -1); -- out-of-bounds +SELECT changeMonth(toDateTime('2000-01-01 11:22:33'), 13); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 1); +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 2); +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 12); +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 0); -- out-of-bounds +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), -1); -- out-of-bounds +SELECT changeMonth(toDateTime64('2000-01-01 11:22:33.4444', 4), 13); -- out-of-bounds -SELECT changeDay(toDate('2000-01-01'), 02); -SELECT changeDay(toDate32('2000-01-01'), 03); -SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 04); -SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 05); +SELECT 'changeDay'; +SELECT '-- Date'; +SELECT changeDay(toDate('2000-01-01'), 1); +SELECT changeDay(toDate('2000-01-01'), 2); +SELECT changeDay(toDate('2000-01-01'), 31); +SELECT changeDay(toDate('2000-01-01'), 0); -- out-of-bounds +SELECT changeDay(toDate('2000-01-01'), -1); -- out-of-bounds +SELECT changeDay(toDate('2000-01-01'), 32); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeDay(toDate32('2000-01-01'), 1); +SELECT changeDay(toDate32('2000-01-01'), 2); +SELECT changeDay(toDate32('2000-01-01'), 31); +SELECT changeDay(toDate32('2000-01-01'), 0); -- out-of-bounds +SELECT changeDay(toDate32('2000-01-01'), -1); -- out-of-bounds +SELECT changeDay(toDate32('2000-01-01'), 32); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 1); +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 2); +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 31); +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 0); -- out-of-bounds +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), -1); -- out-of-bounds +SELECT changeDay(toDateTime('2000-01-01 11:22:33'), 32); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 1); +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 2); +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 31); +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 0); -- out-of-bounds +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), -1); -- out-of-bounds +SELECT changeDay(toDateTime64('2000-01-01 11:22:33.4444', 4), 32); -- out-of-bounds +SELECT '-- Special case: change to 29 Feb in a leap year'; +SELECT changeDay(toDate('2000-02-28'), 29); +SELECT changeDay(toDate32('2000-02-01'), 29); +SELECT changeDay(toDateTime('2000-02-01 11:22:33'), 29); +SELECT changeDay(toDateTime64('2000-02-01 11:22:33.4444', 4), 29); -SELECT changeHour(toDate('2000-01-01'), 12); -SELECT changeHour(toDate32('2000-01-01'), 13); -SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 14); -SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 15); - -SELECT changeMinute(toDate('2000-01-01'), 23); -SELECT changeMinute(toDate32('2000-01-01'), 24); -SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 25); -SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 26); - -SELECT changeSecond(toDate('2000-01-01'), 34); -SELECT changeSecond(toDate32('2000-01-01'), 35); -SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 36); -SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 37); - -SELECT changeYear(toDate('2000-01-01'), 1969.0); -SELECT changeYear(toDate('2000-06-07'), 2149.0); -SELECT changeMonth(toDate('2149-01-01'), 07); -SELECT changeMonth(toDate('2000-01-01'), 13); -SELECT changeDay(toDate('2000-01-01'), 0); -SELECT changeDay(toDate('2000-01-01'), 32); +SELECT 'changeHour'; +SELECT '-- Date'; +SELECT changeHour(toDate('2000-01-01'), 0); +SELECT changeHour(toDate('2000-01-01'), 2); +SELECT changeHour(toDate('2000-01-01'), 23); +SELECT changeHour(toDate('2000-01-01'), -1); -- out-of-bounds +SELECT changeHour(toDate('2000-01-01'), 24); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeHour(toDate32('2000-01-01'), 0); +SELECT changeHour(toDate32('2000-01-01'), 2); +SELECT changeHour(toDate32('2000-01-01'), 23); +SELECT changeHour(toDate32('2000-01-01'), -1); -- out-of-bounds +SELECT changeHour(toDate32('2000-01-01'), 24); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 0); +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 2); +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 23); +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), -1); -- out-of-bounds +SELECT changeHour(toDateTime('2000-01-01 11:22:33'), 24); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 0); +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 2); +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 23); +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), -1); -- out-of-bounds +SELECT changeHour(toDateTime64('2000-01-01 11:22:33.4444', 4), 24); -- out-of-bounds +SELECT '-- With different timezone'; SELECT changeHour(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeHour(toDate('2000-01-01'), 24) SETTINGS session_timezone = 'Asia/Novosibirsk'; + +SELECT 'changeMinute'; +SELECT '-- Date'; +SELECT changeMinute(toDate('2000-01-01'), 0); +SELECT changeMinute(toDate('2000-01-01'), 2); +SELECT changeMinute(toDate('2000-01-01'), 59); +SELECT changeMinute(toDate('2000-01-01'), -1); -- out-of-bounds +SELECT changeMinute(toDate('2000-01-01'), 60); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeMinute(toDate32('2000-01-01'), 0); +SELECT changeMinute(toDate32('2000-01-01'), 2); +SELECT changeMinute(toDate32('2000-01-01'), 59); +SELECT changeMinute(toDate32('2000-01-01'), -1); -- out-of-bounds +SELECT changeMinute(toDate32('2000-01-01'), 60); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 0); +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 2); +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 59); +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), -1); -- out-of-bounds +SELECT changeMinute(toDateTime('2000-01-01 11:22:33'), 60); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 0); +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 2); +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 59); +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), -1); -- out-of-bounds +SELECT changeMinute(toDateTime64('2000-01-01 11:22:33.4444', 4), 60); -- out-of-bounds +SELECT '-- With different timezone'; SELECT changeMinute(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeMinute(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; + +SELECT 'changeSecond'; +SELECT '-- Date'; +SELECT changeSecond(toDate('2000-01-01'), 0); +SELECT changeSecond(toDate('2000-01-01'), 2); +SELECT changeSecond(toDate('2000-01-01'), 59); +SELECT changeSecond(toDate('2000-01-01'), -1); -- out-of-bounds +SELECT changeSecond(toDate('2000-01-01'), 60); -- out-of-bounds +SELECT '-- Date32'; +SELECT changeSecond(toDate32('2000-01-01'), 0); +SELECT changeSecond(toDate32('2000-01-01'), 2); +SELECT changeSecond(toDate32('2000-01-01'), 59); +SELECT changeSecond(toDate32('2000-01-01'), -1); -- out-of-bounds +SELECT changeSecond(toDate32('2000-01-01'), 60); -- out-of-bounds +SELECT '-- DateTime'; +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 0); +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 2); +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 59); +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), -1); -- out-of-bounds +SELECT changeSecond(toDateTime('2000-01-01 11:22:33'), 60); -- out-of-bounds +SELECT '-- DateTime64'; +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 0); +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 2); +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 59); +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), -1); -- out-of-bounds +SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 60); -- out-of-bounds +SELECT '-- With different timezone'; SELECT changeSecond(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; From 6ff3cf82819e61020f3483687934a70ade67be67 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 20 May 2024 15:36:25 +0000 Subject: [PATCH 019/161] Some fixups, pt. V --- .../functions/date-time-functions.md | 419 +++++++++--------- 1 file changed, 210 insertions(+), 209 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 8bf301d76c2..7a39cc74372 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -3520,6 +3520,216 @@ Result: └───────────────────────────────────────────────────────────────────────┘ ``` +## changeYear + +Changes the year component of a date or date time. + +**Syntax** + +``` sql +changeYear(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeYear(toDate('1999-01-01'), 2000), changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); +``` + +Result: + +``` +┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ +│ 2000-01-01 │ 2000-01-01 00:00:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + +## changeMonth + +Changes the month component of a date or date time. + +**Syntax** + +``` sql +changeMonth(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the month. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeMonth(toDate('1999-01-01'), 2), changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2); +``` + +Result: + +``` +┌─changeMonth(toDate('1999-01-01'), 2)─┬─changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2)─┐ +│ 1999-02-01 │ 1999-02-01 00:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeDay + +Changes the day component of a date or date time. + +**Syntax** + +``` sql +changeDay(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the day. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeDay(toDate('1999-01-01'), 5), changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5); +``` + +Result: + +``` +┌─changeDay(toDate('1999-01-01'), 5)─┬─changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5)─┐ +│ 1999-01-05 │ 1999-01-05 00:00:00.000 │ +└────────────────────────────────────┴──────────────────────────────────────────────────────────┘ +``` + +## changeHour + +Changes the hour component of a date or date time. + +**Syntax** + +``` sql +changeHour(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeHour(toDate('1999-01-01'), 14), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14); +``` + +Result: + +``` +┌─changeHour(toDate('1999-01-01'), 14)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14)─┐ +│ 1999-01-01 14:00:00 │ 1999-01-01 14:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeMinute + +Changes the minute component of a date or date time. + +**Syntax** + +``` sql +changeMinute(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the minute. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeMinute(toDate('1999-01-01'), 15), changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeMinute(toDate('1999-01-01'), 15)─┬─changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:15:00 │ 1999-01-01 00:15:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + +## changeSecond + +Changes the second component of a date or date time. + +**Syntax** + +``` sql +changeSecond(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the second. [Integer](../../sql-reference/data-types/int-uint.md). + +**Return value** + +- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. + +Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeSecond(toDate('1999-01-01'), 15), changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeSecond(toDate('1999-01-01'), 15)─┬─changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:00:15 │ 1999-01-01 00:00:15.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + ## timeSlots(StartTime, Duration,\[, Size\]) For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. @@ -4102,212 +4312,3 @@ timeDiff(toDateTime64('1927-01-01 00:00:00', 3), toDate32('1927-01-02')); - Blog: [Working with time series data in ClickHouse](https://clickhouse.com/blog/working-with-time-series-data-and-functions-ClickHouse) -## changeYear - -Changes the year component of a date or date time. - -**Syntax** - -``` sql -changeYear(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeYear(toDate('1999-01-01'), 2000), changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); -``` - -Result: - -``` -┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ -│ 2000-01-01 │ 2000-01-01 00:00:00.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` - -## changeMonth - -Changes the month component of a date or date time. - -**Syntax** - -``` sql -changeMonth(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the month. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeMonth(toDate('1999-01-01'), 2), changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2); -``` - -Result: - -``` -┌─changeMonth(toDate('1999-01-01'), 2)─┬─changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2)─┐ -│ 1999-02-01 │ 1999-02-01 00:00:00.000 │ -└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ -``` - -## changeDay - -Changes the day component of a date or date time. - -**Syntax** - -``` sql -changeDay(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the day. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeDay(toDate('1999-01-01'), 5), changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5); -``` - -Result: - -``` -┌─changeDay(toDate('1999-01-01'), 5)─┬─changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5)─┐ -│ 1999-01-05 │ 1999-01-05 00:00:00.000 │ -└────────────────────────────────────┴──────────────────────────────────────────────────────────┘ -``` - -## changeHour - -Changes the hour component of a date or date time. - -**Syntax** - -``` sql -changeHour(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeHour(toDate('1999-01-01'), 14), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14); -``` - -Result: - -``` -┌─changeHour(toDate('1999-01-01'), 14)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14)─┐ -│ 1999-01-01 14:00:00 │ 1999-01-01 14:00:00.000 │ -└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ -``` - -## changeMinute - -Changes the minute component of a date or date time. - -**Syntax** - -``` sql -changeMinute(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the minute. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeMinute(toDate('1999-01-01'), 15), changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15); -``` - -Result: - -``` -┌─changeMinute(toDate('1999-01-01'), 15)─┬─changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ -│ 1999-01-01 00:15:00 │ 1999-01-01 00:15:00.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` - -## changeSecond - -Changes the second component of a date or date time. - -**Syntax** - -``` sql -changeSecond(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the second. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeSecond(toDate('1999-01-01'), 15), changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15); -``` - -Result: - -``` -┌─changeSecond(toDate('1999-01-01'), 15)─┬─changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ -│ 1999-01-01 00:00:15 │ 1999-01-01 00:00:15.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` From b47524a9a5d5c379227da1ac290938f42d2b0586 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 21 May 2024 05:19:49 +0000 Subject: [PATCH 020/161] Fix spelling --- .../aspell-ignore/en/aspell-dict.txt | 116 +++++++++--------- 1 file changed, 61 insertions(+), 55 deletions(-) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 2a9aa259fdd..ade822f508a 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -29,13 +29,6 @@ Alexey AnyEvent AppleClang Approximative -arrayDotProduct -arrayEnumerateDenseRanked -arrayEnumerateUniqRanked -arrayFirstOrNull -arrayLastOrNull -arrayPartialShuffle -arrayShuffle ArrayJoin ArrowStream AsyncInsertCacheSize @@ -184,7 +177,6 @@ ComplexKeyCache ComplexKeyDirect ComplexKeyHashed Composable -composable Config ConnectionDetails Const @@ -394,8 +386,6 @@ InterserverThreads IsPentagon IsResClassIII IsValid -isNotDistinctFrom -isNullable JBOD JOINed JOINs @@ -464,8 +454,6 @@ KittenHouse Klickhouse Kolmogorov Konstantin -kostik -kostikConsistentHash Korzeniewski Kubernetes LDAP @@ -475,9 +463,8 @@ LLDB LLVM's LOCALTIME LOCALTIMESTAMP -LOONGARCH LONGLONG -LoongArch +LOONGARCH Levenshtein Liao LibFuzzer @@ -495,6 +482,7 @@ LocalThreadActive LogQL Logstash LookML +LoongArch LowCardinality LpDistance LpNorm @@ -569,17 +557,6 @@ MindsDB Mongodb Monotonicity MsgPack -multiSearchAllPositionsCaseInsensitive -multiSearchAllPositionsCaseInsensitiveUTF -multiSearchAnyCaseInsensitive -multiSearchAnyCaseInsensitiveUTF -multiSearchAnyUTF -multiSearchFirstIndexCaseInsensitive -multiSearchFirstIndexCaseInsensitiveUTF -multiSearchFirstIndexUTF -multiSearchFirstPositionCaseInsensitive -multiSearchFirstPositionCaseInsensitiveUTF -multiSearchFirstPositionUTF MultiPolygon Multiline Multiqueries @@ -681,8 +658,8 @@ OSUserTimeNormalized OTLP OUTFILE ObjectId -Observability Oblakov +Observability Octonica Ok OnTime @@ -883,7 +860,6 @@ Simhash SimpleAggregateFunction SimpleState SipHash -sigmoid Smirnov's Smirnov'test Soundex @@ -929,7 +905,6 @@ TAVG TCPConnection TCPThreads TDigest -ThreadMonotonic TINYINT TLSv TMAX @@ -955,7 +930,6 @@ TablesLoaderForegroundThreads TablesLoaderForegroundThreadsActive TablesToDropQueueSize TargetSpecific -tanh Telegraf TemplateIgnoreSpaces TemporaryFilesForAggregation @@ -965,6 +939,7 @@ TemporaryFilesUnknown Testflows Tgz Theil's +ThreadMonotonic ThreadPoolFSReaderThreads ThreadPoolFSReaderThreadsActive ThreadPoolRemoteFSReaderThreads @@ -1025,7 +1000,6 @@ UncompressedCacheBytes UncompressedCacheCells UnidirectionalEdgeIsValid UniqThetaSketch -unshuffled Updatable Uppercased Uptime @@ -1092,6 +1066,7 @@ activerecord addDate addDays addHours +addInterval addMicroseconds addMilliseconds addMinutes @@ -1099,10 +1074,9 @@ addMonths addNanoseconds addQuarters addSeconds +addTupleOfIntervals addWeeks addYears -addInterval -addTupleOfIntervals addr addressToLine addressToLineWithInlines @@ -1144,15 +1118,19 @@ arrayCumSum arrayCumSumNonNegative arrayDifference arrayDistinct +arrayDotProduct arrayElement arrayEnumerate arrayEnumerateDense +arrayEnumerateDenseRanked arrayEnumerateUniq +arrayEnumerateUniqRanked arrayExists arrayFill arrayFilter arrayFirst arrayFirstIndex +arrayFirstOrNull arrayFlatten arrayFold arrayIntersect @@ -1160,10 +1138,12 @@ arrayJaccardIndex arrayJoin arrayLast arrayLastIndex +arrayLastOrNull arrayMap arrayMax arrayMin arrayPartialReverseSort +arrayPartialShuffle arrayPartialSort arrayPopBack arrayPopFront @@ -1183,6 +1163,7 @@ arrayRotateRight arrayShiftLeft arrayShiftRight arrayShingles +arrayShuffle arraySlice arraySort arraySplit @@ -1323,6 +1304,12 @@ cfg cgroup cgroups chadmin +changeDay +changeHour +changeMinute +changeMonth +changeSecond +changeYear changelog changelogs charset @@ -1364,6 +1351,7 @@ collapsingmergetree combinator combinators comparising +composable compressability concat concatAssumeInjective @@ -1725,8 +1713,8 @@ hasSubsequenceCaseInsensitive hasSubsequenceCaseInsensitiveUTF hasSubsequenceUTF hasSubstr -hasToken hasThreadFuzzer +hasToken hasTokenCaseInsensitive hasTokenCaseInsensitiveOrNull hasTokenOrNull @@ -1799,8 +1787,10 @@ isIPAddressInRange isIPv isInfinite isNaN +isNotDistinctFrom isNotNull isNull +isNullable isValidJSON isValidUTF isZeroOrNull @@ -1852,6 +1842,8 @@ kolmogorovSmirnovTest kolmogorovsmirnovtest kolya konsole +kostik +kostikConsistentHash kurtPop kurtSamp kurtosis @@ -1863,9 +1855,9 @@ laravel largestTriangleThreeBuckets latencies ldap -leftUTF leftPad leftPadUTF +leftUTF lemmatization lemmatize lemmatized @@ -1912,8 +1904,8 @@ logTrace logagent loghouse london -loongarch lookups +loongarch lowcardinality lowerUTF lowercased @@ -1984,8 +1976,8 @@ mispredictions mmap mmapped modularization -moduloOrZero moduli +moduloOrZero mongodb monotonicity monthName @@ -2002,10 +1994,21 @@ multiMatchAllIndices multiMatchAny multiMatchAnyIndex multiSearchAllPositions +multiSearchAllPositionsCaseInsensitive +multiSearchAllPositionsCaseInsensitiveUTF multiSearchAllPositionsUTF multiSearchAny +multiSearchAnyCaseInsensitive +multiSearchAnyCaseInsensitiveUTF +multiSearchAnyUTF multiSearchFirstIndex +multiSearchFirstIndexCaseInsensitive +multiSearchFirstIndexCaseInsensitiveUTF +multiSearchFirstIndexUTF multiSearchFirstPosition +multiSearchFirstPositionCaseInsensitive +multiSearchFirstPositionCaseInsensitiveUTF +multiSearchFirstPositionUTF multibyte multidirectory multiline @@ -2340,8 +2343,8 @@ retentions rethrow retransmit retriable -rewritable reverseUTF +rewritable rightPad rightPadUTF rightUTF @@ -2401,8 +2404,9 @@ sharded sharding shortcircuit shortkeys -showCertificate shoutout +showCertificate +sigmoid simdjson simpleJSON simpleJSONExtractBool @@ -2416,8 +2420,8 @@ simpleLinearRegression simpleaggregatefunction simplelinearregression simpod -singlepart singleValueOrNull +singlepart singlevalueornull sinh sipHash @@ -2462,13 +2466,13 @@ statbox stateful stddev stddevPop -stddevSamp -stddevpop -stddevsamp -stddevpopstable stddevPopStable -stddevsampstable +stddevSamp stddevSampStable +stddevpop +stddevpopstable +stddevsamp +stddevsampstable stderr stdin stdout @@ -2529,6 +2533,7 @@ substrings subtitiles subtractDays subtractHours +subtractInterval subtractMicroseconds subtractMilliseconds subtractMinutes @@ -2536,10 +2541,9 @@ subtractMonths subtractNanoseconds subtractQuarters subtractSeconds +subtractTupleOfIntervals subtractWeeks subtractYears -subtractInterval -subtractTupleOfIntervals subtree subtrees subtype @@ -2548,13 +2552,13 @@ sumCount sumKahan sumMap sumMapFiltered +sumMapFilteredWithOverflow +sumMapWithOverflow sumWithOverflow sumcount sumkahan summap summapwithoverflow -sumMapWithOverflow -sumMapFilteredWithOverflow summingmergetree sumwithoverflow superaggregates @@ -2577,6 +2581,7 @@ tabseparatedrawwithnames tabseparatedrawwithnamesandtypes tabseparatedwithnames tabseparatedwithnamesandtypes +tanh tcp tcpPort tcpnodelay @@ -2711,18 +2716,18 @@ tupleDivide tupleDivideByNumber tupleElement tupleHammingDistance +tupleIntDiv +tupleIntDivByNumber +tupleIntDivOrZero +tupleIntDivOrZeroByNumber tupleMinus +tupleModulo +tupleModuloByNumber tupleMultiply tupleMultiplyByNumber tupleNegate tuplePlus tupleToNameValuePairs -tupleIntDiv -tupleIntDivByNumber -tupleIntDivOrZero -tupleIntDivOrZeroByNumber -tupleModulo -tupleModuloByNumber turbostat txt typename @@ -2765,6 +2770,7 @@ unrealiable unreplicated unresolvable unrounded +unshuffled untracked untrusted untuple @@ -2775,8 +2781,8 @@ uptime uptrace uring url -urlencoded urlCluster +urlencoded urls usearch userspace From c026a5b7e938b797d3f8cb016457dab38c002aab Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 21 May 2024 05:20:46 +0000 Subject: [PATCH 021/161] Fix style check --- src/Functions/changeDate.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index bd73154a6f1..9868ac5b914 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -23,13 +23,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int LOGICAL_ERROR; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - namespace { From 336e791ea87c8298ff584ae2de8a140f0947f02a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 21 May 2024 20:50:54 +0000 Subject: [PATCH 022/161] Fix style check, pt. II --- src/Functions/changeDate.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 9868ac5b914..b400680d272 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -23,6 +23,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace { From ab2baf5e9664133097025c149a93a09e84cca152 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 22 May 2024 12:23:14 +0000 Subject: [PATCH 023/161] Fix expected results --- .../0_stateless/02982_changeDate.reference | 17 -------------- .../queries/0_stateless/02982_changeDate.sql | 23 ------------------- 2 files changed, 40 deletions(-) diff --git a/tests/queries/0_stateless/02982_changeDate.reference b/tests/queries/0_stateless/02982_changeDate.reference index 8ce647481bb..4a7f093ca2b 100644 --- a/tests/queries/0_stateless/02982_changeDate.reference +++ b/tests/queries/0_stateless/02982_changeDate.reference @@ -167,20 +167,3 @@ changeSecond -- With different timezone 1970-01-01 07:00:00 1970-01-01 07:00:00 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2000-02-01 -2299-12-31 -1900-01-01 -2106-02-07 13:28:15 -1970-01-01 07:00:00 -2299-12-31 23:59:59.999 -1900-01-01 00:00:00.000 diff --git a/tests/queries/0_stateless/02982_changeDate.sql b/tests/queries/0_stateless/02982_changeDate.sql index 62232079d61..2bc9aa95569 100644 --- a/tests/queries/0_stateless/02982_changeDate.sql +++ b/tests/queries/0_stateless/02982_changeDate.sql @@ -183,26 +183,3 @@ SELECT changeSecond(toDateTime64('2000-01-01 11:22:33.4444', 4), 60); -- out-of- SELECT '-- With different timezone'; SELECT changeSecond(toDate('2000-01-01'), -1) SETTINGS session_timezone = 'Asia/Novosibirsk'; SELECT changeSecond(toDate('2000-01-01'), 60) SETTINGS session_timezone = 'Asia/Novosibirsk'; - -SELECT changeYear(toDate('2000-01-01')); -- { serverError 42 } -SELECT changeYear(toDate('2000-01-01'), 2001, 2002); -- { serverError 42 } -SELECT changeYear(toDate('2000-01-01'), '2001'); -- { serverError 43 } - -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int8)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int16)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int32)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Int64)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt8)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt16)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt32)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS UInt64)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Float32)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Float64)); -SELECT changeMonth(toDate('2000-01-01'), CAST(2 AS Decimal(10, 5))); - -SELECT changeYear(toDate32('2000-01-01'), 2300); -SELECT changeYear(toDate32('2000-01-01'), 1899); -SELECT changeSecond(toDateTime('2106-02-07 13:28:15'), 16) SETTINGS session_timezone = 'Asia/Novosibirsk'; -SELECT changeHour(toDateTime('1970-01-01 23:59:59'), 6) SETTINGS session_timezone = 'Asia/Novosibirsk'; -SELECT changeYear(toDateTime64('2000-01-01 00:00:00.000', 3), 2300) SETTINGS session_timezone = 'Asia/Novosibirsk'; -SELECT changeYear(toDateTime64('2000-01-01 00:00:00.000', 3), 1899) SETTINGS session_timezone = 'Asia/Novosibirsk'; \ No newline at end of file From cdd99a73a0e46801bb01f47df49c47e67fd9bb6f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 23 May 2024 10:12:32 +0000 Subject: [PATCH 024/161] Fix clang-tidy --- src/Functions/changeDate.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index b400680d272..e24391afe12 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -274,15 +274,25 @@ public: } Int64 result; - if (isDateOrDate32(result_type)) + if (isDate(result_type) || isDate32(result_type)) result = date_lut.makeDayNum(year, month, day); else if (isDateTime(result_type)) result = date_lut.makeDateTime(year, month, day, hours, minutes, seconds); else +#ifndef __clang_analyzer__ + /// ^^ This looks funny. It is the least terrible suppression of a false positive reported by clang-analyzer (a sub-class + /// of clang-tidy checks) deep down in 'decimalFromComponents'. Usual suppressions of the form NOLINT* don't work here (they + /// would only affect code in _this_ file), and suppressing the issue in 'decimalFromComponents' may suppress true positives. result = DecimalUtils::decimalFromComponents( date_lut.makeDateTime(year, month, day, hours, minutes, seconds), - static_cast(fraction), + fraction, static_cast(scale)); +#else + { + UNUSED(fraction); + result = 0; + } +#endif if (result < min_date) return min_date; From cae2f023852f3f49c4a0f25e8e11bcb3fe6ed49a Mon Sep 17 00:00:00 2001 From: wudidapaopao <664920313@qq.com> Date: Fri, 14 Jun 2024 14:39:00 +0800 Subject: [PATCH 025/161] Fix unexpected projection name when query with CTE --- src/Analyzer/Resolve/QueryAnalyzer.cpp | 6 +++++- .../02378_analyzer_projection_names.reference | 12 ++++++++++++ .../0_stateless/02378_analyzer_projection_names.sql | 10 ++++++++++ .../0_stateless/03123_analyzer_dist_join_CTE.sql | 2 +- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 5e5ecaaa93a..dc35462a4b0 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -1484,7 +1484,11 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & /// Build additional column qualification parts array std::vector additional_column_qualification_parts; - if (table_expression_node->hasAlias()) + auto * query_node = table_expression_node->as(); + auto * union_node = table_expression_node->as(); + bool is_cte = (query_node && query_node->isCTE()) || (union_node && union_node->isCTE()); + + if (!is_cte && table_expression_node->hasAlias()) additional_column_qualification_parts = {table_expression_node->getAlias()}; else if (auto * table_node = table_expression_node->as()) additional_column_qualification_parts = {table_node->getStorageID().getDatabaseName(), table_node->getStorageID().getTableName()}; diff --git a/tests/queries/0_stateless/02378_analyzer_projection_names.reference b/tests/queries/0_stateless/02378_analyzer_projection_names.reference index fd5bc7d4ae8..5860f1deb0d 100644 --- a/tests/queries/0_stateless/02378_analyzer_projection_names.reference +++ b/tests/queries/0_stateless/02378_analyzer_projection_names.reference @@ -443,6 +443,18 @@ SELECT '--'; -- DESCRIBE (WITH test_table_in_cte AS (SELECT id FROM test_table) SELECT id IN test_table_in_cte FROM test_table); in(id, test_table_in_cte) UInt8 +SELECT '--'; +-- +DESCRIBE (WITH test_table_in_cte_1 AS (SELECT 1 AS c1), test_table_in_cte_2 AS (SELECT 1 AS c1) SELECT * +FROM test_table_in_cte_1 INNER JOIN test_table_in_cte_2 as test_table_in_cte_2 ON test_table_in_cte_1.c1 = test_table_in_cte_2.c1); +c1 UInt8 +c1 UInt8 +SELECT '--'; +-- +DESCRIBE (WITH test_table_in_cte_1 AS (SELECT 1 AS c1), test_table_in_cte_2 AS (SELECT 1 AS c1 UNION ALL SELECT 1 AS c1) SELECT * +FROM test_table_in_cte_1 INNER JOIN test_table_in_cte_2 as test_table_in_cte_2 ON test_table_in_cte_1.c1 = test_table_in_cte_2.c1); +c1 UInt8 +c1 UInt8 SELECT 'Joins'; Joins DESCRIBE (SELECT * FROM test_table_join_1, test_table_join_2); diff --git a/tests/queries/0_stateless/02378_analyzer_projection_names.sql b/tests/queries/0_stateless/02378_analyzer_projection_names.sql index f5ac5f7476f..f41afe6a950 100644 --- a/tests/queries/0_stateless/02378_analyzer_projection_names.sql +++ b/tests/queries/0_stateless/02378_analyzer_projection_names.sql @@ -408,6 +408,16 @@ SELECT '--'; DESCRIBE (WITH test_table_in_cte AS (SELECT id FROM test_table) SELECT id IN test_table_in_cte FROM test_table); +SELECT '--'; + +DESCRIBE (WITH test_table_in_cte_1 AS (SELECT 1 AS c1), test_table_in_cte_2 AS (SELECT 1 AS c1) SELECT * +FROM test_table_in_cte_1 INNER JOIN test_table_in_cte_2 as test_table_in_cte_2 ON test_table_in_cte_1.c1 = test_table_in_cte_2.c1); + +SELECT '--'; + +DESCRIBE (WITH test_table_in_cte_1 AS (SELECT 1 AS c1), test_table_in_cte_2 AS (SELECT 1 AS c1 UNION ALL SELECT 1 AS c1) SELECT * +FROM test_table_in_cte_1 INNER JOIN test_table_in_cte_2 as test_table_in_cte_2 ON test_table_in_cte_1.c1 = test_table_in_cte_2.c1); + SELECT 'Joins'; DESCRIBE (SELECT * FROM test_table_join_1, test_table_join_2); diff --git a/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql b/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql index 4fb8e0b91c4..4275f405d13 100644 --- a/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql +++ b/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql @@ -20,7 +20,7 @@ WITH SELECT toInt64(number) AS a FROM numbers(10) ) -SELECT * +SELECT a.a, b, b.a, c.a FROM dist_t0 AS a LEFT JOIN b AS b ON a.a = b.a LEFT JOIN c AS c ON a.a = c.a From 409b24b8d8d185e7840bc082fd73afc1b13314d2 Mon Sep 17 00:00:00 2001 From: wudidapaopao <664920313@qq.com> Date: Tue, 18 Jun 2024 16:22:00 +0800 Subject: [PATCH 026/161] empty commit From f4c1266975ff1a1aabe71e61ae318d1765931a62 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 30 Jun 2024 18:56:05 +0200 Subject: [PATCH 027/161] Remove incorrect test --- ...87_distributed_in_not_default_db.reference | 2 - .../01487_distributed_in_not_default_db.sql | 39 ------------------- 2 files changed, 41 deletions(-) delete mode 100644 tests/queries/0_stateless/01487_distributed_in_not_default_db.reference delete mode 100644 tests/queries/0_stateless/01487_distributed_in_not_default_db.sql diff --git a/tests/queries/0_stateless/01487_distributed_in_not_default_db.reference b/tests/queries/0_stateless/01487_distributed_in_not_default_db.reference deleted file mode 100644 index 0d66ea1aee9..00000000000 --- a/tests/queries/0_stateless/01487_distributed_in_not_default_db.reference +++ /dev/null @@ -1,2 +0,0 @@ -0 -1 diff --git a/tests/queries/0_stateless/01487_distributed_in_not_default_db.sql b/tests/queries/0_stateless/01487_distributed_in_not_default_db.sql deleted file mode 100644 index ccd2c571290..00000000000 --- a/tests/queries/0_stateless/01487_distributed_in_not_default_db.sql +++ /dev/null @@ -1,39 +0,0 @@ --- Tags: distributed, no-parallel - -CREATE DATABASE IF NOT EXISTS shard_0; -CREATE DATABASE IF NOT EXISTS shard_1; -CREATE DATABASE IF NOT EXISTS main_01487; -CREATE DATABASE IF NOT EXISTS test_01487; - -USE main_01487; - -DROP TABLE IF EXISTS shard_0.l; -DROP TABLE IF EXISTS shard_1.l; -DROP TABLE IF EXISTS d; -DROP TABLE IF EXISTS t; - -CREATE TABLE shard_0.l (value UInt8) ENGINE = MergeTree ORDER BY value; -CREATE TABLE shard_1.l (value UInt8) ENGINE = MergeTree ORDER BY value; -CREATE TABLE t (value UInt8) ENGINE = Memory; - -INSERT INTO shard_0.l VALUES (0); -INSERT INTO shard_1.l VALUES (1); -INSERT INTO t VALUES (0), (1), (2); - -CREATE TABLE d AS t ENGINE = Distributed(test_cluster_two_shards_different_databases, currentDatabase(), t); - -USE test_01487; -DROP DATABASE test_01487; - -SELECT * FROM main_01487.d WHERE value IN (SELECT l.value FROM l) ORDER BY value; - -USE main_01487; - -DROP TABLE IF EXISTS shard_0.l; -DROP TABLE IF EXISTS shard_1.l; -DROP TABLE IF EXISTS d; -DROP TABLE IF EXISTS t; - -DROP DATABASE shard_0; -DROP DATABASE shard_1; -DROP DATABASE main_01487; From 4f61f530bd02fec686b273f4c2519cde77a689f2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 2 Jul 2024 11:22:05 +0200 Subject: [PATCH 028/161] Named collections in clickhouse-local --- programs/local/LocalServer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index b33e1595056..74906d8797c 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -376,6 +376,7 @@ void LocalServer::setupUsers() " " " default" " default" + " 1 " " " " " " From e45a905904b5afd94e960bf48de015017a351527 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 2 Jul 2024 11:52:51 +0200 Subject: [PATCH 029/161] Update LocalServer.cpp --- programs/local/LocalServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 74906d8797c..46b543e49e9 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -376,7 +376,7 @@ void LocalServer::setupUsers() " " " default" " default" - " 1 + " 1" " " " " " " From 2f083fd1a709dd236ffaf37a7a059cafef8c4a41 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 2 Jul 2024 15:08:14 +0000 Subject: [PATCH 030/161] Fix resolving dynamic subcolumns in analyzer --- src/Analyzer/Resolve/IdentifierResolver.cpp | 35 ++++++++++++++++--- .../03198_dynamic_read_subcolumns.reference | 8 +++++ .../03198_dynamic_read_subcolumns.sql | 18 ++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/03198_dynamic_read_subcolumns.reference create mode 100644 tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index 692a31b66ba..4347f07e829 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -678,9 +679,33 @@ QueryTreeNodePtr IdentifierResolver::tryResolveIdentifierFromStorage( bool match_full_identifier = false; const auto & identifier_full_name = identifier_without_column_qualifier.getFullName(); - auto it = table_expression_data.column_name_to_column_node.find(identifier_full_name); - bool can_resolve_directly_from_storage = it != table_expression_data.column_name_to_column_node.end(); - if (can_resolve_directly_from_storage && table_expression_data.subcolumn_names.contains(identifier_full_name)) + + ColumnNodePtr result_column_node; + bool can_resolve_directly_from_storage = false; + bool is_subcolumn = false; + if (auto it = table_expression_data.column_name_to_column_node.find(identifier_full_name); it != table_expression_data.column_name_to_column_node.end()) + { + can_resolve_directly_from_storage = true; + is_subcolumn = table_expression_data.subcolumn_names.contains(identifier_full_name); + result_column_node = it->second; + } + /// Check if it's a dynamic subcolumn + else + { + auto [column_name, dynamic_subcolumn_name] = Nested::splitName(identifier_full_name); + auto jt = table_expression_data.column_name_to_column_node.find(column_name); + if (jt != table_expression_data.column_name_to_column_node.end() && jt->second->getColumnType()->hasDynamicSubcolumns()) + { + if (auto dynamic_subcolumn_type = jt->second->getColumnType()->tryGetSubcolumnType(dynamic_subcolumn_name)) + { + result_column_node = std::make_shared(NameAndTypePair{identifier_full_name, dynamic_subcolumn_type}, jt->second->getColumnSource()); + can_resolve_directly_from_storage = true; + is_subcolumn = true; + } + } + } + + if (can_resolve_directly_from_storage && is_subcolumn) { /** In the case when we have an ARRAY JOIN, we should not resolve subcolumns directly from storage. * For example, consider the following SQL query: @@ -696,11 +721,11 @@ QueryTreeNodePtr IdentifierResolver::tryResolveIdentifierFromStorage( if (can_resolve_directly_from_storage) { match_full_identifier = true; - result_expression = it->second; + result_expression = result_column_node; } else { - it = table_expression_data.column_name_to_column_node.find(identifier_without_column_qualifier.at(0)); + auto it = table_expression_data.column_name_to_column_node.find(identifier_without_column_qualifier.at(0)); if (it != table_expression_data.column_name_to_column_node.end()) result_expression = it->second; } diff --git a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.reference b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.reference new file mode 100644 index 00000000000..59d65220c83 --- /dev/null +++ b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.reference @@ -0,0 +1,8 @@ +foo +\N +\N +foo +\N +\N +6 +6 diff --git a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql new file mode 100644 index 00000000000..7f88b4cd347 --- /dev/null +++ b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql @@ -0,0 +1,18 @@ +SET allow_experimental_dynamic_type = 1; +DROP TABLE IF EXISTS test_dynamic; +CREATE TABLE test_dynamic (id UInt64, d Dynamic) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO test_dynamic VALUES (1, 'foo'), (2, 1111), (3, [1, 2, 3]); +SYSTEM DROP MARK CACHE; +SELECT d.String FROM test_dynamic SETTINGS allow_experimental_analyzer = 1; +SYSTEM DROP MARK CACHE; +SELECT d.String FROM test_dynamic SETTINGS allow_experimental_analyzer = 0; +SYSTEM FLUSH LOGS; +SELECT + ProfileEvents['FileOpen'] +FROM system.query_log +WHERE (type = 2) AND (query LIKE 'SELECT d.String %test_dynamic%') AND (current_database = currentDatabase()) +ORDER BY event_time_microseconds DESC +LIMIT 2; + +DROP TABLE test_dynamic; + From 84fb836ecba44b0674f8e4bdbb1626e1aae02ba6 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Tue, 2 Jul 2024 23:27:02 +0000 Subject: [PATCH 031/161] proper qualification for CTE, fix tests --- src/Analyzer/Resolve/QueryAnalyzer.cpp | 10 +++++----- .../02378_analyzer_projection_names.reference | 4 ++-- .../0_stateless/03123_analyzer_dist_join_CTE.sql | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index dc35462a4b0..58a85debf37 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -1484,14 +1484,14 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & /// Build additional column qualification parts array std::vector additional_column_qualification_parts; - auto * query_node = table_expression_node->as(); - auto * union_node = table_expression_node->as(); - bool is_cte = (query_node && query_node->isCTE()) || (union_node && union_node->isCTE()); - - if (!is_cte && table_expression_node->hasAlias()) + if (table_expression_node->hasAlias()) additional_column_qualification_parts = {table_expression_node->getAlias()}; else if (auto * table_node = table_expression_node->as()) additional_column_qualification_parts = {table_node->getStorageID().getDatabaseName(), table_node->getStorageID().getTableName()}; + else if (auto * query_node = table_expression_node->as(); query_node && query_node->isCTE()) + additional_column_qualification_parts = {"", query_node->getCTEName()}; + else if (auto * union_node = table_expression_node->as(); union_node && union_node->isCTE()) + additional_column_qualification_parts = {"", union_node->getCTEName()}; size_t additional_column_qualification_parts_size = additional_column_qualification_parts.size(); const auto & table_expression_data = scope.getTableExpressionDataOrThrow(table_expression_node); diff --git a/tests/queries/0_stateless/02378_analyzer_projection_names.reference b/tests/queries/0_stateless/02378_analyzer_projection_names.reference index 5860f1deb0d..170f3e9e1ad 100644 --- a/tests/queries/0_stateless/02378_analyzer_projection_names.reference +++ b/tests/queries/0_stateless/02378_analyzer_projection_names.reference @@ -448,13 +448,13 @@ SELECT '--'; DESCRIBE (WITH test_table_in_cte_1 AS (SELECT 1 AS c1), test_table_in_cte_2 AS (SELECT 1 AS c1) SELECT * FROM test_table_in_cte_1 INNER JOIN test_table_in_cte_2 as test_table_in_cte_2 ON test_table_in_cte_1.c1 = test_table_in_cte_2.c1); c1 UInt8 -c1 UInt8 +test_table_in_cte_2.c1 UInt8 SELECT '--'; -- DESCRIBE (WITH test_table_in_cte_1 AS (SELECT 1 AS c1), test_table_in_cte_2 AS (SELECT 1 AS c1 UNION ALL SELECT 1 AS c1) SELECT * FROM test_table_in_cte_1 INNER JOIN test_table_in_cte_2 as test_table_in_cte_2 ON test_table_in_cte_1.c1 = test_table_in_cte_2.c1); c1 UInt8 -c1 UInt8 +test_table_in_cte_2.c1 UInt8 SELECT 'Joins'; Joins DESCRIBE (SELECT * FROM test_table_join_1, test_table_join_2); diff --git a/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql b/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql index 4275f405d13..4fb8e0b91c4 100644 --- a/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql +++ b/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql @@ -20,7 +20,7 @@ WITH SELECT toInt64(number) AS a FROM numbers(10) ) -SELECT a.a, b, b.a, c.a +SELECT * FROM dist_t0 AS a LEFT JOIN b AS b ON a.a = b.a LEFT JOIN c AS c ON a.a = c.a From ac315785d3ab89756b160ef1610bf5e4125c332b Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 2 Jul 2024 23:41:57 +0000 Subject: [PATCH 032/161] Update tests --- .../03040_dynamic_type_alters_1.reference | 108 +++++++++--------- .../03040_dynamic_type_alters_1.sh | 2 +- 2 files changed, 55 insertions(+), 55 deletions(-) diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1.reference b/tests/queries/0_stateless/03040_dynamic_type_alters_1.reference index ca98ec0963c..b41645f0c4a 100644 --- a/tests/queries/0_stateless/03040_dynamic_type_alters_1.reference +++ b/tests/queries/0_stateless/03040_dynamic_type_alters_1.reference @@ -30,16 +30,16 @@ alter modify column 1 0 0 \N \N \N \N \N 1 1 \N \N \N \N \N 2 2 \N \N \N \N \N -3 3 3 3 \N \N \N -4 4 4 4 \N \N \N -5 5 5 5 \N \N \N +3 3 3 \N 3 \N \N +4 4 4 \N 4 \N \N +5 5 5 \N 5 \N \N 6 6 str_6 str_6 \N \N \N 7 7 str_7 str_7 \N \N \N 8 8 str_8 str_8 \N \N \N 9 9 \N \N \N \N \N 10 10 \N \N \N \N \N 11 11 \N \N \N \N \N -12 12 12 12 \N \N \N +12 12 12 \N 12 \N \N 13 13 str_13 str_13 \N \N \N 14 14 \N \N \N \N \N insert after alter modify column 1 @@ -48,16 +48,16 @@ insert after alter modify column 1 0 0 \N \N \N \N \N 1 1 \N \N \N \N \N 2 2 \N \N \N \N \N -3 3 3 3 \N \N \N -4 4 4 4 \N \N \N -5 5 5 5 \N \N \N +3 3 3 \N 3 \N \N +4 4 4 \N 4 \N \N +5 5 5 \N 5 \N \N 6 6 str_6 str_6 \N \N \N 7 7 str_7 str_7 \N \N \N 8 8 str_8 str_8 \N \N \N 9 9 \N \N \N \N \N 10 10 \N \N \N \N \N 11 11 \N \N \N \N \N -12 12 12 12 \N \N \N +12 12 12 \N 12 \N \N 13 13 str_13 str_13 \N \N \N 14 14 \N \N \N \N \N 15 15 \N \N \N \N \N @@ -120,57 +120,57 @@ alter modify column 3 5 UInt64 8 String 9 None -0 0 0 \N \N \N \N \N \N -1 1 1 \N \N \N \N \N \N -2 2 2 \N \N \N \N \N \N -3 3 3 \N \N \N 3 \N \N -4 4 4 \N \N \N 4 \N \N -5 5 5 \N \N \N 5 \N \N -6 6 6 \N \N str_6 \N \N \N -7 7 7 \N \N str_7 \N \N \N -8 8 8 \N \N str_8 \N \N \N -9 9 9 \N \N \N \N \N \N -10 10 10 \N \N \N \N \N \N -11 11 11 \N \N \N \N \N \N -12 12 12 \N \N \N 12 \N \N -13 13 13 \N \N str_13 \N \N \N -14 14 14 \N \N \N \N \N \N -15 15 15 \N \N \N \N \N \N -16 16 16 \N \N 16 \N \N \N -17 17 17 \N \N str_17 \N \N \N -18 18 18 \N \N 1970-01-19 \N \N \N -19 19 19 \N \N \N \N \N \N -20 20 20 \N \N \N 20 \N \N -21 21 21 \N \N str_21 \N \N \N -22 22 22 \N \N \N \N 1970-01-23 \N +0 0 \N \N \N \N \N \N \N +1 1 \N \N \N \N \N \N \N +2 2 \N \N \N \N \N \N \N +3 3 \N \N \N \N 3 \N \N +4 4 \N \N \N \N 4 \N \N +5 5 \N \N \N \N 5 \N \N +6 6 \N \N \N str_6 \N \N \N +7 7 \N \N \N str_7 \N \N \N +8 8 \N \N \N str_8 \N \N \N +9 9 \N \N \N \N \N \N \N +10 10 \N \N \N \N \N \N \N +11 11 \N \N \N \N \N \N \N +12 12 \N \N \N \N 12 \N \N +13 13 \N \N \N str_13 \N \N \N +14 14 \N \N \N \N \N \N \N +15 15 \N \N \N \N \N \N \N +16 16 \N \N \N 16 \N \N \N +17 17 \N \N \N str_17 \N \N \N +18 18 \N \N \N 1970-01-19 \N \N \N +19 19 \N \N \N \N \N \N \N +20 20 \N \N \N \N 20 \N \N +21 21 \N \N \N str_21 \N \N \N +22 22 \N \N \N \N \N 1970-01-23 \N insert after alter modify column 3 1 Date 5 UInt64 8 String 12 None -0 0 0 \N \N \N \N \N \N -1 1 1 \N \N \N \N \N \N -2 2 2 \N \N \N \N \N \N -3 3 3 \N \N \N 3 \N \N -4 4 4 \N \N \N 4 \N \N -5 5 5 \N \N \N 5 \N \N -6 6 6 \N \N str_6 \N \N \N -7 7 7 \N \N str_7 \N \N \N -8 8 8 \N \N str_8 \N \N \N -9 9 9 \N \N \N \N \N \N -10 10 10 \N \N \N \N \N \N -11 11 11 \N \N \N \N \N \N -12 12 12 \N \N \N 12 \N \N -13 13 13 \N \N str_13 \N \N \N -14 14 14 \N \N \N \N \N \N -15 15 15 \N \N \N \N \N \N -16 16 16 \N \N 16 \N \N \N -17 17 17 \N \N str_17 \N \N \N -18 18 18 \N \N 1970-01-19 \N \N \N -19 19 19 \N \N \N \N \N \N -20 20 20 \N \N \N 20 \N \N -21 21 21 \N \N str_21 \N \N \N -22 22 22 \N \N \N \N 1970-01-23 \N +0 0 \N \N \N \N \N \N \N +1 1 \N \N \N \N \N \N \N +2 2 \N \N \N \N \N \N \N +3 3 \N \N \N \N 3 \N \N +4 4 \N \N \N \N 4 \N \N +5 5 \N \N \N \N 5 \N \N +6 6 \N \N \N str_6 \N \N \N +7 7 \N \N \N str_7 \N \N \N +8 8 \N \N \N str_8 \N \N \N +9 9 \N \N \N \N \N \N \N +10 10 \N \N \N \N \N \N \N +11 11 \N \N \N \N \N \N \N +12 12 \N \N \N \N 12 \N \N +13 13 \N \N \N str_13 \N \N \N +14 14 \N \N \N \N \N \N \N +15 15 \N \N \N \N \N \N \N +16 16 \N \N \N 16 \N \N \N +17 17 \N \N \N str_17 \N \N \N +18 18 \N \N \N 1970-01-19 \N \N \N +19 19 \N \N \N \N \N \N \N +20 20 \N \N \N \N 20 \N \N +21 21 \N \N \N str_21 \N \N \N +22 22 \N \N \N \N \N 1970-01-23 \N 23 \N \N \N \N \N \N \N \N 24 24 24 \N \N \N \N \N \N 25 str_25 \N str_25 \N \N \N \N \N diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1.sh b/tests/queries/0_stateless/03040_dynamic_type_alters_1.sh index 7a73be20a4d..080a7e583bf 100755 --- a/tests/queries/0_stateless/03040_dynamic_type_alters_1.sh +++ b/tests/queries/0_stateless/03040_dynamic_type_alters_1.sh @@ -7,7 +7,7 @@ CLICKHOUSE_LOG_COMMENT= # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_analyzer=1" +CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_analyzer=0" function run() { From 6128f53f63332d57e9e7f34e54dcd3b9f5c89df5 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Wed, 3 Jul 2024 02:56:47 +0000 Subject: [PATCH 033/161] fix --- src/Analyzer/Resolve/IdentifierResolver.cpp | 2 +- src/Analyzer/Resolve/QueryAnalyzer.cpp | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index 692a31b66ba..59cf82989e7 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -1129,7 +1129,7 @@ QueryTreeNodePtr IdentifierResolver::tryResolveIdentifierFromJoin(const Identifi resolved_identifier = left_resolved_identifier; } } - else if (scope.joins_count == 1 && scope.context->getSettingsRef().single_join_prefer_left_table) + else if (scope.joins_count && scope.context->getSettingsRef().single_join_prefer_left_table) { resolved_side = JoinTableSide::Left; resolved_identifier = left_resolved_identifier; diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 58a85debf37..25ab3752f11 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -1489,9 +1489,9 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & else if (auto * table_node = table_expression_node->as()) additional_column_qualification_parts = {table_node->getStorageID().getDatabaseName(), table_node->getStorageID().getTableName()}; else if (auto * query_node = table_expression_node->as(); query_node && query_node->isCTE()) - additional_column_qualification_parts = {"", query_node->getCTEName()}; + additional_column_qualification_parts = {query_node->getCTEName()}; else if (auto * union_node = table_expression_node->as(); union_node && union_node->isCTE()) - additional_column_qualification_parts = {"", union_node->getCTEName()}; + additional_column_qualification_parts = {union_node->getCTEName()}; size_t additional_column_qualification_parts_size = additional_column_qualification_parts.size(); const auto & table_expression_data = scope.getTableExpressionDataOrThrow(table_expression_node); @@ -4455,10 +4455,11 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table if (auto * scope_query_node = scope.scope_node->as()) { auto left_table_expression = extractLeftTableExpression(scope_query_node->getJoinTree()); + bool is_cte = (query_node && query_node->isCTE()) || (union_node && union_node->isCTE()); if (table_expression_node.get() == left_table_expression.get() && - scope.joins_count == 1 && - scope.context->getSettingsRef().single_join_prefer_left_table) - table_expression_data.should_qualify_columns = false; + scope.joins_count && + (scope.context->getSettingsRef().single_join_prefer_left_table || is_cte)) + table_expression_data.should_qualify_columns = false; } scope.table_expression_node_to_data.emplace(table_expression_node, std::move(table_expression_data)); From 65a34b8f63cc0cd6fda3d454e7505a01f0a7a75a Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 3 Jul 2024 07:27:44 +0000 Subject: [PATCH 034/161] Fix test flakiness --- tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql index 7f88b4cd347..a9e3d88b7c7 100644 --- a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql +++ b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-settings + SET allow_experimental_dynamic_type = 1; DROP TABLE IF EXISTS test_dynamic; CREATE TABLE test_dynamic (id UInt64, d Dynamic) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; From be3d707a5b519e9d810bfcf65b10379b00323762 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 3 Jul 2024 16:37:01 +0000 Subject: [PATCH 035/161] Update test --- .../0_stateless/03198_dynamic_read_subcolumns.reference | 9 +++++++++ .../0_stateless/03198_dynamic_read_subcolumns.sql | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.reference b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.reference index 59d65220c83..f9fe0c2ad77 100644 --- a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.reference +++ b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.reference @@ -1,3 +1,12 @@ +QUERY id: 0 + PROJECTION COLUMNS + d.String Nullable(String) + PROJECTION + LIST id: 1, nodes: 1 + COLUMN id: 2, column_name: d.String, result_type: Nullable(String), source_id: 3 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.test_dynamic + SETTINGS allow_experimental_analyzer=1 foo \N \N diff --git a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql index a9e3d88b7c7..27fca179580 100644 --- a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql +++ b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql @@ -1,9 +1,10 @@ --- Tags: no-random-settings +-- Tags: no-random-settings, no-s3-storage SET allow_experimental_dynamic_type = 1; DROP TABLE IF EXISTS test_dynamic; CREATE TABLE test_dynamic (id UInt64, d Dynamic) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; INSERT INTO test_dynamic VALUES (1, 'foo'), (2, 1111), (3, [1, 2, 3]); +EXPLAIN QUERY TREE SELECT d.String FROM test_dynamic SETTINGS allow_experimental_analyzer = 1; SYSTEM DROP MARK CACHE; SELECT d.String FROM test_dynamic SETTINGS allow_experimental_analyzer = 1; SYSTEM DROP MARK CACHE; From a2de85427473eb3ec982a3fc79cb85fe0ab3cbcb Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 3 Jul 2024 16:37:54 +0000 Subject: [PATCH 036/161] Remove allow_experimental_analyzer from the test --- tests/queries/0_stateless/03040_dynamic_type_alters_1.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1.sh b/tests/queries/0_stateless/03040_dynamic_type_alters_1.sh index 080a7e583bf..1f2a6a31ad7 100755 --- a/tests/queries/0_stateless/03040_dynamic_type_alters_1.sh +++ b/tests/queries/0_stateless/03040_dynamic_type_alters_1.sh @@ -7,7 +7,7 @@ CLICKHOUSE_LOG_COMMENT= # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_analyzer=0" +CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --use_variant_as_common_type=1" function run() { From e11b48c7b5ef1bb25369f8a687b99b16b8ab198d Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Wed, 3 Jul 2024 23:04:13 +0000 Subject: [PATCH 037/161] fix --- src/Analyzer/Resolve/QueryAnalyzer.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 9e4d3fead6c..fc7e56c281f 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -4458,10 +4458,8 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table if (auto * scope_query_node = scope.scope_node->as()) { auto left_table_expression = extractLeftTableExpression(scope_query_node->getJoinTree()); - bool is_cte = (query_node && query_node->isCTE()) || (union_node && union_node->isCTE()); if (table_expression_node.get() == left_table_expression.get() && - scope.joins_count && - (scope.context->getSettingsRef().single_join_prefer_left_table || is_cte)) + scope.joins_count && scope.context->getSettingsRef().single_join_prefer_left_table) table_expression_data.should_qualify_columns = false; } From 20f1256a2fce9ea9604d58bb4065302af06422ec Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 4 Jul 2024 18:29:09 +0000 Subject: [PATCH 038/161] Debuging 02956_rocksdb_bulk_sink --- src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp b/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp index 31812406d34..6f0f5f19970 100644 --- a/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp +++ b/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp @@ -199,9 +199,10 @@ void EmbeddedRocksDBBulkSink::consume(Chunk chunk_) if (chunks_to_write.empty()) return; + size_t num_chunks = chunks_to_write.size(); auto [serialized_key_column, serialized_value_column] = serializeChunks(std::move(chunks_to_write)); auto sst_file_path = getTemporarySSTFilePath(); - LOG_DEBUG(getLogger("EmbeddedRocksDBBulkSink"), "Writing {} rows to SST file {}", serialized_key_column->size(), sst_file_path); + LOG_DEBUG(getLogger("EmbeddedRocksDBBulkSink"), "Writing {} rows from {} chunks to SST file {}", serialized_key_column->size(), num_chunks, sst_file_path); if (auto status = buildSSTFile(sst_file_path, *serialized_key_column, *serialized_value_column); !status.ok()) throw Exception(ErrorCodes::ROCKSDB_ERROR, "RocksDB write error: {}", status.ToString()); From f0966f519b5bbc88c63f604ab97603b504023869 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 4 Jul 2024 21:00:51 +0000 Subject: [PATCH 039/161] Revert changes in test reference --- .../03040_dynamic_type_alters_1.reference | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1.reference b/tests/queries/0_stateless/03040_dynamic_type_alters_1.reference index b41645f0c4a..ca98ec0963c 100644 --- a/tests/queries/0_stateless/03040_dynamic_type_alters_1.reference +++ b/tests/queries/0_stateless/03040_dynamic_type_alters_1.reference @@ -30,16 +30,16 @@ alter modify column 1 0 0 \N \N \N \N \N 1 1 \N \N \N \N \N 2 2 \N \N \N \N \N -3 3 3 \N 3 \N \N -4 4 4 \N 4 \N \N -5 5 5 \N 5 \N \N +3 3 3 3 \N \N \N +4 4 4 4 \N \N \N +5 5 5 5 \N \N \N 6 6 str_6 str_6 \N \N \N 7 7 str_7 str_7 \N \N \N 8 8 str_8 str_8 \N \N \N 9 9 \N \N \N \N \N 10 10 \N \N \N \N \N 11 11 \N \N \N \N \N -12 12 12 \N 12 \N \N +12 12 12 12 \N \N \N 13 13 str_13 str_13 \N \N \N 14 14 \N \N \N \N \N insert after alter modify column 1 @@ -48,16 +48,16 @@ insert after alter modify column 1 0 0 \N \N \N \N \N 1 1 \N \N \N \N \N 2 2 \N \N \N \N \N -3 3 3 \N 3 \N \N -4 4 4 \N 4 \N \N -5 5 5 \N 5 \N \N +3 3 3 3 \N \N \N +4 4 4 4 \N \N \N +5 5 5 5 \N \N \N 6 6 str_6 str_6 \N \N \N 7 7 str_7 str_7 \N \N \N 8 8 str_8 str_8 \N \N \N 9 9 \N \N \N \N \N 10 10 \N \N \N \N \N 11 11 \N \N \N \N \N -12 12 12 \N 12 \N \N +12 12 12 12 \N \N \N 13 13 str_13 str_13 \N \N \N 14 14 \N \N \N \N \N 15 15 \N \N \N \N \N @@ -120,57 +120,57 @@ alter modify column 3 5 UInt64 8 String 9 None -0 0 \N \N \N \N \N \N \N -1 1 \N \N \N \N \N \N \N -2 2 \N \N \N \N \N \N \N -3 3 \N \N \N \N 3 \N \N -4 4 \N \N \N \N 4 \N \N -5 5 \N \N \N \N 5 \N \N -6 6 \N \N \N str_6 \N \N \N -7 7 \N \N \N str_7 \N \N \N -8 8 \N \N \N str_8 \N \N \N -9 9 \N \N \N \N \N \N \N -10 10 \N \N \N \N \N \N \N -11 11 \N \N \N \N \N \N \N -12 12 \N \N \N \N 12 \N \N -13 13 \N \N \N str_13 \N \N \N -14 14 \N \N \N \N \N \N \N -15 15 \N \N \N \N \N \N \N -16 16 \N \N \N 16 \N \N \N -17 17 \N \N \N str_17 \N \N \N -18 18 \N \N \N 1970-01-19 \N \N \N -19 19 \N \N \N \N \N \N \N -20 20 \N \N \N \N 20 \N \N -21 21 \N \N \N str_21 \N \N \N -22 22 \N \N \N \N \N 1970-01-23 \N +0 0 0 \N \N \N \N \N \N +1 1 1 \N \N \N \N \N \N +2 2 2 \N \N \N \N \N \N +3 3 3 \N \N \N 3 \N \N +4 4 4 \N \N \N 4 \N \N +5 5 5 \N \N \N 5 \N \N +6 6 6 \N \N str_6 \N \N \N +7 7 7 \N \N str_7 \N \N \N +8 8 8 \N \N str_8 \N \N \N +9 9 9 \N \N \N \N \N \N +10 10 10 \N \N \N \N \N \N +11 11 11 \N \N \N \N \N \N +12 12 12 \N \N \N 12 \N \N +13 13 13 \N \N str_13 \N \N \N +14 14 14 \N \N \N \N \N \N +15 15 15 \N \N \N \N \N \N +16 16 16 \N \N 16 \N \N \N +17 17 17 \N \N str_17 \N \N \N +18 18 18 \N \N 1970-01-19 \N \N \N +19 19 19 \N \N \N \N \N \N +20 20 20 \N \N \N 20 \N \N +21 21 21 \N \N str_21 \N \N \N +22 22 22 \N \N \N \N 1970-01-23 \N insert after alter modify column 3 1 Date 5 UInt64 8 String 12 None -0 0 \N \N \N \N \N \N \N -1 1 \N \N \N \N \N \N \N -2 2 \N \N \N \N \N \N \N -3 3 \N \N \N \N 3 \N \N -4 4 \N \N \N \N 4 \N \N -5 5 \N \N \N \N 5 \N \N -6 6 \N \N \N str_6 \N \N \N -7 7 \N \N \N str_7 \N \N \N -8 8 \N \N \N str_8 \N \N \N -9 9 \N \N \N \N \N \N \N -10 10 \N \N \N \N \N \N \N -11 11 \N \N \N \N \N \N \N -12 12 \N \N \N \N 12 \N \N -13 13 \N \N \N str_13 \N \N \N -14 14 \N \N \N \N \N \N \N -15 15 \N \N \N \N \N \N \N -16 16 \N \N \N 16 \N \N \N -17 17 \N \N \N str_17 \N \N \N -18 18 \N \N \N 1970-01-19 \N \N \N -19 19 \N \N \N \N \N \N \N -20 20 \N \N \N \N 20 \N \N -21 21 \N \N \N str_21 \N \N \N -22 22 \N \N \N \N \N 1970-01-23 \N +0 0 0 \N \N \N \N \N \N +1 1 1 \N \N \N \N \N \N +2 2 2 \N \N \N \N \N \N +3 3 3 \N \N \N 3 \N \N +4 4 4 \N \N \N 4 \N \N +5 5 5 \N \N \N 5 \N \N +6 6 6 \N \N str_6 \N \N \N +7 7 7 \N \N str_7 \N \N \N +8 8 8 \N \N str_8 \N \N \N +9 9 9 \N \N \N \N \N \N +10 10 10 \N \N \N \N \N \N +11 11 11 \N \N \N \N \N \N +12 12 12 \N \N \N 12 \N \N +13 13 13 \N \N str_13 \N \N \N +14 14 14 \N \N \N \N \N \N +15 15 15 \N \N \N \N \N \N +16 16 16 \N \N 16 \N \N \N +17 17 17 \N \N str_17 \N \N \N +18 18 18 \N \N 1970-01-19 \N \N \N +19 19 19 \N \N \N \N \N \N +20 20 20 \N \N \N 20 \N \N +21 21 21 \N \N str_21 \N \N \N +22 22 22 \N \N \N \N 1970-01-23 \N 23 \N \N \N \N \N \N \N \N 24 24 24 \N \N \N \N \N \N 25 str_25 \N str_25 \N \N \N \N \N From 8fded210bc7320d144bdccfa05feabf0b07ce67e Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Fri, 5 Jul 2024 14:49:19 +0000 Subject: [PATCH 040/161] revert to one join --- src/Analyzer/Resolve/IdentifierResolver.cpp | 2 +- src/Analyzer/Resolve/QueryAnalyzer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index 59cf82989e7..692a31b66ba 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -1129,7 +1129,7 @@ QueryTreeNodePtr IdentifierResolver::tryResolveIdentifierFromJoin(const Identifi resolved_identifier = left_resolved_identifier; } } - else if (scope.joins_count && scope.context->getSettingsRef().single_join_prefer_left_table) + else if (scope.joins_count == 1 && scope.context->getSettingsRef().single_join_prefer_left_table) { resolved_side = JoinTableSide::Left; resolved_identifier = left_resolved_identifier; diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index fc7e56c281f..134b2759c93 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -4459,7 +4459,7 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table { auto left_table_expression = extractLeftTableExpression(scope_query_node->getJoinTree()); if (table_expression_node.get() == left_table_expression.get() && - scope.joins_count && scope.context->getSettingsRef().single_join_prefer_left_table) + scope.joins_count == 1 && scope.context->getSettingsRef().single_join_prefer_left_table) table_expression_data.should_qualify_columns = false; } From d2d72794a14459e60be1c3a88d39440af1f7a8f9 Mon Sep 17 00:00:00 2001 From: serxa Date: Fri, 5 Jul 2024 15:06:50 +0000 Subject: [PATCH 041/161] proper destruction order of AsyncLoader::Pool fields --- src/Common/AsyncLoader.cpp | 4 ++-- src/Common/AsyncLoader.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index cfb273b9058..6264eb03106 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -49,6 +49,7 @@ void logAboutProgress(LoggerPtr log, size_t processed, size_t total, AtomicStopw AsyncLoader::Pool::Pool(const AsyncLoader::PoolInitializer & init) : name(init.name) , priority(init.priority) + , max_threads(init.max_threads > 0 ? init.max_threads : getNumberOfPhysicalCPUCores()) , thread_pool(std::make_unique( init.metric_threads, init.metric_active_threads, @@ -56,17 +57,16 @@ AsyncLoader::Pool::Pool(const AsyncLoader::PoolInitializer & init) /* max_threads = */ std::numeric_limits::max(), // Unlimited number of threads, we do worker management ourselves /* max_free_threads = */ 0, // We do not require free threads /* queue_size = */0)) // Unlimited queue to avoid blocking during worker spawning - , max_threads(init.max_threads > 0 ? init.max_threads : getNumberOfPhysicalCPUCores()) {} AsyncLoader::Pool::Pool(Pool&& o) noexcept : name(o.name) , priority(o.priority) - , thread_pool(std::move(o.thread_pool)) , ready_queue(std::move(o.ready_queue)) , max_threads(o.max_threads) , workers(o.workers) , suspended_workers(o.suspended_workers.load()) // All these constructors are needed because std::atomic is neither copy-constructible, nor move-constructible. We never move pools after init, so it is safe. + , thread_pool(std::move(o.thread_pool)) {} void cancelOnDependencyFailure(const LoadJobPtr & self, const LoadJobPtr & dependency, std::exception_ptr & cancel) diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h index 42707a4ee91..05b809aceae 100644 --- a/src/Common/AsyncLoader.h +++ b/src/Common/AsyncLoader.h @@ -365,11 +365,11 @@ private: { const String name; const Priority priority; - std::unique_ptr thread_pool; // NOTE: we avoid using a `ThreadPool` queue to be able to move jobs between pools. std::map ready_queue; // FIFO queue of jobs to be executed in this pool. Map is used for faster erasing. Key is `ready_seqno` size_t max_threads; // Max number of workers to be spawn size_t workers = 0; // Number of currently executing workers std::atomic suspended_workers{0}; // Number of workers that are blocked by `wait()` call on a job executing in the same pool (for deadlock resolution) + std::unique_ptr thread_pool; // NOTE: we avoid using a `ThreadPool` queue to be able to move jobs between pools. explicit Pool(const PoolInitializer & init); Pool(Pool&& o) noexcept; From 1f0c6061556cd64225a58933177df0228b4c185a Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Fri, 5 Jul 2024 18:48:01 +0000 Subject: [PATCH 042/161] fix tests --- .../02378_analyzer_projection_names.reference | 4 ++-- ...056_analyzer_double_subquery_alias.reference | 1 - .../03056_analyzer_double_subquery_alias.sql | 17 ----------------- 3 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 tests/queries/0_stateless/03056_analyzer_double_subquery_alias.reference delete mode 100644 tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql diff --git a/tests/queries/0_stateless/02378_analyzer_projection_names.reference b/tests/queries/0_stateless/02378_analyzer_projection_names.reference index 170f3e9e1ad..532414f117c 100644 --- a/tests/queries/0_stateless/02378_analyzer_projection_names.reference +++ b/tests/queries/0_stateless/02378_analyzer_projection_names.reference @@ -447,13 +447,13 @@ SELECT '--'; -- DESCRIBE (WITH test_table_in_cte_1 AS (SELECT 1 AS c1), test_table_in_cte_2 AS (SELECT 1 AS c1) SELECT * FROM test_table_in_cte_1 INNER JOIN test_table_in_cte_2 as test_table_in_cte_2 ON test_table_in_cte_1.c1 = test_table_in_cte_2.c1); -c1 UInt8 +test_table_in_cte_1.c1 UInt8 test_table_in_cte_2.c1 UInt8 SELECT '--'; -- DESCRIBE (WITH test_table_in_cte_1 AS (SELECT 1 AS c1), test_table_in_cte_2 AS (SELECT 1 AS c1 UNION ALL SELECT 1 AS c1) SELECT * FROM test_table_in_cte_1 INNER JOIN test_table_in_cte_2 as test_table_in_cte_2 ON test_table_in_cte_1.c1 = test_table_in_cte_2.c1); -c1 UInt8 +test_table_in_cte_1.c1 UInt8 test_table_in_cte_2.c1 UInt8 SELECT 'Joins'; Joins diff --git a/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.reference b/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.reference deleted file mode 100644 index 72749c905a3..00000000000 --- a/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.reference +++ /dev/null @@ -1 +0,0 @@ -1 1 1 diff --git a/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql b/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql deleted file mode 100644 index de471c1a091..00000000000 --- a/tests/queries/0_stateless/03056_analyzer_double_subquery_alias.sql +++ /dev/null @@ -1,17 +0,0 @@ --- https://github.com/ClickHouse/ClickHouse/issues/22627 -SET allow_experimental_analyzer=1; -WITH - x AS - ( - SELECT 1 AS a - ), - xx AS - ( - SELECT * - FROM x - , x AS x1 - , x AS x2 - ) -SELECT * -FROM xx -WHERE a = 1; From f00a8a42a0c26f3805ddf10e09b11fd0e4d5325e Mon Sep 17 00:00:00 2001 From: Blargian Date: Fri, 5 Jul 2024 20:55:46 +0200 Subject: [PATCH 043/161] update partitionId --- .../functions/other-functions.md | 51 +++++++++++++++++++ .../aspell-ignore/en/aspell-dict.txt | 1 + 2 files changed, 52 insertions(+) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 58fc1eba02e..8d8b54b64f5 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2984,6 +2984,57 @@ Result: └─────────┘ ``` +## partitionId + +Returns computed [partition](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) IDs of its arguments. + +:::note +This function is slow and should not be called for large amount of rows. +::: + +**Syntax** + +```sql +partitionId(x[, y, ...]); +``` + +**Arguments** + +- `x` — Column for which to return the partition ID. +- `y, ...` — Remaining N columns for which to return the partition ID (optional). + +**Return Type** + +- Partition ID that the row belongs to. [String](../data-types/string.md). + +**Example** + +Query: + +```sql +DROP TABLE IF EXISTS mt; +CREATE TABLE mt (n UInt64) ENGINE=MergeTree ORDER BY n PARTITION BY n % 2; +INSERT INTO mt SELECT * FROM numbers(10); +SELECT partitionId(*) FROM mt; +``` +Result: + +```response + ┌─partitionId(n)─┐ + 1. │ 0 │ + 2. │ 2 │ + 3. │ 4 │ + 4. │ 6 │ + 5. │ 8 │ + 6. │ 10 │ + 7. │ 12 │ + 8. │ 14 │ + 9. │ 16 │ +10. │ 18 │ + └────────────────┘ +``` + + ## shardNum Returns the index of a shard which processes a part of data in a distributed query. Indices are started from `1`. diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 229eccefa48..d6406e2bb76 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -2184,6 +2184,7 @@ parseReadableSizeOrZero parseTimeDelta parseable parsers +partitionId pathFull pclmulqdq pcre From e2885b1cfa976e9fe072e6453d8142364c013aa9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Jul 2024 02:16:51 +0200 Subject: [PATCH 044/161] Add a test --- .../03201_local_named_collections.reference | 1 + .../03201_local_named_collections.sh | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/queries/0_stateless/03201_local_named_collections.reference create mode 100755 tests/queries/0_stateless/03201_local_named_collections.sh diff --git a/tests/queries/0_stateless/03201_local_named_collections.reference b/tests/queries/0_stateless/03201_local_named_collections.reference new file mode 100644 index 00000000000..af5626b4a11 --- /dev/null +++ b/tests/queries/0_stateless/03201_local_named_collections.reference @@ -0,0 +1 @@ +Hello, world! diff --git a/tests/queries/0_stateless/03201_local_named_collections.sh b/tests/queries/0_stateless/03201_local_named_collections.sh new file mode 100755 index 00000000000..54ca76a52d9 --- /dev/null +++ b/tests/queries/0_stateless/03201_local_named_collections.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --multiquery " +DROP TABLE IF EXISTS test; +CREATE TABLE test (s String) ORDER BY (); +INSERT INTO test VALUES ('Hello, world!'); +" + +${CLICKHOUSE_LOCAL} --multiquery " +CREATE NAMED COLLECTION mydb AS host = '${CLICKHOUSE_HOST}', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', db = '${CLICKHOUSE_DATABASE}'; +SELECT * FROM remote(mydb, table = 'test'); +" + +${CLICKHOUSE_CLIENT} --multiquery " +DROP TABLE test; +" From 02ff6a70737b77a7711a9427eaa487cf705f7378 Mon Sep 17 00:00:00 2001 From: Blargian Date: Sat, 6 Jul 2024 11:36:13 +0200 Subject: [PATCH 045/161] Update example for partitionId --- .../functions/other-functions.md | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 8d8b54b64f5..b1424c8b19a 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -3013,25 +3013,24 @@ Query: ```sql DROP TABLE IF EXISTS mt; -CREATE TABLE mt (n UInt64) ENGINE=MergeTree ORDER BY n PARTITION BY n % 2; -INSERT INTO mt SELECT * FROM numbers(10); -SELECT partitionId(*) FROM mt; +CREATE TABLE mt (i int, j int) ENGINE=MergeTree PARTITION BY i ORDER BY j SETTINGS index_granularity = 1; +INSERT INTO mt VALUES (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); +SELECT * FROM mt WHERE _partition_id = partitionId(1); +SELECT * FROM mt WHERE _partition_id = partitionId(2); ``` Result: ```response - ┌─partitionId(n)─┐ - 1. │ 0 │ - 2. │ 2 │ - 3. │ 4 │ - 4. │ 6 │ - 5. │ 8 │ - 6. │ 10 │ - 7. │ 12 │ - 8. │ 14 │ - 9. │ 16 │ -10. │ 18 │ - └────────────────┘ + ┌─i─┬─j─┐ +1. │ 1 │ 1 │ +2. │ 1 │ 2 │ +3. │ 1 │ 3 │ + └───┴───┘ + ┌─i─┬─j─┐ +1. │ 2 │ 4 │ +2. │ 2 │ 5 │ +3. │ 2 │ 6 │ + └───┴───┘ ``` From f1d373bb1c95a1b852e88b7890b44789e7ae4026 Mon Sep 17 00:00:00 2001 From: Blargian Date: Sat, 6 Jul 2024 11:38:32 +0200 Subject: [PATCH 046/161] update example formatting --- docs/en/sql-reference/functions/other-functions.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index b1424c8b19a..c0225f009b7 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -3013,8 +3013,18 @@ Query: ```sql DROP TABLE IF EXISTS mt; -CREATE TABLE mt (i int, j int) ENGINE=MergeTree PARTITION BY i ORDER BY j SETTINGS index_granularity = 1; +CREATE TABLE mt +( + `i` int, + `j` int +) + ENGINE = MergeTree +PARTITION BY i +ORDER BY j +SETTINGS index_granularity = 1; + INSERT INTO mt VALUES (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); + SELECT * FROM mt WHERE _partition_id = partitionId(1); SELECT * FROM mt WHERE _partition_id = partitionId(2); ``` From b9c662e1da504c4964b67d6408ad06fe9bcc0f95 Mon Sep 17 00:00:00 2001 From: Blargian Date: Sat, 6 Jul 2024 11:39:11 +0200 Subject: [PATCH 047/161] update example formatting --- docs/en/sql-reference/functions/other-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index c0225f009b7..86a03ed6df8 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -3018,7 +3018,7 @@ CREATE TABLE mt `i` int, `j` int ) - ENGINE = MergeTree +ENGINE = MergeTree PARTITION BY i ORDER BY j SETTINGS index_granularity = 1; From d821dfe9033357ae6038f3f7b16d87fd91e05da9 Mon Sep 17 00:00:00 2001 From: avogar Date: Sat, 6 Jul 2024 15:03:13 +0000 Subject: [PATCH 048/161] Fix tests --- src/Interpreters/getColumnFromBlock.cpp | 2 +- tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/getColumnFromBlock.cpp b/src/Interpreters/getColumnFromBlock.cpp index 2e70a58b5a1..89166bb2b3e 100644 --- a/src/Interpreters/getColumnFromBlock.cpp +++ b/src/Interpreters/getColumnFromBlock.cpp @@ -40,7 +40,7 @@ ColumnPtr tryGetSubcolumnFromBlock(const Block & block, const DataTypePtr & requ auto subcolumn_name = requested_subcolumn.getSubcolumnName(); /// If requested subcolumn is dynamic, we should first perform cast and then /// extract the subcolumn, because the data of dynamic subcolumn can change after cast. - if (elem->type->hasDynamicSubcolumns() && !elem->type->equals(*requested_column_type)) + if ((elem->type->hasDynamicSubcolumns() || requested_column_type->hasDynamicSubcolumns()) && !elem->type->equals(*requested_column_type)) { auto casted_column = castColumn({elem->column, elem->type, ""}, requested_column_type); auto elem_column = requested_column_type->tryGetSubcolumn(subcolumn_name, casted_column); diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh index 65517061b99..35d4f3a02cf 100755 --- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh +++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh @@ -7,7 +7,7 @@ CLICKHOUSE_LOG_COMMENT= # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1" +CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 optimize_functions_to_subcolumns=0" function test() From fc3ae09a11c45346f18ec39094adc448dcb9ec45 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Sat, 6 Jul 2024 21:24:56 +0200 Subject: [PATCH 049/161] Fix typo in the test --- tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh index 35d4f3a02cf..a1cb35e4609 100755 --- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh +++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh @@ -7,7 +7,7 @@ CLICKHOUSE_LOG_COMMENT= # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 optimize_functions_to_subcolumns=0" +CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --optimize_functions_to_subcolumns=0" function test() From 5f28c025ce39b1f9e9ea2e48b3cee78ad9a432be Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Jul 2024 12:51:29 +0200 Subject: [PATCH 050/161] Fix test (connections use coroutines) --- tests/queries/0_stateless/03201_local_named_collections.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03201_local_named_collections.sh b/tests/queries/0_stateless/03201_local_named_collections.sh index 54ca76a52d9..2054a09df06 100755 --- a/tests/queries/0_stateless/03201_local_named_collections.sh +++ b/tests/queries/0_stateless/03201_local_named_collections.sh @@ -13,7 +13,7 @@ INSERT INTO test VALUES ('Hello, world!'); ${CLICKHOUSE_LOCAL} --multiquery " CREATE NAMED COLLECTION mydb AS host = '${CLICKHOUSE_HOST}', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', db = '${CLICKHOUSE_DATABASE}'; SELECT * FROM remote(mydb, table = 'test'); -" +" | grep --text -F -v "ASan doesn't fully support makecontext/swapcontext functions" ${CLICKHOUSE_CLIENT} --multiquery " DROP TABLE test; From 30bb03d6c327d15c7161ca18b35b9edf04310de4 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 8 Jul 2024 11:13:44 +0200 Subject: [PATCH 051/161] Update test --- tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh index a1cb35e4609..65517061b99 100755 --- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh +++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh @@ -7,7 +7,7 @@ CLICKHOUSE_LOG_COMMENT= # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --optimize_functions_to_subcolumns=0" +CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1" function test() From 7ab4af85e5d5bea114f1c74dd5cbe4b5a6176772 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 8 Jul 2024 12:36:02 +0200 Subject: [PATCH 052/161] Delete flaky case from 02956_rocksdb_bulk_sink --- tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh b/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh index f7111d0afe2..4e6e123bba2 100755 --- a/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh +++ b/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh @@ -25,13 +25,6 @@ ${CLICKHOUSE_CLIENT} --query "INSERT INTO rocksdb_worm SELECT number, number+1 F ${CLICKHOUSE_CLIENT} --query "SELECT sum(value) FROM system.rocksdb WHERE database = currentDatabase() AND table = 'rocksdb_worm' AND name = 'no.file.opens';" # should be 1 ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM rocksdb_worm;" -# Testing insert with multiple sinks and fixed block size -${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE rocksdb_worm;" -# Must set both max_threads and max_insert_threads to 2 to make sure there is only two sinks -${CLICKHOUSE_CLIENT} --query "INSERT INTO rocksdb_worm SELECT number, number+1 FROM numbers_mt(1000000) SETTINGS max_threads = 2, max_insert_threads = 2, max_block_size = 10000, min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0, insert_deduplication_token = '', optimize_trivial_insert_select = 1;" -${CLICKHOUSE_CLIENT} --query "SELECT sum(value) FROM system.rocksdb WHERE database = currentDatabase() AND table = 'rocksdb_worm' AND name = 'no.file.opens';" # should be 2 because default bulk sink size is ~1M rows / SST file -${CLICKHOUSE_CLIENT} --query "SELECT count() FROM rocksdb_worm;" - # Testing insert with duplicated keys ${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE rocksdb_worm;" ${CLICKHOUSE_CLIENT} --query "INSERT INTO rocksdb_worm SELECT number % 1000, number+1 FROM numbers_mt(1000000) SETTINGS max_block_size = 100000, max_insert_threads = 1, optimize_trivial_insert_select = 1;" From c3b72386a2f26ad40431eca0c1985b14663b43ca Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 8 Jul 2024 14:14:13 +0200 Subject: [PATCH 053/161] remove partitionID function, update tests and remove from documentation --- .../functions/other-functions.md | 60 ---------------- src/Functions/partitionId.cpp | 71 ------------------- tests/integration/test_lost_part/test.py | 2 +- .../01748_partition_id_pruning.sql | 6 +- 4 files changed, 4 insertions(+), 135 deletions(-) delete mode 100644 src/Functions/partitionId.cpp diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 86a03ed6df8..58fc1eba02e 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2984,66 +2984,6 @@ Result: └─────────┘ ``` -## partitionId - -Returns computed [partition](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) IDs of its arguments. - -:::note -This function is slow and should not be called for large amount of rows. -::: - -**Syntax** - -```sql -partitionId(x[, y, ...]); -``` - -**Arguments** - -- `x` — Column for which to return the partition ID. -- `y, ...` — Remaining N columns for which to return the partition ID (optional). - -**Return Type** - -- Partition ID that the row belongs to. [String](../data-types/string.md). - -**Example** - -Query: - -```sql -DROP TABLE IF EXISTS mt; -CREATE TABLE mt -( - `i` int, - `j` int -) -ENGINE = MergeTree -PARTITION BY i -ORDER BY j -SETTINGS index_granularity = 1; - -INSERT INTO mt VALUES (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); - -SELECT * FROM mt WHERE _partition_id = partitionId(1); -SELECT * FROM mt WHERE _partition_id = partitionId(2); -``` -Result: - -```response - ┌─i─┬─j─┐ -1. │ 1 │ 1 │ -2. │ 1 │ 2 │ -3. │ 1 │ 3 │ - └───┴───┘ - ┌─i─┬─j─┐ -1. │ 2 │ 4 │ -2. │ 2 │ 5 │ -3. │ 2 │ 6 │ - └───┴───┘ -``` - - ## shardNum Returns the index of a shard which processes a part of data in a distributed query. Indices are started from `1`. diff --git a/src/Functions/partitionId.cpp b/src/Functions/partitionId.cpp deleted file mode 100644 index e2e9038cd8b..00000000000 --- a/src/Functions/partitionId.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - - -/** partitionId(x, y, ...) is a function that computes partition ids of arguments. - * The function is slow and should not be called for large amount of rows. - */ -class FunctionPartitionId : public IFunction -{ -public: - static constexpr auto name = "partitionId"; - - static FunctionPtr create(ContextPtr) { return std::make_shared(); } - - String getName() const override { return name; } - - bool isVariadic() const override { return true; } - - size_t getNumberOfArguments() const override { return 0; } - - bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const override { return true; } - - bool useDefaultImplementationForNulls() const override { return true; } - bool useDefaultImplementationForConstants() const override { return true; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (arguments.empty()) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires at least one argument.", getName()); - - return std::make_shared(); - } - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override - { - Block sample_block(arguments); - size_t size = arguments.size(); - - auto result_column = ColumnString::create(); - for (size_t j = 0; j < input_rows_count; ++j) - { - Row row(size); - for (size_t i = 0; i < size; ++i) - arguments[i].column->get(j, row[i]); - MergeTreePartition partition(std::move(row)); - result_column->insert(partition.getID(sample_block)); - } - return result_column; - } -}; - -REGISTER_FUNCTION(PartitionId) -{ - factory.registerFunction(); -} - -} diff --git a/tests/integration/test_lost_part/test.py b/tests/integration/test_lost_part/test.py index b8e67551d79..5d0ec383a39 100644 --- a/tests/integration/test_lost_part/test.py +++ b/tests/integration/test_lost_part/test.py @@ -266,7 +266,7 @@ def test_lost_last_part(start_cluster): "ALTER TABLE mt3 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"} ) - partition_id = node1.query("select partitionId('x')").strip() + partition_id = node1.query("select _partition_id").strip() remove_part_from_disk(node1, "mt3", f"{partition_id}_0_0_0") # other way to detect broken parts diff --git a/tests/queries/0_stateless/01748_partition_id_pruning.sql b/tests/queries/0_stateless/01748_partition_id_pruning.sql index b637528bc6c..0e942573652 100644 --- a/tests/queries/0_stateless/01748_partition_id_pruning.sql +++ b/tests/queries/0_stateless/01748_partition_id_pruning.sql @@ -8,17 +8,17 @@ insert into x values (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); set max_rows_to_read = 3; -select * from x where _partition_id = partitionId(1); +select * from x where _partition_id = '1'; set max_rows_to_read = 5; -- one row for subquery + subquery -select * from x where _partition_id in (select partitionId(number + 1) from numbers(1)); +select * from x where _partition_id in (select number + 1 from numbers(1)); -- trivial count optimization test set max_rows_to_read = 2; -- one row for subquery + subquery itself -- TODO: Relax the limits because we might build prepared set twice with _minmax_count_projection set max_rows_to_read = 3; -select count() from x where _partition_id in (select partitionId(number + 1) from numbers(1)); +select count() from x where _partition_id in (select number + 1 from numbers(1)); drop table x; From ee4162f5ec3c34e2e35eec5fca012d75aee84401 Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Mon, 8 Jul 2024 15:23:58 +0200 Subject: [PATCH 054/161] Update 02415_all_new_functions_must_be_documented.reference --- .../02415_all_new_functions_must_be_documented.reference | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index a152066a460..eb5ae5b4f44 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -534,7 +534,6 @@ parseDateTimeInJodaSyntaxOrZero parseDateTimeOrNull parseDateTimeOrZero parseTimeDelta -partitionId path pathFull pi From 6098bc20d90fc3ab45128f62db70624efa0c05dc Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 8 Jul 2024 10:49:46 -0300 Subject: [PATCH 055/161] remove test --- .../__init__.py | 0 .../configs/config.xml | 4 - .../configs/host_regexp.xml | 11 --- .../configs/listen_host.xml | 5 -- .../coredns_config/Corefile | 8 -- .../coredns_config/example.com | 1 - .../scripts/stress_test.py | 62 ------------- .../test.py | 88 ------------------- 8 files changed, 179 deletions(-) delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/__init__.py delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/__init__.py b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml deleted file mode 100644 index 42a1f962705..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml +++ /dev/null @@ -1,4 +0,0 @@ - - 1 - 250 - diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml deleted file mode 100644 index 9329c8dbde2..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - test1\.example\.com$ - - default - - - diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml deleted file mode 100644 index 9c27c612f63..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml +++ /dev/null @@ -1,5 +0,0 @@ - - :: - 0.0.0.0 - 1 - diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile deleted file mode 100644 index 3edf37dafa5..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile +++ /dev/null @@ -1,8 +0,0 @@ -. { - hosts /example.com { - reload "20ms" - fallthrough - } - forward . 127.0.0.11 - log -} diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com deleted file mode 100644 index 9beb415c290..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com +++ /dev/null @@ -1 +0,0 @@ -filled in runtime, but needs to exist in order to be volume mapped in docker \ No newline at end of file diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py deleted file mode 100644 index 70419f95dd3..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py +++ /dev/null @@ -1,62 +0,0 @@ -import pycurl -import threading -from io import BytesIO -import sys - -client_ip = sys.argv[1] -server_ip = sys.argv[2] - -mutex = threading.Lock() -success_counter = 0 -number_of_threads = 100 -number_of_iterations = 50 - - -def perform_request(): - buffer = BytesIO() - crl = pycurl.Curl() - crl.setopt(pycurl.INTERFACE, client_ip) - crl.setopt(crl.WRITEDATA, buffer) - crl.setopt(crl.URL, f"http://{server_ip}:8123/?query=select+1&user=test_dns") - - crl.perform() - - # End curl session - crl.close() - - str_response = buffer.getvalue().decode("iso-8859-1") - expected_response = "1\n" - - mutex.acquire() - - global success_counter - - if str_response == expected_response: - success_counter += 1 - - mutex.release() - - -def perform_multiple_requests(n): - for request_number in range(n): - perform_request() - - -threads = [] - - -for i in range(number_of_threads): - thread = threading.Thread( - target=perform_multiple_requests, args=(number_of_iterations,) - ) - thread.start() - threads.append(thread) - -for thread in threads: - thread.join() - - -if success_counter == number_of_threads * number_of_iterations: - exit(0) - -exit(1) diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py deleted file mode 100644 index d73e8813e79..00000000000 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py +++ /dev/null @@ -1,88 +0,0 @@ -import pytest -import socket -from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check -from time import sleep -import os - -DOCKER_COMPOSE_PATH = get_docker_compose_path() -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -cluster = ClickHouseCluster(__file__) - -ch_server = cluster.add_instance( - "clickhouse-server", - with_coredns=True, - main_configs=["configs/config.xml", "configs/listen_host.xml"], - user_configs=["configs/host_regexp.xml"], -) - -client = cluster.add_instance( - "clickhouse-client", -) - - -@pytest.fixture(scope="module") -def started_cluster(): - global cluster - try: - cluster.start() - yield cluster - - finally: - cluster.shutdown() - - -def check_ptr_record(ip, hostname): - try: - host, aliaslist, ipaddrlist = socket.gethostbyaddr(ip) - if hostname.lower() == host.lower(): - return True - except socket.herror: - pass - return False - - -def setup_dns_server(ip): - domains_string = "test3.example.com test2.example.com test1.example.com" - example_file_path = f'{ch_server.env_variables["COREDNS_CONFIG_DIR"]}/example.com' - run_and_check(f"echo '{ip} {domains_string}' > {example_file_path}", shell=True) - - # DNS server takes time to reload the configuration. - for try_num in range(10): - if all(check_ptr_record(ip, host) for host in domains_string.split()): - break - sleep(1) - - -def setup_ch_server(dns_server_ip): - ch_server.exec_in_container( - (["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"]) - ) - ch_server.exec_in_container( - (["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"]) - ) - ch_server.query("SYSTEM DROP DNS CACHE") - - -def build_endpoint_v4(ip): - return f"'http://{ip}:8123/?query=SELECT+1&user=test_dns'" - - -def build_endpoint_v6(ip): - return build_endpoint_v4(f"[{ip}]") - - -def test_host_regexp_multiple_ptr_v4(started_cluster): - server_ip = cluster.get_instance_ip("clickhouse-server") - client_ip = cluster.get_instance_ip("clickhouse-client") - dns_server_ip = cluster.get_instance_ip(cluster.coredns_host) - - setup_dns_server(client_ip) - setup_ch_server(dns_server_ip) - - current_dir = os.path.dirname(__file__) - client.copy_file_to_container( - os.path.join(current_dir, "scripts", "stress_test.py"), "stress_test.py" - ) - - client.exec_in_container(["python3", f"stress_test.py", client_ip, server_ip]) From 1e924768f95250acbfeed67841e40f7ec2fcf9c5 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 8 Jul 2024 16:14:51 +0200 Subject: [PATCH 056/161] fix 02956_rocksdb_bulk_sink.reference --- tests/queries/0_stateless/02956_rocksdb_bulk_sink.reference | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/02956_rocksdb_bulk_sink.reference b/tests/queries/0_stateless/02956_rocksdb_bulk_sink.reference index 74c71827e6e..2b887148ffb 100644 --- a/tests/queries/0_stateless/02956_rocksdb_bulk_sink.reference +++ b/tests/queries/0_stateless/02956_rocksdb_bulk_sink.reference @@ -3,7 +3,6 @@ 1000 1 1000 -2 1000000 1000 0 999001 From 0f38756f4e02061ddb4c144c2d80e574ccb97cbd Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 8 Jul 2024 16:33:23 +0200 Subject: [PATCH 057/161] Disable optimize_functions_to_subcolumns in 03036_dynamic_read_subcolumns --- tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh index 65517061b99..a1cb35e4609 100755 --- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh +++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh @@ -7,7 +7,7 @@ CLICKHOUSE_LOG_COMMENT= # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1" +CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --optimize_functions_to_subcolumns=0" function test() From 22154aa079132184183659ebee80abcc855be403 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 8 Jul 2024 14:36:31 +0000 Subject: [PATCH 058/161] Disable optimize_functions_to_subcolumns in 03202_dynamic_null_map_subcolumn --- tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh index aa06e48376c..d357419f03a 100755 --- a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh +++ b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh @@ -7,7 +7,7 @@ CLICKHOUSE_LOG_COMMENT= # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1" +CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 optimize_functions_to_subcolumns=0" function test() From 7cf38826afa53deccee9aeb904d98bf98ae78d20 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Mon, 8 Jul 2024 17:48:49 +0200 Subject: [PATCH 059/161] Fix detection of number of CPUs in containers In the case when the 'parent' cgroup was used (i.e. name of cgroup was empty, which is common for containers) ClickHouse was ignoring the CPU limits set for the container. --- base/base/cgroupsv2.cpp | 17 ++++++++++------- base/base/cgroupsv2.h | 6 +++--- base/base/getMemoryAmount.cpp | 6 ++++-- src/Common/CgroupsMemoryUsageObserver.cpp | 6 ++++-- src/Common/getNumberOfPhysicalCPUCores.cpp | 10 +++++----- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/base/base/cgroupsv2.cpp b/base/base/cgroupsv2.cpp index f20b9daf22e..466ebbc3ffb 100644 --- a/base/base/cgroupsv2.cpp +++ b/base/base/cgroupsv2.cpp @@ -33,8 +33,9 @@ bool cgroupsV2MemoryControllerEnabled() /// According to https://docs.kernel.org/admin-guide/cgroup-v2.html, file "cgroup.controllers" defines which controllers are available /// for the current + child cgroups. The set of available controllers can be restricted from level to level using file /// "cgroups.subtree_control". It is therefore sufficient to check the bottom-most nested "cgroup.controllers" file. - std::string cgroup = cgroupV2OfProcess(); - auto cgroup_dir = cgroup.empty() ? default_cgroups_mount : (default_cgroups_mount / cgroup); + auto cgroup_dir = currentCGroupV2Path(); + if (cgroup_dir.empty()) + return false; std::ifstream controllers_file(cgroup_dir / "cgroup.controllers"); if (!controllers_file.is_open()) return false; @@ -46,7 +47,7 @@ bool cgroupsV2MemoryControllerEnabled() #endif } -std::string cgroupV2OfProcess() +std::filesystem::path currentCGroupV2Path() { #if defined(OS_LINUX) chassert(cgroupsV2Enabled()); @@ -54,17 +55,19 @@ std::string cgroupV2OfProcess() /// A simpler way to get the membership is: std::ifstream cgroup_name_file("/proc/self/cgroup"); if (!cgroup_name_file.is_open()) - return ""; + return {}; /// With cgroups v2, there will be a *single* line with prefix "0::/" /// (see https://docs.kernel.org/admin-guide/cgroup-v2.html) std::string cgroup; std::getline(cgroup_name_file, cgroup); static const std::string v2_prefix = "0::/"; if (!cgroup.starts_with(v2_prefix)) - return ""; + return {}; + + // the 'root' cgroup can have empty path, which is valid cgroup = cgroup.substr(v2_prefix.length()); - return cgroup; + return default_cgroups_mount / cgroup; #else - return ""; + return {}; #endif } diff --git a/base/base/cgroupsv2.h b/base/base/cgroupsv2.h index 70219d87cd1..2c58682ce31 100644 --- a/base/base/cgroupsv2.h +++ b/base/base/cgroupsv2.h @@ -16,7 +16,7 @@ bool cgroupsV2Enabled(); /// Assumes that cgroupsV2Enabled() is enabled. bool cgroupsV2MemoryControllerEnabled(); -/// Which cgroup does the process belong to? -/// Returns an empty string if the cgroup cannot be determined. +/// Detects which cgroup the process belong and returns the path to it in sysfs (for cgroups v2). +/// Returns an empty path if the cgroup cannot be determined. /// Assumes that cgroupsV2Enabled() is enabled. -std::string cgroupV2OfProcess(); +std::filesystem::path currentCGroupV2Path(); diff --git a/base/base/getMemoryAmount.cpp b/base/base/getMemoryAmount.cpp index f47cba9833d..9bd5ad75445 100644 --- a/base/base/getMemoryAmount.cpp +++ b/base/base/getMemoryAmount.cpp @@ -23,8 +23,10 @@ std::optional getCgroupsV2MemoryLimit() if (!cgroupsV2MemoryControllerEnabled()) return {}; - std::string cgroup = cgroupV2OfProcess(); - auto current_cgroup = cgroup.empty() ? default_cgroups_mount : (default_cgroups_mount / cgroup); + auto current_cgroup = currentCGroupV2Path(); + + if (current_cgroup.empty()) + return {}; /// Open the bottom-most nested memory limit setting file. If there is no such file at the current /// level, try again at the parent level as memory settings are inherited. diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp index d36c7fd08aa..e034319b21f 100644 --- a/src/Common/CgroupsMemoryUsageObserver.cpp +++ b/src/Common/CgroupsMemoryUsageObserver.cpp @@ -129,8 +129,10 @@ std::optional getCgroupsV2Path() if (!cgroupsV2MemoryControllerEnabled()) return {}; - String cgroup = cgroupV2OfProcess(); - auto current_cgroup = cgroup.empty() ? default_cgroups_mount : (default_cgroups_mount / cgroup); + auto current_cgroup = currentCGroupV2Path(); + + if (current_cgroup.empty()) + return {}; /// Return the bottom-most nested current memory file. If there is no such file at the current /// level, try again at the parent level as memory settings are inherited. diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index 7e18a93e6ed..b16c635f23e 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -37,12 +37,12 @@ uint32_t getCGroupLimitedCPUCores(unsigned default_cpu_count) /// cgroupsv2 if (cgroupsV2Enabled()) { - /// First, we identify the cgroup the process belongs - std::string cgroup = cgroupV2OfProcess(); - if (cgroup.empty()) + /// First, we identify the path of the cgroup the process belongs + auto cgroup_path = currentCGroupV2Path(); + if (cgroup_path.empty()) return default_cpu_count; - auto current_cgroup = cgroup.empty() ? default_cgroups_mount : (default_cgroups_mount / cgroup); + auto current_cgroup = cgroup_path; // Looking for cpu.max in directories from the current cgroup to the top level // It does not stop on the first time since the child could have a greater value than parent @@ -62,7 +62,7 @@ uint32_t getCGroupLimitedCPUCores(unsigned default_cpu_count) } current_cgroup = current_cgroup.parent_path(); } - current_cgroup = default_cgroups_mount / cgroup; + current_cgroup = cgroup_path; // Looking for cpuset.cpus.effective in directories from the current cgroup to the top level while (current_cgroup != default_cgroups_mount.parent_path()) { From db1817a633d1fc1dca13eefa75303f0cb78145b7 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 8 Jul 2024 19:33:52 +0000 Subject: [PATCH 060/161] Some minor fixups --- .../settings/merge-tree-settings.md | 2 +- .../functions/date-time-functions.md | 434 +++++++++--------- src/Functions/changeDate.cpp | 110 ++--- .../aspell-ignore/en/aspell-dict.txt | 3 + 4 files changed, 285 insertions(+), 264 deletions(-) diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 22c8c704ba2..7278b91f90d 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -1030,7 +1030,7 @@ A table with no primary key represents the extreme case of a single equivalence The fewer and the larger the equivalence classes are, the higher the degree of freedom when re-shuffling rows. -The heuristics applied to find the best row order within each equivalence class is suggested by D. Lemir, O. Kaser in [Reordering columns for smaller indexes](https://doi.org/10.1016/j.ins.2011.02.002) and based on sorting the rows within each equivalence class by ascending cardinality of the non-primary key columns. +The heuristics applied to find the best row order within each equivalence class is suggested by D. Lemire, O. Kaser in [Reordering columns for smaller indexes](https://doi.org/10.1016/j.ins.2011.02.002) and based on sorting the rows within each equivalence class by ascending cardinality of the non-primary key columns. It performs three steps: 1. Find all equivalence classes based on the row values in primary key columns. 2. For each equivalence class, calculate (usually estimate) the cardinalities of the non-primary-key columns. diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 1edd8d407eb..4f5e5a5d716 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -2698,6 +2698,204 @@ Like function `YYYYMMDDhhmmssToDate()` but produces a [DateTime64](../data-types Accepts an additional, optional `precision` parameter after the `timezone` parameter. +## changeYear + +Changes the year component of a date or date time. + +**Syntax** +``` sql + +changeYear(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- The same type as `date_or_datetime`. + +**Example** + +``` sql +SELECT changeYear(toDate('1999-01-01'), 2000), changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); +``` + +Result: + +``` +┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ +│ 2000-01-01 │ 2000-01-01 00:00:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + +## changeMonth + +Changes the month component of a date or date time. + +**Syntax** + +``` sql +changeMonth(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the month. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Returns a value of same type as `date_or_datetime`. + +**Example** + +``` sql +SELECT changeMonth(toDate('1999-01-01'), 2), changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2); +``` + +Result: + +``` +┌─changeMonth(toDate('1999-01-01'), 2)─┬─changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2)─┐ +│ 1999-02-01 │ 1999-02-01 00:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeDay + +Changes the day component of a date or date time. + +**Syntax** + +``` sql +changeDay(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the day. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Returns a value of same type as `date_or_datetime`. + +**Example** + +``` sql +SELECT changeDay(toDate('1999-01-01'), 5), changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5); +``` + +Result: + +``` +┌─changeDay(toDate('1999-01-01'), 5)─┬─changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5)─┐ +│ 1999-01-05 │ 1999-01-05 00:00:00.000 │ +└────────────────────────────────────┴──────────────────────────────────────────────────────────┘ +``` + +## changeHour + +Changes the hour component of a date or date time. + +**Syntax** + +``` sql +changeHour(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Returns a value of same type as `date_or_datetime`. If the input is a [Date](../data-types/date.md), return [DateTime](../data-types/datetime.md). If the input is a [Date32](../data-types/date32.md), return [DateTime64](../data-types/datetime64.md). + +**Example** + +``` sql +SELECT changeHour(toDate('1999-01-01'), 14), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14); +``` + +Result: + +``` +┌─changeHour(toDate('1999-01-01'), 14)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14)─┐ +│ 1999-01-01 14:00:00 │ 1999-01-01 14:00:00.000 │ +└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +## changeMinute + +Changes the minute component of a date or date time. + +**Syntax** + +``` sql +changeMinute(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the minute. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Returns a value of same type as `date_or_datetime`. If the input is a [Date](../data-types/date.md), return [DateTime](../data-types/datetime.md). If the input is a [Date32](../data-types/date32.md), return [DateTime64](../data-types/datetime64.md). + +**Example** + +``` sql + SELECT changeMinute(toDate('1999-01-01'), 15), changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeMinute(toDate('1999-01-01'), 15)─┬─changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:15:00 │ 1999-01-01 00:15:00.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + +## changeSecond + +Changes the second component of a date or date time. + +**Syntax** + +``` sql +changeSecond(date_or_datetime, value) +``` + +**Arguments** + +- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) +- `value` - a new value of the second. [Integer](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Returns a value of same type as `date_or_datetime`. If the input is a [Date](../data-types/date.md), return [DateTime](../data-types/datetime.md). If the input is a [Date32](../data-types/date32.md), return [DateTime64](../data-types/datetime64.md). + +**Example** + +``` sql +SELECT changeSecond(toDate('1999-01-01'), 15), changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15); +``` + +Result: + +``` +┌─changeSecond(toDate('1999-01-01'), 15)─┬─changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ +│ 1999-01-01 00:00:15 │ 1999-01-01 00:00:15.000 │ +└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ +``` + ## addYears Adds a specified number of years to a date, a date with time or a string-encoded date / date with time. @@ -2714,6 +2912,7 @@ addYears(date, num) - `num`: Number of years to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` years. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2751,6 +2950,7 @@ addQuarters(date, num) - `num`: Number of quarters to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` quarters. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2788,6 +2988,7 @@ addMonths(date, num) - `num`: Number of months to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` months. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2825,6 +3026,7 @@ addWeeks(date, num) - `num`: Number of weeks to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` weeks. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2862,6 +3064,7 @@ addDays(date, num) - `num`: Number of days to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` days. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2899,6 +3102,7 @@ addHours(date, num) - `num`: Number of hours to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** +o - Returns `date` plus `num` hours. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2936,6 +3140,7 @@ addMinutes(date, num) - `num`: Number of minutes to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` minutes. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -2973,6 +3178,7 @@ addSeconds(date, num) - `num`: Number of seconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` plus `num` seconds. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3010,6 +3216,7 @@ addMilliseconds(date_time, num) - `num`: Number of milliseconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` plus `num` milliseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3045,6 +3252,7 @@ addMicroseconds(date_time, num) - `num`: Number of microseconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` plus `num` microseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3080,6 +3288,7 @@ addNanoseconds(date_time, num) - `num`: Number of nanoseconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` plus `num` nanoseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3115,6 +3324,7 @@ addInterval(interval_1, interval_2) - `interval_2`: Second interval to be added. [interval](../data-types/special-data-types/interval.md). **Returned value** + - Returns a tuple of intervals. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). :::note @@ -3161,6 +3371,7 @@ addTupleOfIntervals(interval_1, interval_2) - `intervals`: Tuple of intervals to add to `date`. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). **Returned value** + - Returns `date` with added `intervals`. [date](../data-types/date.md)/[date32](../data-types/date32.md)/[datetime](../data-types/datetime.md)/[datetime64](../data-types/datetime64.md). **Example** @@ -3195,6 +3406,7 @@ subtractYears(date, num) - `num`: Number of years to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` years. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3232,6 +3444,7 @@ subtractQuarters(date, num) - `num`: Number of quarters to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` quarters. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3269,6 +3482,7 @@ subtractMonths(date, num) - `num`: Number of months to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` months. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3306,6 +3520,7 @@ subtractWeeks(date, num) - `num`: Number of weeks to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` weeks. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3343,6 +3558,7 @@ subtractDays(date, num) - `num`: Number of days to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` days. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3380,6 +3596,7 @@ subtractHours(date, num) - `num`: Number of hours to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` hours. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[Datetime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3417,6 +3634,7 @@ subtractMinutes(date, num) - `num`: Number of minutes to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` minutes. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3454,6 +3672,7 @@ subtractSeconds(date, num) - `num`: Number of seconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date` minus `num` seconds. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3491,6 +3710,7 @@ subtractMilliseconds(date_time, num) - `num`: Number of milliseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` minus `num` milliseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3526,6 +3746,7 @@ subtractMicroseconds(date_time, num) - `num`: Number of microseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` minus `num` microseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3561,6 +3782,7 @@ subtractNanoseconds(date_time, num) - `num`: Number of nanoseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). **Returned value** + - Returns `date_time` minus `num` nanoseconds. [DateTime64](../data-types/datetime64.md). **Example** @@ -3596,6 +3818,7 @@ subtractInterval(interval_1, interval_2) - `interval_2`: Second interval to be negated. [interval](../data-types/special-data-types/interval.md). **Returned value** + - Returns a tuple of intervals. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). :::note @@ -3642,6 +3865,7 @@ subtractTupleOfIntervals(interval_1, interval_2) - `intervals`: Tuple of intervals to subtract from `date`. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). **Returned value** + - Returns `date` with subtracted `intervals`. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). **Example** @@ -3660,216 +3884,6 @@ Result: └───────────────────────────────────────────────────────────────────────┘ ``` -## changeYear - -Changes the year component of a date or date time. - -**Syntax** - -``` sql -changeYear(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the year. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeYear(toDate('1999-01-01'), 2000), changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000); -``` - -Result: - -``` -┌─changeYear(toDate('1999-01-01'), 2000)─┬─changeYear(toDateTime64('1999-01-01 00:00:00.000', 3), 2000)─┐ -│ 2000-01-01 │ 2000-01-01 00:00:00.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` - -## changeMonth - -Changes the month component of a date or date time. - -**Syntax** - -``` sql -changeMonth(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the month. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeMonth(toDate('1999-01-01'), 2), changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2); -``` - -Result: - -``` -┌─changeMonth(toDate('1999-01-01'), 2)─┬─changeMonth(toDateTime64('1999-01-01 00:00:00.000', 3), 2)─┐ -│ 1999-02-01 │ 1999-02-01 00:00:00.000 │ -└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ -``` - -## changeDay - -Changes the day component of a date or date time. - -**Syntax** - -``` sql -changeDay(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the day. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeDay(toDate('1999-01-01'), 5), changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5); -``` - -Result: - -``` -┌─changeDay(toDate('1999-01-01'), 5)─┬─changeDay(toDateTime64('1999-01-01 00:00:00.000', 3), 5)─┐ -│ 1999-01-05 │ 1999-01-05 00:00:00.000 │ -└────────────────────────────────────┴──────────────────────────────────────────────────────────┘ -``` - -## changeHour - -Changes the hour component of a date or date time. - -**Syntax** - -``` sql -changeHour(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the hour. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeHour(toDate('1999-01-01'), 14), changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14); -``` - -Result: - -``` -┌─changeHour(toDate('1999-01-01'), 14)─┬─changeHour(toDateTime64('1999-01-01 00:00:00.000', 3), 14)─┐ -│ 1999-01-01 14:00:00 │ 1999-01-01 14:00:00.000 │ -└──────────────────────────────────────┴────────────────────────────────────────────────────────────┘ -``` - -## changeMinute - -Changes the minute component of a date or date time. - -**Syntax** - -``` sql -changeMinute(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the minute. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeMinute(toDate('1999-01-01'), 15), changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15); -``` - -Result: - -``` -┌─changeMinute(toDate('1999-01-01'), 15)─┬─changeMinute(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ -│ 1999-01-01 00:15:00 │ 1999-01-01 00:15:00.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` - -## changeSecond - -Changes the second component of a date or date time. - -**Syntax** - -``` sql -changeSecond(date_or_datetime, value) -``` - -**Arguments** - -- `date_or_datetime` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md) -- `value` - a new value of the second. [Integer](../../sql-reference/data-types/int-uint.md). - -**Return value** - -- The same type as `date_or_datetime`. If the input is a Date, return DateTime. If the input is a Date32, return DateTime64. - -Type: [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). - -**Example** - -``` sql - SELECT changeSecond(toDate('1999-01-01'), 15), changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15); -``` - -Result: - -``` -┌─changeSecond(toDate('1999-01-01'), 15)─┬─changeSecond(toDateTime64('1999-01-01 00:00:00.000', 3), 15)─┐ -│ 1999-01-01 00:00:15 │ 1999-01-01 00:00:15.000 │ -└────────────────────────────────────────┴──────────────────────────────────────────────────────────────┘ -``` - ## timeSlots(StartTime, Duration,\[, Size\]) For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index e24391afe12..5965f3d1d00 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -41,11 +41,6 @@ enum class Component Second }; -bool isTimeComponentChange(Component type) -{ - return type == Component::Hour || type == Component::Minute || type == Component::Second; -} - } template @@ -65,11 +60,11 @@ public: {"date_or_datetime", static_cast(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date or date with time"}, {"value", static_cast(&isNativeInteger), nullptr, "Integer"} }; - validateFunctionArgumentTypes(*this, arguments, args); + validateFunctionArguments(*this, arguments, args); const auto & input_type = arguments[0].type; - if (isTimeComponentChange(Traits::component)) + if constexpr (Traits::component == Component::Hour || Traits::component == Component::Minute || Traits::component == Component::Second) { if (isDate(input_type)) return std::make_shared(); @@ -85,13 +80,13 @@ public: const auto & input_type = arguments[0].type; if (isDate(input_type)) { - if (isTimeComponentChange(Traits::component)) + if constexpr (Traits::component == Component::Hour || Traits::component == Component::Minute || Traits::component == Component::Second) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } if (isDate32(input_type)) { - if (isTimeComponentChange(Traits::component)) + if constexpr (Traits::component == Component::Hour || Traits::component == Component::Minute || Traits::component == Component::Second) return execute(arguments, input_type, result_type, input_rows_count); return execute(arguments, input_type, result_type, input_rows_count); } @@ -109,83 +104,92 @@ public: bool is_const = (isColumnConst(*arguments[0].column) && isColumnConst(*arguments[1].column)); size_t result_rows_count = (is_const ? 1 : input_rows_count); - typename ResultDataType::ColumnType::MutablePtr result_column; + typename ResultDataType::ColumnType::MutablePtr result_col; if constexpr (std::is_same_v) { auto scale = DataTypeDateTime64::default_scale; if constexpr (std::is_same_v) scale = typeid_cast(*result_type).getScale(); - result_column = ResultDataType::ColumnType::create(result_rows_count, scale); + result_col = ResultDataType::ColumnType::create(result_rows_count, scale); } else - result_column = ResultDataType::ColumnType::create(result_rows_count); + result_col = ResultDataType::ColumnType::create(result_rows_count); - auto input_column = arguments[0].column->convertToFullIfNeeded(); - const auto & input_column_data = typeid_cast(*input_column).getData(); + auto date_time_col = arguments[0].column->convertToFullIfNeeded(); + const auto & date_time_col_data = typeid_cast(*date_time_col).getData(); - auto new_value_column = castColumn(arguments[1], std::make_shared()); - new_value_column = new_value_column->convertToFullIfNeeded(); - const auto & new_value_column_data = typeid_cast(*new_value_column).getData(); + auto value_col = castColumn(arguments[1], std::make_shared()); + value_col = value_col->convertToFullIfNeeded(); + const auto & value_col_data = typeid_cast(*value_col).getData(); - auto & result_data = result_column->getData(); + auto & result_col_data = result_col->getData(); - for (size_t i = 0; i < result_rows_count; ++i) + if constexpr (std::is_same_v) { - if constexpr (std::is_same_v) + const auto scale = typeid_cast(*result_type).getScale(); + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + + Int64 deg = 1; + for (size_t j = 0; j < scale; ++j) + deg *= 10; + + for (size_t i = 0; i < result_rows_count; ++i) { - const auto scale = typeid_cast(*result_type).getScale(); - const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + Int64 time = date_lut.toNumYYYYMMDDhhmmss(date_time_col_data[i] / deg); + Int64 fraction = date_time_col_data[i] % deg; - Int64 deg = 1; - for (size_t j = 0; j < scale; ++j) - deg *= 10; - - Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i] / deg); - Int64 fraction = input_column_data[i] % deg; - - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, scale, fraction); + result_col_data[i] = getChangedDate(time, value_col_data[i], result_type, date_lut, scale, fraction); } - else if constexpr (std::is_same_v && std::is_same_v) + } + else if constexpr (std::is_same_v && std::is_same_v) + { + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + for (size_t i = 0; i < result_rows_count; ++i) { - const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; - - result_data[i] = getChangedDate(time, new_value_column_data[i], result_type, date_lut, 3, 0); + Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(date_time_col_data[i]))) * 1'000'000; + result_col_data[i] = getChangedDate(time, value_col_data[i], result_type, date_lut, 3, 0); } - else if constexpr (std::is_same_v && std::is_same_v) + } + else if constexpr (std::is_same_v && std::is_same_v) + { + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + for (size_t i = 0; i < result_rows_count; ++i) { - const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; - - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(date_time_col_data[i]))) * 1'000'000; + result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); } - else if constexpr (std::is_same_v) + } + else if constexpr (std::is_same_v) + { + const auto & date_lut = typeid_cast(*result_type).getTimeZone(); + for (size_t i = 0; i < result_rows_count; ++i) { - const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - Int64 time = date_lut.toNumYYYYMMDDhhmmss(input_column_data[i]); - - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + Int64 time = date_lut.toNumYYYYMMDDhhmmss(date_time_col_data[i]); + result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); } - else + } + else + { + const auto & date_lut = DateLUT::instance(); + for (size_t i = 0; i < result_rows_count; ++i) { - const auto & date_lut = DateLUT::instance(); Int64 time; if (isDate(input_type)) - time = static_cast(date_lut.toNumYYYYMMDD(DayNum(input_column_data[i]))) * 1'000'000; + time = static_cast(date_lut.toNumYYYYMMDD(DayNum(date_time_col_data[i]))) * 1'000'000; else - time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(input_column_data[i]))) * 1'000'000; + time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(date_time_col_data[i]))) * 1'000'000; if (isDate(result_type)) - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); else - result_data[i] = static_cast(getChangedDate(time, new_value_column_data[i], result_type, date_lut)); + result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); } } if (is_const) - return ColumnConst::create(std::move(result_column), input_rows_count); + return ColumnConst::create(std::move(result_col), input_rows_count); - return result_column; + return result_col; } Int64 getChangedDate(Int64 time, Float64 new_value, const DataTypePtr & result_type, const DateLUTImpl & date_lut, Int64 scale = 0, Int64 fraction = 0) const diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 5689f94d2ae..5e31a09effb 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -467,6 +467,7 @@ LOCALTIME LOCALTIMESTAMP LONGLONG LOONGARCH +Lemire Levenshtein Liao LibFuzzer @@ -1962,6 +1963,8 @@ loghouse london lookups loongarch +lowCardinalityIndices +lowCardinalityKeys lowcardinality lowerUTF lowercased From 85f5fe832a23fa25e9784728b9f26d712057148a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 8 Jul 2024 20:46:56 +0000 Subject: [PATCH 061/161] Fix integration tests --- tests/integration/test_backward_compatibility/test_functions.py | 2 ++ tests/integration/test_lost_part/test.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_backward_compatibility/test_functions.py b/tests/integration/test_backward_compatibility/test_functions.py index fc03a77030e..551a0550c0c 100644 --- a/tests/integration/test_backward_compatibility/test_functions.py +++ b/tests/integration/test_backward_compatibility/test_functions.py @@ -168,6 +168,8 @@ def test_string_functions(start_cluster): "filesystemAvailable", # Exclude it for now. Looks like the result depends on the build type. "farmHash64", + # Removed in 24.7 + "partitionId", ] functions = filter(lambda x: x not in excludes, functions) diff --git a/tests/integration/test_lost_part/test.py b/tests/integration/test_lost_part/test.py index 5d0ec383a39..5070b3c5b68 100644 --- a/tests/integration/test_lost_part/test.py +++ b/tests/integration/test_lost_part/test.py @@ -266,7 +266,7 @@ def test_lost_last_part(start_cluster): "ALTER TABLE mt3 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"} ) - partition_id = node1.query("select _partition_id").strip() + partition_id = node1.query("select distinct _partition_id from mt3 where p = 'x'").strip() remove_part_from_disk(node1, "mt3", f"{partition_id}_0_0_0") # other way to detect broken parts From 595a12ee884840da76e2b50ae723511e00915b85 Mon Sep 17 00:00:00 2001 From: nauu Date: Tue, 9 Jul 2024 09:48:51 +0800 Subject: [PATCH 062/161] to avoid ambiguity, replace FilesystemCacheFailToReserveSpaceBecauseOfLockContention with FilesystemCacheFailToReserveSpaceBecauseOfCacheResize. to avoid ambiguity, replace FilesystemCacheFailToReserveSpaceBecauseOfLockContention with FilesystemCacheResize. --- src/Common/ProfileEvents.cpp | 1 + src/Interpreters/Cache/FileCache.cpp | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 439965a92fb..e80afc95e8d 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -508,6 +508,7 @@ The server successfully detected this situation and will download merged part fr M(FileSegmentHolderCompleteMicroseconds, "File segments holder complete() time") \ M(FileSegmentFailToIncreasePriority, "Number of times the priority was not increased due to a high contention on the cache lock") \ M(FilesystemCacheFailToReserveSpaceBecauseOfLockContention, "Number of times space reservation was skipped due to a high contention on the cache lock") \ + M(FilesystemCacheFailToReserveSpaceBecauseOfCacheResize, "Number of times space reservation was skipped due to the cache is being resized") \ M(FilesystemCacheHoldFileSegments, "Filesystem cache file segments count, which were hold") \ M(FilesystemCacheUnusedHoldFileSegments, "Filesystem cache file segments count, which were hold, but not used (because of seek or LIMIT n, etc)") \ M(FilesystemCacheFreeSpaceKeepingThreadRun, "Number of times background thread executed free space keeping job") \ diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 0d33e39ffa3..a3848fa3a75 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -30,6 +30,7 @@ namespace ProfileEvents extern const Event FilesystemCacheFailToReserveSpaceBecauseOfLockContention; extern const Event FilesystemCacheFreeSpaceKeepingThreadRun; extern const Event FilesystemCacheFreeSpaceKeepingThreadWorkMilliseconds; + extern const Event FilesystemCacheFailToReserveSpaceBecauseOfCacheResize; } namespace DB @@ -813,7 +814,7 @@ bool FileCache::tryReserve( /// ok compared to the number of cases this check will help. if (cache_is_being_resized.load(std::memory_order_relaxed)) { - ProfileEvents::increment(ProfileEvents::FilesystemCacheFailToReserveSpaceBecauseOfLockContention); + ProfileEvents::increment(ProfileEvents::FilesystemCacheFailToReserveSpaceBecauseOfCacheResize); return false; } From 0eb081f899bd0fa9c185add84c9c59e212dfd6ca Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 9 Jul 2024 10:30:07 +0000 Subject: [PATCH 063/161] Fix style --- tests/integration/test_lost_part/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_lost_part/test.py b/tests/integration/test_lost_part/test.py index 5070b3c5b68..e88b84775b4 100644 --- a/tests/integration/test_lost_part/test.py +++ b/tests/integration/test_lost_part/test.py @@ -266,7 +266,9 @@ def test_lost_last_part(start_cluster): "ALTER TABLE mt3 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"} ) - partition_id = node1.query("select distinct _partition_id from mt3 where p = 'x'").strip() + partition_id = node1.query( + "select distinct _partition_id from mt3 where p = 'x'" + ).strip() remove_part_from_disk(node1, "mt3", f"{partition_id}_0_0_0") # other way to detect broken parts From 859c63298ef044d92c585921579f90dd6e56deda Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 9 Jul 2024 10:51:40 +0000 Subject: [PATCH 064/161] upd 02956_rocksdb_bulk_sink --- .../queries/0_stateless/02956_rocksdb_bulk_sink.reference | 1 + tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/tests/queries/0_stateless/02956_rocksdb_bulk_sink.reference b/tests/queries/0_stateless/02956_rocksdb_bulk_sink.reference index 2b887148ffb..1f140df1d6b 100644 --- a/tests/queries/0_stateless/02956_rocksdb_bulk_sink.reference +++ b/tests/queries/0_stateless/02956_rocksdb_bulk_sink.reference @@ -3,6 +3,7 @@ 1000 1 1000 +1 1000000 1000 0 999001 diff --git a/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh b/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh index 4e6e123bba2..95c136584f0 100755 --- a/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh +++ b/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh @@ -25,6 +25,13 @@ ${CLICKHOUSE_CLIENT} --query "INSERT INTO rocksdb_worm SELECT number, number+1 F ${CLICKHOUSE_CLIENT} --query "SELECT sum(value) FROM system.rocksdb WHERE database = currentDatabase() AND table = 'rocksdb_worm' AND name = 'no.file.opens';" # should be 1 ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM rocksdb_worm;" +# Testing insert with multiple sinks and fixed block size +${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE rocksdb_worm;" +# Must set both max_threads and max_insert_threads to 2 to make sure there is only two sinks +${CLICKHOUSE_CLIENT} --query "INSERT INTO rocksdb_worm SELECT number, number+1 FROM numbers_mt(1000000) SETTINGS max_threads = 2, max_insert_threads = 2, max_block_size = 10000, min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0, insert_deduplication_token = '', optimize_trivial_insert_select = 1;" +${CLICKHOUSE_CLIENT} --query "SELECT sum(value) IN (1, 2) FROM system.rocksdb WHERE database = currentDatabase() AND table = 'rocksdb_worm' AND name = 'no.file.opens';" # should be not more than 2 because default bulk sink size is ~1M rows / SST file. +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM rocksdb_worm;" + # Testing insert with duplicated keys ${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE rocksdb_worm;" ${CLICKHOUSE_CLIENT} --query "INSERT INTO rocksdb_worm SELECT number % 1000, number+1 FROM numbers_mt(1000000) SETTINGS max_block_size = 100000, max_insert_threads = 1, optimize_trivial_insert_select = 1;" From a339dd020f97e0e31d20e97966aa18c6afb4c3c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 9 Jul 2024 18:52:34 +0200 Subject: [PATCH 065/161] Reduce dependencies on MergeTreeSettings.h --- programs/server/Server.cpp | 1 + src/Access/AccessControl.cpp | 1 - src/Databases/DatabaseOrdinary.cpp | 1 + src/Interpreters/InterpreterCreateQuery.cpp | 4 +- src/Interpreters/MutationsInterpreter.cpp | 2 + .../QueryPlan/ReadFromMergeTree.cpp | 1 + src/Server/ReplicasStatusHandler.cpp | 1 + src/Storages/AlterCommands.cpp | 2 +- src/Storages/MergeTree/AsyncBlockIDsCache.cpp | 5 +- src/Storages/MergeTree/DataPartsExchange.cpp | 1 + src/Storages/MergeTree/IMergeTreeDataPart.cpp | 1 + .../MergeTree/IMergeTreeDataPartWriter.h | 6 ++- .../MergeTree/IMergedBlockOutputStream.cpp | 1 + .../MergeTree/IMergedBlockOutputStream.h | 8 +-- .../MergeTree/MergeFromLogEntryTask.cpp | 1 + src/Storages/MergeTree/MergeTask.cpp | 11 ++++ src/Storages/MergeTree/MergeTask.h | 11 +--- src/Storages/MergeTree/MergeTreeData.cpp | 13 +++++ src/Storages/MergeTree/MergeTreeData.h | 19 ++----- .../MergeTree/MergeTreeDataMergerMutator.cpp | 1 + .../MergeTreeDataPartWriterOnDisk.cpp | 1 + .../MergeTree/MergeTreeDataPartWriterWide.cpp | 1 + .../MergeTree/MergeTreeDataSelectExecutor.cpp | 1 + .../MergeTree/MergeTreeDataWriter.cpp | 1 + .../MergeTree/MergeTreeIOSettings.cpp | 37 +++++++++++++ src/Storages/MergeTree/MergeTreeIOSettings.h | 30 +++-------- .../MergeTreeIndexGranularityInfo.cpp | 3 +- .../MergeTree/MergeTreePartsMover.cpp | 3 +- .../MergeTree/MergeTreeSequentialSource.cpp | 1 + src/Storages/MergeTree/MergeTreeSettings.cpp | 10 ++++ src/Storages/MergeTree/MergeTreeSettings.h | 19 +++---- .../MergeTree/MutateFromLogEntryTask.cpp | 1 + src/Storages/MergeTree/MutateTask.cpp | 4 ++ .../PartMovesBetweenShardsOrchestrator.cpp | 1 + .../ReplicatedMergeMutateTaskBase.cpp | 3 +- .../ReplicatedMergeTreeAttachThread.cpp | 1 + .../ReplicatedMergeTreeCleanupThread.cpp | 5 +- ...ReplicatedMergeTreeMergeStrategyPicker.cpp | 3 +- .../ReplicatedMergeTreePartCheckThread.cpp | 1 + .../MergeTree/ReplicatedMergeTreeQueue.cpp | 1 + .../ReplicatedMergeTreeRestartingThread.cpp | 1 + .../MergeTree/ReplicatedMergeTreeSink.cpp | 1 + .../ReplicatedMergeTreeTableMetadata.cpp | 1 + .../MergeTree/StorageFromMergeTreeDataPart.h | 48 +++-------------- src/Storages/MergeTree/checkDataPart.cpp | 1 + .../MergeTree/registerStorageMergeTree.cpp | 1 + src/Storages/StorageFromMergeTreeDataPart.cpp | 54 +++++++++++++++++++ src/Storages/StorageMergeTree.cpp | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 1 + .../System/StorageSystemMergeTreeSettings.cpp | 1 + .../System/StorageSystemMergeTreeSettings.h | 1 - 51 files changed, 210 insertions(+), 119 deletions(-) create mode 100644 src/Storages/MergeTree/MergeTreeIOSettings.cpp create mode 100644 src/Storages/StorageFromMergeTreeDataPart.cpp diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 3b88bb36954..053ddaf8d8b 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -68,6 +68,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 353358fac65..95a467bbbe5 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 7d4bb07e8ef..46830861c75 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 0ee2bb6c0e9..15e987750c2 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -34,10 +33,11 @@ #include #include +#include #include #include -#include #include +#include #include #include diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 6d3a4f30b34..f2eec8ff8f5 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index dc6d96a721e..2829f43a0e8 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index 67823117758..5a29612c263 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 35a5e95e643..eef83025097 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -33,9 +33,9 @@ #include #include #include +#include #include #include -#include #include diff --git a/src/Storages/MergeTree/AsyncBlockIDsCache.cpp b/src/Storages/MergeTree/AsyncBlockIDsCache.cpp index 9d64592ed64..6606b00e287 100644 --- a/src/Storages/MergeTree/AsyncBlockIDsCache.cpp +++ b/src/Storages/MergeTree/AsyncBlockIDsCache.cpp @@ -1,7 +1,8 @@ +#include +#include +#include #include #include -#include -#include #include diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index b7fdcc63e1b..8e73021d3e7 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index c2e0e778220..70044db7336 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index d9e9a433827..adaa4eddb98 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -1,11 +1,10 @@ #pragma once +#include #include -#include #include #include #include -#include #include #include @@ -13,6 +12,9 @@ namespace DB { +struct MergeTreeSettings; +using MergeTreeSettingsPtr = std::shared_ptr; + Block getBlockAndPermute(const Block & block, const Names & names, const IColumn::Permutation * permutation); Block permuteBlockIfNeeded(const Block & block, const IColumn::Permutation * permutation); diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 89c813ab233..70e838e666a 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.h b/src/Storages/MergeTree/IMergedBlockOutputStream.h index a9b058418ea..cfcfb177e05 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -1,16 +1,18 @@ #pragma once #include -#include -#include -#include #include #include +#include +#include #include namespace DB { +struct MergeTreeSettings; +using MergeTreeSettingsPtr = std::shared_ptr; + class IMergedBlockOutputStream { public: diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index 79efb0ca8b3..4c947487d21 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index c8f1a08128b..dafb562f0d8 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -422,6 +423,16 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() return false; } +bool MergeTask::enabledBlockNumberColumn(GlobalRuntimeContextPtr global_ctx) +{ + return global_ctx->data->getSettings()->enable_block_number_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); +} + +bool MergeTask::enabledBlockOffsetColumn(GlobalRuntimeContextPtr global_ctx) +{ + return global_ctx->data->getSettings()->enable_block_offset_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); +} + void MergeTask::addGatheringColumn(GlobalRuntimeContextPtr global_ctx, const String & name, const DataTypePtr & type) { if (global_ctx->storage_columns.contains(name)) diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index 56909d1b7a0..8b0f2130e8e 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -408,15 +408,8 @@ private: Stages::const_iterator stages_iterator = stages.begin(); - static bool enabledBlockNumberColumn(GlobalRuntimeContextPtr global_ctx) - { - return global_ctx->data->getSettings()->enable_block_number_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); - } - - static bool enabledBlockOffsetColumn(GlobalRuntimeContextPtr global_ctx) - { - return global_ctx->data->getSettings()->enable_block_offset_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); - } + static bool enabledBlockNumberColumn(GlobalRuntimeContextPtr global_ctx); + static bool enabledBlockOffsetColumn(GlobalRuntimeContextPtr global_ctx); static void addGatheringColumn(GlobalRuntimeContextPtr global_ctx, const String & name, const DataTypePtr & type); }; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 467a5c82141..59a4032f235 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -73,6 +73,7 @@ #include #include #include +#include #include #include #include @@ -6172,6 +6173,11 @@ bool MergeTreeData::hasProjection() const return false; } +bool MergeTreeData::areAsynchronousInsertsEnabled() const +{ + return getSettings()->async_insert; +} + MergeTreeData::ProjectionPartsVector MergeTreeData::getAllProjectionPartsVector(MergeTreeData::DataPartStateVector * out_states) const { ProjectionPartsVector res; @@ -7393,6 +7399,13 @@ std::pair MergeTreeData::cloneAn return std::make_pair(dst_data_part, std::move(temporary_directory_lock)); } +bool MergeTreeData::canUseAdaptiveGranularity() const +{ + const auto settings = getSettings(); + return settings->index_granularity_bytes != 0 + && (settings->enable_mixed_granularity_parts || !has_non_adaptive_index_granularity_parts); +} + String MergeTreeData::getFullPathOnDisk(const DiskPtr & disk) const { return disk->getPath() + relative_data_path; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index c8b721038c6..7076b680521 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -65,6 +64,7 @@ using BackupEntries = std::vector; +struct MergeTreeSettings; struct WriteSettings; /// Auxiliary struct holding information about the future merged or mutated part. @@ -441,7 +441,7 @@ public: bool hasProjection() const override; - bool areAsynchronousInsertsEnabled() const override { return getSettings()->async_insert; } + bool areAsynchronousInsertsEnabled() const override; bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override; @@ -860,12 +860,7 @@ public: /// Returns true if table can create new parts with adaptive granularity /// Has additional constraint in replicated version - virtual bool canUseAdaptiveGranularity() const - { - const auto settings = getSettings(); - return settings->index_granularity_bytes != 0 && - (settings->enable_mixed_granularity_parts || !has_non_adaptive_index_granularity_parts); - } + virtual bool canUseAdaptiveGranularity() const; /// Get constant pointer to storage settings. /// Copy this pointer into your scope and you will @@ -1735,14 +1730,6 @@ struct CurrentlySubmergingEmergingTagger ~CurrentlySubmergingEmergingTagger(); }; - -/// TODO: move it somewhere -[[ maybe_unused ]] static bool needSyncPart(size_t input_rows, size_t input_bytes, const MergeTreeSettings & settings) -{ - return ((settings.min_rows_to_fsync_after_merge && input_rows >= settings.min_rows_to_fsync_after_merge) - || (settings.min_compressed_bytes_to_fsync_after_merge && input_bytes >= settings.min_compressed_bytes_to_fsync_after_merge)); -} - /// Look at MutationCommands if it contains mutations for AlterConversions, update the counter. /// Return true if the counter had been updated bool updateAlterConversionsMutations(const MutationCommands & commands, std::atomic & alter_conversions_mutations, bool remove); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 2d49e1df19b..3c223b8d748 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index 5c9191dbb54..6dc7e649b06 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 74ea89a8864..3fbabe1dd52 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 2e287ff3042..758d6ad6140 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 5c8aa32949d..bf05f3afaf4 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.cpp b/src/Storages/MergeTree/MergeTreeIOSettings.cpp new file mode 100644 index 00000000000..58c3bd28d6a --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeIOSettings.cpp @@ -0,0 +1,37 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +MergeTreeWriterSettings::MergeTreeWriterSettings( + const Settings & global_settings, + const WriteSettings & query_write_settings_, + const MergeTreeSettingsPtr & storage_settings, + bool can_use_adaptive_granularity_, + bool rewrite_primary_key_, + bool blocks_are_granules_size_) + : min_compress_block_size( + storage_settings->min_compress_block_size ? storage_settings->min_compress_block_size : global_settings.min_compress_block_size) + , max_compress_block_size( + storage_settings->max_compress_block_size ? storage_settings->max_compress_block_size : global_settings.max_compress_block_size) + , marks_compression_codec(storage_settings->marks_compression_codec) + , marks_compress_block_size(storage_settings->marks_compress_block_size) + , compress_primary_key(storage_settings->compress_primary_key) + , primary_key_compression_codec(storage_settings->primary_key_compression_codec) + , primary_key_compress_block_size(storage_settings->primary_key_compress_block_size) + , can_use_adaptive_granularity(can_use_adaptive_granularity_) + , rewrite_primary_key(rewrite_primary_key_) + , blocks_are_granules_size(blocks_are_granules_size_) + , query_write_settings(query_write_settings_) + , max_threads_for_annoy_index_creation(global_settings.max_threads_for_annoy_index_creation) + , low_cardinality_max_dictionary_size(global_settings.low_cardinality_max_dictionary_size) + , low_cardinality_use_single_dictionary_for_part(global_settings.low_cardinality_use_single_dictionary_for_part != 0) + , use_compact_variant_discriminators_serialization(storage_settings->use_compact_variant_discriminators_serialization) +{ +} + +} diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index 04171656fcf..c79ca1e66ee 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -1,15 +1,17 @@ #pragma once #include -#include -#include -#include #include #include +#include namespace DB { +struct MergeTreeSettings; +using MergeTreeSettingsPtr = std::shared_ptr; +struct Settings; + class MMappedFileCache; using MMappedFileCachePtr = std::shared_ptr; @@ -58,27 +60,7 @@ struct MergeTreeWriterSettings const MergeTreeSettingsPtr & storage_settings, bool can_use_adaptive_granularity_, bool rewrite_primary_key_, - bool blocks_are_granules_size_ = false) - : min_compress_block_size( - storage_settings->min_compress_block_size ? storage_settings->min_compress_block_size : global_settings.min_compress_block_size) - , max_compress_block_size( - storage_settings->max_compress_block_size ? storage_settings->max_compress_block_size - : global_settings.max_compress_block_size) - , marks_compression_codec(storage_settings->marks_compression_codec) - , marks_compress_block_size(storage_settings->marks_compress_block_size) - , compress_primary_key(storage_settings->compress_primary_key) - , primary_key_compression_codec(storage_settings->primary_key_compression_codec) - , primary_key_compress_block_size(storage_settings->primary_key_compress_block_size) - , can_use_adaptive_granularity(can_use_adaptive_granularity_) - , rewrite_primary_key(rewrite_primary_key_) - , blocks_are_granules_size(blocks_are_granules_size_) - , query_write_settings(query_write_settings_) - , max_threads_for_annoy_index_creation(global_settings.max_threads_for_annoy_index_creation) - , low_cardinality_max_dictionary_size(global_settings.low_cardinality_max_dictionary_size) - , low_cardinality_use_single_dictionary_for_part(global_settings.low_cardinality_use_single_dictionary_for_part != 0) - , use_compact_variant_discriminators_serialization(storage_settings->use_compact_variant_discriminators_serialization) - { - } + bool blocks_are_granules_size_ = false); size_t min_compress_block_size; size_t max_compress_block_size; diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index 62bb31dff68..067a692a3b5 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -1,5 +1,6 @@ -#include #include +#include +#include namespace fs = std::filesystem; diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 1db70162bff..9223d6fd5b1 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -1,5 +1,6 @@ -#include #include +#include +#include #include #include diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 02f8d6f4f6a..ecf7ac08294 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 5d6f08d3c53..c968ad84936 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -115,6 +115,16 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def, ContextPtr conte #undef ADD_IF_ABSENT } +bool MergeTreeSettings::isReadonlySetting(const String & name) +{ + return name == "index_granularity" || name == "index_granularity_bytes" || name == "enable_mixed_granularity_parts"; +} + +bool MergeTreeSettings::isPartFormatSetting(const String & name) +{ + return name == "min_bytes_for_wide_part" || name == "min_rows_for_wide_part"; +} + void MergeTreeSettings::sanityCheck(size_t background_pool_tasks) const { if (number_of_free_entries_in_pool_to_execute_mutation > background_pool_tasks) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index c0afd781c7e..f5ada81cf55 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -268,17 +268,8 @@ struct MergeTreeSettings : public BaseSettings, public /// NOTE: will rewrite the AST to add immutable settings. void loadFromQuery(ASTStorage & storage_def, ContextPtr context, bool is_attach); - /// We check settings after storage creation - static bool isReadonlySetting(const String & name) - { - return name == "index_granularity" || name == "index_granularity_bytes" - || name == "enable_mixed_granularity_parts"; - } - - static bool isPartFormatSetting(const String & name) - { - return name == "min_bytes_for_wide_part" || name == "min_rows_for_wide_part"; - } + static bool isReadonlySetting(const String & name); + static bool isPartFormatSetting(const String & name); /// Check that the values are sane taking also query-level settings into account. void sanityCheck(size_t background_pool_tasks) const; @@ -295,4 +286,10 @@ namespace MergeTreeColumnSettings void validate(const SettingsChanges & changes); } +[[maybe_unused]] static bool needSyncPart(size_t input_rows, size_t input_bytes, const MergeTreeSettings & settings) +{ + return ( + (settings.min_rows_to_fsync_after_merge && input_rows >= settings.min_rows_to_fsync_after_merge) + || (settings.min_compressed_bytes_to_fsync_after_merge && input_bytes >= settings.min_compressed_bytes_to_fsync_after_merge)); +} } diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 4c96cbf2c97..73084f487b9 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index a552ee89aee..35333af4b03 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -19,11 +20,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -32,6 +35,7 @@ #include + namespace ProfileEvents { extern const Event MutateTaskProjectionsCalculationMicroseconds; diff --git a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp index 4228d7b70b6..10c7bef72fc 100644 --- a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp +++ b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp index 2fc5238827d..24929365b72 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp @@ -1,8 +1,9 @@ #include -#include #include +#include #include +#include #include diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index 336d19692d4..6e22a3515bc 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 328c03a5b94..7aef249c366 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -1,13 +1,14 @@ +#include +#include #include #include -#include -#include #include #include #include #include +#include namespace DB diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp index 192f0d23f96..3e64a4c7c52 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp @@ -1,6 +1,7 @@ +#include +#include #include #include -#include #include #include diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index d7601e6e638..dc242a7b084 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 30ba95c46f0..627bda3f8bf 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 35f355d1d9b..268cead2a0e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 4b4f4c33e7d..0fd7d93b9aa 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 287a4d20543..c4bae5352cb 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index a94508ad41f..731c30f4b79 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -1,16 +1,12 @@ #pragma once +#include #include #include #include #include -#include -#include -#include -#include #include -#include -#include + namespace DB @@ -21,6 +17,8 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +class QueryPlan; + /// A Storage that allows reading from a single MergeTree data part. class StorageFromMergeTreeDataPart final : public IStorage { @@ -46,20 +44,7 @@ public: String getName() const override { return "FromMergeTreeDataPart"; } - StorageSnapshotPtr getStorageSnapshot( - const StorageMetadataPtr & metadata_snapshot, ContextPtr /*query_context*/) const override - { - const auto & storage_columns = metadata_snapshot->getColumns(); - if (!hasDynamicSubcolumns(storage_columns)) - return std::make_shared(*this, metadata_snapshot); - - auto data_parts = storage.getDataPartsVectorForInternalUsage(); - - auto object_columns = getConcreteObjectColumns( - data_parts.begin(), data_parts.end(), storage_columns, [](const auto & part) -> const auto & { return part->getColumns(); }); - - return std::make_shared(*this, metadata_snapshot, std::move(object_columns)); - } + StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr /*query_context*/) const override; void read( QueryPlan & query_plan, @@ -69,21 +54,7 @@ public: ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, - size_t num_streams) override - { - query_plan.addStep(MergeTreeDataSelectExecutor(storage) - .readFromParts( - parts, - alter_conversions, - column_names, - storage_snapshot, - query_info, - context, - max_block_size, - num_streams, - nullptr, - analysis_result_ptr)); - } + size_t num_streams) override; bool supportsPrewhere() const override { return true; } @@ -102,12 +73,7 @@ public: return storage.getPartitionIDFromQuery(ast, context); } - bool materializeTTLRecalculateOnly() const - { - if (parts.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "parts must not be empty for materializeTTLRecalculateOnly"); - return parts.front()->storage.getSettings()->materialize_ttl_recalculate_only; - } + bool materializeTTLRecalculateOnly() const; bool hasLightweightDeletedMask() const override { diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 525960d5314..774fd95ebc6 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index d234103e52b..0ada6fae220 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/StorageFromMergeTreeDataPart.cpp b/src/Storages/StorageFromMergeTreeDataPart.cpp new file mode 100644 index 00000000000..37e142d246a --- /dev/null +++ b/src/Storages/StorageFromMergeTreeDataPart.cpp @@ -0,0 +1,54 @@ +#include +#include +#include +#include + +namespace DB +{ + +bool StorageFromMergeTreeDataPart::materializeTTLRecalculateOnly() const +{ + if (parts.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "parts must not be empty for materializeTTLRecalculateOnly"); + return parts.front()->storage.getSettings()->materialize_ttl_recalculate_only; +} + +void StorageFromMergeTreeDataPart::read( + QueryPlan & query_plan, + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum /*processed_stage*/, + size_t max_block_size, + size_t num_streams) +{ + query_plan.addStep(MergeTreeDataSelectExecutor(storage).readFromParts( + parts, + alter_conversions, + column_names, + storage_snapshot, + query_info, + context, + max_block_size, + num_streams, + nullptr, + analysis_result_ptr)); +} + +StorageSnapshotPtr +StorageFromMergeTreeDataPart::getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr /*query_context*/) const +{ + const auto & storage_columns = metadata_snapshot->getColumns(); + if (!hasDynamicSubcolumns(storage_columns)) + return std::make_shared(*this, metadata_snapshot); + + auto data_parts = storage.getDataPartsVectorForInternalUsage(); + + auto object_columns = getConcreteObjectColumns( + data_parts.begin(), data_parts.end(), storage_columns, [](const auto & part) -> const auto & { return part->getColumns(); }); + + return std::make_shared(*this, metadata_snapshot, std::move(object_columns)); +} + +} diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 20ad064f1fc..65afd9c3230 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 13b7cc582a9..af0dd59213c 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemMergeTreeSettings.cpp b/src/Storages/System/StorageSystemMergeTreeSettings.cpp index 7781e3789a4..db8fd55cfc5 100644 --- a/src/Storages/System/StorageSystemMergeTreeSettings.cpp +++ b/src/Storages/System/StorageSystemMergeTreeSettings.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/System/StorageSystemMergeTreeSettings.h b/src/Storages/System/StorageSystemMergeTreeSettings.h index e2913a7e55b..4d4150dcec2 100644 --- a/src/Storages/System/StorageSystemMergeTreeSettings.h +++ b/src/Storages/System/StorageSystemMergeTreeSettings.h @@ -1,6 +1,5 @@ #pragma once -#include #include From b60151c1d8e8e2d0fe5ea0946bdbdbfae9abecc1 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 10 Jul 2024 10:13:49 +0000 Subject: [PATCH 066/161] Remove optimize_functions_to_subcolumns setting from test --- tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh index d357419f03a..aa06e48376c 100755 --- a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh +++ b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh @@ -7,7 +7,7 @@ CLICKHOUSE_LOG_COMMENT= # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 optimize_functions_to_subcolumns=0" +CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1" function test() From 314174013b95c4bff17ed05b835391c5c4290834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 10 Jul 2024 18:10:19 +0200 Subject: [PATCH 067/161] Reduce dependencies on ServerSettings.h --- programs/local/LocalServer.h | 7 ++++--- src/Backups/DDLAdjustingForBackupVisitor.cpp | 6 +++--- src/BridgeHelper/LibraryBridgeHelper.cpp | 1 + src/Databases/DatabaseOrdinary.cpp | 1 + src/Databases/DatabaseReplicated.cpp | 1 + src/Dictionaries/HTTPDictionarySource.cpp | 3 ++- src/Dictionaries/XDBCDictionarySource.cpp | 1 + src/Disks/IO/ReadBufferFromWebServer.cpp | 6 ++++-- src/Disks/ObjectStorages/Web/WebObjectStorage.cpp | 2 ++ src/IO/ConnectionTimeouts.cpp | 4 +++- src/IO/ConnectionTimeouts.h | 2 +- src/IO/S3/copyS3File.cpp | 6 +++--- src/Interpreters/InterpreterSystemQuery.cpp | 1 + src/Server/HTTP/sendExceptionToHTTPClient.cpp | 1 + src/Server/HTTPHandler.cpp | 1 + src/Server/InterserverIOHTTPHandler.cpp | 1 + src/Server/NotFoundHandler.cpp | 1 + src/Server/ReplicasStatusHandler.cpp | 1 + src/Server/StaticRequestHandler.cpp | 5 +++-- src/Server/WebUIRequestHandler.cpp | 3 ++- src/Storages/MergeTree/registerStorageMergeTree.cpp | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 1 + src/Storages/StorageURL.cpp | 1 + src/Storages/StorageXDBC.cpp | 1 + src/TableFunctions/ITableFunctionXDBC.cpp | 7 ++++--- 25 files changed, 45 insertions(+), 20 deletions(-) diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index da2466650a7..0715f358313 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -3,11 +3,12 @@ #include #include -#include -#include -#include +#include #include #include +#include +#include +#include #include #include diff --git a/src/Backups/DDLAdjustingForBackupVisitor.cpp b/src/Backups/DDLAdjustingForBackupVisitor.cpp index 7e5ce91629b..910831195a3 100644 --- a/src/Backups/DDLAdjustingForBackupVisitor.cpp +++ b/src/Backups/DDLAdjustingForBackupVisitor.cpp @@ -1,11 +1,11 @@ #include +#include +#include #include #include #include -#include -#include - #include +#include namespace DB diff --git a/src/BridgeHelper/LibraryBridgeHelper.cpp b/src/BridgeHelper/LibraryBridgeHelper.cpp index 84bfe096e79..57baf7e5a33 100644 --- a/src/BridgeHelper/LibraryBridgeHelper.cpp +++ b/src/BridgeHelper/LibraryBridgeHelper.cpp @@ -1,5 +1,6 @@ #include "LibraryBridgeHelper.h" +#include #include namespace DB diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 46830861c75..6555c4444e2 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 4ca9afc49eb..7b8955f7111 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index dae8ec06d30..663c63dd6c6 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -1,12 +1,13 @@ #include "HTTPDictionarySource.h" +#include #include #include #include #include #include #include -#include #include +#include #include #include #include diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index 1ebfc4a29b0..65a9f806722 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -15,6 +15,7 @@ #include "readInvalidateQuery.h" #include "registerDictionaries.h" #include +#include #include #include #include "config.h" diff --git a/src/Disks/IO/ReadBufferFromWebServer.cpp b/src/Disks/IO/ReadBufferFromWebServer.cpp index 7c5de8a13de..af4fae6b723 100644 --- a/src/Disks/IO/ReadBufferFromWebServer.cpp +++ b/src/Disks/IO/ReadBufferFromWebServer.cpp @@ -1,9 +1,11 @@ #include "ReadBufferFromWebServer.h" -#include +#include +#include #include #include -#include +#include + #include diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp index e837e056acc..7f7a3fe1a62 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp @@ -3,6 +3,8 @@ #include #include +#include + #include #include #include diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index da6214ae477..c2eb67c6f3b 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -1,7 +1,9 @@ +#include #include -#include #include +#include + namespace DB { diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index b86ec44d21c..b84db46d656 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include @@ -10,6 +9,7 @@ namespace DB { +struct ServerSettings; struct Settings; #define APPLY_FOR_ALL_CONNECTION_TIMEOUT_MEMBERS(M) \ diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index bb654c3f5c9..db10b4c2086 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -290,9 +290,9 @@ namespace if (!total_size) throw Exception(ErrorCodes::LOGICAL_ERROR, "Chosen multipart upload for an empty file. This must not happen"); - auto max_part_number = request_settings.max_part_number; - auto min_upload_part_size = request_settings.min_upload_part_size; - auto max_upload_part_size = request_settings.max_upload_part_size; + UInt64 max_part_number = request_settings.max_part_number; + UInt64 min_upload_part_size = request_settings.min_upload_part_size; + UInt64 max_upload_part_size = request_settings.max_upload_part_size; if (!max_part_number) throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "max_part_number must not be 0"); diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index d3526941b33..c284acfa308 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/HTTP/sendExceptionToHTTPClient.cpp b/src/Server/HTTP/sendExceptionToHTTPClient.cpp index 78650758e35..022a763a9a2 100644 --- a/src/Server/HTTP/sendExceptionToHTTPClient.cpp +++ b/src/Server/HTTP/sendExceptionToHTTPClient.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 370af79e456..75901e6b901 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/InterserverIOHTTPHandler.cpp b/src/Server/InterserverIOHTTPHandler.cpp index 45c28babe3a..e46021c8e68 100644 --- a/src/Server/InterserverIOHTTPHandler.cpp +++ b/src/Server/InterserverIOHTTPHandler.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include diff --git a/src/Server/NotFoundHandler.cpp b/src/Server/NotFoundHandler.cpp index 38f56921c89..304021ee8dc 100644 --- a/src/Server/NotFoundHandler.cpp +++ b/src/Server/NotFoundHandler.cpp @@ -1,6 +1,7 @@ #include #include +#include #include namespace DB diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index 5a29612c263..f43357db0a8 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Server/StaticRequestHandler.cpp b/src/Server/StaticRequestHandler.cpp index 331b7a84857..f3981dea9fb 100644 --- a/src/Server/StaticRequestHandler.cpp +++ b/src/Server/StaticRequestHandler.cpp @@ -4,13 +4,14 @@ #include "HTTPHandlerFactory.h" #include "HTTPResponseHeaderWriter.h" +#include #include #include #include -#include #include -#include +#include #include +#include #include diff --git a/src/Server/WebUIRequestHandler.cpp b/src/Server/WebUIRequestHandler.cpp index 68d3ff0b325..a3d098014e7 100644 --- a/src/Server/WebUIRequestHandler.cpp +++ b/src/Server/WebUIRequestHandler.cpp @@ -5,8 +5,9 @@ #include #include -#include +#include #include +#include #include #include diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 0ada6fae220..343c2c1008e 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index af0dd59213c..dffc5f08aa2 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 895da028fc2..cd1282bb02c 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index fb8fa2d6da4..b1bcae0f7e9 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/src/TableFunctions/ITableFunctionXDBC.cpp b/src/TableFunctions/ITableFunctionXDBC.cpp index a5c16b3a5aa..043712f97b8 100644 --- a/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/src/TableFunctions/ITableFunctionXDBC.cpp @@ -1,8 +1,9 @@ +#include #include -#include -#include -#include #include +#include +#include +#include #include #include #include From d2f89a044a41eab1ffd1bf6241525e022c5143cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 10 Jul 2024 18:20:39 +0200 Subject: [PATCH 068/161] Reduce dependencies on CoordinationSettings.h --- src/Coordination/FourLetterCommand.cpp | 1 + src/Coordination/KeeperDispatcher.cpp | 3 ++- src/Coordination/KeeperDispatcher.h | 1 - src/Coordination/KeeperServer.cpp | 8 +++++--- src/Coordination/KeeperServer.h | 4 +++- src/Coordination/KeeperStateMachine.cpp | 1 + src/Coordination/KeeperStateMachine.h | 1 - src/Coordination/KeeperStateManager.h | 5 ++--- src/Coordination/KeeperStorage.cpp | 3 ++- src/Coordination/tests/gtest_coordination.cpp | 1 + src/Server/KeeperReadinessHandler.cpp | 1 + src/Server/KeeperTCPHandler.cpp | 1 + 12 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/Coordination/FourLetterCommand.cpp b/src/Coordination/FourLetterCommand.cpp index 8de9f8dfa1c..d0ecbe744a0 100644 --- a/src/Coordination/FourLetterCommand.cpp +++ b/src/Coordination/FourLetterCommand.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 6f57fa6d2e2..097103ff378 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -5,7 +5,7 @@ #include #include -#include "Common/ZooKeeper/IKeeper.h" +#include #include #include #include @@ -13,6 +13,7 @@ #include #include #include +#include #include diff --git a/src/Coordination/KeeperDispatcher.h b/src/Coordination/KeeperDispatcher.h index 2e0c73131d5..4e2b9d9e247 100644 --- a/src/Coordination/KeeperDispatcher.h +++ b/src/Coordination/KeeperDispatcher.h @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 09a1132e96d..dc9658e895f 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -3,9 +3,7 @@ #include "config.h" -#include -#include -#include +#include #include #include #include @@ -30,6 +28,10 @@ #include #include +#include +#include +#include + #pragma clang diagnostic ignored "-Wdeprecated-declarations" #include diff --git a/src/Coordination/KeeperServer.h b/src/Coordination/KeeperServer.h index 5e45a552cba..2eb630b4dd6 100644 --- a/src/Coordination/KeeperServer.h +++ b/src/Coordination/KeeperServer.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -17,6 +16,9 @@ namespace DB using RaftAppendResult = nuraft::ptr>>; +struct KeeperConfigurationAndSettings; +using KeeperConfigurationAndSettingsPtr = std::shared_ptr; + class KeeperServer { private: diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index e7cae714ba6..413d600717a 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index 7ea14aa2d30..6357fd170df 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include diff --git a/src/Coordination/KeeperStateManager.h b/src/Coordination/KeeperStateManager.h index 60f6dbe7b62..e5ab5fb4807 100644 --- a/src/Coordination/KeeperStateManager.h +++ b/src/Coordination/KeeperStateManager.h @@ -1,14 +1,13 @@ #pragma once -#include #include #include -#include +#include +#include #include #include #include "Coordination/KeeperStateMachine.h" #include "Coordination/RaftServerConfig.h" -#include namespace DB { diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 1542eb0d71a..efb57ae96e2 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -20,11 +20,12 @@ #include #include +#include #include #include +#include #include #include -#include #include #include diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index a329bec8e2a..b65e9358a72 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/KeeperReadinessHandler.cpp b/src/Server/KeeperReadinessHandler.cpp index c973be040c8..1972c3263b2 100644 --- a/src/Server/KeeperReadinessHandler.cpp +++ b/src/Server/KeeperReadinessHandler.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp index 47064b467e7..71ff320e2ff 100644 --- a/src/Server/KeeperTCPHandler.cpp +++ b/src/Server/KeeperTCPHandler.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include From b08be12252cddccb62279bae34a2978d9dc7832c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 10 Jul 2024 19:19:10 +0200 Subject: [PATCH 069/161] Remove dependency on SettingsFields from FormatSettings --- src/Common/ThreadFuzzer.cpp | 5 +++-- src/DataTypes/EnumValues.h | 2 +- src/Formats/FormatFactory.cpp | 2 +- src/Formats/FormatSettings.h | 6 +++--- src/Processors/Formats/Impl/PrettyBlockOutputFormat.h | 3 ++- src/Storages/StorageSet.cpp | 1 + src/Storages/StorageSet.h | 1 - 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/Common/ThreadFuzzer.cpp b/src/Common/ThreadFuzzer.cpp index 7b96d1f075b..719ac9593ff 100644 --- a/src/Common/ThreadFuzzer.cpp +++ b/src/Common/ThreadFuzzer.cpp @@ -12,13 +12,14 @@ #include #include -#include +#include #include #include +#include +#include #include -#include #include "config.h" // USE_JEMALLOC diff --git a/src/DataTypes/EnumValues.h b/src/DataTypes/EnumValues.h index 889878bc60f..9762ff5870f 100644 --- a/src/DataTypes/EnumValues.h +++ b/src/DataTypes/EnumValues.h @@ -7,7 +7,7 @@ namespace DB { -namespace ErrorCodesEnumValues +namespace ErrorCodes { extern const int BAD_ARGUMENTS; } diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 6de106893f3..484ea084d63 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -175,7 +175,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.parquet.write_page_index = settings.output_format_parquet_write_page_index; format_settings.parquet.local_read_min_bytes_for_seek = settings.input_format_parquet_local_file_min_bytes_for_seek; format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; - format_settings.pretty.color = settings.output_format_pretty_color; + format_settings.pretty.color = settings.output_format_pretty_color.valueOr(2); format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 446dc17a187..0f9b5e6f681 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -1,10 +1,9 @@ #pragma once -#include #include +#include #include #include -#include namespace DB { @@ -287,7 +286,8 @@ struct FormatSettings UInt64 max_value_width = 10000; UInt64 max_value_width_apply_for_single_value = false; bool highlight_digit_groups = true; - SettingFieldUInt64Auto color{"auto"}; + /// Set to 2 for auto + UInt64 color = 2; bool output_format_pretty_row_numbers = false; UInt64 output_format_pretty_single_large_number_tip_threshold = 1'000'000; diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h index 4c52300fbd1..698efecd4b2 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -74,7 +74,8 @@ void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, cons const Block & sample, const FormatSettings & format_settings) { - bool color = !no_escapes && format_settings.pretty.color.valueOr(format_settings.is_writing_to_terminal); + bool color = !no_escapes + && (format_settings.pretty.color == 1 || (format_settings.pretty.color == 2 && format_settings.is_writing_to_terminal)); return std::make_shared(buf, sample, format_settings, mono_block, color); }); if (!mono_block) diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 5b7f9fc0ac2..22bd01e0071 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Storages/StorageSet.h b/src/Storages/StorageSet.h index 67a9528ff5e..5b6d899b48d 100644 --- a/src/Storages/StorageSet.h +++ b/src/Storages/StorageSet.h @@ -2,7 +2,6 @@ #include #include -#include namespace DB From 71afe96e0f40cdbf260929d3460fca6f250c2fba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 10 Jul 2024 19:43:50 +0200 Subject: [PATCH 070/161] Less deps for SettingsEnums.h --- src/Core/SettingsEnums.h | 8 +------- src/Core/ShortCircuitFunctionEvaluation.h | 15 +++++++++++++++ src/Interpreters/ExpressionActionsSettings.h | 2 +- .../ObjectStorageQueueMetadata.h | 6 ++---- 4 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 src/Core/ShortCircuitFunctionEvaluation.h diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 67fbce31be8..8c2f5364448 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -292,13 +293,6 @@ enum class StreamingHandleErrorMode : uint8_t DECLARE_SETTING_ENUM(StreamingHandleErrorMode) -enum class ShortCircuitFunctionEvaluation : uint8_t -{ - ENABLE, // Use short-circuit function evaluation for functions that are suitable for it. - FORCE_ENABLE, // Use short-circuit function evaluation for all functions. - DISABLE, // Disable short-circuit function evaluation. -}; - DECLARE_SETTING_ENUM(ShortCircuitFunctionEvaluation) enum class TransactionsWaitCSNMode : uint8_t diff --git a/src/Core/ShortCircuitFunctionEvaluation.h b/src/Core/ShortCircuitFunctionEvaluation.h new file mode 100644 index 00000000000..3785efe8015 --- /dev/null +++ b/src/Core/ShortCircuitFunctionEvaluation.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace DB +{ + +enum class ShortCircuitFunctionEvaluation : uint8_t +{ + ENABLE, // Use short-circuit function evaluation for functions that are suitable for it. + FORCE_ENABLE, // Use short-circuit function evaluation for all functions. + DISABLE, // Disable short-circuit function evaluation. +}; + +} diff --git a/src/Interpreters/ExpressionActionsSettings.h b/src/Interpreters/ExpressionActionsSettings.h index 326de983748..384a61a54a2 100644 --- a/src/Interpreters/ExpressionActionsSettings.h +++ b/src/Interpreters/ExpressionActionsSettings.h @@ -1,7 +1,7 @@ #pragma once +#include #include -#include #include diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h index 05060931b5a..e5fae047ac5 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h @@ -2,21 +2,19 @@ #include "config.h" #include -#include -#include #include -#include +#include #include #include #include #include +#include namespace fs = std::filesystem; namespace Poco { class Logger; } namespace DB { -struct ObjectStorageQueueSettings; class StorageObjectStorageQueue; struct ObjectStorageQueueTableMetadata; struct StorageInMemoryMetadata; From 4729161f4187e887e10757731b2381673234b43a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 10 Jul 2024 18:41:48 +0000 Subject: [PATCH 071/161] Collect logs from minio --- docker/test/stateful/run.sh | 5 +++++ docker/test/stateless/run.sh | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 2215ac2b37c..ce637b9a146 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -20,7 +20,10 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test /usr/share/clickhouse-test/config/install.sh azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence & + ./setup_minio.sh stateful +mc admin trace clickminio > /test_output/rubbish.log & +MC_ADMIN_PID=$! config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml @@ -251,6 +254,8 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||: fi +# Kill minio admin client to stop collecting logs +kill $MC_ADMIN_PID rg -Fa "" /var/log/clickhouse-server/clickhouse-server.log ||: zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst ||: diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 029c5a03151..a01abe5929f 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -46,6 +46,9 @@ source /utils.lib /usr/share/clickhouse-test/config/install.sh ./setup_minio.sh stateless +mc admin trace clickminio > /test_output/rubbish.log & +MC_ADMIN_PID=$! + ./setup_hdfs_minicluster.sh config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml @@ -325,6 +328,9 @@ if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||: fi +# Kill minio admin client to stop collecting logs +kill $MC_ADMIN_PID + rg -Fa "" /var/log/clickhouse-server/clickhouse-server.log ||: rg -A50 -Fa "============" /var/log/clickhouse-server/stderr.log ||: zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst & From a9a227ddc614834eb2943a20bf0f2fa722549fec Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 11 Jul 2024 07:53:54 +0200 Subject: [PATCH 072/161] Fix "Sending a batch of X files to Y (0.00 rows, 0.00 B bytes)." in case of batch restoring Previously it was always zeros. Signed-off-by: Azat Khuzhin --- .../Distributed/DistributedAsyncInsertBatch.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp index 06d4c185840..e1facec5b40 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp @@ -196,6 +196,16 @@ void DistributedAsyncInsertBatch::readText(ReadBuffer & in) UInt64 idx; in >> idx >> "\n"; files.push_back(std::filesystem::absolute(fmt::format("{}/{}.bin", parent.path, idx)).string()); + + ReadBufferFromFile header_buffer(files.back()); + const DistributedAsyncInsertHeader & header = DistributedAsyncInsertHeader::read(header_buffer, parent.log); + total_bytes += total_bytes; + + if (header.rows) + { + total_rows += header.rows; + total_bytes += header.bytes; + } } recovered = true; From 37c66c8976da45e1984e22e55a3932e02e7937cc Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 11 Jul 2024 08:04:44 +0200 Subject: [PATCH 073/161] Fix 03030_system_flush_distributed_settings flakiness Signed-off-by: Azat Khuzhin --- .../03030_system_flush_distributed_settings.reference | 2 +- .../03030_system_flush_distributed_settings.sql | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/03030_system_flush_distributed_settings.reference b/tests/queries/0_stateless/03030_system_flush_distributed_settings.reference index 5caff40c4a0..3a05c8b3ee8 100644 --- a/tests/queries/0_stateless/03030_system_flush_distributed_settings.reference +++ b/tests/queries/0_stateless/03030_system_flush_distributed_settings.reference @@ -1 +1 @@ -10000 +30000 diff --git a/tests/queries/0_stateless/03030_system_flush_distributed_settings.sql b/tests/queries/0_stateless/03030_system_flush_distributed_settings.sql index ac64135b593..fac673a4fe4 100644 --- a/tests/queries/0_stateless/03030_system_flush_distributed_settings.sql +++ b/tests/queries/0_stateless/03030_system_flush_distributed_settings.sql @@ -6,15 +6,17 @@ drop table if exists dist_out; create table ephemeral (key Int, value Int) engine=Null(); create table dist_in as ephemeral engine=Distributed(test_shard_localhost, currentDatabase(), ephemeral, key) settings background_insert_batch=1; -create table data (key Int, uniq_values Int) engine=Memory(); -create materialized view mv to data as select key, uniqExact(value) uniq_values from ephemeral group by key; +create table data (key Int, uniq_values Int) engine=TinyLog(); +create materialized view mv to data as select key, uniqExact(value::String) uniq_values from ephemeral group by key; system stop distributed sends dist_in; create table dist_out as data engine=Distributed(test_shard_localhost, currentDatabase(), data); set prefer_localhost_replica=0; SET optimize_trivial_insert_select = 1; -insert into dist_in select number/100, number from system.numbers limit 1e6 settings max_memory_usage='20Mi'; +-- due to pushing to MV with aggregation the query needs ~300MiB +-- but it will be done in background via "system flush distributed" +insert into dist_in select number/100, number from system.numbers limit 3e6 settings max_block_size=3e6, max_memory_usage='100Mi'; system flush distributed dist_in; -- { serverError MEMORY_LIMIT_EXCEEDED } system flush distributed dist_in settings max_memory_usage=0; select count() from dist_out; From 321a58c51931e8d39b8ba111108fed20ba0541cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Thu, 11 Jul 2024 08:33:10 +0000 Subject: [PATCH 074/161] Fix relative path to `mc` --- docker/test/stateful/run.sh | 2 +- docker/test/stateless/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index ce637b9a146..2efbed6aa4f 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -22,7 +22,7 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence & ./setup_minio.sh stateful -mc admin trace clickminio > /test_output/rubbish.log & +./mc admin trace clickminio > /test_output/rubbish.log & MC_ADMIN_PID=$! config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index a01abe5929f..d47728cddb9 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -46,7 +46,7 @@ source /utils.lib /usr/share/clickhouse-test/config/install.sh ./setup_minio.sh stateless -mc admin trace clickminio > /test_output/rubbish.log & +m./c admin trace clickminio > /test_output/rubbish.log & MC_ADMIN_PID=$! ./setup_hdfs_minicluster.sh From 9e599576ab01b66b4bd12a56a464fb77abd34a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 11 Jul 2024 14:00:05 +0200 Subject: [PATCH 075/161] Hide Settings object from Context.h --- programs/client/Client.cpp | 5 +- .../library-bridge/LibraryBridgeHandlers.cpp | 1 + programs/odbc-bridge/ColumnInfoHandler.cpp | 1 + .../odbc-bridge/IdentifierQuoteHandler.cpp | 1 + programs/odbc-bridge/MainHandler.cpp | 1 + programs/odbc-bridge/SchemaAllowedHandler.cpp | 2 + .../AggregateFunctionFactory.cpp | 1 + src/Analyzer/InDepthQueryTreeVisitor.h | 1 - .../AggregateFunctionOfGroupByKeysPass.cpp | 2 + ...egateFunctionsArithmericOperationsPass.cpp | 2 + src/Analyzer/Passes/ArrayExistsToHasPass.cpp | 2 + src/Analyzer/Passes/AutoFinalOnQueryPass.cpp | 2 + .../Passes/ConvertOrLikeChainPass.cpp | 2 + src/Analyzer/Passes/ConvertQueryToCNFPass.cpp | 2 + src/Analyzer/Passes/CountDistinctPass.cpp | 2 + src/Analyzer/Passes/CrossToInnerJoinPass.cpp | 3 +- .../Passes/FunctionToSubcolumnsPass.cpp | 2 + src/Analyzer/Passes/FuseFunctionsPass.cpp | 1 + .../Passes/GroupingFunctionsResolvePass.cpp | 1 + src/Analyzer/Passes/IfChainToMultiIfPass.cpp | 1 + .../Passes/IfTransformStringsToEnumPass.cpp | 1 + .../Passes/LogicalExpressionOptimizerPass.cpp | 1 + src/Analyzer/Passes/MultiIfToIfPass.cpp | 1 + .../Passes/NormalizeCountVariantsPass.cpp | 1 + ...ateOrDateTimeConverterWithPreimagePass.cpp | 1 + .../OptimizeGroupByFunctionKeysPass.cpp | 1 + .../OptimizeGroupByInjectiveFunctionsPass.cpp | 1 + ...ptimizeRedundantFunctionsInOrderByPass.cpp | 1 + .../RewriteAggregateFunctionWithIfPass.cpp | 2 + .../RewriteSumFunctionWithSumAndCountPass.cpp | 1 + src/Analyzer/Passes/SumIfToCountIfPass.cpp | 15 +- .../UniqInjectiveFunctionsEliminationPass.cpp | 2 + src/Analyzer/Passes/UniqToCountPass.cpp | 2 + src/Analyzer/QueryTreeBuilder.cpp | 2 + .../Resolve/IdentifierResolveScope.cpp | 3 +- src/Analyzer/Resolve/IdentifierResolver.cpp | 2 + src/Analyzer/Resolve/QueryAnalyzer.cpp | 2 + src/Analyzer/TableNode.cpp | 2 + src/Analyzer/Utils.cpp | 2 + src/Backups/BackupEntriesCollector.cpp | 1 + src/Backups/BackupIO_S3.cpp | 1 + src/Backups/BackupsWorker.cpp | 1 + src/Backups/RestorerFromBackup.cpp | 3 +- src/Backups/WithRetries.cpp | 4 +- src/Bridge/IBridge.cpp | 1 + src/BridgeHelper/LibraryBridgeHelper.cpp | 1 + src/Client/ClientBase.h | 1 + src/Client/ConnectionEstablisher.cpp | 1 + src/Client/ConnectionPool.cpp | 12 + src/Client/ConnectionPool.h | 13 +- src/Client/HedgedConnections.cpp | 1 + src/Client/HedgedConnectionsFactory.h | 3 +- src/Client/MultiplexedConnections.cpp | 1 + src/Common/DateLUT.cpp | 2 + .../NamedCollectionsFactory.cpp | 1 + .../NamedCollectionsMetadataStorage.cpp | 1 + src/Core/ExternalTable.cpp | 1 + src/Daemon/BaseDaemon.cpp | 1 + src/DataTypes/DataTypeDecimalBase.cpp | 1 + src/DataTypes/DataTypeFactory.cpp | 1 + src/Databases/DDLDependencyVisitor.cpp | 1 + src/Databases/DatabaseAtomic.cpp | 3 +- src/Databases/DatabaseDictionary.cpp | 1 + src/Databases/DatabaseFactory.cpp | 1 + src/Databases/DatabaseFilesystem.cpp | 1 + src/Databases/DatabaseHDFS.cpp | 1 + src/Databases/DatabaseOnDisk.cpp | 1 + src/Databases/DatabaseReplicated.cpp | 1 + src/Databases/DatabaseReplicatedWorker.cpp | 1 + src/Databases/DatabaseS3.cpp | 1 + src/Databases/MySQL/DatabaseMySQL.cpp | 1 + .../MySQL/FetchTablesColumnsList.cpp | 1 + src/Databases/MySQL/FetchTablesColumnsList.h | 3 +- src/Databases/MySQL/MaterializeMetadata.h | 1 + .../PostgreSQL/DatabasePostgreSQL.cpp | 1 + src/Databases/SQLite/DatabaseSQLite.cpp | 1 + src/Dictionaries/DirectDictionary.cpp | 1 + .../ExecutablePoolDictionarySource.cpp | 2 + src/Dictionaries/HashedArrayDictionary.cpp | 1 + src/Dictionaries/IPAddressDictionary.cpp | 7 +- .../PolygonDictionaryImplementations.cpp | 1 + .../PostgreSQLDictionarySource.cpp | 1 + src/Dictionaries/RegExpTreeDictionary.cpp | 1 + src/Dictionaries/XDBCDictionarySource.cpp | 1 + .../registerCacheDictionaries.cpp | 1 + src/Dictionaries/registerHashedDictionary.cpp | 2 +- .../registerRangeHashedDictionary.cpp | 1 + src/Disks/IO/ReadBufferFromWebServer.cpp | 1 + .../AzureBlobStorageCommon.cpp | 1 + .../AzureBlobStorage/AzureBlobStorageCommon.h | 3 +- .../ObjectStorages/ObjectStorageFactory.cpp | 1 + .../ObjectStorages/S3/S3ObjectStorage.cpp | 1 + src/Disks/ObjectStorages/S3/diskSettings.cpp | 5 +- src/Formats/FormatFactory.cpp | 1 + src/Formats/NumpyDataTypes.h | 1 + src/Formats/ReadSchemaUtils.cpp | 1 + src/Functions/CastOverloadResolver.cpp | 1 + src/Functions/FunctionBinaryArithmetic.h | 1 + .../FunctionCustomWeekToDateOrDate32.h | 1 + .../FunctionDateOrDateTimeToDateOrDate32.h | 1 + ...tionDateOrDateTimeToDateTimeOrDateTime64.h | 2 + src/Functions/FunctionFactory.cpp | 1 + .../FunctionGenerateRandomStructure.h | 1 + src/Functions/FunctionJoinGet.cpp | 1 + src/Functions/FunctionTokens.h | 1 + src/Functions/FunctionUnixTimestamp64.h | 1 + src/Functions/FunctionsCodingIP.cpp | 1 + src/Functions/FunctionsConversion.cpp | 1 + .../FunctionsMultiStringFuzzySearch.h | 1 + src/Functions/FunctionsMultiStringSearch.h | 1 + src/Functions/FunctionsStringSearch.h | 1 + src/Functions/FunctionsTextClassification.h | 1 + src/Functions/PerformanceAdaptors.h | 1 + ...alUserDefinedExecutableFunctionsLoader.cpp | 1 + .../UserDefinedSQLObjectsBackup.cpp | 1 + .../UserDefinedSQLObjectsDiskStorage.cpp | 2 + .../UserDefinedSQLObjectsZooKeeperStorage.cpp | 2 +- src/Functions/array/array.cpp | 1 + src/Functions/array/range.cpp | 1 + src/Functions/castOrDefault.cpp | 1 + src/Functions/formatDateTime.cpp | 1 + src/Functions/formatQuery.cpp | 1 + src/Functions/getClientHTTPHeader.cpp | 1 + src/Functions/getSetting.cpp | 1 + src/Functions/hasColumnInTable.cpp | 1 + src/Functions/if.cpp | 1 + .../keyvaluepair/extractKeyValuePairs.cpp | 1 + src/Functions/lemmatize.cpp | 1 + src/Functions/map.cpp | 1 + src/Functions/multiIf.cpp | 1 + src/Functions/neighbor.cpp | 1 + src/Functions/now.cpp | 1 + src/Functions/now64.cpp | 1 + src/Functions/nowInBlock.cpp | 1 + src/Functions/parseDateTime.cpp | 1 + src/Functions/pointInPolygon.cpp | 1 + src/Functions/runningAccumulate.cpp | 1 + src/Functions/runningDifference.h | 1 + src/Functions/serverConstants.cpp | 1 + src/Functions/sleep.h | 1 + src/Functions/snowflake.cpp | 1 + src/Functions/snowflakeIDToDateTime.cpp | 1 + src/Functions/stem.cpp | 1 + src/Functions/synonyms.cpp | 1 + src/Functions/throwIf.cpp | 1 + src/Functions/toTimezone.cpp | 1 + src/Functions/toTypeName.cpp | 1 + src/Functions/tuple.h | 1 + src/Functions/visibleWidth.cpp | 1 + src/Functions/widthBucket.cpp | 2 + src/IO/ConnectionTimeouts.cpp | 1 + src/IO/S3/Client.cpp | 3 +- src/IO/S3Common.cpp | 1 + src/IO/S3Common.h | 4 +- src/IO/S3Settings.cpp | 4 +- src/Interpreters/ArrayJoinAction.cpp | 1 + .../Cache/WriteBufferToFileSegment.cpp | 2 + src/Interpreters/Context.cpp | 258 ++++++++++++------ src/Interpreters/Context.h | 44 +-- src/Interpreters/DDLTask.cpp | 1 + src/Interpreters/DDLTask.h | 1 + .../ExternalDictionariesLoader.cpp | 1 + src/Interpreters/FilesystemCacheLog.h | 1 - src/Interpreters/GraceHashJoin.cpp | 1 + src/Interpreters/IInterpreter.cpp | 1 + .../IInterpreterUnionOrSelectQuery.h | 2 + src/Interpreters/IdentifierSemantic.cpp | 2 + .../InJoinSubqueriesPreprocessor.cpp | 1 + src/Interpreters/InterpreterCheckQuery.cpp | 2 + .../InterpreterCreateIndexQuery.cpp | 1 + .../InterpreterDropIndexQuery.cpp | 1 + .../InterpreterKillQueryQuery.cpp | 1 + src/Interpreters/InterpreterRenameQuery.cpp | 1 + .../InterpreterShowColumnsQuery.cpp | 1 + .../InterpreterTransactionControlQuery.cpp | 1 + .../MutationsNonDeterministicHelpers.cpp | 1 + .../NormalizeSelectWithUnionQueryVisitor.h | 5 +- .../PredicateExpressionsOptimizer.cpp | 1 + src/Interpreters/QueryLog.cpp | 1 + src/Interpreters/QueryLog.h | 4 +- .../RewriteCountVariantsVisitor.cpp | 1 + src/Interpreters/Session.cpp | 1 + src/Interpreters/ThreadStatusExt.cpp | 1 + .../evaluateConstantExpression.cpp | 2 + src/Interpreters/executeDDLQueryOnCluster.cpp | 1 + .../formatWithPossiblyHidingSecrets.h | 2 + .../getCustomKeyFilterForParallelReplicas.cpp | 2 + src/Interpreters/loadMetadata.cpp | 1 + .../parseColumnsListForTableFunction.cpp | 13 + .../parseColumnsListForTableFunction.h | 13 +- .../removeOnClusterClauseIfNeeded.cpp | 1 + src/Planner/CollectSets.cpp | 1 + src/Planner/Planner.cpp | 1 + src/Planner/PlannerActionsVisitor.cpp | 2 + src/Planner/PlannerExpressionAnalysis.cpp | 2 + src/Planner/PlannerJoinTree.cpp | 2 + src/Planner/PlannerJoins.cpp | 2 + src/Planner/PlannerSorting.cpp | 2 + src/Planner/PlannerWindowFunctions.cpp | 2 + src/Planner/Utils.cpp | 2 + src/Planner/findParallelReplicasQuery.cpp | 1 + src/Processors/Executors/PipelineExecutor.cpp | 1 + .../Impl/ConstantExpressionTemplate.cpp | 1 + .../Formats/Impl/ValuesBlockInputFormat.cpp | 1 + .../Formats/InputFormatErrorsLogger.cpp | 1 + .../Optimizations/optimizePrewhere.cpp | 2 + .../optimizeUseNormalProjection.cpp | 2 + src/Processors/QueryPlan/ReadFromRemote.cpp | 1 + src/Processors/QueryPlan/SortingStep.cpp | 4 +- src/Processors/Sources/MySQLSource.cpp | 2 + src/Processors/Sources/MySQLSource.h | 3 +- src/Processors/Sources/RecursiveCTESource.cpp | 2 + .../TTL/TTLAggregationAlgorithm.cpp | 3 +- .../getSourceFromASTInsertQuery.cpp | 3 +- src/QueryPipeline/RemoteQueryExecutor.cpp | 7 + src/QueryPipeline/RemoteQueryExecutor.h | 2 +- src/Server/GRPCServer.cpp | 1 + .../setReadOnlyIfHTTPMethodIdempotent.cpp | 1 + src/Server/HTTPHandler.cpp | 1 + src/Server/MySQLHandler.cpp | 1 + src/Server/MySQLHandler.h | 1 + src/Server/PostgreSQLHandler.cpp | 1 + src/Server/ProxyV1Handler.cpp | 1 + src/Storages/Distributed/DistributedSink.cpp | 1 + src/Storages/FileLog/FileLogSettings.cpp | 1 + src/Storages/MaterializedView/RefreshTask.cpp | 1 + ...pproximateNearestNeighborIndexesCommon.cpp | 1 + src/Storages/MergeTree/KeyCondition.cpp | 1 + .../MergeTree/MergeTreeDataWriter.cpp | 1 + .../MergeTree/MergeTreeIndexAnnoy.cpp | 1 + .../MergeTree/MergeTreePrefetchedReadPool.cpp | 1 + src/Storages/MergeTree/MergeTreeReadPool.cpp | 9 + src/Storages/MergeTree/MergeTreeReadPool.h | 7 +- .../MergeTree/MergeTreeWhereOptimizer.cpp | 1 + .../MergeTree/MergedBlockOutputStream.cpp | 1 + .../MergedColumnOnlyOutputStream.cpp | 1 + src/Storages/MergeTree/RPNBuilder.cpp | 1 + .../MergeTree/ReplicatedMergeTreeSink.h | 1 + src/Storages/MySQL/MySQLSettings.cpp | 1 + src/Storages/NamedCollectionsHelpers.cpp | 1 + src/Storages/NamedCollectionsHelpers.h | 6 +- .../ObjectStorage/Azure/Configuration.cpp | 1 + .../DataLakes/IcebergMetadata.cpp | 2 +- .../ObjectStorage/HDFS/Configuration.cpp | 1 + .../ObjectStorage/ReadBufferIterator.cpp | 1 + .../ObjectStorage/S3/Configuration.cpp | 1 + .../ObjectStorage/StorageObjectStorage.cpp | 1 + .../StorageObjectStorageSink.cpp | 1 + .../StorageObjectStorageSource.cpp | 1 + .../registerStorageObjectStorage.cpp | 3 +- src/Storages/ReadInOrderOptimizer.cpp | 1 + .../RocksDB/EmbeddedRocksDBBulkSink.cpp | 1 + src/Storages/RocksDB/StorageSystemRocksDB.cpp | 1 + src/Storages/SetSettings.cpp | 1 + src/Storages/StorageFactory.cpp | 1 + src/Storages/StorageGenerateRandom.cpp | 1 + src/Storages/StorageInMemoryMetadata.cpp | 2 + src/Storages/StorageLog.cpp | 1 + src/Storages/StorageMySQL.cpp | 1 + src/Storages/StorageStripeLog.cpp | 2 + src/Storages/StorageURL.cpp | 1 + src/Storages/StorageXDBC.cpp | 1 + src/Storages/System/StorageSystemColumns.cpp | 1 + .../System/StorageSystemDDLWorkerQueue.cpp | 1 + .../System/StorageSystemDetachedParts.cpp | 1 + src/Storages/System/StorageSystemErrors.cpp | 1 + src/Storages/System/StorageSystemEvents.cpp | 1 + .../System/StorageSystemMergeTreeSettings.cpp | 1 + src/Storages/System/StorageSystemPartsBase.h | 1 + .../System/StorageSystemRemoteDataPaths.cpp | 1 + src/Storages/System/StorageSystemSettings.cpp | 2 + .../StorageSystemSettingsProfileElements.cpp | 1 + .../System/StorageSystemStackTrace.cpp | 1 + src/Storages/System/StorageSystemUsers.cpp | 6 +- src/Storages/TTLDescription.cpp | 1 + src/Storages/buildQueryTreeForShard.cpp | 1 + src/Storages/getStructureOfRemoteTable.cpp | 1 + .../transformQueryForExternalDatabase.cpp | 1 + src/TableFunctions/ITableFunctionXDBC.cpp | 1 + src/TableFunctions/TableFunctionExplain.cpp | 1 + src/TableFunctions/TableFunctionFactory.cpp | 1 + src/TableFunctions/TableFunctionFile.cpp | 1 + .../TableFunctionFileCluster.cpp | 1 + src/TableFunctions/TableFunctionFormat.cpp | 2 + src/TableFunctions/TableFunctionMySQL.cpp | 2 + .../TableFunctionPostgreSQL.cpp | 1 + src/TableFunctions/TableFunctionRemote.cpp | 1 + 287 files changed, 600 insertions(+), 184 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 6343dc85d00..2400dab4b33 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -24,9 +24,8 @@ #include #include #include - +#include #include -#include #include #include @@ -49,6 +48,8 @@ #include #include +#include + namespace fs = std::filesystem; using namespace std::literals; diff --git a/programs/library-bridge/LibraryBridgeHandlers.cpp b/programs/library-bridge/LibraryBridgeHandlers.cpp index 8d116e537aa..53f4b31ca06 100644 --- a/programs/library-bridge/LibraryBridgeHandlers.cpp +++ b/programs/library-bridge/LibraryBridgeHandlers.cpp @@ -6,6 +6,7 @@ #include "ExternalDictionaryLibraryHandlerFactory.h" #include +#include #include #include #include diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 5ff985b3d12..a738d5e8c12 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -2,6 +2,7 @@ #if USE_ODBC +#include #include #include #include diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/programs/odbc-bridge/IdentifierQuoteHandler.cpp index cf5acdc4534..2f00e1171bb 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.cpp +++ b/programs/odbc-bridge/IdentifierQuoteHandler.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "getIdentifierQuote.h" #include "validateODBCConnectionString.h" #include "ODBCPooledConnectionFactory.h" diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 2cf1576ccd7..404f66eab3b 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/programs/odbc-bridge/SchemaAllowedHandler.cpp b/programs/odbc-bridge/SchemaAllowedHandler.cpp index c7025ca4311..dfb4333240b 100644 --- a/programs/odbc-bridge/SchemaAllowedHandler.cpp +++ b/programs/odbc-bridge/SchemaAllowedHandler.cpp @@ -2,8 +2,10 @@ #if USE_ODBC +#include #include #include +#include #include #include #include diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index 6555ae63128..b5c6440a69c 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -6,6 +6,7 @@ #include #include #include +#include static constexpr size_t MAX_AGGREGATE_FUNCTION_NAME_LENGTH = 1000; diff --git a/src/Analyzer/InDepthQueryTreeVisitor.h b/src/Analyzer/InDepthQueryTreeVisitor.h index 62ddc06659c..67e3b75d8b5 100644 --- a/src/Analyzer/InDepthQueryTreeVisitor.h +++ b/src/Analyzer/InDepthQueryTreeVisitor.h @@ -3,7 +3,6 @@ #include #include -#include #include #include diff --git a/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp b/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp index a419bbcc133..b68635ab953 100644 --- a/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp +++ b/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp @@ -10,6 +10,8 @@ #include #include +#include + namespace DB { diff --git a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp index 098e2858abc..6578a6c5bd7 100644 --- a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp +++ b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp @@ -11,6 +11,8 @@ #include #include +#include + namespace DB { diff --git a/src/Analyzer/Passes/ArrayExistsToHasPass.cpp b/src/Analyzer/Passes/ArrayExistsToHasPass.cpp index 62db502e1dc..bf4c1232d79 100644 --- a/src/Analyzer/Passes/ArrayExistsToHasPass.cpp +++ b/src/Analyzer/Passes/ArrayExistsToHasPass.cpp @@ -11,6 +11,8 @@ #include #include +#include + namespace DB { diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp index 70aa1a41548..f7dd542dbf4 100644 --- a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp @@ -8,6 +8,8 @@ #include #include +#include + namespace DB { diff --git a/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp b/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp index eb897ef8746..2b2ac95d7b9 100644 --- a/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp +++ b/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp @@ -22,6 +22,8 @@ #include #include +#include + namespace DB { diff --git a/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp b/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp index 5951e8fc5ea..f66d092429f 100644 --- a/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp +++ b/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp @@ -10,6 +10,8 @@ #include #include +#include + #include #include diff --git a/src/Analyzer/Passes/CountDistinctPass.cpp b/src/Analyzer/Passes/CountDistinctPass.cpp index 23fde89a348..3f9185a07ca 100644 --- a/src/Analyzer/Passes/CountDistinctPass.cpp +++ b/src/Analyzer/Passes/CountDistinctPass.cpp @@ -11,6 +11,8 @@ #include #include +#include + namespace DB { diff --git a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp index 3e2a2055fdb..eb615879893 100644 --- a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp +++ b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp @@ -9,13 +9,14 @@ #include #include #include +#include #include #include #include #include -#include +#include namespace DB diff --git a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp index 90051779a26..b968f43c6a6 100644 --- a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp +++ b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp @@ -21,6 +21,8 @@ #include #include +#include + namespace DB { diff --git a/src/Analyzer/Passes/FuseFunctionsPass.cpp b/src/Analyzer/Passes/FuseFunctionsPass.cpp index ac4678fb362..0175e304a2b 100644 --- a/src/Analyzer/Passes/FuseFunctionsPass.cpp +++ b/src/Analyzer/Passes/FuseFunctionsPass.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include diff --git a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp index e259ffa64a6..91f06183523 100644 --- a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp +++ b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp @@ -1,6 +1,7 @@ #include #include +#include #include diff --git a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp index beb2247607e..cff98adb6af 100644 --- a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp +++ b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include diff --git a/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp b/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp index 29b392a0951..d966f129d08 100644 --- a/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp +++ b/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index ac221bd66e7..244a8184ba8 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include diff --git a/src/Analyzer/Passes/MultiIfToIfPass.cpp b/src/Analyzer/Passes/MultiIfToIfPass.cpp index c42ea61b34a..0b441032b02 100644 --- a/src/Analyzer/Passes/MultiIfToIfPass.cpp +++ b/src/Analyzer/Passes/MultiIfToIfPass.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include diff --git a/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp b/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp index f9066b0363c..3a8b6e75d40 100644 --- a/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp +++ b/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include diff --git a/src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp b/src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp index a8364f1ab7a..feb8bcc792d 100644 --- a/src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp +++ b/src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp index fd8c3e6ee6c..bfb6aa1dd07 100644 --- a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp index a30ad2a1590..fc989003c02 100644 --- a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp index b13af7f00ae..4f0271b4914 100644 --- a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp +++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp index 05377f8514e..c1adf05ac76 100644 --- a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp +++ b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp @@ -6,6 +6,8 @@ #include #include +#include + #include #include diff --git a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp index 524ee807721..a42b4750f47 100644 --- a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp +++ b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include namespace DB diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.cpp b/src/Analyzer/Passes/SumIfToCountIfPass.cpp index e5ee8a0d0b2..1524629dc81 100644 --- a/src/Analyzer/Passes/SumIfToCountIfPass.cpp +++ b/src/Analyzer/Passes/SumIfToCountIfPass.cpp @@ -1,19 +1,16 @@ #include -#include -#include - #include #include -#include - -#include - -#include - #include #include #include +#include +#include +#include +#include +#include +#include namespace DB { diff --git a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp index 947952ac3a8..cb8ef35fc13 100644 --- a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp +++ b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp @@ -9,6 +9,8 @@ #include #include +#include + namespace DB { diff --git a/src/Analyzer/Passes/UniqToCountPass.cpp b/src/Analyzer/Passes/UniqToCountPass.cpp index 83705f54b38..3f73322d1fa 100644 --- a/src/Analyzer/Passes/UniqToCountPass.cpp +++ b/src/Analyzer/Passes/UniqToCountPass.cpp @@ -9,6 +9,8 @@ #include #include +#include + namespace DB { diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 6a5db4bc1de..d68918e7027 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -40,6 +40,8 @@ #include #include +#include + #include #include diff --git a/src/Analyzer/Resolve/IdentifierResolveScope.cpp b/src/Analyzer/Resolve/IdentifierResolveScope.cpp index ae363b57047..eb3e2179440 100644 --- a/src/Analyzer/Resolve/IdentifierResolveScope.cpp +++ b/src/Analyzer/Resolve/IdentifierResolveScope.cpp @@ -1,8 +1,9 @@ #include -#include #include #include +#include +#include namespace DB { diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index 692a31b66ba..9dd8aa1a05f 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -25,6 +25,8 @@ #include #include +#include + namespace DB { namespace ErrorCodes diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 576c4943ccb..bd9b16b7649 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -64,6 +64,8 @@ #include #include +#include + namespace ProfileEvents { extern const Event ScalarSubqueriesGlobalCacheHit; diff --git a/src/Analyzer/TableNode.cpp b/src/Analyzer/TableNode.cpp index 1c4d022542d..d5a4b06f652 100644 --- a/src/Analyzer/TableNode.cpp +++ b/src/Analyzer/TableNode.cpp @@ -10,6 +10,8 @@ #include +#include + namespace DB { diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index cf4a3f77e34..d10bbd9bd23 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 64ab2112326..3ee7a05600e 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 56544312c26..aef37021227 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -1,6 +1,7 @@ #include #if USE_AWS_S3 +#include #include #include #include diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 69d9c52ebd9..15a7d7c1eca 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 454a0468e9f..3056f9fe421 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -23,8 +23,9 @@ #include #include #include -#include +#include +#include #include #include diff --git a/src/Backups/WithRetries.cpp b/src/Backups/WithRetries.cpp index 66851fa42ce..181e6331ac9 100644 --- a/src/Backups/WithRetries.cpp +++ b/src/Backups/WithRetries.cpp @@ -1,5 +1,7 @@ -#include #include +#include + +#include namespace DB { diff --git a/src/Bridge/IBridge.cpp b/src/Bridge/IBridge.cpp index c25d7bd2fed..de48a4f2b84 100644 --- a/src/Bridge/IBridge.cpp +++ b/src/Bridge/IBridge.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/src/BridgeHelper/LibraryBridgeHelper.cpp b/src/BridgeHelper/LibraryBridgeHelper.cpp index 57baf7e5a33..12ca5564e80 100644 --- a/src/BridgeHelper/LibraryBridgeHelper.cpp +++ b/src/BridgeHelper/LibraryBridgeHelper.cpp @@ -1,6 +1,7 @@ #include "LibraryBridgeHelper.h" #include +#include #include namespace DB diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index a260fd5761e..d2b10d7ed2e 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index 303105751ad..ebfabbb15d4 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace ProfileEvents { diff --git a/src/Client/ConnectionPool.cpp b/src/Client/ConnectionPool.cpp index 5cabb1465d1..ed2e7c3c725 100644 --- a/src/Client/ConnectionPool.cpp +++ b/src/Client/ConnectionPool.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -85,4 +86,15 @@ ConnectionPoolFactory & ConnectionPoolFactory::instance() return ret; } +IConnectionPool::Entry ConnectionPool::get(const DB::ConnectionTimeouts& timeouts, const DB::Settings& settings, + bool force_connected) +{ + Entry entry = Base::get(settings.connection_pool_max_wait_ms.totalMilliseconds()); + + if (force_connected) + entry->forceConnected(timeouts); + + return entry; +} + } diff --git a/src/Client/ConnectionPool.h b/src/Client/ConnectionPool.h index 725a5e91ac0..d83ecdd75f9 100644 --- a/src/Client/ConnectionPool.h +++ b/src/Client/ConnectionPool.h @@ -4,12 +4,13 @@ #include #include #include -#include #include namespace DB { +struct Settings; + /** Interface for connection pools. * * Usage (using the usual `ConnectionPool` example) @@ -102,15 +103,7 @@ public: Entry get(const ConnectionTimeouts & timeouts, /// NOLINT const Settings & settings, - bool force_connected = true) override - { - Entry entry = Base::get(settings.connection_pool_max_wait_ms.totalMilliseconds()); - - if (force_connected) - entry->forceConnected(timeouts); - - return entry; - } + bool force_connected = true) override; std::string getDescription() const { diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index 4effc3adafa..dd8348ea04f 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include diff --git a/src/Client/HedgedConnectionsFactory.h b/src/Client/HedgedConnectionsFactory.h index 51da85ea178..63e39819f58 100644 --- a/src/Client/HedgedConnectionsFactory.h +++ b/src/Client/HedgedConnectionsFactory.h @@ -8,13 +8,14 @@ #include #include #include -#include #include #include namespace DB { +struct Settings; + /** Class for establishing hedged connections with replicas. * The process of establishing connection is divided on stages, on each stage if * replica doesn't respond for a long time, we start establishing connection with diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index bcef286ecbc..207bf6c9e07 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Common/DateLUT.cpp b/src/Common/DateLUT.cpp index 3a20fb1a125..f9bbe2fbf40 100644 --- a/src/Common/DateLUT.cpp +++ b/src/Common/DateLUT.cpp @@ -2,7 +2,9 @@ #include #include +#include #include +#include #include #include diff --git a/src/Common/NamedCollections/NamedCollectionsFactory.cpp b/src/Common/NamedCollections/NamedCollectionsFactory.cpp index 14105a8651d..f8ddcba30b8 100644 --- a/src/Common/NamedCollections/NamedCollectionsFactory.cpp +++ b/src/Common/NamedCollections/NamedCollectionsFactory.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include namespace DB diff --git a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp index 32fdb25abd3..7b3dfd5b90f 100644 --- a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp +++ b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp index bc72c996384..af681cd5639 100644 --- a/src/Core/ExternalTable.cpp +++ b/src/Core/ExternalTable.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index 48f76769a09..9b132c3e199 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/src/DataTypes/DataTypeDecimalBase.cpp b/src/DataTypes/DataTypeDecimalBase.cpp index 62218694924..2adfda5bb27 100644 --- a/src/DataTypes/DataTypeDecimalBase.cpp +++ b/src/DataTypes/DataTypeDecimalBase.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/DataTypes/DataTypeFactory.cpp b/src/DataTypes/DataTypeFactory.cpp index 8c8f9999ada..07dc4395bfe 100644 --- a/src/DataTypes/DataTypeFactory.cpp +++ b/src/DataTypes/DataTypeFactory.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp index c85e8f5688a..d81dc7a76d8 100644 --- a/src/Databases/DDLDependencyVisitor.cpp +++ b/src/Databases/DDLDependencyVisitor.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index ccab72cfbae..35f66cb4583 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -12,10 +12,11 @@ #include #include #include -#include "Common/logger_useful.h" +#include #include #include #include +#include namespace fs = std::filesystem; diff --git a/src/Databases/DatabaseDictionary.cpp b/src/Databases/DatabaseDictionary.cpp index adb9a659fcd..52196e75c4a 100644 --- a/src/Databases/DatabaseDictionary.cpp +++ b/src/Databases/DatabaseDictionary.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 3a4c1423f7c..05a5e057c55 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace fs = std::filesystem; diff --git a/src/Databases/DatabaseFilesystem.cpp b/src/Databases/DatabaseFilesystem.cpp index b27a816a60d..31701e665a1 100644 --- a/src/Databases/DatabaseFilesystem.cpp +++ b/src/Databases/DatabaseFilesystem.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include diff --git a/src/Databases/DatabaseHDFS.cpp b/src/Databases/DatabaseHDFS.cpp index 060991d1290..eccaae5f22e 100644 --- a/src/Databases/DatabaseHDFS.cpp +++ b/src/Databases/DatabaseHDFS.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index b8154372116..07a250e72c7 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -28,6 +28,7 @@ #include #include #include +#include namespace fs = std::filesystem; diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 7b8955f7111..7ce2859e962 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 31d6f7876a8..1ef88dc03bc 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace fs = std::filesystem; diff --git a/src/Databases/DatabaseS3.cpp b/src/Databases/DatabaseS3.cpp index 1589cc1c75d..2b2d978a846 100644 --- a/src/Databases/DatabaseS3.cpp +++ b/src/Databases/DatabaseS3.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 1c82131af0d..cacb91ce780 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -29,6 +29,7 @@ # include # include # include +# include # include # include # include diff --git a/src/Databases/MySQL/FetchTablesColumnsList.cpp b/src/Databases/MySQL/FetchTablesColumnsList.cpp index e78f4aa2234..0828438cad2 100644 --- a/src/Databases/MySQL/FetchTablesColumnsList.cpp +++ b/src/Databases/MySQL/FetchTablesColumnsList.cpp @@ -2,6 +2,7 @@ #if USE_MYSQL #include +#include #include #include #include diff --git a/src/Databases/MySQL/FetchTablesColumnsList.h b/src/Databases/MySQL/FetchTablesColumnsList.h index 736a0ffd607..ec12cdc73b0 100644 --- a/src/Databases/MySQL/FetchTablesColumnsList.h +++ b/src/Databases/MySQL/FetchTablesColumnsList.h @@ -12,11 +12,12 @@ #include #include -#include namespace DB { +struct Settings; + std::map fetchTablesColumnsList( mysqlxx::PoolWithFailover & pool, const String & database_name, diff --git a/src/Databases/MySQL/MaterializeMetadata.h b/src/Databases/MySQL/MaterializeMetadata.h index e78b7132c8d..36d3fb569d8 100644 --- a/src/Databases/MySQL/MaterializeMetadata.h +++ b/src/Databases/MySQL/MaterializeMetadata.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 136fb7fd6d2..b4f8291cef9 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include namespace fs = std::filesystem; diff --git a/src/Databases/SQLite/DatabaseSQLite.cpp b/src/Databases/SQLite/DatabaseSQLite.cpp index e758ea35de5..132a978140c 100644 --- a/src/Databases/SQLite/DatabaseSQLite.cpp +++ b/src/Databases/SQLite/DatabaseSQLite.cpp @@ -3,6 +3,7 @@ #if USE_SQLITE #include +#include #include #include #include diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index 2c0f7653aff..f3b32402c20 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -1,6 +1,7 @@ #include "DirectDictionary.h" #include +#include #include #include diff --git a/src/Dictionaries/ExecutablePoolDictionarySource.cpp b/src/Dictionaries/ExecutablePoolDictionarySource.cpp index d8111afdc19..217a67453e4 100644 --- a/src/Dictionaries/ExecutablePoolDictionarySource.cpp +++ b/src/Dictionaries/ExecutablePoolDictionarySource.cpp @@ -8,6 +8,8 @@ #include #include +#include + #include #include #include diff --git a/src/Dictionaries/HashedArrayDictionary.cpp b/src/Dictionaries/HashedArrayDictionary.cpp index 9b194acf87f..d7d50dfb0a6 100644 --- a/src/Dictionaries/HashedArrayDictionary.cpp +++ b/src/Dictionaries/HashedArrayDictionary.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Dictionaries/IPAddressDictionary.cpp b/src/Dictionaries/IPAddressDictionary.cpp index 206516edab0..41fafcc162b 100644 --- a/src/Dictionaries/IPAddressDictionary.cpp +++ b/src/Dictionaries/IPAddressDictionary.cpp @@ -1,11 +1,11 @@ #include "IPAddressDictionary.h" -#include -#include + #include #include #include #include #include +#include #include #include #include @@ -23,6 +23,9 @@ #include #include +#include +#include + namespace DB { diff --git a/src/Dictionaries/PolygonDictionaryImplementations.cpp b/src/Dictionaries/PolygonDictionaryImplementations.cpp index e21fadb0e7e..84abeed2731 100644 --- a/src/Dictionaries/PolygonDictionaryImplementations.cpp +++ b/src/Dictionaries/PolygonDictionaryImplementations.cpp @@ -8,6 +8,7 @@ #include #include +#include #include diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index c7401386e40..9e482287cb9 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "DictionarySourceFactory.h" #include "registerDictionaries.h" diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index 830f9d09364..a5aa622e01d 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index 65a9f806722..b84cfa109ee 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include "DictionarySourceFactory.h" #include "DictionaryStructure.h" #include "readInvalidateQuery.h" diff --git a/src/Dictionaries/registerCacheDictionaries.cpp b/src/Dictionaries/registerCacheDictionaries.cpp index b79261955ff..f2d027575f5 100644 --- a/src/Dictionaries/registerCacheDictionaries.cpp +++ b/src/Dictionaries/registerCacheDictionaries.cpp @@ -2,6 +2,7 @@ #include "CacheDictionaryStorage.h" #include "SSDCacheDictionaryStorage.h" #include +#include #include #include diff --git a/src/Dictionaries/registerHashedDictionary.cpp b/src/Dictionaries/registerHashedDictionary.cpp index 5fc4f5d5cb6..068b71170cc 100644 --- a/src/Dictionaries/registerHashedDictionary.cpp +++ b/src/Dictionaries/registerHashedDictionary.cpp @@ -1,8 +1,8 @@ +#include #include #include #include #include - #include namespace DB diff --git a/src/Dictionaries/registerRangeHashedDictionary.cpp b/src/Dictionaries/registerRangeHashedDictionary.cpp index 8123b811198..bc142a6bddc 100644 --- a/src/Dictionaries/registerRangeHashedDictionary.cpp +++ b/src/Dictionaries/registerRangeHashedDictionary.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Disks/IO/ReadBufferFromWebServer.cpp b/src/Disks/IO/ReadBufferFromWebServer.cpp index af4fae6b723..837452aa9f2 100644 --- a/src/Disks/IO/ReadBufferFromWebServer.cpp +++ b/src/Disks/IO/ReadBufferFromWebServer.cpp @@ -1,6 +1,7 @@ #include "ReadBufferFromWebServer.h" #include +#include #include #include #include diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index d9dfedadd48..0aa3b9c40b5 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h index 19ba48ea225..bb2f0270924 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h @@ -11,7 +11,6 @@ #include #include -#include #include #include #include @@ -19,6 +18,8 @@ namespace DB { +struct Settings; + namespace AzureBlobStorage { diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp index d9535358daf..d42da6dc9c4 100644 --- a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp +++ b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index a6781e0ab35..a6672e14e10 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 62df98f51e6..de78ce4594d 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -1,14 +1,14 @@ #include -#include -#include #if USE_AWS_S3 +#include #include #include #include #include #include +#include #include #include #include @@ -17,6 +17,7 @@ #include #include #include +#include #include #include diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 484ea084d63..ef1fe1ca38c 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include diff --git a/src/Formats/NumpyDataTypes.h b/src/Formats/NumpyDataTypes.h index 062f743c0ea..6ccdf65f457 100644 --- a/src/Formats/NumpyDataTypes.h +++ b/src/Formats/NumpyDataTypes.h @@ -2,6 +2,7 @@ #include #include #include +#include namespace ErrorCodes { diff --git a/src/Formats/ReadSchemaUtils.cpp b/src/Formats/ReadSchemaUtils.cpp index 735b536986d..617595c19a2 100644 --- a/src/Formats/ReadSchemaUtils.cpp +++ b/src/Formats/ReadSchemaUtils.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Functions/CastOverloadResolver.cpp b/src/Functions/CastOverloadResolver.cpp index 0f54ff52ba2..10a08c6e35f 100644 --- a/src/Functions/CastOverloadResolver.cpp +++ b/src/Functions/CastOverloadResolver.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 5d19ba44d9b..51bc7583cfb 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/FunctionCustomWeekToDateOrDate32.h b/src/Functions/FunctionCustomWeekToDateOrDate32.h index c904bfaec30..6ae8c847b65 100644 --- a/src/Functions/FunctionCustomWeekToDateOrDate32.h +++ b/src/Functions/FunctionCustomWeekToDateOrDate32.h @@ -1,4 +1,5 @@ #pragma once +#include #include namespace DB diff --git a/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h b/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h index 6eb3e534b62..28caef36879 100644 --- a/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h +++ b/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h @@ -1,4 +1,5 @@ #pragma once +#include #include namespace DB diff --git a/src/Functions/FunctionDateOrDateTimeToDateTimeOrDateTime64.h b/src/Functions/FunctionDateOrDateTimeToDateTimeOrDateTime64.h index 9f1066fd687..d5a1f28b7cd 100644 --- a/src/Functions/FunctionDateOrDateTimeToDateTimeOrDateTime64.h +++ b/src/Functions/FunctionDateOrDateTimeToDateTimeOrDateTime64.h @@ -1,4 +1,6 @@ #pragma once + +#include #include namespace DB diff --git a/src/Functions/FunctionFactory.cpp b/src/Functions/FunctionFactory.cpp index 004ef745a93..de6d5a9e1c1 100644 --- a/src/Functions/FunctionFactory.cpp +++ b/src/Functions/FunctionFactory.cpp @@ -4,6 +4,7 @@ #include #include +#include #include diff --git a/src/Functions/FunctionGenerateRandomStructure.h b/src/Functions/FunctionGenerateRandomStructure.h index 894096a6e07..b9a83eb261e 100644 --- a/src/Functions/FunctionGenerateRandomStructure.h +++ b/src/Functions/FunctionGenerateRandomStructure.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index 0a2859fe864..a67f3a977db 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Functions/FunctionTokens.h b/src/Functions/FunctionTokens.h index 0ca47126198..f1435ca5651 100644 --- a/src/Functions/FunctionTokens.h +++ b/src/Functions/FunctionTokens.h @@ -17,6 +17,7 @@ #include #include #include +#include namespace DB diff --git a/src/Functions/FunctionUnixTimestamp64.h b/src/Functions/FunctionUnixTimestamp64.h index e282bcfbfe2..652b3fe60ec 100644 --- a/src/Functions/FunctionUnixTimestamp64.h +++ b/src/Functions/FunctionUnixTimestamp64.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/FunctionsCodingIP.cpp b/src/Functions/FunctionsCodingIP.cpp index e01967274f4..a134e39fbcd 100644 --- a/src/Functions/FunctionsCodingIP.cpp +++ b/src/Functions/FunctionsCodingIP.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index f3e54d2fbd9..1342e3f2c5d 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index 18b411e9839..a92a6570279 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index c0ed90aa042..03db2651fd0 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/FunctionsStringSearch.h b/src/Functions/FunctionsStringSearch.h index 924a66c1130..fba6336ebff 100644 --- a/src/Functions/FunctionsStringSearch.h +++ b/src/Functions/FunctionsStringSearch.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/FunctionsTextClassification.h b/src/Functions/FunctionsTextClassification.h index 8e0f236366d..90e8af06ccc 100644 --- a/src/Functions/FunctionsTextClassification.h +++ b/src/Functions/FunctionsTextClassification.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h index ef2c788bf43..48daf13ba68 100644 --- a/src/Functions/PerformanceAdaptors.h +++ b/src/Functions/PerformanceAdaptors.h @@ -4,6 +4,7 @@ #include #include +#include #include #include diff --git a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp index 2c031158c48..4a5ff19763e 100644 --- a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp +++ b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp @@ -2,6 +2,7 @@ #include #include +#include #include diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp index c15958f81c9..da79e17f18a 100644 --- a/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace DB diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp index b406cc8d317..4c004d2537c 100644 --- a/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp @@ -9,6 +9,8 @@ #include #include +#include + #include #include #include diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp index 766d63eafb0..01e7e3995fa 100644 --- a/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp @@ -14,7 +14,7 @@ #include #include #include - +#include namespace DB { diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index 03b51808799..dfe589fb74f 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/array/range.cpp b/src/Functions/array/range.cpp index 1ac698a2745..b8b03ab5e74 100644 --- a/src/Functions/array/range.cpp +++ b/src/Functions/array/range.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/castOrDefault.cpp b/src/Functions/castOrDefault.cpp index 995b5fa91e7..1ece17ca1e3 100644 --- a/src/Functions/castOrDefault.cpp +++ b/src/Functions/castOrDefault.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index 1c5dc05c5aa..64b9d4d8c71 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/formatQuery.cpp b/src/Functions/formatQuery.cpp index 655ea2e7cde..d10b3f9a5b7 100644 --- a/src/Functions/formatQuery.cpp +++ b/src/Functions/formatQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Functions/getClientHTTPHeader.cpp b/src/Functions/getClientHTTPHeader.cpp index ebd070a90b9..140f39d03b8 100644 --- a/src/Functions/getClientHTTPHeader.cpp +++ b/src/Functions/getClientHTTPHeader.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/src/Functions/getSetting.cpp b/src/Functions/getSetting.cpp index d6b8946e760..aed6b2119e4 100644 --- a/src/Functions/getSetting.cpp +++ b/src/Functions/getSetting.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/src/Functions/hasColumnInTable.cpp b/src/Functions/hasColumnInTable.cpp index 8ea16f688ee..00714997b4a 100644 --- a/src/Functions/hasColumnInTable.cpp +++ b/src/Functions/hasColumnInTable.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index 4c08cd3b931..30eaa26fa20 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp index 94f02861af0..02e8412bbf3 100644 --- a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp +++ b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include diff --git a/src/Functions/lemmatize.cpp b/src/Functions/lemmatize.cpp index e1168e077bb..76e5ed7432e 100644 --- a/src/Functions/lemmatize.cpp +++ b/src/Functions/lemmatize.cpp @@ -3,6 +3,7 @@ #if USE_NLP #include +#include #include #include #include diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 66cd10a3f0b..0d42ce2abe1 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index d3bf5618f66..c4b675fcf6c 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/neighbor.cpp b/src/Functions/neighbor.cpp index 62f129109f9..49b8980e7e2 100644 --- a/src/Functions/neighbor.cpp +++ b/src/Functions/neighbor.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Functions/now.cpp b/src/Functions/now.cpp index 827b800a243..4e3f7be4f7e 100644 --- a/src/Functions/now.cpp +++ b/src/Functions/now.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/now64.cpp b/src/Functions/now64.cpp index d6f8474c984..988694708cc 100644 --- a/src/Functions/now64.cpp +++ b/src/Functions/now64.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include diff --git a/src/Functions/nowInBlock.cpp b/src/Functions/nowInBlock.cpp index 74f420986c8..b79653f501c 100644 --- a/src/Functions/nowInBlock.cpp +++ b/src/Functions/nowInBlock.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 339eb4cb26c..2bd8d370ed6 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 6b413829bd1..32d5905a513 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/runningAccumulate.cpp b/src/Functions/runningAccumulate.cpp index d585affd91b..c94cbe67216 100644 --- a/src/Functions/runningAccumulate.cpp +++ b/src/Functions/runningAccumulate.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Functions/runningDifference.h b/src/Functions/runningDifference.h index fe477d13744..d51ea805f3c 100644 --- a/src/Functions/runningDifference.h +++ b/src/Functions/runningDifference.h @@ -1,6 +1,7 @@ #pragma once #include #include +#include #include #include #include diff --git a/src/Functions/serverConstants.cpp b/src/Functions/serverConstants.cpp index e7e423058f1..b74edfc3121 100644 --- a/src/Functions/serverConstants.cpp +++ b/src/Functions/serverConstants.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/sleep.h b/src/Functions/sleep.h index 22748e86888..62ee19fa904 100644 --- a/src/Functions/sleep.h +++ b/src/Functions/sleep.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/snowflake.cpp b/src/Functions/snowflake.cpp index 31ea6a28ece..0d4d65ee4b0 100644 --- a/src/Functions/snowflake.cpp +++ b/src/Functions/snowflake.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include diff --git a/src/Functions/snowflakeIDToDateTime.cpp b/src/Functions/snowflakeIDToDateTime.cpp index 9a1d5b8a74b..bd805a2e607 100644 --- a/src/Functions/snowflakeIDToDateTime.cpp +++ b/src/Functions/snowflakeIDToDateTime.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include diff --git a/src/Functions/stem.cpp b/src/Functions/stem.cpp index d326d17a7a8..5b845cf332b 100644 --- a/src/Functions/stem.cpp +++ b/src/Functions/stem.cpp @@ -3,6 +3,7 @@ #if USE_NLP #include +#include #include #include #include diff --git a/src/Functions/synonyms.cpp b/src/Functions/synonyms.cpp index 6b3e205785c..28dd83627d9 100644 --- a/src/Functions/synonyms.cpp +++ b/src/Functions/synonyms.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/throwIf.cpp b/src/Functions/throwIf.cpp index c6ff5b9151f..becc6d2f772 100644 --- a/src/Functions/throwIf.cpp +++ b/src/Functions/throwIf.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/toTimezone.cpp b/src/Functions/toTimezone.cpp index a0d90351898..be839f1acb3 100644 --- a/src/Functions/toTimezone.cpp +++ b/src/Functions/toTimezone.cpp @@ -9,6 +9,7 @@ #include #include +#include namespace DB { diff --git a/src/Functions/toTypeName.cpp b/src/Functions/toTypeName.cpp index 63bcc58d65e..abe02148d4a 100644 --- a/src/Functions/toTypeName.cpp +++ b/src/Functions/toTypeName.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/tuple.h b/src/Functions/tuple.h index 94529d86861..a49f346ef2a 100644 --- a/src/Functions/tuple.h +++ b/src/Functions/tuple.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include diff --git a/src/Functions/visibleWidth.cpp b/src/Functions/visibleWidth.cpp index 9a3edd9fbec..ebd4a1ff713 100644 --- a/src/Functions/visibleWidth.cpp +++ b/src/Functions/visibleWidth.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include diff --git a/src/Functions/widthBucket.cpp b/src/Functions/widthBucket.cpp index 62ed460ca9d..e804808b699 100644 --- a/src/Functions/widthBucket.cpp +++ b/src/Functions/widthBucket.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index c2eb67c6f3b..662a2c067cf 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -1,4 +1,5 @@ #include +#include #include #include diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 55441cfb86b..3b958dea046 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -23,10 +23,11 @@ #include #include - #include #include +#include + #include diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 490bf8c2d0c..3f2c1f95e71 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -12,6 +12,7 @@ #if USE_AWS_S3 +#include #include #include #include diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index 2dca08871d3..fcfd0cfffae 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include @@ -30,6 +30,8 @@ namespace ErrorCodes extern const int S3_ERROR; } +struct Settings; + class S3Exception : public Exception { public: diff --git a/src/IO/S3Settings.cpp b/src/IO/S3Settings.cpp index a5a50c873cb..f99e8c0105b 100644 --- a/src/IO/S3Settings.cpp +++ b/src/IO/S3Settings.cpp @@ -1,9 +1,11 @@ #include +#include #include -#include #include +#include + namespace DB { diff --git a/src/Interpreters/ArrayJoinAction.cpp b/src/Interpreters/ArrayJoinAction.cpp index 54ff53764e7..df7a0b48057 100644 --- a/src/Interpreters/ArrayJoinAction.cpp +++ b/src/Interpreters/ArrayJoinAction.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp index dd038948adf..e6ebf6ad50c 100644 --- a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp +++ b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp @@ -12,6 +12,8 @@ #include #include +#include + namespace DB { diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index adb20f4854f..fe674a747ef 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -824,8 +825,69 @@ void ContextSharedMutex::lockSharedImpl() ProfileEvents::increment(ProfileEvents::ContextLockWaitMicroseconds, watch.elapsedMicroseconds()); } -ContextData::ContextData() = default; -ContextData::ContextData(const ContextData &) = default; +ContextData::ContextData() +{ + settings = std::make_unique(); +} +ContextData::ContextData(const ContextData &o) : + shared(o.shared), + client_info(o.client_info), + external_tables_initializer_callback(o.external_tables_initializer_callback), + input_initializer_callback(o.input_initializer_callback), + input_blocks_reader(o.input_blocks_reader), + user_id(o.user_id), + current_roles(o.current_roles), + settings_constraints_and_current_profiles(o.settings_constraints_and_current_profiles), + access(o.access), + need_recalculate_access(o.need_recalculate_access), + current_database(o.current_database), + settings(std::make_unique(*o.settings)), + progress_callback(o.progress_callback), + file_progress_callback(o.file_progress_callback), + process_list_elem(o.process_list_elem), + has_process_list_elem(o.has_process_list_elem), + insertion_table_info(o.insertion_table_info), + is_distributed(o.is_distributed), + default_format(o.default_format), + insert_format(o.insert_format), + external_tables_mapping(o.external_tables_mapping), + scalars(o.scalars), + special_scalars(o.special_scalars), + next_task_callback(o.next_task_callback), + merge_tree_read_task_callback(o.merge_tree_read_task_callback), + merge_tree_all_ranges_callback(o.merge_tree_all_ranges_callback), + parallel_replicas_group_uuid(o.parallel_replicas_group_uuid), + client_protocol_version(o.client_protocol_version), + query_access_info(std::make_shared(*o.query_access_info)), + query_factories_info(o.query_factories_info), + query_privileges_info(o.query_privileges_info), + async_read_counters(o.async_read_counters), + view_source(o.view_source), + table_function_results(o.table_function_results), + query_context(o.query_context), + session_context(o.session_context), + global_context(o.global_context), + buffer_context(o.buffer_context), + is_internal_query(o.is_internal_query), + temp_data_on_disk(o.temp_data_on_disk), + classifier(o.classifier), + prepared_sets_cache(o.prepared_sets_cache), + offset_parallel_replicas_enabled(o.offset_parallel_replicas_enabled), + kitchen_sink(o.kitchen_sink), + part_uuids(o.part_uuids), + ignored_part_uuids(o.ignored_part_uuids), + query_parameters(o.query_parameters), + host_context(o.host_context), + metadata_transaction(o.metadata_transaction), + merge_tree_transaction(o.merge_tree_transaction), + merge_tree_transaction_holder(o.merge_tree_transaction_holder), + remote_read_query_throttler(o.remote_read_query_throttler), + remote_write_query_throttler(o.remote_write_query_throttler), + local_read_query_throttler(o.local_read_query_throttler), + local_write_query_throttler(o.local_write_query_throttler), + backups_query_throttler(o.backups_query_throttler) +{ +} Context::Context() = default; Context::Context(const Context & rhs) : ContextData(rhs), std::enable_shared_from_this(rhs) {} @@ -865,7 +927,6 @@ ContextMutablePtr Context::createCopy(const ContextPtr & other) { SharedLockGuard lock(other->mutex); auto new_context = std::shared_ptr(new Context(*other)); - new_context->query_access_info = std::make_shared(*other->query_access_info); return new_context; } @@ -971,7 +1032,7 @@ Strings Context::getWarnings() const } /// Make setting's name ordered std::set obsolete_settings; - for (const auto & setting : settings) + for (const auto & setting : *settings) { if (setting.isValueChanged() && setting.isObsolete()) obsolete_settings.emplace(setting.getName()); @@ -1486,7 +1547,7 @@ std::shared_ptr Context::getAccess() const /// If setUserID() was never called then this must be the global context with the full access. bool full_access = !user_id; - return ContextAccessParams{user_id, full_access, /* use_default_roles= */ false, current_roles, settings, current_database, client_info}; + return ContextAccessParams{user_id, full_access, /* use_default_roles= */ false, current_roles, *settings, current_database, client_info}; }; /// Check if the current access rights are still valid, otherwise get parameters for recalculating access rights. @@ -1568,7 +1629,7 @@ void Context::setCurrentProfilesWithLock(const SettingsProfilesInfo & profiles_i checkSettingsConstraintsWithLock(profiles_info.settings, SettingSource::PROFILE); applySettingsChangesWithLock(profiles_info.settings, lock); settings_constraints_and_current_profiles = profiles_info.getConstraintsAndProfileIDs(settings_constraints_and_current_profiles); - contextSanityClampSettingsWithLock(*this, settings, lock); + contextSanityClampSettingsWithLock(*this, *settings, lock); } void Context::setCurrentProfile(const String & profile_name, bool check_constraints) @@ -2209,15 +2270,15 @@ bool Context::displaySecretsInShowAndSelect() const Settings Context::getSettings() const { SharedLockGuard lock(mutex); - return settings; + return *settings; } void Context::setSettings(const Settings & settings_) { std::lock_guard lock(mutex); - settings = settings_; + *settings = settings_; need_recalculate_access = true; - contextSanityClampSettings(*this, settings); + contextSanityClampSettings(*this, *settings); } void Context::setSettingWithLock(std::string_view name, const String & value, const std::lock_guard & lock) @@ -2227,10 +2288,10 @@ void Context::setSettingWithLock(std::string_view name, const String & value, co setCurrentProfileWithLock(value, true /*check_constraints*/, lock); return; } - settings.set(name, value); + settings->set(name, value); if (ContextAccessParams::dependsOnSettingName(name)) need_recalculate_access = true; - contextSanityClampSettingsWithLock(*this, settings, lock); + contextSanityClampSettingsWithLock(*this, *settings, lock); } void Context::setSettingWithLock(std::string_view name, const Field & value, const std::lock_guard & lock) @@ -2240,7 +2301,7 @@ void Context::setSettingWithLock(std::string_view name, const Field & value, con setCurrentProfileWithLock(value.safeGet(), true /*check_constraints*/, lock); return; } - settings.set(name, value); + settings->set(name, value); if (ContextAccessParams::dependsOnSettingName(name)) need_recalculate_access = true; } @@ -2250,7 +2311,7 @@ void Context::applySettingChangeWithLock(const SettingChange & change, const std try { setSettingWithLock(change.name, change.value, lock); - contextSanityClampSettingsWithLock(*this, settings, lock); + contextSanityClampSettingsWithLock(*this, *settings, lock); } catch (Exception & e) { @@ -2265,7 +2326,7 @@ void Context::applySettingsChangesWithLock(const SettingsChanges & changes, cons { for (const SettingChange & change : changes) applySettingChangeWithLock(change, lock); - applySettingsQuirks(settings); + applySettingsQuirks(*settings); } void Context::setSetting(std::string_view name, const String & value) @@ -2278,7 +2339,7 @@ void Context::setSetting(std::string_view name, const Field & value) { std::lock_guard lock(mutex); setSettingWithLock(name, value, lock); - contextSanityClampSettingsWithLock(*this, settings, lock); + contextSanityClampSettingsWithLock(*this, *settings, lock); } void Context::setServerSetting(std::string_view name, const Field & value) @@ -2311,37 +2372,37 @@ void Context::applySettingsChanges(const SettingsChanges & changes) void Context::checkSettingsConstraintsWithLock(const SettingsProfileElements & profile_elements, SettingSource source) { - getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(settings, profile_elements, source); + getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(*settings, profile_elements, source); if (getApplicationType() == ApplicationType::LOCAL || getApplicationType() == ApplicationType::SERVER) - doSettingsSanityCheckClamp(settings, getLogger("SettingsSanity")); + doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); } void Context::checkSettingsConstraintsWithLock(const SettingChange & change, SettingSource source) { - getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(settings, change, source); + getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(*settings, change, source); if (getApplicationType() == ApplicationType::LOCAL || getApplicationType() == ApplicationType::SERVER) - doSettingsSanityCheckClamp(settings, getLogger("SettingsSanity")); + doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); } void Context::checkSettingsConstraintsWithLock(const SettingsChanges & changes, SettingSource source) { - getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(settings, changes, source); + getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(*settings, changes, source); if (getApplicationType() == ApplicationType::LOCAL || getApplicationType() == ApplicationType::SERVER) - doSettingsSanityCheckClamp(settings, getLogger("SettingsSanity")); + doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); } void Context::checkSettingsConstraintsWithLock(SettingsChanges & changes, SettingSource source) { - getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(settings, changes, source); + getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(*settings, changes, source); if (getApplicationType() == ApplicationType::LOCAL || getApplicationType() == ApplicationType::SERVER) - doSettingsSanityCheckClamp(settings, getLogger("SettingsSanity")); + doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); } void Context::clampToSettingsConstraintsWithLock(SettingsChanges & changes, SettingSource source) { - getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.clamp(settings, changes, source); + getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.clamp(*settings, changes, source); if (getApplicationType() == ApplicationType::LOCAL || getApplicationType() == ApplicationType::SERVER) - doSettingsSanityCheckClamp(settings, getLogger("SettingsSanity")); + doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); } void Context::checkMergeTreeSettingsConstraintsWithLock(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const @@ -2364,8 +2425,8 @@ void Context::checkSettingsConstraints(const SettingChange & change, SettingSour void Context::checkSettingsConstraints(const SettingsChanges & changes, SettingSource source) { SharedLockGuard lock(mutex); - getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(settings, changes, source); - doSettingsSanityCheckClamp(settings, getLogger("SettingsSanity")); + getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(*settings, changes, source); + doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); } void Context::checkSettingsConstraints(SettingsChanges & changes, SettingSource source) @@ -2390,7 +2451,7 @@ void Context::resetSettingsToDefaultValue(const std::vector & names) { std::lock_guard lock(mutex); for (const String & name: names) - settings.setDefaultValue(name); + settings->setDefaultValue(name); } std::shared_ptr Context::getSettingsConstraintsAndCurrentProfilesWithLock() const @@ -2610,14 +2671,14 @@ void Context::makeQueryContextForMerge(const MergeTreeSettings & merge_tree_sett { makeQueryContext(); classifier.reset(); // It is assumed that there are no active queries running using this classifier, otherwise this will lead to crashes - settings.workload = merge_tree_settings.merge_workload.value.empty() ? getMergeWorkload() : merge_tree_settings.merge_workload; + settings->workload = merge_tree_settings.merge_workload.value.empty() ? getMergeWorkload() : merge_tree_settings.merge_workload; } void Context::makeQueryContextForMutate(const MergeTreeSettings & merge_tree_settings) { makeQueryContext(); classifier.reset(); // It is assumed that there are no active queries running using this classifier, otherwise this will lead to crashes - settings.workload = merge_tree_settings.mutation_workload.value.empty() ? getMutationWorkload() : merge_tree_settings.mutation_workload; + settings->workload = merge_tree_settings.mutation_workload.value.empty() ? getMutationWorkload() : merge_tree_settings.mutation_workload; } void Context::makeSessionContext() @@ -3900,7 +3961,7 @@ void Context::reloadClusterConfig() const } const auto & config = cluster_config ? *cluster_config : getConfigRef(); - auto new_clusters = std::make_shared(config, settings, getMacros()); + auto new_clusters = std::make_shared(config, *settings, getMacros()); { std::lock_guard lock(shared->clusters_mutex); @@ -3935,7 +3996,7 @@ std::shared_ptr Context::getClustersImpl(std::lock_guard & if (!shared->clusters) { const auto & config = shared->clusters_config ? *shared->clusters_config : getConfigRef(); - shared->clusters = std::make_shared(config, settings, getMacros()); + shared->clusters = std::make_shared(config, *settings, getMacros()); } return shared->clusters; @@ -3967,9 +4028,9 @@ void Context::setClustersConfig(const ConfigurationPtr & config, bool enable_dis shared->clusters_config = config; if (!shared->clusters) - shared->clusters = std::make_shared(*shared->clusters_config, settings, getMacros(), config_name); + shared->clusters = std::make_shared(*shared->clusters_config, *settings, getMacros(), config_name); else - shared->clusters->updateClusters(*shared->clusters_config, settings, config_name, old_clusters_config); + shared->clusters->updateClusters(*shared->clusters_config, *settings, config_name, old_clusters_config); ++shared->clusters_version; } @@ -4231,7 +4292,7 @@ std::shared_ptr Context::getBackupLog() const std::shared_ptr Context::getBlobStorageLog() const { - bool enable_blob_storage_log = settings.enable_blob_storage_log; + bool enable_blob_storage_log = settings->enable_blob_storage_log; if (hasQueryContext()) enable_blob_storage_log = getQueryContext()->getSettingsRef().enable_blob_storage_log; @@ -4672,8 +4733,8 @@ void Context::setDefaultProfiles(const Poco::Util::AbstractConfiguration & confi shared->system_profile_name = config.getString("system_profile", shared->default_profile_name); setCurrentProfile(shared->system_profile_name); - applySettingsQuirks(settings, getLogger("SettingsQuirks")); - doSettingsSanityCheckClamp(settings, getLogger("SettingsSanity")); + applySettingsQuirks(*settings, getLogger("SettingsQuirks")); + doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); shared->buffer_profile_name = config.getString("buffer_profile", shared->system_profile_name); buffer_context = Context::createCopy(shared_from_this()); @@ -5242,11 +5303,11 @@ AsynchronousInsertQueue * Context::tryGetAsynchronousInsertQueue() const void Context::setAsynchronousInsertQueue(const std::shared_ptr & ptr) { - AsynchronousInsertQueue::validateSettings(settings, getLogger("Context")); + AsynchronousInsertQueue::validateSettings(*settings, getLogger("Context")); SharedLockGuard lock(shared->mutex); - if (std::chrono::milliseconds(settings.async_insert_poll_timeout_ms) == std::chrono::milliseconds::zero()) + if (std::chrono::milliseconds(settings->async_insert_poll_timeout_ms) == std::chrono::milliseconds::zero()) throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Setting async_insert_poll_timeout_ms can't be zero"); shared->async_insert_queue = ptr; @@ -5393,69 +5454,69 @@ ReadSettings Context::getReadSettings() const { ReadSettings res; - std::string_view read_method_str = settings.local_filesystem_read_method.value; + std::string_view read_method_str = settings->local_filesystem_read_method.value; if (auto opt_method = magic_enum::enum_cast(read_method_str)) res.local_fs_method = *opt_method; else throw Exception(ErrorCodes::UNKNOWN_READ_METHOD, "Unknown read method '{}' for local filesystem", read_method_str); - read_method_str = settings.remote_filesystem_read_method.value; + read_method_str = settings->remote_filesystem_read_method.value; if (auto opt_method = magic_enum::enum_cast(read_method_str)) res.remote_fs_method = *opt_method; else throw Exception(ErrorCodes::UNKNOWN_READ_METHOD, "Unknown read method '{}' for remote filesystem", read_method_str); - res.local_fs_prefetch = settings.local_filesystem_read_prefetch; - res.remote_fs_prefetch = settings.remote_filesystem_read_prefetch; + res.local_fs_prefetch = settings->local_filesystem_read_prefetch; + res.remote_fs_prefetch = settings->remote_filesystem_read_prefetch; - res.load_marks_asynchronously = settings.load_marks_asynchronously; + res.load_marks_asynchronously = settings->load_marks_asynchronously; - res.enable_filesystem_read_prefetches_log = settings.enable_filesystem_read_prefetches_log; + res.enable_filesystem_read_prefetches_log = settings->enable_filesystem_read_prefetches_log; - res.remote_fs_read_max_backoff_ms = settings.remote_fs_read_max_backoff_ms; - res.remote_fs_read_backoff_max_tries = settings.remote_fs_read_backoff_max_tries; - res.enable_filesystem_cache = settings.enable_filesystem_cache; - res.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache; - res.enable_filesystem_cache_log = settings.enable_filesystem_cache_log; - res.filesystem_cache_segments_batch_size = settings.filesystem_cache_segments_batch_size; - res.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds = settings.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds; + res.remote_fs_read_max_backoff_ms = settings->remote_fs_read_max_backoff_ms; + res.remote_fs_read_backoff_max_tries = settings->remote_fs_read_backoff_max_tries; + res.enable_filesystem_cache = settings->enable_filesystem_cache; + res.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = settings->read_from_filesystem_cache_if_exists_otherwise_bypass_cache; + res.enable_filesystem_cache_log = settings->enable_filesystem_cache_log; + res.filesystem_cache_segments_batch_size = settings->filesystem_cache_segments_batch_size; + res.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds = settings->filesystem_cache_reserve_space_wait_lock_timeout_milliseconds; - res.filesystem_cache_max_download_size = settings.filesystem_cache_max_download_size; - res.skip_download_if_exceeds_query_cache = settings.skip_download_if_exceeds_query_cache; + res.filesystem_cache_max_download_size = settings->filesystem_cache_max_download_size; + res.skip_download_if_exceeds_query_cache = settings->skip_download_if_exceeds_query_cache; res.page_cache = getPageCache(); - res.use_page_cache_for_disks_without_file_cache = settings.use_page_cache_for_disks_without_file_cache; - res.read_from_page_cache_if_exists_otherwise_bypass_cache = settings.read_from_page_cache_if_exists_otherwise_bypass_cache; - res.page_cache_inject_eviction = settings.page_cache_inject_eviction; + res.use_page_cache_for_disks_without_file_cache = settings->use_page_cache_for_disks_without_file_cache; + res.read_from_page_cache_if_exists_otherwise_bypass_cache = settings->read_from_page_cache_if_exists_otherwise_bypass_cache; + res.page_cache_inject_eviction = settings->page_cache_inject_eviction; - res.remote_read_min_bytes_for_seek = settings.remote_read_min_bytes_for_seek; + res.remote_read_min_bytes_for_seek = settings->remote_read_min_bytes_for_seek; /// Zero read buffer will not make progress. - if (!settings.max_read_buffer_size) + if (!settings->max_read_buffer_size) { throw Exception(ErrorCodes::INVALID_SETTING_VALUE, - "Invalid value '{}' for max_read_buffer_size", settings.max_read_buffer_size); + "Invalid value '{}' for max_read_buffer_size", settings->max_read_buffer_size); } res.local_fs_buffer_size - = settings.max_read_buffer_size_local_fs ? settings.max_read_buffer_size_local_fs : settings.max_read_buffer_size; + = settings->max_read_buffer_size_local_fs ? settings->max_read_buffer_size_local_fs : settings->max_read_buffer_size; res.remote_fs_buffer_size - = settings.max_read_buffer_size_remote_fs ? settings.max_read_buffer_size_remote_fs : settings.max_read_buffer_size; - res.prefetch_buffer_size = settings.prefetch_buffer_size; - res.direct_io_threshold = settings.min_bytes_to_use_direct_io; - res.mmap_threshold = settings.min_bytes_to_use_mmap_io; - res.priority = Priority{settings.read_priority}; + = settings->max_read_buffer_size_remote_fs ? settings->max_read_buffer_size_remote_fs : settings->max_read_buffer_size; + res.prefetch_buffer_size = settings->prefetch_buffer_size; + res.direct_io_threshold = settings->min_bytes_to_use_direct_io; + res.mmap_threshold = settings->min_bytes_to_use_mmap_io; + res.priority = Priority{settings->read_priority}; res.remote_throttler = getRemoteReadThrottler(); res.local_throttler = getLocalReadThrottler(); - res.http_max_tries = settings.http_max_tries; - res.http_retry_initial_backoff_ms = settings.http_retry_initial_backoff_ms; - res.http_retry_max_backoff_ms = settings.http_retry_max_backoff_ms; - res.http_skip_not_found_url_for_globs = settings.http_skip_not_found_url_for_globs; - res.http_make_head_request = settings.http_make_head_request; + res.http_max_tries = settings->http_max_tries; + res.http_retry_initial_backoff_ms = settings->http_retry_initial_backoff_ms; + res.http_retry_max_backoff_ms = settings->http_retry_max_backoff_ms; + res.http_skip_not_found_url_for_globs = settings->http_skip_not_found_url_for_globs; + res.http_make_head_request = settings->http_make_head_request; res.mmap_cache = getMMappedFileCache().get(); @@ -5466,13 +5527,13 @@ WriteSettings Context::getWriteSettings() const { WriteSettings res; - res.enable_filesystem_cache_on_write_operations = settings.enable_filesystem_cache_on_write_operations; - res.enable_filesystem_cache_log = settings.enable_filesystem_cache_log; - res.throw_on_error_from_cache = settings.throw_on_error_from_cache_on_write_operations; - res.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds = settings.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds; + res.enable_filesystem_cache_on_write_operations = settings->enable_filesystem_cache_on_write_operations; + res.enable_filesystem_cache_log = settings->enable_filesystem_cache_log; + res.throw_on_error_from_cache = settings->throw_on_error_from_cache_on_write_operations; + res.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds = settings->filesystem_cache_reserve_space_wait_lock_timeout_milliseconds; - res.s3_allow_parallel_part_upload = settings.s3_allow_parallel_part_upload; - res.azure_allow_parallel_part_upload = settings.azure_allow_parallel_part_upload; + res.s3_allow_parallel_part_upload = settings->s3_allow_parallel_part_upload; + res.azure_allow_parallel_part_upload = settings->azure_allow_parallel_part_upload; res.remote_throttler = getRemoteWriteThrottler(); res.local_throttler = getLocalWriteThrottler(); @@ -5520,7 +5581,7 @@ bool Context::canUseParallelReplicasOnFollower() const bool Context::canUseParallelReplicasCustomKey() const { - return settings.max_parallel_replicas > 1 && getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY; + return settings->max_parallel_replicas > 1 && getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY; } bool Context::canUseParallelReplicasCustomKeyForCluster(const Cluster & cluster) const @@ -5530,7 +5591,7 @@ bool Context::canUseParallelReplicasCustomKeyForCluster(const Cluster & cluster) bool Context::canUseOffsetParallelReplicas() const { - return offset_parallel_replicas_enabled && settings.max_parallel_replicas > 1 + return offset_parallel_replicas_enabled && settings->max_parallel_replicas > 1 && getParallelReplicasMode() != Context::ParallelReplicasMode::READ_TASKS; } @@ -5542,13 +5603,13 @@ void Context::disableOffsetParallelReplicas() ClusterPtr Context::getClusterForParallelReplicas() const { /// check cluster for parallel replicas - if (settings.cluster_for_parallel_replicas.value.empty()) + if (settings->cluster_for_parallel_replicas.value.empty()) throw Exception( ErrorCodes::CLUSTER_DOESNT_EXIST, "Reading in parallel from replicas is enabled but cluster to execute query is not provided. Please set " "'cluster_for_parallel_replicas' setting"); - return getCluster(settings.cluster_for_parallel_replicas); + return getCluster(settings->cluster_for_parallel_replicas); } void Context::setPreparedSetsCache(const PreparedSetsCachePtr & cache) @@ -5576,4 +5637,39 @@ const ServerSettings & Context::getServerSettings() const return shared->server_settings; } +uint64_t HTTPContext::getMaxHstsAge() const +{ + return context->getSettingsRef().hsts_max_age; +} + + uint64_t HTTPContext::getMaxUriSize() const +{ + return context->getSettingsRef().http_max_uri_size; +} + +uint64_t HTTPContext::getMaxFields() const +{ +return context->getSettingsRef().http_max_fields; +} + +uint64_t HTTPContext::getMaxFieldNameSize() const +{ +return context->getSettingsRef().http_max_field_name_size; +} + +uint64_t HTTPContext::getMaxFieldValueSize() const +{ +return context->getSettingsRef().http_max_field_value_size; +} + +Poco::Timespan HTTPContext::getReceiveTimeout() const +{ +return context->getSettingsRef().http_receive_timeout; +} + +Poco::Timespan HTTPContext::getSendTimeout() const +{ +return context->getSettingsRef().http_send_timeout; +} + } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 8c5492bcbc8..284cac50769 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -11,9 +11,11 @@ #include #include #include -#include #include +#include #include +#include +#include #include #include #include @@ -129,6 +131,7 @@ class ShellCommand; class ICompressionCodec; class AccessControl; class GSSAcceptorContext; +struct Settings; struct SettingsConstraintsAndProfileIDs; class SettingsProfileElements; class RemoteHostFilter; @@ -276,7 +279,7 @@ protected: mutable std::shared_ptr access; mutable bool need_recalculate_access = true; String current_database; - Settings settings; /// Setting for query execution. + std::unique_ptr settings{}; /// Setting for query execution. using ProgressCallback = std::function; ProgressCallback progress_callback; /// Callback for tracking progress of query execution. @@ -952,7 +955,7 @@ public: void makeSessionContext(); void makeGlobalContext(); - const Settings & getSettingsRef() const { return settings; } + const Settings & getSettingsRef() const { return *settings; } void setProgressCallback(ProgressCallback callback); /// Used in executeQuery() to pass it to the QueryPipeline. @@ -1428,40 +1431,19 @@ struct HTTPContext : public IHTTPContext : context(Context::createCopy(context_)) {} - uint64_t getMaxHstsAge() const override - { - return context->getSettingsRef().hsts_max_age; - } + uint64_t getMaxHstsAge() const override; - uint64_t getMaxUriSize() const override - { - return context->getSettingsRef().http_max_uri_size; - } + uint64_t getMaxUriSize() const override; - uint64_t getMaxFields() const override - { - return context->getSettingsRef().http_max_fields; - } + uint64_t getMaxFields() const override; - uint64_t getMaxFieldNameSize() const override - { - return context->getSettingsRef().http_max_field_name_size; - } + uint64_t getMaxFieldNameSize() const override; - uint64_t getMaxFieldValueSize() const override - { - return context->getSettingsRef().http_max_field_value_size; - } + uint64_t getMaxFieldValueSize() const override; - Poco::Timespan getReceiveTimeout() const override - { - return context->getSettingsRef().http_receive_timeout; - } + Poco::Timespan getReceiveTimeout() const override; - Poco::Timespan getSendTimeout() const override - { - return context->getSettingsRef().http_send_timeout; - } + Poco::Timespan getSendTimeout() const override; ContextPtr context; }; diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 6c346836ed8..fa197d59c13 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/DDLTask.h b/src/Interpreters/DDLTask.h index 0b0460b26c8..515a35d8671 100644 --- a/src/Interpreters/DDLTask.h +++ b/src/Interpreters/DDLTask.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 1685c06d387..5f6d7e33927 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include "config.h" diff --git a/src/Interpreters/FilesystemCacheLog.h b/src/Interpreters/FilesystemCacheLog.h index 27c616ff40c..6d92fbd7a43 100644 --- a/src/Interpreters/FilesystemCacheLog.h +++ b/src/Interpreters/FilesystemCacheLog.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp index 6970048269b..a241d1cd258 100644 --- a/src/Interpreters/GraceHashJoin.cpp +++ b/src/Interpreters/GraceHashJoin.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/IInterpreter.cpp b/src/Interpreters/IInterpreter.cpp index 148e73e43ce..a291199f24b 100644 --- a/src/Interpreters/IInterpreter.cpp +++ b/src/Interpreters/IInterpreter.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.h b/src/Interpreters/IInterpreterUnionOrSelectQuery.h index e4425a73505..4ecfc464e90 100644 --- a/src/Interpreters/IInterpreterUnionOrSelectQuery.h +++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.h @@ -6,6 +6,8 @@ #include #include +#include + namespace DB { diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index 36972aeb03d..24e594661e9 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -2,6 +2,8 @@ #include +#include + #include #include diff --git a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp index 3b3ef928b42..22ecb6a5db2 100644 --- a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp +++ b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace DB diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp index 4a84a7bf570..5c4fb7a1443 100644 --- a/src/Interpreters/InterpreterCheckQuery.cpp +++ b/src/Interpreters/InterpreterCheckQuery.cpp @@ -12,6 +12,8 @@ #include #include +#include + #include #include diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index a439cb672c8..1c6abe842d4 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterDropIndexQuery.cpp b/src/Interpreters/InterpreterDropIndexQuery.cpp index f052aa201f1..8777532e4d0 100644 --- a/src/Interpreters/InterpreterDropIndexQuery.cpp +++ b/src/Interpreters/InterpreterDropIndexQuery.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 6d6b1085ffb..7eb487ba7b3 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index 32c475d138f..c6e52590f89 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include diff --git a/src/Interpreters/InterpreterShowColumnsQuery.cpp b/src/Interpreters/InterpreterShowColumnsQuery.cpp index f32ebceaa63..d8fff4e6026 100644 --- a/src/Interpreters/InterpreterShowColumnsQuery.cpp +++ b/src/Interpreters/InterpreterShowColumnsQuery.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterTransactionControlQuery.cpp b/src/Interpreters/InterpreterTransactionControlQuery.cpp index 13872fbe3f5..e92d6f9c5e7 100644 --- a/src/Interpreters/InterpreterTransactionControlQuery.cpp +++ b/src/Interpreters/InterpreterTransactionControlQuery.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Interpreters/MutationsNonDeterministicHelpers.cpp b/src/Interpreters/MutationsNonDeterministicHelpers.cpp index 7a4cb91acc0..bcff3e18b25 100644 --- a/src/Interpreters/MutationsNonDeterministicHelpers.cpp +++ b/src/Interpreters/MutationsNonDeterministicHelpers.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.h b/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.h index e8194f0dfe1..b2f55003da5 100644 --- a/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.h +++ b/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.h @@ -1,11 +1,10 @@ #pragma once -#include - +#include #include #include -#include +#include namespace DB { diff --git a/src/Interpreters/PredicateExpressionsOptimizer.cpp b/src/Interpreters/PredicateExpressionsOptimizer.cpp index 8dc8c1c92cc..af67e62be1a 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Interpreters/QueryLog.cpp b/src/Interpreters/QueryLog.cpp index e63a2ae31aa..527159dc981 100644 --- a/src/Interpreters/QueryLog.cpp +++ b/src/Interpreters/QueryLog.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/QueryLog.h b/src/Interpreters/QueryLog.h index bbaa7179757..40904bb939c 100644 --- a/src/Interpreters/QueryLog.h +++ b/src/Interpreters/QueryLog.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -22,6 +22,8 @@ namespace ProfileEvents namespace DB { +struct Settings; + /** Allows to log information about queries execution: * - info about start of query execution; diff --git a/src/Interpreters/RewriteCountVariantsVisitor.cpp b/src/Interpreters/RewriteCountVariantsVisitor.cpp index f207bc51527..4a541c3765a 100644 --- a/src/Interpreters/RewriteCountVariantsVisitor.cpp +++ b/src/Interpreters/RewriteCountVariantsVisitor.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index bb8c415602f..2ef6bec94df 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 6ec6a64b13d..1f7c6b1fe68 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index 4e1a2bcf5ee..ac80ad2fb68 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include #include + #include diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 9c3f85128cf..5d237d28089 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/formatWithPossiblyHidingSecrets.h b/src/Interpreters/formatWithPossiblyHidingSecrets.h index 25e1e7a5616..039bcbc2bca 100644 --- a/src/Interpreters/formatWithPossiblyHidingSecrets.h +++ b/src/Interpreters/formatWithPossiblyHidingSecrets.h @@ -2,6 +2,8 @@ #include "Access/ContextAccess.h" #include "Interpreters/Context.h" +#include + namespace DB { struct SecretHidingFormatSettings diff --git a/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp b/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp index 43be7c5f043..6f902c7cd7e 100644 --- a/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp +++ b/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index 9c3922b8bda..b5148174043 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include diff --git a/src/Interpreters/parseColumnsListForTableFunction.cpp b/src/Interpreters/parseColumnsListForTableFunction.cpp index 3529863a623..b9fdaabede1 100644 --- a/src/Interpreters/parseColumnsListForTableFunction.cpp +++ b/src/Interpreters/parseColumnsListForTableFunction.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -21,6 +22,18 @@ extern const int ILLEGAL_COLUMN; } +DataTypeValidationSettings::DataTypeValidationSettings(const DB::Settings& settings) + : allow_suspicious_low_cardinality_types(settings.allow_suspicious_low_cardinality_types) + , allow_experimental_object_type(settings.allow_experimental_object_type) + , allow_suspicious_fixed_string_types(settings.allow_suspicious_fixed_string_types) + , allow_experimental_variant_type(settings.allow_experimental_variant_type) + , allow_suspicious_variant_types(settings.allow_suspicious_variant_types) + , validate_nested_types(settings.validate_experimental_and_suspicious_types_inside_nested_types) + , allow_experimental_dynamic_type(settings.allow_experimental_dynamic_type) +{ +} + + void validateDataType(const DataTypePtr & type_to_check, const DataTypeValidationSettings & settings) { auto validate_callback = [&](const IDataType & data_type) diff --git a/src/Interpreters/parseColumnsListForTableFunction.h b/src/Interpreters/parseColumnsListForTableFunction.h index e2d2bc97ff7..6f15c585e4f 100644 --- a/src/Interpreters/parseColumnsListForTableFunction.h +++ b/src/Interpreters/parseColumnsListForTableFunction.h @@ -2,28 +2,19 @@ #include #include -#include namespace DB { class Context; +struct Settings; struct DataTypeValidationSettings { DataTypeValidationSettings() = default; - explicit DataTypeValidationSettings(const Settings & settings) - : allow_suspicious_low_cardinality_types(settings.allow_suspicious_low_cardinality_types) - , allow_experimental_object_type(settings.allow_experimental_object_type) - , allow_suspicious_fixed_string_types(settings.allow_suspicious_fixed_string_types) - , allow_experimental_variant_type(settings.allow_experimental_variant_type) - , allow_suspicious_variant_types(settings.allow_suspicious_variant_types) - , validate_nested_types(settings.validate_experimental_and_suspicious_types_inside_nested_types) - , allow_experimental_dynamic_type(settings.allow_experimental_dynamic_type) - { - } + explicit DataTypeValidationSettings(const Settings & settings); bool allow_suspicious_low_cardinality_types = true; bool allow_experimental_object_type = true; diff --git a/src/Interpreters/removeOnClusterClauseIfNeeded.cpp b/src/Interpreters/removeOnClusterClauseIfNeeded.cpp index 44167fe7242..c23c600f95a 100644 --- a/src/Interpreters/removeOnClusterClauseIfNeeded.cpp +++ b/src/Interpreters/removeOnClusterClauseIfNeeded.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Planner/CollectSets.cpp b/src/Planner/CollectSets.cpp index 52a0d748d63..9cac4e61380 100644 --- a/src/Planner/CollectSets.cpp +++ b/src/Planner/CollectSets.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 2d42ed73223..fc10a73fae9 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 7a12d5d690d..a0c7f9b2516 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -30,6 +30,8 @@ #include #include +#include + namespace DB { diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index ceb506d1bbb..68e0ef9785e 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -23,6 +23,8 @@ #include #include +#include + namespace DB { diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 47582d2904f..2877dc55b0d 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index e752c57b08b..ba8af7a7f13 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -43,6 +43,8 @@ #include #include +#include + namespace DB { diff --git a/src/Planner/PlannerSorting.cpp b/src/Planner/PlannerSorting.cpp index 611a26f78fa..1c6e820fed1 100644 --- a/src/Planner/PlannerSorting.cpp +++ b/src/Planner/PlannerSorting.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include diff --git a/src/Planner/PlannerWindowFunctions.cpp b/src/Planner/PlannerWindowFunctions.cpp index ce74d82c08d..225852de5a7 100644 --- a/src/Planner/PlannerWindowFunctions.cpp +++ b/src/Planner/PlannerWindowFunctions.cpp @@ -4,6 +4,8 @@ #include #include +#include + #include #include diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index 18a6d297838..3c54c57a28c 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -33,6 +33,8 @@ #include #include +#include + #include #include #include diff --git a/src/Planner/findParallelReplicasQuery.cpp b/src/Planner/findParallelReplicasQuery.cpp index f2bc1f060d8..c89a70be541 100644 --- a/src/Planner/findParallelReplicasQuery.cpp +++ b/src/Planner/findParallelReplicasQuery.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 49ec9999521..82cad471a29 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #ifndef NDEBUG #include diff --git a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp index fe82d1b1c53..06e8668cd7c 100644 --- a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp +++ b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index 1493779ec2d..c23d717d52f 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Processors/Formats/InputFormatErrorsLogger.cpp b/src/Processors/Formats/InputFormatErrorsLogger.cpp index 814c4679cf9..b5aa936df07 100644 --- a/src/Processors/Formats/InputFormatErrorsLogger.cpp +++ b/src/Processors/Formats/InputFormatErrorsLogger.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp index 1badd315200..3cb7f8a5d83 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -8,6 +9,7 @@ #include #include #include + namespace DB { diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp index 0af3869ccf1..44c1bf0f7d5 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp @@ -7,9 +7,11 @@ #include #include #include +#include #include #include #include + #include namespace DB::QueryPlanOptimizations diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 84c2515e8ca..fe724a8a198 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index 1c40f84d23d..48fad9f5fdb 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -10,10 +9,13 @@ #include #include #include +#include #include #include +#include + namespace CurrentMetrics { extern const Metric TemporaryFilesForSort; diff --git a/src/Processors/Sources/MySQLSource.cpp b/src/Processors/Sources/MySQLSource.cpp index 985a82a7b17..5d533a7747e 100644 --- a/src/Processors/Sources/MySQLSource.cpp +++ b/src/Processors/Sources/MySQLSource.cpp @@ -2,7 +2,9 @@ #if USE_MYSQL #include + #include +#include #include #include #include diff --git a/src/Processors/Sources/MySQLSource.h b/src/Processors/Sources/MySQLSource.h index 4ae5af22dab..b99e48a7759 100644 --- a/src/Processors/Sources/MySQLSource.h +++ b/src/Processors/Sources/MySQLSource.h @@ -6,11 +6,12 @@ #include #include #include -#include namespace DB { +struct Settings; + struct StreamSettings { /// Check if setting is enabled, otherwise use common `max_block_size` setting. diff --git a/src/Processors/Sources/RecursiveCTESource.cpp b/src/Processors/Sources/RecursiveCTESource.cpp index 221198c622a..40e72e9cbb1 100644 --- a/src/Processors/Sources/RecursiveCTESource.cpp +++ b/src/Processors/Sources/RecursiveCTESource.cpp @@ -17,6 +17,8 @@ #include #include +#include + namespace DB { diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.cpp b/src/Processors/TTL/TTLAggregationAlgorithm.cpp index 45e8a96412e..2d7a37d0abe 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.cpp +++ b/src/Processors/TTL/TTLAggregationAlgorithm.cpp @@ -1,5 +1,6 @@ -#include +#include #include +#include namespace DB { diff --git a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp index a2a42f27c3f..c33e87815b1 100644 --- a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp @@ -14,7 +14,8 @@ #include #include #include "IO/CompressionMethod.h" -#include "Parsers/ASTLiteral.h" +#include +#include namespace DB diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index bde8ce78f55..30662b1365c 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -899,4 +900,10 @@ void RemoteQueryExecutor::setProfileInfoCallback(ProfileInfoCallback callback) std::lock_guard guard(was_cancelled_mutex); profile_info_callback = std::move(callback); } + +bool RemoteQueryExecutor::needToSkipUnavailableShard() const +{ + return context->getSettingsRef().skip_unavailable_shards && (0 == connections->size()); +} + } diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h index 6b1539bd08e..04a59cc3b7e 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.h +++ b/src/QueryPipeline/RemoteQueryExecutor.h @@ -218,7 +218,7 @@ public: IConnections & getConnections() { return *connections; } - bool needToSkipUnavailableShard() const { return context->getSettingsRef().skip_unavailable_shards && (0 == connections->size()); } + bool needToSkipUnavailableShard() const; bool isReplicaUnavailable() const { return extension && extension->parallel_reading_coordinator && connections->size() == 0; } diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index cb36df1efc0..4546de1ccae 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/HTTP/setReadOnlyIfHTTPMethodIdempotent.cpp b/src/Server/HTTP/setReadOnlyIfHTTPMethodIdempotent.cpp index d42bd77e339..48a3b39a3e7 100644 --- a/src/Server/HTTP/setReadOnlyIfHTTPMethodIdempotent.cpp +++ b/src/Server/HTTP/setReadOnlyIfHTTPMethodIdempotent.cpp @@ -1,5 +1,6 @@ #include +#include #include #include diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 75901e6b901..c8a58527f2c 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index c0f015bfcd5..b6d795b1e69 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/MySQLHandler.h b/src/Server/MySQLHandler.h index 2f891ebf810..f632ef51c00 100644 --- a/src/Server/MySQLHandler.h +++ b/src/Server/MySQLHandler.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "IO/ReadBufferFromString.h" #include "IServer.h" diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 8ba8421e6f0..cde31b4c58a 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #if USE_SSL # include diff --git a/src/Server/ProxyV1Handler.cpp b/src/Server/ProxyV1Handler.cpp index d5e6ab23360..5b33a119cef 100644 --- a/src/Server/ProxyV1Handler.cpp +++ b/src/Server/ProxyV1Handler.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index e556bda2561..a2bd7237854 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include diff --git a/src/Storages/FileLog/FileLogSettings.cpp b/src/Storages/FileLog/FileLogSettings.cpp index 8e245285b9a..cf2d6b64d7c 100644 --- a/src/Storages/FileLog/FileLogSettings.cpp +++ b/src/Storages/FileLog/FileLogSettings.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp index bc8cb0ce69a..29631b95542 100644 --- a/src/Storages/MaterializedView/RefreshTask.cpp +++ b/src/Storages/MaterializedView/RefreshTask.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/ApproximateNearestNeighborIndexesCommon.cpp b/src/Storages/MergeTree/ApproximateNearestNeighborIndexesCommon.cpp index 69e54dd5f0c..dfd742c9cdd 100644 --- a/src/Storages/MergeTree/ApproximateNearestNeighborIndexesCommon.cpp +++ b/src/Storages/MergeTree/ApproximateNearestNeighborIndexesCommon.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 7e4b1db4c89..35fa2a8236f 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index bf05f3afaf4..73244b714bf 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include diff --git a/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp b/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp index e492ca0aec2..c20e31524a0 100644 --- a/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp index 2c249f7b63b..b6f626c992b 100644 --- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace ProfileEvents diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index dc1ba030f45..1daf46b6e25 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -335,4 +336,12 @@ void MergeTreeReadPool::fillPerThreadInfo(size_t threads, size_t sum_marks) } } +MergeTreeReadPool::BackoffSettings::BackoffSettings(const DB::Settings& settings) + : min_read_latency_ms(settings.read_backoff_min_latency_ms.totalMilliseconds()), + max_throughput(settings.read_backoff_max_throughput), + min_interval_between_events_ms(settings.read_backoff_min_interval_between_events_ms.totalMilliseconds()), + min_events(settings.read_backoff_min_events), + min_concurrency(settings.read_backoff_min_concurrency) +{} + } diff --git a/src/Storages/MergeTree/MergeTreeReadPool.h b/src/Storages/MergeTree/MergeTreeReadPool.h index cb0e8a9657f..cd12b9db9fd 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/src/Storages/MergeTree/MergeTreeReadPool.h @@ -64,12 +64,7 @@ public: size_t min_concurrency = 1; /// Constants above is just an example. - explicit BackoffSettings(const Settings & settings) - : min_read_latency_ms(settings.read_backoff_min_latency_ms.totalMilliseconds()), - max_throughput(settings.read_backoff_max_throughput), - min_interval_between_events_ms(settings.read_backoff_min_interval_between_events_ms.totalMilliseconds()), - min_events(settings.read_backoff_min_events), - min_concurrency(settings.read_backoff_min_concurrency) {} + explicit BackoffSettings(const Settings & settings); BackoffSettings() : min_read_latency_ms(0) {} }; diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index a9a5fddace4..e7b3585ecda 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 9d696b70d9f..38869aebaa5 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 5ae6517a236..c167ac87317 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp index 4a18d606bb7..93087b0d3fe 100644 --- a/src/Storages/MergeTree/RPNBuilder.cpp +++ b/src/Storages/MergeTree/RPNBuilder.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h index 39623c20584..13cb98e65a1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h @@ -5,6 +5,7 @@ #include #include #include +#include #include diff --git a/src/Storages/MySQL/MySQLSettings.cpp b/src/Storages/MySQL/MySQLSettings.cpp index fd53174f4f6..1c88e4a94e9 100644 --- a/src/Storages/MySQL/MySQLSettings.cpp +++ b/src/Storages/MySQL/MySQLSettings.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index ba90f21c907..8babb2bd315 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index b4aea096c59..f444a581eb6 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -9,14 +9,14 @@ #include +namespace DB +{ + namespace ErrorCodes { extern const int BAD_ARGUMENTS; } -namespace DB -{ - /// Helper function to get named collection for table engine. /// Table engines have collection name as first argument of ast and other arguments are key-value overrides. MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index 91cc02de0f0..f0a0a562b92 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 0484f86542c..6d18b13df01 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -3,7 +3,7 @@ #if USE_AWS_S3 && USE_AVRO #include - +#include #include #include #include diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.cpp b/src/Storages/ObjectStorage/HDFS/Configuration.cpp index 155f51adf61..e8071be6f02 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.cpp +++ b/src/Storages/ObjectStorage/HDFS/Configuration.cpp @@ -2,6 +2,7 @@ #if USE_HDFS #include +#include #include #include #include diff --git a/src/Storages/ObjectStorage/ReadBufferIterator.cpp b/src/Storages/ObjectStorage/ReadBufferIterator.cpp index 78cdc442f64..d739f7a28c9 100644 --- a/src/Storages/ObjectStorage/ReadBufferIterator.cpp +++ b/src/Storages/ObjectStorage/ReadBufferIterator.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index b33d55105e9..094ca069e7a 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -1,6 +1,7 @@ #include #if USE_AWS_S3 +#include #include #include #include diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 683473006e3..fe675a17d54 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp index f2f6eac333c..000e73c70ce 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include namespace DB diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 6940f10cb91..f25eb0001b9 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace fs = std::filesystem; diff --git a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp index bf595b2f5d4..b5f4cf5bb54 100644 --- a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp @@ -1,9 +1,10 @@ +#include +#include #include #include #include #include #include -#include namespace DB { diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index e0ef967d491..9c8c4c2fe79 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp b/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp index 90792c59d38..b53f694bc5a 100644 --- a/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp +++ b/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/RocksDB/StorageSystemRocksDB.cpp b/src/Storages/RocksDB/StorageSystemRocksDB.cpp index 5105b190fd9..744d43e1c77 100644 --- a/src/Storages/RocksDB/StorageSystemRocksDB.cpp +++ b/src/Storages/RocksDB/StorageSystemRocksDB.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/SetSettings.cpp b/src/Storages/SetSettings.cpp index baa3d231067..4e6dd6a0519 100644 --- a/src/Storages/SetSettings.cpp +++ b/src/Storages/SetSettings.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 9d12a1569d8..060b271d8f4 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/StorageGenerateRandom.cpp b/src/Storages/StorageGenerateRandom.cpp index 754bc096958..f3eb88f3207 100644 --- a/src/Storages/StorageGenerateRandom.cpp +++ b/src/Storages/StorageGenerateRandom.cpp @@ -30,6 +30,7 @@ #include #include +#include #include #include diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index a5bae0acce5..2226de3e64f 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -3,6 +3,8 @@ #include #include +#include + #include #include #include diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index de0324d7998..ee0a37b50e3 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index da391909dff..f2e19ad4c80 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 8df87d6290f..82de84220f1 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -7,6 +7,8 @@ #include #include +#include + #include #include #include diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index cd1282bb02c..9cec8c75ebe 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index b1bcae0f7e9..fa23a574ade 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 8dd8d3b6154..6a7810b97f9 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemDDLWorkerQueue.cpp b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp index f96b839a322..ae1a233ea81 100644 --- a/src/Storages/System/StorageSystemDDLWorkerQueue.cpp +++ b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemDetachedParts.cpp b/src/Storages/System/StorageSystemDetachedParts.cpp index f48a8c67971..59eba9b5791 100644 --- a/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/src/Storages/System/StorageSystemDetachedParts.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index 2da268305f8..2955aebeb7e 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemEvents.cpp b/src/Storages/System/StorageSystemEvents.cpp index 822d5c77788..f74c316e3fc 100644 --- a/src/Storages/System/StorageSystemEvents.cpp +++ b/src/Storages/System/StorageSystemEvents.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemMergeTreeSettings.cpp b/src/Storages/System/StorageSystemMergeTreeSettings.cpp index db8fd55cfc5..18730010e53 100644 --- a/src/Storages/System/StorageSystemMergeTreeSettings.cpp +++ b/src/Storages/System/StorageSystemMergeTreeSettings.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Storages/System/StorageSystemPartsBase.h b/src/Storages/System/StorageSystemPartsBase.h index be945162b39..8671fd850f8 100644 --- a/src/Storages/System/StorageSystemPartsBase.h +++ b/src/Storages/System/StorageSystemPartsBase.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include diff --git a/src/Storages/System/StorageSystemRemoteDataPaths.cpp b/src/Storages/System/StorageSystemRemoteDataPaths.cpp index a5c496db7e7..00b4824cc3d 100644 --- a/src/Storages/System/StorageSystemRemoteDataPaths.cpp +++ b/src/Storages/System/StorageSystemRemoteDataPaths.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index b437108b00e..f1e9d542fec 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -1,4 +1,6 @@ #include + +#include #include #include #include diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp index 2af3e6dfd05..40a669351b1 100644 --- a/src/Storages/System/StorageSystemSettingsProfileElements.cpp +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemStackTrace.cpp b/src/Storages/System/StorageSystemStackTrace.cpp index ba7433fb9ae..cc221bbea69 100644 --- a/src/Storages/System/StorageSystemStackTrace.cpp +++ b/src/Storages/System/StorageSystemStackTrace.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index 541bd852140..b4d83058c82 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -18,8 +18,12 @@ #include #include #include -#include "base/types.h" + #include + +#include +#include + #include diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index f831465277d..ac091e7cf3c 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index 84ba92bba00..69e8f0a7880 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/getStructureOfRemoteTable.cpp b/src/Storages/getStructureOfRemoteTable.cpp index 6ea7bdc312d..56071abaa95 100644 --- a/src/Storages/getStructureOfRemoteTable.cpp +++ b/src/Storages/getStructureOfRemoteTable.cpp @@ -1,4 +1,5 @@ #include "getStructureOfRemoteTable.h" +#include #include #include #include diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index fc85bde11d9..f1b73e939b8 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/TableFunctions/ITableFunctionXDBC.cpp b/src/TableFunctions/ITableFunctionXDBC.cpp index 043712f97b8..83d0ac4ef41 100644 --- a/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/src/TableFunctions/ITableFunctionXDBC.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/TableFunctions/TableFunctionExplain.cpp b/src/TableFunctions/TableFunctionExplain.cpp index df2835dd630..552b9fde986 100644 --- a/src/TableFunctions/TableFunctionExplain.cpp +++ b/src/TableFunctions/TableFunctionExplain.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/TableFunctions/TableFunctionFactory.cpp b/src/TableFunctions/TableFunctionFactory.cpp index ce3daff0785..6ecdeb47779 100644 --- a/src/TableFunctions/TableFunctionFactory.cpp +++ b/src/TableFunctions/TableFunctionFactory.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 28bf72e07fb..1b6d86f8fa5 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -4,6 +4,7 @@ #include "registerTableFunctions.h" #include +#include #include #include #include diff --git a/src/TableFunctions/TableFunctionFileCluster.cpp b/src/TableFunctions/TableFunctionFileCluster.cpp index 3e53349b022..514fa3143d0 100644 --- a/src/TableFunctions/TableFunctionFileCluster.cpp +++ b/src/TableFunctions/TableFunctionFileCluster.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/TableFunctions/TableFunctionFormat.cpp b/src/TableFunctions/TableFunctionFormat.cpp index ad2a142a140..5a0f47653d5 100644 --- a/src/TableFunctions/TableFunctionFormat.cpp +++ b/src/TableFunctions/TableFunctionFormat.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index b027cf42880..ffd318f36f7 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -3,6 +3,8 @@ #if USE_MYSQL #include + +#include #include #include #include diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index 8d94988cd65..c6c8655b394 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index d7774bb6478..e60c31b2d77 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include "registerTableFunctions.h" From 56f497bf7f748bb36b187477e93195567b8fefdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 11 Jul 2024 14:09:00 +0200 Subject: [PATCH 076/161] Remove some unnecessary copies --- src/Dictionaries/RegExpTreeDictionary.cpp | 6 +++--- src/Dictionaries/XDBCDictionarySource.cpp | 2 +- src/Functions/FunctionUnixTimestamp64.h | 2 +- .../ExternalUserDefinedExecutableFunctionsLoader.cpp | 2 +- src/Functions/formatDateTime.cpp | 6 +++--- src/Functions/now.cpp | 2 +- src/Functions/now64.cpp | 2 +- src/Functions/nowInBlock.cpp | 2 +- src/Functions/parseDateTime.cpp | 4 ++-- src/Functions/snowflakeIDToDateTime.cpp | 4 ++-- src/Functions/toTimezone.cpp | 2 +- src/Interpreters/ActionsVisitor.cpp | 2 +- src/Interpreters/InterpreterAlterQuery.cpp | 7 +++---- src/Interpreters/InterpreterExplainQuery.cpp | 2 +- src/Interpreters/removeOnClusterClauseIfNeeded.cpp | 4 ++-- src/Planner/PlannerJoins.cpp | 3 ++- .../Optimizations/optimizeUseAggregateProjection.cpp | 9 +++++---- .../Optimizations/optimizeUseNormalProjection.cpp | 8 ++++---- src/Server/GRPCServer.cpp | 7 +++---- src/Storages/Hive/StorageHive.cpp | 4 ++-- .../ApproximateNearestNeighborIndexesCommon.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp | 2 +- src/Storages/NamedCollectionsHelpers.cpp | 4 ++-- .../ObjectStorage/StorageObjectStorageSource.cpp | 2 +- src/Storages/StorageExecutable.cpp | 4 ++-- src/Storages/StorageMergeTree.cpp | 11 ++++++----- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/StorageStripeLog.cpp | 2 +- 29 files changed, 56 insertions(+), 55 deletions(-) diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index a5aa622e01d..2522de60183 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -1003,9 +1003,9 @@ void registerDictionaryRegExpTree(DictionaryFactory & factory) dict_struct, std::move(source_ptr), configuration, - context->getSettings().regexp_dict_allow_hyperscan, - context->getSettings().regexp_dict_flag_case_insensitive, - context->getSettings().regexp_dict_flag_dotall); + context->getSettingsRef().regexp_dict_allow_hyperscan, + context->getSettingsRef().regexp_dict_flag_case_insensitive, + context->getSettingsRef().regexp_dict_flag_dotall); }; factory.registerLayout("regexp_tree", create_layout, true); diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index b84cfa109ee..590d000f16e 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -241,7 +241,7 @@ void registerDictionarySourceXDBC(DictionarySourceFactory & factory) #if USE_ODBC BridgeHelperPtr bridge = std::make_shared>( global_context, - global_context->getSettings().http_receive_timeout, + global_context->getSettingsRef().http_receive_timeout, config.getString(config_prefix + ".odbc.connection_string"), config.getBool(config_prefix + ".settings.odbc_bridge_use_connection_pooling", global_context->getSettingsRef().odbc_bridge_use_connection_pooling)); diff --git a/src/Functions/FunctionUnixTimestamp64.h b/src/Functions/FunctionUnixTimestamp64.h index 652b3fe60ec..7b8da733f6d 100644 --- a/src/Functions/FunctionUnixTimestamp64.h +++ b/src/Functions/FunctionUnixTimestamp64.h @@ -111,7 +111,7 @@ public: FunctionFromUnixTimestamp64(size_t target_scale_, const char * name_, ContextPtr context) : target_scale(target_scale_) , name(name_) - , allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments) + , allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) {} String getName() const override { return name; } diff --git a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp index 4a5ff19763e..a20e1807487 100644 --- a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp +++ b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp @@ -185,7 +185,7 @@ ExternalLoader::LoadableMutablePtr ExternalUserDefinedExecutableFunctionsLoader: pool_size = config.getUInt64(key_in_config + ".pool_size", 16); max_command_execution_time = config.getUInt64(key_in_config + ".max_command_execution_time", 10); - size_t max_execution_time_seconds = static_cast(getContext()->getSettings().max_execution_time.totalSeconds()); + size_t max_execution_time_seconds = static_cast(getContext()->getSettingsRef().max_execution_time.totalSeconds()); if (max_execution_time_seconds != 0 && max_command_execution_time > max_execution_time_seconds) max_command_execution_time = max_execution_time_seconds; } diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index 64b9d4d8c71..c3a5fe036c3 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -783,9 +783,9 @@ public: static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } explicit FunctionFormatDateTimeImpl(ContextPtr context) - : mysql_M_is_month_name(context->getSettings().formatdatetime_parsedatetime_m_is_month_name) - , mysql_f_prints_single_zero(context->getSettings().formatdatetime_f_prints_single_zero) - , mysql_format_ckl_without_leading_zeros(context->getSettings().formatdatetime_format_without_leading_zeros) + : mysql_M_is_month_name(context->getSettingsRef().formatdatetime_parsedatetime_m_is_month_name) + , mysql_f_prints_single_zero(context->getSettingsRef().formatdatetime_f_prints_single_zero) + , mysql_format_ckl_without_leading_zeros(context->getSettingsRef().formatdatetime_format_without_leading_zeros) { } diff --git a/src/Functions/now.cpp b/src/Functions/now.cpp index 4e3f7be4f7e..352c05f8f32 100644 --- a/src/Functions/now.cpp +++ b/src/Functions/now.cpp @@ -91,7 +91,7 @@ public: size_t getNumberOfArguments() const override { return 0; } static FunctionOverloadResolverPtr create(ContextPtr context) { return std::make_unique(context); } explicit NowOverloadResolver(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) {} DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override diff --git a/src/Functions/now64.cpp b/src/Functions/now64.cpp index 988694708cc..23ab8cad003 100644 --- a/src/Functions/now64.cpp +++ b/src/Functions/now64.cpp @@ -119,7 +119,7 @@ public: size_t getNumberOfArguments() const override { return 0; } static FunctionOverloadResolverPtr create(ContextPtr context) { return std::make_unique(context); } explicit Now64OverloadResolver(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) {} DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override diff --git a/src/Functions/nowInBlock.cpp b/src/Functions/nowInBlock.cpp index b79653f501c..291e43bdce5 100644 --- a/src/Functions/nowInBlock.cpp +++ b/src/Functions/nowInBlock.cpp @@ -32,7 +32,7 @@ public: return std::make_shared(context); } explicit FunctionNowInBlock(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) {} String getName() const override diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 2bd8d370ed6..d2353c19a61 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -565,8 +565,8 @@ namespace static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } explicit FunctionParseDateTimeImpl(ContextPtr context) - : mysql_M_is_month_name(context->getSettings().formatdatetime_parsedatetime_m_is_month_name) - , mysql_parse_ckl_without_leading_zeros(context->getSettings().parsedatetime_parse_without_leading_zeros) + : mysql_M_is_month_name(context->getSettingsRef().formatdatetime_parsedatetime_m_is_month_name) + , mysql_parse_ckl_without_leading_zeros(context->getSettingsRef().parsedatetime_parse_without_leading_zeros) { } diff --git a/src/Functions/snowflakeIDToDateTime.cpp b/src/Functions/snowflakeIDToDateTime.cpp index bd805a2e607..6c57c72fff1 100644 --- a/src/Functions/snowflakeIDToDateTime.cpp +++ b/src/Functions/snowflakeIDToDateTime.cpp @@ -39,7 +39,7 @@ public: static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } explicit FunctionSnowflakeIDToDateTime(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) {} String getName() const override { return name; } @@ -110,7 +110,7 @@ public: static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } explicit FunctionSnowflakeIDToDateTime64(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) {} String getName() const override { return name; } diff --git a/src/Functions/toTimezone.cpp b/src/Functions/toTimezone.cpp index be839f1acb3..cbcdf9bc772 100644 --- a/src/Functions/toTimezone.cpp +++ b/src/Functions/toTimezone.cpp @@ -89,7 +89,7 @@ public: size_t getNumberOfArguments() const override { return 2; } static FunctionOverloadResolverPtr create(ContextPtr context) { return std::make_unique(context); } explicit ToTimeZoneOverloadResolver(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) {} DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index c3285d73145..74d83590ec9 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -446,7 +446,7 @@ FutureSetPtr makeExplicitSet( else block = createBlockForSet(left_arg_type, right_arg, set_element_types, context); - return prepared_sets.addFromTuple(set_key, block, context->getSettings()); + return prepared_sets.addFromTuple(set_key, block, context->getSettingsRef()); } class ScopeStack::Index diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index c70a3397f4e..a7a524ef836 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -175,10 +175,9 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) else throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong parameter type in ALTER query"); - if (!getContext()->getSettings().allow_experimental_statistics && ( - command_ast->type == ASTAlterCommand::ADD_STATISTICS || - command_ast->type == ASTAlterCommand::DROP_STATISTICS || - command_ast->type == ASTAlterCommand::MATERIALIZE_STATISTICS)) + if (!getContext()->getSettingsRef().allow_experimental_statistics + && (command_ast->type == ASTAlterCommand::ADD_STATISTICS || command_ast->type == ASTAlterCommand::DROP_STATISTICS + || command_ast->type == ASTAlterCommand::MATERIALIZE_STATISTICS)) throw Exception(ErrorCodes::INCORRECT_QUERY, "Alter table with statistics is now disabled. Turn on allow_experimental_statistics"); } diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index 7c7b4b3f95a..6ec5937f287 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -69,7 +69,7 @@ namespace static void visit(ASTSelectQuery & select, ASTPtr & node, Data & data) { /// we need to read statistic when `allow_statistics_optimize` is enabled. - bool only_analyze = !data.getContext()->getSettings().allow_statistics_optimize; + bool only_analyze = !data.getContext()->getSettingsRef().allow_statistics_optimize; InterpreterSelectQuery interpreter( node, data.getContext(), SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze(only_analyze).modify()); diff --git a/src/Interpreters/removeOnClusterClauseIfNeeded.cpp b/src/Interpreters/removeOnClusterClauseIfNeeded.cpp index c23c600f95a..948f5944590 100644 --- a/src/Interpreters/removeOnClusterClauseIfNeeded.cpp +++ b/src/Interpreters/removeOnClusterClauseIfNeeded.cpp @@ -47,10 +47,10 @@ ASTPtr removeOnClusterClauseIfNeeded(const ASTPtr & query, ContextPtr context, c return query; if ((isUserDefinedFunctionQuery(query) - && context->getSettings().ignore_on_cluster_for_replicated_udf_queries + && context->getSettingsRef().ignore_on_cluster_for_replicated_udf_queries && context->getUserDefinedSQLObjectsStorage().isReplicated()) || (isAccessControlQuery(query) - && context->getSettings().ignore_on_cluster_for_replicated_access_entities_queries + && context->getSettingsRef().ignore_on_cluster_for_replicated_access_entities_queries && context->getAccessControl().containsStorage(ReplicatedAccessStorage::STORAGE_TYPE))) { LOG_DEBUG(getLogger("removeOnClusterClauseIfNeeded"), "ON CLUSTER clause was ignored for query {}", query->getID()); diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index ba8af7a7f13..70ccc64e098 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -807,7 +807,8 @@ static std::shared_ptr tryCreateJoin(JoinAlgorithm algorithm, auto query_context = planner_context->getQueryContext(); if (table_join->allowParallelHashJoin()) - return std::make_shared(query_context, table_join, query_context->getSettings().max_threads, right_table_expression_header); + return std::make_shared( + query_context, table_join, query_context->getSettingsRef().max_threads, right_table_expression_header); return std::make_shared(table_join, right_table_expression_header, query_context->getSettingsRef().join_any_take_last_row); } diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp index 70327bc95b4..f6052846aec 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp @@ -515,10 +515,11 @@ AggregateProjectionCandidates getAggregateProjectionCandidates( if (!candidates.minmax_projection) { - auto it = std::find_if(agg_projections.begin(), agg_projections.end(), [&](const auto * projection) - { - return projection->name == context->getSettings().preferred_optimize_projection_name.value; - }); + auto it = std::find_if( + agg_projections.begin(), + agg_projections.end(), + [&](const auto * projection) + { return projection->name == context->getSettingsRef().preferred_optimize_projection_name.value; }); if (it != agg_projections.end()) { diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp index 44c1bf0f7d5..291800c2c86 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp @@ -112,10 +112,10 @@ std::optional optimizeUseNormalProjections(Stack & stack, QueryPlan::Nod return {}; ContextPtr context = reading->getContext(); - auto it = std::find_if(normal_projections.begin(), normal_projections.end(), [&](const auto * projection) - { - return projection->name == context->getSettings().preferred_optimize_projection_name.value; - }); + auto it = std::find_if( + normal_projections.begin(), + normal_projections.end(), + [&](const auto * projection) { return projection->name == context->getSettingsRef().preferred_optimize_projection_name.value; }); if (it != normal_projections.end()) { diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 4546de1ccae..d8a4d7f0e1f 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -1083,8 +1083,8 @@ namespace read_buffer = wrapReadBufferWithCompressionMethod(std::move(read_buffer), input_compression_method); assert(!pipeline); - auto source = query_context->getInputFormat( - input_format, *read_buffer, header, query_context->getSettings().max_insert_block_size); + auto source + = query_context->getInputFormat(input_format, *read_buffer, header, query_context->getSettingsRef().max_insert_block_size); pipeline = std::make_unique(std::move(source)); pipeline_executor = std::make_unique(*pipeline); @@ -1152,8 +1152,7 @@ namespace external_table_context->applySettingsChanges(settings_changes); } auto in = external_table_context->getInputFormat( - format, *buf, metadata_snapshot->getSampleBlock(), - external_table_context->getSettings().max_insert_block_size); + format, *buf, metadata_snapshot->getSampleBlock(), external_table_context->getSettingsRef().max_insert_block_size); QueryPipelineBuilder cur_pipeline; cur_pipeline.init(Pipe(std::move(in))); diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 28d8128e052..9063437c065 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -321,7 +321,7 @@ public: Chunk generateChunkFromMetadata() { - size_t num_rows = std::min(current_file_remained_rows, UInt64(getContext()->getSettings().max_block_size)); + size_t num_rows = std::min(current_file_remained_rows, UInt64(getContext()->getSettingsRef().max_block_size)); current_file_remained_rows -= num_rows; Block source_block; @@ -964,7 +964,7 @@ HiveFiles StorageHive::collectHiveFiles( /// Hive files to collect HiveFiles hive_files; Int64 hit_parttions_num = 0; - Int64 hive_max_query_partitions = context_->getSettings().max_partitions_to_read; + Int64 hive_max_query_partitions = context_->getSettingsRef().max_partitions_to_read; /// Mutext to protect hive_files, which maybe appended in multiple threads std::mutex hive_files_mutex; ThreadPool pool{CurrentMetrics::StorageHiveThreads, CurrentMetrics::StorageHiveThreadsActive, CurrentMetrics::StorageHiveThreadsScheduled, max_threads}; diff --git a/src/Storages/MergeTree/ApproximateNearestNeighborIndexesCommon.cpp b/src/Storages/MergeTree/ApproximateNearestNeighborIndexesCommon.cpp index dfd742c9cdd..7354243732c 100644 --- a/src/Storages/MergeTree/ApproximateNearestNeighborIndexesCommon.cpp +++ b/src/Storages/MergeTree/ApproximateNearestNeighborIndexesCommon.cpp @@ -55,7 +55,7 @@ ApproximateNearestNeighborInformation::Metric stringToMetric(std::string_view me ApproximateNearestNeighborCondition::ApproximateNearestNeighborCondition(const SelectQueryInfo & query_info, ContextPtr context) : block_with_constants(KeyCondition::getBlockWithConstants(query_info.query, query_info.syntax_analyzer_result, context)) , index_granularity(context->getMergeTreeSettings().index_granularity) - , max_limit_for_ann_queries(context->getSettings().max_limit_for_ann_queries) + , max_limit_for_ann_queries(context->getSettingsRef().max_limit_for_ann_queries) , index_is_useful(checkQueryStructure(query_info)) {} diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 59a4032f235..c1551668262 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -474,7 +474,7 @@ StoragePolicyPtr MergeTreeData::getStoragePolicy() const ConditionSelectivityEstimator MergeTreeData::getConditionSelectivityEstimatorByPredicate( const StorageSnapshotPtr & storage_snapshot, const ActionsDAGPtr & filter_dag, ContextPtr local_context) const { - if (!local_context->getSettings().allow_statistics_optimize) + if (!local_context->getSettingsRef().allow_statistics_optimize) return {}; const auto & parts = assert_cast(*storage_snapshot->data).parts; diff --git a/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp b/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp index c20e31524a0..c66aa2373e7 100644 --- a/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp @@ -231,7 +231,7 @@ MergeTreeIndexConditionAnnoy::MergeTreeIndexConditionAnnoy( ContextPtr context) : ann_condition(query, context) , distance_function(distance_function_) - , search_k(context->getSettings().annoy_index_search_k_nodes) + , search_k(context->getSettingsRef().annoy_index_search_k_nodes) {} bool MergeTreeIndexConditionAnnoy::mayBeTrueOnGranule(MergeTreeIndexGranulePtr /*idx_granule*/) const diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index 8babb2bd315..b58aed573ae 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -118,7 +118,7 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( if (asts.size() == 1) return collection_copy; - const auto allow_override_by_default = context->getSettings().allow_named_collection_override_by_default; + const auto allow_override_by_default = context->getSettingsRef().allow_named_collection_override_by_default; for (auto * it = std::next(asts.begin()); it != asts.end(); ++it) { @@ -163,7 +163,7 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( Poco::Util::AbstractConfiguration::Keys keys; config.keys(config_prefix, keys); - const auto allow_override_by_default = context->getSettings().allow_named_collection_override_by_default; + const auto allow_override_by_default = context->getSettingsRef().allow_named_collection_override_by_default; for (const auto & key : keys) { if (collection_copy->isOverridable(key, allow_override_by_default)) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index f25eb0001b9..2ba9afe88d5 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -420,7 +420,7 @@ std::unique_ptr StorageObjectStorageSource::createReadBuffer( /// FIXME: Changing this setting to default value breaks something around parquet reading read_settings.remote_read_min_bytes_for_seek = read_settings.remote_fs_buffer_size; - const bool object_too_small = object_size <= 2 * context_->getSettings().max_download_buffer_size; + const bool object_too_small = object_size <= 2 * context_->getSettingsRef().max_download_buffer_size; const bool use_prefetch = object_too_small && read_settings.remote_fs_method == RemoteFSReadMethod::threadpool; read_settings.remote_fs_method = use_prefetch ? RemoteFSReadMethod::threadpool : RemoteFSReadMethod::read; /// User's object may change, don't cache it. diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp index e475211deb3..66beb403086 100644 --- a/src/Storages/StorageExecutable.cpp +++ b/src/Storages/StorageExecutable.cpp @@ -147,7 +147,7 @@ void StorageExecutable::read( for (auto & input_query : input_queries) { QueryPipelineBuilder builder; - if (context->getSettings().allow_experimental_analyzer) + if (context->getSettingsRef().allow_experimental_analyzer) builder = InterpreterSelectQueryAnalyzer(input_query, context, {}).buildQueryPipeline(); else builder = InterpreterSelectWithUnionQuery(input_query, context, {}).buildQueryPipeline(); @@ -225,7 +225,7 @@ void registerStorageExecutable(StorageFactory & factory) { size_t max_command_execution_time = 10; - size_t max_execution_time_seconds = static_cast(args.getContext()->getSettings().max_execution_time.totalSeconds()); + size_t max_execution_time_seconds = static_cast(args.getContext()->getSettingsRef().max_execution_time.totalSeconds()); if (max_execution_time_seconds != 0 && max_command_execution_time > max_execution_time_seconds) max_command_execution_time = max_execution_time_seconds; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 65afd9c3230..a27e14abe67 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -657,12 +657,13 @@ void StorageMergeTree::mutate(const MutationCommands & commands, ContextPtr quer { /// It's important to serialize order of mutations with alter queries because /// they can depend on each other. - if (auto alter_lock = tryLockForAlter(query_context->getSettings().lock_acquire_timeout); alter_lock == std::nullopt) + if (auto alter_lock = tryLockForAlter(query_context->getSettingsRef().lock_acquire_timeout); alter_lock == std::nullopt) { - throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, - "Cannot start mutation in {}ms because some metadata-changing ALTER (MODIFY|RENAME|ADD|DROP) is currently executing. " - "You can change this timeout with `lock_acquire_timeout` setting", - query_context->getSettings().lock_acquire_timeout.totalMilliseconds()); + throw Exception( + ErrorCodes::TIMEOUT_EXCEEDED, + "Cannot start mutation in {}ms because some metadata-changing ALTER (MODIFY|RENAME|ADD|DROP) is currently executing. " + "You can change this timeout with `lock_acquire_timeout` setting", + query_context->getSettingsRef().lock_acquire_timeout.totalMilliseconds()); } version = startMutation(commands, query_context); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index dffc5f08aa2..a73b80e142f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7349,7 +7349,7 @@ void StorageReplicatedMergeTree::fetchPartition( if (try_no) LOG_INFO(log, "Some of parts ({}) are missing. Will try to fetch covering parts.", missing_parts.size()); - if (try_no >= query_context->getSettings().max_fetch_partition_retries_count) + if (try_no >= query_context->getSettingsRef().max_fetch_partition_retries_count) throw Exception(ErrorCodes::TOO_MANY_RETRIES_TO_FETCH_PARTS, "Too many retries to fetch parts from {}:{}", from_zookeeper_name, best_replica_path); diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 82de84220f1..b50a92ecb80 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -291,7 +291,7 @@ StorageStripeLog::StorageStripeLog( , data_file_path(table_path + "data.bin") , index_file_path(table_path + "index.mrk") , file_checker(disk, table_path + "sizes.json") - , max_compress_block_size(context_->getSettings().max_compress_block_size) + , max_compress_block_size(context_->getSettingsRef().max_compress_block_size) , log(getLogger("StorageStripeLog")) { StorageInMemoryMetadata storage_metadata; From e0486436d811d7c309bf99e70acbc34968e369a5 Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Thu, 11 Jul 2024 16:12:48 +0200 Subject: [PATCH 077/161] added a stateless test to check if an analyser bug described in https://github.com/ClickHouse/ClickHouse/issues/66333 is still happening --- ...re_2_conditions_old_analyzer_bug.reference | 4 ++ ...nd_where_2_conditions_old_analyzer_bug.sql | 42 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference create mode 100644 tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql diff --git a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference new file mode 100644 index 00000000000..38f7434e427 --- /dev/null +++ b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference @@ -0,0 +1,4 @@ +this query used to be broken in old analyser: +c1 c2 yes true 1 +this query worked: +c1 c2 yes true 1 diff --git a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql new file mode 100644 index 00000000000..8c82c5c3b53 --- /dev/null +++ b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql @@ -0,0 +1,42 @@ +CREATE OR REPLACE TABLE bugcheck1 +ENGINE = MergeTree +ORDER BY tuple() +AS SELECT + 'c1' as column_a, + 'c2' as column_b; + +select 'this query used to be broken in old analyser:'; +SELECT + *, + multiIf(column_b IN ( + SELECT 'c2' as someproduct + ), 'yes', 'no') AS condition_1, + multiIf(column_b = 'c2', 'true', 'false') AS condition_2, + --condition in WHERE is true + (condition_1 IN ('yes')) AND (condition_2 in ('true')) as cond +FROM +( + SELECT column_a, column_b + FROM bugcheck1 +) +WHERE (condition_1 IN ('yes')) AND (condition_2 in ('true')) +settings allow_experimental_analyzer=0; + +select 'this query worked:'; +SELECT + *, + multiIf(column_b IN ( + SELECT 'c2' as someproduct + ), 'yes', 'no') AS condition_1, + multiIf(column_b = 'c2', 'true', 'false') AS condition_2, + --condition in WHERE is true + (condition_1 IN ('yes')) AND (condition_2 in ('true')) as cond +FROM +( + SELECT column_a, column_b + FROM bugcheck1 +) +--the next line is the only difference: +-- WHERE (condition_1 IN ('yes')) AND (condition_2 in ('true')) +WHERE (condition_2 in ('true')) +settings allow_experimental_analyzer=0; From cdb3b3f2aaec64b8d5b3ec1ee6e38545e3a2b186 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 11 Jul 2024 16:42:29 +0200 Subject: [PATCH 078/161] Add query elapsed time for non-default format in play UI Signed-off-by: Azat Khuzhin --- programs/server/play.html | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/programs/server/play.html b/programs/server/play.html index 507a96382a7..b5bcc687c27 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -516,6 +516,9 @@ /// Save query in history only if it is different. let previous_query = ''; + /// Start of the last query + let last_query_start = 0; + const current_url = new URL(window.location); const opened_locally = location.protocol == 'file:'; @@ -567,6 +570,8 @@ '&password=' + encodeURIComponent(password) } + last_query_start = performance.now(); + const xhr = new XMLHttpRequest; xhr.open('POST', url, true); @@ -579,7 +584,8 @@ if (posted_request_num != request_num) { return; } else if (this.readyState === XMLHttpRequest.DONE) { - renderResponse(this.status, this.response); + const elapsed_msec = performance.now() - last_query_start; + renderResponse(this.status, this.response, elapsed_msec); /// The query is saved in browser history (in state JSON object) /// as well as in URL fragment identifier. @@ -587,7 +593,8 @@ const state = { query: query, status: this.status, - response: this.response.length > 100000 ? null : this.response /// Lower than the browser's limit. + response: this.response.length > 100000 ? null : this.response, /// Lower than the browser's limit. + elapsed_msec: elapsed_msec, }; const title = "ClickHouse Query: " + query; @@ -617,7 +624,7 @@ xhr.send(query); } - function renderResponse(status, response) { + function renderResponse(status, response, elapsed_msec) { document.getElementById('hourglass').style.display = 'none'; if (status === 200) { @@ -632,6 +639,7 @@ renderChart(json); } else { renderUnparsedResult(response); + stats.innerText = `Elapsed: ${(elapsed_msec/1000).toFixed(3)} sec.`; } document.getElementById('check-mark').style.display = 'inline'; } else { @@ -651,7 +659,7 @@ clear(); return; } - renderResponse(event.state.status, event.state.response); + renderResponse(event.state.status, event.state.response, event.state.elapsed_msec); }; if (window.location.hash) { From a41bbc5b3d1aa7fe40ce19f783f7c82237043fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 11 Jul 2024 17:33:14 +0200 Subject: [PATCH 079/161] Less indirect dependencies on Settings.h --- .../FunctionGenerateRandomStructure.cpp | 6 ++++++ .../FunctionGenerateRandomStructure.h | 6 +----- src/Functions/FunctionUnixTimestamp64.cpp | 13 +++++++++++++ src/Functions/FunctionUnixTimestamp64.h | 7 +------ src/Functions/tuple.cpp | 7 +++++++ src/Functions/tuple.h | 7 +------ src/Interpreters/ActionsVisitor.cpp | 1 + .../ClusterProxy/SelectStreamFactory.cpp | 1 + src/Interpreters/DDLWorker.cpp | 1 + .../ExecuteScalarSubqueriesVisitor.cpp | 1 + src/Interpreters/GlobalSubqueriesVisitor.h | 1 + .../IInterpreterUnionOrSelectQuery.cpp | 19 +++++++++++++++++++ .../IInterpreterUnionOrSelectQuery.h | 18 +----------------- src/Interpreters/InterpreterAlterQuery.cpp | 3 ++- src/Interpreters/InterpreterCreateQuery.cpp | 3 ++- src/Interpreters/InterpreterDeleteQuery.cpp | 1 + src/Interpreters/InterpreterDescribeQuery.cpp | 1 + src/Interpreters/InterpreterExplainQuery.cpp | 1 + src/Interpreters/InterpreterFactory.cpp | 1 + src/Interpreters/InterpreterInsertQuery.cpp | 1 + .../InterpreterSelectIntersectExceptQuery.cpp | 1 + src/Interpreters/InterpreterSelectQuery.cpp | 1 + .../InterpreterSelectWithUnionQuery.cpp | 1 + src/Interpreters/JoinedTables.cpp | 1 + src/Interpreters/MutationsInterpreter.cpp | 1 + src/Interpreters/PreparedSets.cpp | 1 + .../getHeaderForProcessingStage.cpp | 1 + src/Interpreters/interpretSubquery.cpp | 1 + .../QueryPlan/DistributedCreateLocalPlan.cpp | 1 + .../Optimizations/optimizeReadInOrder.cpp | 3 ++- .../optimizeUseAggregateProjection.cpp | 1 + .../Optimizations/projectionsCommon.cpp | 1 + .../QueryPlan/ReadFromMergeTree.cpp | 1 + .../QueryPlan/ReadFromStreamLikeEngine.cpp | 1 + .../QueryPlan/ReadFromSystemNumbersStep.cpp | 1 + .../Transforms/buildPushingToViewsChain.cpp | 1 + src/Storages/AlterCommands.cpp | 1 + src/Storages/IStorage.cpp | 1 + src/Storages/IStorageCluster.cpp | 1 + src/Storages/LiveView/StorageLiveView.cpp | 1 + src/Storages/MergeTree/MergeTask.cpp | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 1 + .../MergeTree/MergeTreeDataSelectExecutor.cpp | 1 + src/Storages/MergeTree/MergeTreeSink.cpp | 1 + .../MergeTree/MutatePlainMergeTreeTask.cpp | 1 + src/Storages/MergeTree/MutateTask.cpp | 1 + .../MergeTree/ReplicatedMergeTreeSink.cpp | 11 +++++++++++ .../MergeTree/ReplicatedMergeTreeSink.h | 10 +--------- .../MergeTree/registerStorageMergeTree.cpp | 1 + .../ReadFinalForExternalReplicaStorage.cpp | 1 + .../RocksDB/StorageEmbeddedRocksDB.cpp | 1 + src/Storages/StorageBuffer.cpp | 1 + src/Storages/StorageExecutable.cpp | 1 + src/Storages/StorageExternalDistributed.cpp | 2 +- src/Storages/StorageFile.cpp | 2 ++ src/Storages/StorageJoin.cpp | 1 + src/Storages/StorageKeeperMap.cpp | 1 + src/Storages/StorageMaterializedView.cpp | 1 + src/Storages/StorageMemory.cpp | 3 +-- src/Storages/StorageMerge.cpp | 1 + src/Storages/StorageMergeTree.cpp | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 1 + src/Storages/StorageURLCluster.cpp | 1 + src/Storages/StorageView.cpp | 2 ++ .../System/StorageSystemZooKeeper.cpp | 1 + src/Storages/WindowView/StorageWindowView.cpp | 1 + src/TableFunctions/TableFunctionView.cpp | 1 + .../TableFunctionViewIfPermitted.cpp | 1 + 68 files changed, 124 insertions(+), 49 deletions(-) create mode 100644 src/Functions/FunctionUnixTimestamp64.cpp diff --git a/src/Functions/FunctionGenerateRandomStructure.cpp b/src/Functions/FunctionGenerateRandomStructure.cpp index 6dc68134502..0c6f4287ecb 100644 --- a/src/Functions/FunctionGenerateRandomStructure.cpp +++ b/src/Functions/FunctionGenerateRandomStructure.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -354,6 +355,11 @@ namespace } } + FunctionPtr FunctionGenerateRandomStructure::create(DB::ContextPtr context) + { + return std::make_shared(context->getSettingsRef().allow_suspicious_low_cardinality_types.value); + } + DataTypePtr FunctionGenerateRandomStructure::getReturnTypeImpl(const DataTypes & arguments) const { if (arguments.size() > 2) diff --git a/src/Functions/FunctionGenerateRandomStructure.h b/src/Functions/FunctionGenerateRandomStructure.h index b9a83eb261e..5901565696f 100644 --- a/src/Functions/FunctionGenerateRandomStructure.h +++ b/src/Functions/FunctionGenerateRandomStructure.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include @@ -18,10 +17,7 @@ public: { } - static FunctionPtr create(ContextPtr context) - { - return std::make_shared(context->getSettingsRef().allow_suspicious_low_cardinality_types.value); - } + static FunctionPtr create(ContextPtr context); String getName() const override { return name; } diff --git a/src/Functions/FunctionUnixTimestamp64.cpp b/src/Functions/FunctionUnixTimestamp64.cpp new file mode 100644 index 00000000000..a22c035a0ba --- /dev/null +++ b/src/Functions/FunctionUnixTimestamp64.cpp @@ -0,0 +1,13 @@ +#include +#include + +namespace DB +{ + +FunctionFromUnixTimestamp64::FunctionFromUnixTimestamp64(size_t target_scale_, const char * name_, ContextPtr context) + : target_scale(target_scale_) + , name(name_) + , allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) +{} + +} diff --git a/src/Functions/FunctionUnixTimestamp64.h b/src/Functions/FunctionUnixTimestamp64.h index 7b8da733f6d..bbd7e586a67 100644 --- a/src/Functions/FunctionUnixTimestamp64.h +++ b/src/Functions/FunctionUnixTimestamp64.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #include @@ -108,11 +107,7 @@ private: const bool allow_nonconst_timezone_arguments; public: - FunctionFromUnixTimestamp64(size_t target_scale_, const char * name_, ContextPtr context) - : target_scale(target_scale_) - , name(name_) - , allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) - {} + FunctionFromUnixTimestamp64(size_t target_scale_, const char * name_, ContextPtr context); String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } diff --git a/src/Functions/tuple.cpp b/src/Functions/tuple.cpp index 71f50abba6c..84772f79d08 100644 --- a/src/Functions/tuple.cpp +++ b/src/Functions/tuple.cpp @@ -1,8 +1,15 @@ #include +#include + namespace DB { +FunctionPtr FunctionTuple::create(DB::ContextPtr context) +{ + return std::make_shared(context->getSettingsRef().enable_named_columns_in_function_tuple); +} + REGISTER_FUNCTION(Tuple) { factory.registerFunction(FunctionDocumentation{ diff --git a/src/Functions/tuple.h b/src/Functions/tuple.h index a49f346ef2a..03ba7ec9ef2 100644 --- a/src/Functions/tuple.h +++ b/src/Functions/tuple.h @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -22,11 +21,7 @@ class FunctionTuple : public IFunction public: static constexpr auto name = "tuple"; - /// maybe_unused: false-positive - [[maybe_unused]] static FunctionPtr create(ContextPtr context) - { - return std::make_shared(context->getSettingsRef().enable_named_columns_in_function_tuple); - } + static FunctionPtr create(ContextPtr context); explicit FunctionTuple(bool enable_named_columns_ = false) : enable_named_columns(enable_named_columns_) { } diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 74d83590ec9..56d80e2d4fe 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 94a02eacdaa..e35d31d2350 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 5639eed552e..697fd0f406b 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index a70ff3c6c53..4bd1c47d5a0 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index 64b6eb5dce9..986134a6998 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp index fed29b410db..d8f6df05ca4 100644 --- a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp +++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp @@ -1,4 +1,6 @@ #include + +#include #include #include #include @@ -15,6 +17,23 @@ namespace DB { +IInterpreterUnionOrSelectQuery::IInterpreterUnionOrSelectQuery(const DB::ASTPtr& query_ptr_, + const DB::ContextMutablePtr& context_, const DB::SelectQueryOptions& options_) + : query_ptr(query_ptr_) + , context(context_) + , options(options_) + , max_streams(context->getSettingsRef().max_threads) +{ + if (options.shard_num) + context->addSpecialScalar( + "_shard_num", + Block{{DataTypeUInt32().createColumnConst(1, *options.shard_num), std::make_shared(), "_shard_num"}}); + if (options.shard_count) + context->addSpecialScalar( + "_shard_count", + Block{{DataTypeUInt32().createColumnConst(1, *options.shard_count), std::make_shared(), "_shard_count"}}); +} + QueryPipelineBuilder IInterpreterUnionOrSelectQuery::buildQueryPipeline() { QueryPlan query_plan; diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.h b/src/Interpreters/IInterpreterUnionOrSelectQuery.h index 4ecfc464e90..ac8e90b5f52 100644 --- a/src/Interpreters/IInterpreterUnionOrSelectQuery.h +++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.h @@ -6,8 +6,6 @@ #include #include -#include - namespace DB { @@ -19,21 +17,7 @@ public: { } - IInterpreterUnionOrSelectQuery(const ASTPtr & query_ptr_, const ContextMutablePtr & context_, const SelectQueryOptions & options_) - : query_ptr(query_ptr_) - , context(context_) - , options(options_) - , max_streams(context->getSettingsRef().max_threads) - { - if (options.shard_num) - context->addSpecialScalar( - "_shard_num", - Block{{DataTypeUInt32().createColumnConst(1, *options.shard_num), std::make_shared(), "_shard_num"}}); - if (options.shard_count) - context->addSpecialScalar( - "_shard_count", - Block{{DataTypeUInt32().createColumnConst(1, *options.shard_count), std::make_shared(), "_shard_count"}}); - } + IInterpreterUnionOrSelectQuery(const ASTPtr & query_ptr_, const ContextMutablePtr & context_, const SelectQueryOptions & options_); virtual void buildQueryPlan(QueryPlan & query_plan) = 0; QueryPipelineBuilder buildQueryPipeline(); diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index a7a524ef836..8f9c6bc533e 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include #include #include #include @@ -24,7 +26,6 @@ #include #include #include -#include #include #include diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 15e987750c2..84d7f0a587c 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -5,7 +5,7 @@ #include #include -#include "Common/Exception.h" +#include #include #include #include @@ -14,6 +14,7 @@ #include #include #include +#include #include #include diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index 39d5d9e9cef..84fe0a9f4ae 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp index 87b0eb89302..39fc85a5e23 100644 --- a/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/src/Interpreters/InterpreterDescribeQuery.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index 6ec5937f287..d7addcd6e34 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 387d056ffe7..12b3b510098 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -60,6 +60,7 @@ #include #include #include +#include namespace ProfileEvents diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index f396db70d21..ef298d4d45a 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp b/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp index 6eac2db20c9..b0da4caa3ac 100644 --- a/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp +++ b/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index fae204912fc..f1ec674e0f1 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -84,6 +84,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index cc1d7dd6531..e953a07b41d 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index 457ed3ef4a6..0de2bf9cb1f 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -1,5 +1,6 @@ #include +#include #include #include diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index f2eec8ff8f5..480c6736bc5 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index b04c8b1b725..43a04f8aff0 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp index 06c5d424d2f..cf18cbbb54a 100644 --- a/src/Interpreters/getHeaderForProcessingStage.cpp +++ b/src/Interpreters/getHeaderForProcessingStage.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/interpretSubquery.cpp b/src/Interpreters/interpretSubquery.cpp index 6b42345f1d6..340f6d1d805 100644 --- a/src/Interpreters/interpretSubquery.cpp +++ b/src/Interpreters/interpretSubquery.cpp @@ -1,4 +1,5 @@ #include +#include #include #include diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index 1f4f271fa6e..14f58585d3d 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index e1ef38022b5..28eb4da2e17 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -22,9 +22,10 @@ #include #include #include -#include "Storages/KeyDescription.h" +#include #include #include +#include #include diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp index f6052846aec..9e2c2dcc96d 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp index af1578d6af8..4d60a1135a6 100644 --- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp +++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 2829f43a0e8..6e86d14f5cc 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Processors/QueryPlan/ReadFromStreamLikeEngine.cpp b/src/Processors/QueryPlan/ReadFromStreamLikeEngine.cpp index 4a257bba922..51117ad3c9b 100644 --- a/src/Processors/QueryPlan/ReadFromStreamLikeEngine.cpp +++ b/src/Processors/QueryPlan/ReadFromStreamLikeEngine.cpp @@ -1,5 +1,6 @@ #include +#include #include #include diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index eb974259c5e..a3ae035afdd 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index 25fbf13b0e7..c616c882ed9 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index eef83025097..7891042bb96 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 1f7ac23eb82..3b9c6ff28fa 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index 9c5b29ae265..63467603d16 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 57a1ea302f9..f1323d45c26 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -27,6 +27,7 @@ limitations under the License. */ #include #include #include +#include #include #include diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index dafb562f0d8..fc64fae9a58 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c1551668262..729eab316be 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 758d6ad6140..c5214f0f3d3 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 05751e0fa6f..5c71994d68d 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace ProfileEvents { diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index 20f387137e7..9aec074deae 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 35333af4b03..0671dc6fa60 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 0fd7d93b9aa..83e8a7a3bc5 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -575,6 +576,16 @@ bool ReplicatedMergeTreeSinkImpl::writeExistingPart(MergeTreeData::Mutabl } } +template +bool ReplicatedMergeTreeSinkImpl::lastBlockIsDuplicate() const +{ + /// If MV is responsible for deduplication, block is not considered duplicating. + if (context->getSettingsRef().deduplicate_blocks_in_dependent_materialized_views) + return false; + + return last_block_is_duplicate; +} + template std::vector ReplicatedMergeTreeSinkImpl::detectConflictsInAsyncBlockIDs(const std::vector & ids) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h index 13cb98e65a1..8a37d85a750 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h @@ -5,7 +5,6 @@ #include #include #include -#include #include @@ -61,14 +60,7 @@ public: bool writeExistingPart(MergeTreeData::MutableDataPartPtr & part); /// For proper deduplication in MaterializedViews - bool lastBlockIsDuplicate() const override - { - /// If MV is responsible for deduplication, block is not considered duplicating. - if (context->getSettingsRef().deduplicate_blocks_in_dependent_materialized_views) - return false; - - return last_block_is_duplicate; - } + bool lastBlockIsDuplicate() const override; struct DelayedChunk; private: diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 343c2c1008e..3f0603f6900 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/ReadFinalForExternalReplicaStorage.cpp b/src/Storages/ReadFinalForExternalReplicaStorage.cpp index e1d52eefc20..5070b4a17d8 100644 --- a/src/Storages/ReadFinalForExternalReplicaStorage.cpp +++ b/src/Storages/ReadFinalForExternalReplicaStorage.cpp @@ -2,6 +2,7 @@ #if USE_MYSQL || USE_LIBPQXX +#include #include #include #include diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index b9d3e071b6c..365915cfd68 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index a3f6b6afc5d..c403287c43c 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -38,6 +38,7 @@ #include #include #include +#include namespace ProfileEvents diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp index 66beb403086..381c20c616d 100644 --- a/src/Storages/StorageExecutable.cpp +++ b/src/Storages/StorageExecutable.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index beb93afc972..10037c2513a 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -1,6 +1,6 @@ #include "StorageExternalDistributed.h" - +#include #include #include #include diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 7f39ff615f0..24631944efd 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -53,6 +53,8 @@ #include #include +#include + #include #include diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index 47e41cccc96..a0d6cf11b64 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 20f99070000..ce760a8d63f 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index ec1559b71a4..57d95a98f11 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index f69c4adb552..dd4b53d7522 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include #include @@ -37,7 +37,6 @@ #include #include - namespace DB { diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 2dbe82c92d8..f5bc183931f 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index a27e14abe67..2117d7851cb 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a73b80e142f..9ce42f74ded 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -22,6 +22,7 @@ #include #include +#include #include diff --git a/src/Storages/StorageURLCluster.cpp b/src/Storages/StorageURLCluster.cpp index 2e7c63d0097..69041482fd8 100644 --- a/src/Storages/StorageURLCluster.cpp +++ b/src/Storages/StorageURLCluster.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 016de94c17c..5f768bce978 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -19,6 +19,8 @@ #include +#include + #include #include #include diff --git a/src/Storages/System/StorageSystemZooKeeper.cpp b/src/Storages/System/StorageSystemZooKeeper.cpp index cb46cd19517..ba9fac6d289 100644 --- a/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/src/Storages/System/StorageSystemZooKeeper.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 77e6ee9cb24..b842cdda022 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -48,6 +48,7 @@ #include #include #include +#include #include #include diff --git a/src/TableFunctions/TableFunctionView.cpp b/src/TableFunctions/TableFunctionView.cpp index 2a50fb2d006..57501df6d4d 100644 --- a/src/TableFunctions/TableFunctionView.cpp +++ b/src/TableFunctions/TableFunctionView.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/TableFunctions/TableFunctionViewIfPermitted.cpp b/src/TableFunctions/TableFunctionViewIfPermitted.cpp index b1691803988..935be6c1987 100644 --- a/src/TableFunctions/TableFunctionViewIfPermitted.cpp +++ b/src/TableFunctions/TableFunctionViewIfPermitted.cpp @@ -1,3 +1,4 @@ +#include #include #include #include From 861dcbbffbef382c542e6e9485ec21a8e5fa5ffd Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 11 Jul 2024 16:04:00 +0000 Subject: [PATCH 080/161] Do not check parent scope for group_by_use_nulls outside of subquery. --- .../Resolve/IdentifierResolveScope.cpp | 3 +- src/Analyzer/Resolve/IdentifierResolveScope.h | 4 +- src/Analyzer/Resolve/QueryAnalyzer.cpp | 30 +++++++------ ...2535_analyzer_group_by_use_nulls.reference | 41 +++++++++++++++++ .../02535_analyzer_group_by_use_nulls.sql | 45 +++++++++++++++++++ 5 files changed, 108 insertions(+), 15 deletions(-) diff --git a/src/Analyzer/Resolve/IdentifierResolveScope.cpp b/src/Analyzer/Resolve/IdentifierResolveScope.cpp index ae363b57047..32b3107ac16 100644 --- a/src/Analyzer/Resolve/IdentifierResolveScope.cpp +++ b/src/Analyzer/Resolve/IdentifierResolveScope.cpp @@ -11,9 +11,10 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -IdentifierResolveScope::IdentifierResolveScope(QueryTreeNodePtr scope_node_, IdentifierResolveScope * parent_scope_) +IdentifierResolveScope::IdentifierResolveScope(QueryTreeNodePtr scope_node_, IdentifierResolveScope * parent_scope_, bool is_query_) : scope_node(std::move(scope_node_)) , parent_scope(parent_scope_) + , is_query(is_query_) { if (parent_scope) { diff --git a/src/Analyzer/Resolve/IdentifierResolveScope.h b/src/Analyzer/Resolve/IdentifierResolveScope.h index ab2e27cc14d..917e032321d 100644 --- a/src/Analyzer/Resolve/IdentifierResolveScope.h +++ b/src/Analyzer/Resolve/IdentifierResolveScope.h @@ -128,7 +128,7 @@ constexpr auto PROJECTION_NAME_PLACEHOLDER = "__projection_name_placeholder"; struct IdentifierResolveScope { /// Construct identifier resolve scope using scope node, and parent scope - IdentifierResolveScope(QueryTreeNodePtr scope_node_, IdentifierResolveScope * parent_scope_); + IdentifierResolveScope(QueryTreeNodePtr scope_node_, IdentifierResolveScope * parent_scope_, bool is_query_); QueryTreeNodePtr scope_node; @@ -188,6 +188,8 @@ struct IdentifierResolveScope /// Join retutns NULLs instead of default values bool join_use_nulls = false; + bool is_query; + /// JOINs count size_t joins_count = 0; diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 576c4943ccb..63249ad2aeb 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -117,7 +117,7 @@ QueryAnalyzer::~QueryAnalyzer() = default; void QueryAnalyzer::resolve(QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression, ContextPtr context) { - IdentifierResolveScope scope(node, nullptr /*parent_scope*/); + IdentifierResolveScope scope(node, nullptr /*parent_scope*/, true /*is_query*/); if (!scope.context) scope.context = context; @@ -509,7 +509,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden /// exception with this settings enabled(https://github.com/ClickHouse/ClickHouse/issues/52494). subquery_context->setSetting("use_structure_from_insertion_table_in_table_functions", false); - auto options = SelectQueryOptions(QueryProcessingStage::Complete, scope.subquery_depth, true /*is_subquery*/); + auto options = SelectQueryOptions(QueryProcessingStage::Complete, scope.subquery_depth, true /*is_query*/); options.only_analyze = only_analyze; auto interpreter = std::make_unique(node->toAST(), subquery_context, subquery_context->getViewSource(), options); @@ -2163,7 +2163,7 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I if (apply_transformer->getApplyTransformerType() == ApplyColumnTransformerType::LAMBDA) { auto lambda_expression_to_resolve = expression_node->clone(); - IdentifierResolveScope lambda_scope(expression_node, &scope /*parent_scope*/); + IdentifierResolveScope lambda_scope(expression_node, &scope /*parent_scope*/, false /*is_query*/); node_projection_names = resolveLambda(expression_node, lambda_expression_to_resolve, {node}, lambda_scope); auto & lambda_expression_to_resolve_typed = lambda_expression_to_resolve->as(); node = lambda_expression_to_resolve_typed.getExpression(); @@ -3036,7 +3036,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi auto lambda_expression_clone = lambda_expression_untyped->clone(); - IdentifierResolveScope lambda_scope(lambda_expression_clone, &scope /*parent_scope*/); + IdentifierResolveScope lambda_scope(lambda_expression_clone, &scope /*parent_scope*/, false /*is_query*/); ProjectionNames lambda_projection_names = resolveLambda(lambda_expression_untyped, lambda_expression_clone, function_arguments, lambda_scope); auto & resolved_lambda = lambda_expression_clone->as(); @@ -3291,7 +3291,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi lambda_arguments.push_back(std::make_shared(std::move(column_name_and_type), lambda_to_resolve)); } - IdentifierResolveScope lambda_scope(lambda_to_resolve, &scope /*parent_scope*/); + IdentifierResolveScope lambda_scope(lambda_to_resolve, &scope /*parent_scope*/, false /*is_query*/); lambda_projection_names = resolveLambda(lambda_argument, lambda_to_resolve, lambda_arguments, lambda_scope); if (auto * lambda_list_node_result = lambda_to_resolve_typed.getExpression()->as()) @@ -3512,7 +3512,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( auto node_type = node->getNodeType(); if (!allow_table_expression && (node_type == QueryTreeNodeType::QUERY || node_type == QueryTreeNodeType::UNION)) { - IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/); + IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/, false /*is_query*/); subquery_scope.subquery_depth = scope.subquery_depth + 1; evaluateScalarSubqueryIfNeeded(node, subquery_scope); @@ -3619,7 +3619,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( else union_node->setIsCTE(false); - IdentifierResolveScope subquery_scope(resolved_identifier_node, &scope /*parent_scope*/); + IdentifierResolveScope subquery_scope(resolved_identifier_node, &scope /*parent_scope*/, true /*is_query*/); subquery_scope.subquery_depth = scope.subquery_depth + 1; /// CTE is being resolved, it's required to forbid to resolve to it again @@ -3752,7 +3752,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( [[fallthrough]]; case QueryTreeNodeType::UNION: { - IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/); + IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/, true /*is_query*/); subquery_scope.subquery_depth = scope.subquery_depth + 1; std::string projection_name = "_subquery_" + std::to_string(subquery_counter); @@ -3826,6 +3826,10 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( node->convertToNullable(); break; } + + /// Check parent scopes until find current query scope. + if (scope_ptr->is_query) + break; } } @@ -4112,7 +4116,7 @@ void QueryAnalyzer::resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpo bool is_column_constant = interpolate_node_typed.getExpression()->getNodeType() == QueryTreeNodeType::CONSTANT; auto & interpolation_to_resolve = interpolate_node_typed.getInterpolateExpression(); - IdentifierResolveScope interpolate_scope(interpolation_to_resolve, &scope /*parent_scope*/); + IdentifierResolveScope interpolate_scope(interpolation_to_resolve, &scope /*parent_scope*/, false /*is_query*/); auto fake_column_node = std::make_shared(NameAndTypePair(column_to_interpolate_name, interpolate_node_typed.getExpression()->getResultType()), interpolate_node_typed.getExpression()); if (is_column_constant) @@ -4410,7 +4414,7 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table */ alias_column_to_resolve = column_name_to_column_node[alias_column_to_resolve_name]; - IdentifierResolveScope alias_column_resolve_scope(alias_column_to_resolve, nullptr /*parent_scope*/); + IdentifierResolveScope alias_column_resolve_scope(alias_column_to_resolve, nullptr /*parent_scope*/, false /*is_query*/); alias_column_resolve_scope.column_name_to_column_node = std::move(column_name_to_column_node); alias_column_resolve_scope.context = scope.context; @@ -5003,7 +5007,7 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS left_subquery->getProjection().getNodes().push_back(projection_node->clone()); left_subquery->getJoinTree() = left_table_expression; - IdentifierResolveScope left_subquery_scope(left_subquery, nullptr /*parent_scope*/); + IdentifierResolveScope left_subquery_scope(left_subquery, nullptr /*parent_scope*/, true /*is_query*/); resolveQuery(left_subquery, left_subquery_scope); const auto & resolved_nodes = left_subquery->getProjection().getNodes(); @@ -5612,7 +5616,7 @@ void QueryAnalyzer::resolveUnion(const QueryTreeNodePtr & union_node, Identifier auto & non_recursive_query_mutable_context = non_recursive_query_is_query_node ? non_recursive_query->as().getMutableContext() : non_recursive_query->as().getMutableContext(); - IdentifierResolveScope non_recursive_subquery_scope(non_recursive_query, &scope /*parent_scope*/); + IdentifierResolveScope non_recursive_subquery_scope(non_recursive_query, &scope /*parent_scope*/, true /*is_query*/); non_recursive_subquery_scope.subquery_depth = scope.subquery_depth + 1; if (non_recursive_query_is_query_node) @@ -5643,7 +5647,7 @@ void QueryAnalyzer::resolveUnion(const QueryTreeNodePtr & union_node, Identifier { auto & query_node = queries_nodes[i]; - IdentifierResolveScope subquery_scope(query_node, &scope /*parent_scope*/); + IdentifierResolveScope subquery_scope(query_node, &scope /*parent_scope*/, true /*is_subquery*/); if (recursive_cte_table_node) subquery_scope.expression_argument_name_to_node[union_node_typed.getCTEName()] = recursive_cte_table_node; diff --git a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference index 63610604ddd..858fbe98838 100644 --- a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference +++ b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference @@ -264,3 +264,44 @@ SETTINGS group_by_use_nulls = 1, max_bytes_before_external_sort=10; 9 \N 9 \N 0 20 \N 1 25 +CREATE TABLE test +ENGINE = ReplacingMergeTree +PRIMARY KEY id +AS SELECT number AS id FROM numbers(100); +SELECT id +FROM test +GROUP BY id + WITH CUBE +HAVING id IN ( + SELECT id + FROM test +) +FORMAT `NUll` +SETTINGS allow_experimental_analyzer = 1, group_by_use_nulls = true; +SELECT id +FROM test +FINAL +GROUP BY id + WITH CUBE +HAVING id IN ( + SELECT DISTINCT id + FROM test + FINAL +) +FORMAT `NUll` +SETTINGS allow_experimental_analyzer = 1, group_by_use_nulls = true; +SELECT id +FROM test +FINAL +GROUP BY + GROUPING SETS ((id)) +ORDER BY + id IN ( + SELECT DISTINCT id + FROM test + FINAL + LIMIT 4 + ) ASC +LIMIT 256 BY id +FORMAT `NUll` +SETTINGS allow_experimental_analyzer = 1, group_by_use_nulls=true; diff --git a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql index a4d4f2f8bc9..4ae5df9629a 100644 --- a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql +++ b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql @@ -83,3 +83,48 @@ GROUP BY ) ORDER BY 1, tuple(val) SETTINGS group_by_use_nulls = 1, max_bytes_before_external_sort=10; + +CREATE TABLE test +ENGINE = ReplacingMergeTree +PRIMARY KEY id +AS SELECT number AS id FROM numbers(100); + +SELECT id +FROM test +GROUP BY id + WITH CUBE +HAVING id IN ( + SELECT id + FROM test +) +FORMAT `NUll` +SETTINGS allow_experimental_analyzer = 1, group_by_use_nulls = true; + +SELECT id +FROM test +FINAL +GROUP BY id + WITH CUBE +HAVING id IN ( + SELECT DISTINCT id + FROM test + FINAL +) +FORMAT `NUll` +SETTINGS allow_experimental_analyzer = 1, group_by_use_nulls = true; + +SELECT id +FROM test +FINAL +GROUP BY + GROUPING SETS ((id)) +ORDER BY + id IN ( + SELECT DISTINCT id + FROM test + FINAL + LIMIT 4 + ) ASC +LIMIT 256 BY id +FORMAT `NUll` +SETTINGS allow_experimental_analyzer = 1, group_by_use_nulls=true; From 966e4d2d6327987b05668ce625e9f145c28e264a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 11 Jul 2024 21:46:08 +0200 Subject: [PATCH 081/161] Remove noisy message --- src/Interpreters/DatabaseCatalog.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 964baea1891..bb2dd158710 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -1281,10 +1281,6 @@ void DatabaseCatalog::rescheduleDropTableTask() auto min_drop_time = getMinDropTime(); time_t schedule_after_ms = min_drop_time > current_time ? (min_drop_time - current_time) * 1000 : 0; - LOG_TRACE( - log, - "Have {} tables in queue to drop. Schedule background task in {} seconds", - tables_marked_dropped.size(), schedule_after_ms / 1000); (*drop_task)->scheduleAfter(schedule_after_ms); } From 364686731f1437e708f12d6e0f44d371b977dc9c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 11 Jul 2024 21:51:09 +0200 Subject: [PATCH 082/161] Fix test --- tests/queries/0_stateless/03201_local_named_collections.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03201_local_named_collections.sh b/tests/queries/0_stateless/03201_local_named_collections.sh index 2054a09df06..809b4b52f41 100755 --- a/tests/queries/0_stateless/03201_local_named_collections.sh +++ b/tests/queries/0_stateless/03201_local_named_collections.sh @@ -13,7 +13,7 @@ INSERT INTO test VALUES ('Hello, world!'); ${CLICKHOUSE_LOCAL} --multiquery " CREATE NAMED COLLECTION mydb AS host = '${CLICKHOUSE_HOST}', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', db = '${CLICKHOUSE_DATABASE}'; SELECT * FROM remote(mydb, table = 'test'); -" | grep --text -F -v "ASan doesn't fully support makecontext/swapcontext functions" +" 2>&1 | grep --text -F -v "ASan doesn't fully support makecontext/swapcontext functions" ${CLICKHOUSE_CLIENT} --multiquery " DROP TABLE test; From ba419e43a6546f9ad9522f2f688270ece821a43c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 11 Jul 2024 22:16:04 +0200 Subject: [PATCH 083/161] Style --- src/Storages/MergeTree/MutateTask.cpp | 1 - src/Storages/MergeTree/StorageFromMergeTreeDataPart.h | 6 ------ src/Storages/StorageFromMergeTreeDataPart.cpp | 5 +++++ 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 0671dc6fa60..f74cccd518b 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -36,7 +36,6 @@ #include - namespace ProfileEvents { extern const Event MutateTaskProjectionsCalculationMicroseconds; diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 731c30f4b79..82ebbf09988 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -8,15 +8,9 @@ #include - namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - class QueryPlan; /// A Storage that allows reading from a single MergeTree data part. diff --git a/src/Storages/StorageFromMergeTreeDataPart.cpp b/src/Storages/StorageFromMergeTreeDataPart.cpp index 37e142d246a..481d9e66901 100644 --- a/src/Storages/StorageFromMergeTreeDataPart.cpp +++ b/src/Storages/StorageFromMergeTreeDataPart.cpp @@ -6,6 +6,11 @@ namespace DB { +namespace ErrorCodes +{ +extern const int LOGICAL_ERROR; +} + bool StorageFromMergeTreeDataPart::materializeTTLRecalculateOnly() const { if (parts.empty()) From 9a8e4f2993ff5c0add987989d925754a5aa08c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 11 Jul 2024 22:33:41 +0200 Subject: [PATCH 084/161] No libraries --- src/IO/S3Common.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 3f2c1f95e71..a794cdbcf05 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -6,13 +6,13 @@ #include #include #include +#include #include #include "config.h" #if USE_AWS_S3 -#include #include #include #include From 1caaef0676707c8c19664e6ff00cbf782fb508cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 11 Jul 2024 23:05:24 +0200 Subject: [PATCH 085/161] Fast build --- src/Interpreters/InterpreterDropQuery.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index ba788a2e1f3..b68b3ddcd48 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "config.h" From e559899193ed7cec978d596bdf92ad6876ec66dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 12 Jul 2024 00:00:47 +0200 Subject: [PATCH 086/161] Random missing headers --- programs/library-bridge/LibraryBridge.cpp | 2 ++ programs/odbc-bridge/ODBCBridge.cpp | 2 ++ src/Common/ProgressIndication.cpp | 1 + src/Common/ProgressIndication.h | 7 ++++--- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/programs/library-bridge/LibraryBridge.cpp b/programs/library-bridge/LibraryBridge.cpp index 8a07ca57104..d35e7a9b832 100644 --- a/programs/library-bridge/LibraryBridge.cpp +++ b/programs/library-bridge/LibraryBridge.cpp @@ -1,5 +1,7 @@ #include "LibraryBridge.h" +#include + int mainEntryClickHouseLibraryBridge(int argc, char ** argv) { DB::LibraryBridge app; diff --git a/programs/odbc-bridge/ODBCBridge.cpp b/programs/odbc-bridge/ODBCBridge.cpp index e91cc3158df..155e26c8a27 100644 --- a/programs/odbc-bridge/ODBCBridge.cpp +++ b/programs/odbc-bridge/ODBCBridge.cpp @@ -1,5 +1,7 @@ #include "ODBCBridge.h" +#include + int mainEntryClickHouseODBCBridge(int argc, char ** argv) { DB::ODBCBridge app; diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 0b482cb09be..8f0fb3cac6c 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -1,6 +1,7 @@ #include "ProgressIndication.h" #include #include +#include #include #include #include diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index ae39fb49bcc..188ec4ed764 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -1,14 +1,15 @@ #pragma once -#include -#include -#include #include #include #include #include #include +#include +#include +#include +#include namespace DB { From 25157fd17d27429cf27d4cd1c9e6e87be37d6798 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 12 Jul 2024 11:01:42 +0200 Subject: [PATCH 087/161] Disable optimize_functions_to_subcolumns in test 03202_dynamic_null_map_subcolumn --- tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh index aa06e48376c..968c9e5271f 100755 --- a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh +++ b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh @@ -7,7 +7,7 @@ CLICKHOUSE_LOG_COMMENT= # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1" +CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --optimize_functions_to_subcolumns=0" function test() From d654880ad84ef21a4d0811f2f2f71cbca20504b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 12 Jul 2024 11:34:20 +0200 Subject: [PATCH 088/161] Fix tidy --- utils/keeper-data-dumper/main.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/keeper-data-dumper/main.cpp b/utils/keeper-data-dumper/main.cpp index 25be8549dd6..db6dac52190 100644 --- a/utils/keeper-data-dumper/main.cpp +++ b/utils/keeper-data-dumper/main.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include From 713546e5102829f71b2c2f27021478c63da0ea3e Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 12 Jul 2024 11:44:34 +0200 Subject: [PATCH 089/161] Update src/Common/ProfileEvents.cpp --- src/Common/ProfileEvents.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index e80afc95e8d..ba9c4cbfdb2 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -508,7 +508,7 @@ The server successfully detected this situation and will download merged part fr M(FileSegmentHolderCompleteMicroseconds, "File segments holder complete() time") \ M(FileSegmentFailToIncreasePriority, "Number of times the priority was not increased due to a high contention on the cache lock") \ M(FilesystemCacheFailToReserveSpaceBecauseOfLockContention, "Number of times space reservation was skipped due to a high contention on the cache lock") \ - M(FilesystemCacheFailToReserveSpaceBecauseOfCacheResize, "Number of times space reservation was skipped due to the cache is being resized") \ + M(FilesystemCacheFailToReserveSpaceBecauseOfCacheResize, "Number of times space reservation was skipped due to the cache is being resized" ) \ M(FilesystemCacheHoldFileSegments, "Filesystem cache file segments count, which were hold") \ M(FilesystemCacheUnusedHoldFileSegments, "Filesystem cache file segments count, which were hold, but not used (because of seek or LIMIT n, etc)") \ M(FilesystemCacheFreeSpaceKeepingThreadRun, "Number of times background thread executed free space keeping job") \ From b4959b25dfc1a0cf590734bf2b9bc16bc5728188 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 12 Jul 2024 11:44:44 +0200 Subject: [PATCH 090/161] Update src/Common/ProfileEvents.cpp --- src/Common/ProfileEvents.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index ba9c4cbfdb2..e80afc95e8d 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -508,7 +508,7 @@ The server successfully detected this situation and will download merged part fr M(FileSegmentHolderCompleteMicroseconds, "File segments holder complete() time") \ M(FileSegmentFailToIncreasePriority, "Number of times the priority was not increased due to a high contention on the cache lock") \ M(FilesystemCacheFailToReserveSpaceBecauseOfLockContention, "Number of times space reservation was skipped due to a high contention on the cache lock") \ - M(FilesystemCacheFailToReserveSpaceBecauseOfCacheResize, "Number of times space reservation was skipped due to the cache is being resized" ) \ + M(FilesystemCacheFailToReserveSpaceBecauseOfCacheResize, "Number of times space reservation was skipped due to the cache is being resized") \ M(FilesystemCacheHoldFileSegments, "Filesystem cache file segments count, which were hold") \ M(FilesystemCacheUnusedHoldFileSegments, "Filesystem cache file segments count, which were hold, but not used (because of seek or LIMIT n, etc)") \ M(FilesystemCacheFreeSpaceKeepingThreadRun, "Number of times background thread executed free space keeping job") \ From 7781067ba18ed4817770a804f4407c670dea5550 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 11 Jul 2024 16:53:38 +0000 Subject: [PATCH 091/161] Fix pushdown for join with external table Fix possible incorrect result for queries joining and filtering table external engine (like PostgreSQL), due to too agressive filter pushdown. Since now, conditions from where section won't be send to external database in case of outer join with external table. --- .../transformQueryForExternalDatabase.cpp | 2 +- ...nsformQueryForExternalDatabaseAnalyzer.cpp | 29 ++++++++-- ...ransformQueryForExternalDatabaseAnalyzer.h | 2 +- .../test_storage_postgresql/test.py | 54 +++++++++++++++++++ 4 files changed, 82 insertions(+), 5 deletions(-) diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index fc85bde11d9..7aac138296d 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -419,7 +419,7 @@ String transformQueryForExternalDatabase( throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "No column names for query '{}' to external table '{}.{}'", query_info.query_tree->formatASTForErrorMessage(), database, table); - auto clone_query = getASTForExternalDatabaseFromQueryTree(query_info.query_tree); + auto clone_query = getASTForExternalDatabaseFromQueryTree(query_info.query_tree, query_info.table_expression); return transformQueryForExternalDatabaseImpl( clone_query, diff --git a/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp b/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp index 5e0bfdd5f2a..dc1749b3196 100644 --- a/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp +++ b/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -10,7 +11,7 @@ #include #include #include - +#include #include @@ -20,6 +21,7 @@ namespace DB namespace ErrorCodes { extern const int UNSUPPORTED_METHOD; + extern const int LOGICAL_ERROR; } namespace @@ -55,7 +57,7 @@ public: } -ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tree) +ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tree, const QueryTreeNodePtr & table_expression) { auto new_tree = query_tree->clone(); @@ -63,6 +65,21 @@ ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tre visitor.visit(new_tree); const auto * query_node = new_tree->as(); + const auto & join_tree = query_node->getJoinTree(); + bool allow_where = true; + if (const auto * join_node = join_tree->as()) + { + if (join_node->getStrictness() != JoinStrictness::All) + allow_where = false; + + if (join_node->getKind() == JoinKind::Left) + allow_where = join_node->getLeftTableExpression()->isEqual(*table_expression); + else if (join_node->getKind() == JoinKind::Right) + allow_where = join_node->getRightTableExpression()->isEqual(*table_expression); + else + allow_where = (join_node->getKind() == JoinKind::Inner); + } + auto query_node_ast = query_node->toAST({ .add_cast_for_constants = false, .fully_qualified_identifiers = false }); const IAST * ast = query_node_ast.get(); @@ -76,7 +93,13 @@ ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tre if (union_ast->list_of_selects->children.size() != 1) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "QueryNode AST is not a single ASTSelectQuery, got {}", union_ast->list_of_selects->children.size()); - return union_ast->list_of_selects->children.at(0); + ASTPtr select_query = union_ast->list_of_selects->children.at(0); + auto * select_query_typed = select_query->as(); + if (!select_query_typed) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected ASTSelectQuery, got {}", select_query ? select_query->formatForErrorMessage() : "nullptr"); + if (!allow_where) + select_query_typed->setExpression(ASTSelectQuery::Expression::WHERE, nullptr); + return select_query; } } diff --git a/src/Storages/transformQueryForExternalDatabaseAnalyzer.h b/src/Storages/transformQueryForExternalDatabaseAnalyzer.h index f8983619d1f..7d8bf99646b 100644 --- a/src/Storages/transformQueryForExternalDatabaseAnalyzer.h +++ b/src/Storages/transformQueryForExternalDatabaseAnalyzer.h @@ -6,6 +6,6 @@ namespace DB { -ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tree); +ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tree, const QueryTreeNodePtr & table_expression); } diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index d9f3a9917ab..ffcff36c47e 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -834,6 +834,60 @@ def test_literal_escaping(started_cluster): cursor.execute(f"DROP TABLE escaping") +def test_filter_pushdown(started_cluster): + cursor = started_cluster.postgres_conn.cursor() + cursor.execute("CREATE SCHEMA test_filter_pushdown") + cursor.execute( + "CREATE TABLE test_filter_pushdown.test_table (id integer, value integer)" + ) + cursor.execute( + "INSERT INTO test_filter_pushdown.test_table VALUES (1, 10), (1, 110), (2, 0), (3, 33), (4, 0)" + ) + + node1.query( + """ + CREATE TABLE test_filter_pushdown_pg_table (id UInt32, value UInt32) + ENGINE PostgreSQL('postgres1:5432', 'postgres', 'test_table', 'postgres', 'mysecretpassword', 'test_filter_pushdown'); + """ + ) + + node1.query( + """ + CREATE TABLE test_filter_pushdown_local_table (id UInt32, value UInt32) ENGINE Memory AS SELECT * FROM test_filter_pushdown_pg_table + """ + ) + + node1.query( + "CREATE TABLE ch_table (id UInt32, pg_id UInt32) ENGINE MergeTree ORDER BY id" + ) + node1.query("INSERT INTO ch_table VALUES (1, 1), (2, 2), (3, 1), (4, 2), (5, 999)") + + def compare_results(query, **kwargs): + result1 = node1.query( + query.format(pg_table="test_filter_pushdown_pg_table", **kwargs) + ) + result2 = node1.query( + query.format(pg_table="test_filter_pushdown_local_table", **kwargs) + ) + assert result1 == result2 + + for kind in ["INNER", "LEFT", "RIGHT", "FULL"]: + for value in [0, 10]: + compare_results( + "SELECT * FROM ch_table {kind} JOIN {pg_table} as p ON ch_table.pg_id = p.id WHERE value = {value} ORDER BY ALL", + kind=kind, + value=value, + ) + + compare_results( + "SELECT * FROM {pg_table} as p {kind} JOIN ch_table ON ch_table.pg_id = p.id WHERE value = {value} ORDER BY ALL", + kind=kind, + value=value, + ) + + cursor.execute("DROP SCHEMA test_filter_pushdown CASCADE") + + if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") From 2e0f553ce768ace955e270a036ae83e94b8b0ac4 Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Fri, 12 Jul 2024 12:27:35 +0200 Subject: [PATCH 092/161] squash! added a stateless test to check if an analyser bug described in https://github.com/ClickHouse/ClickHouse/issues/66333 is still happening --- ...re_2_conditions_old_analyzer_bug.reference | 4 +- ...nd_where_2_conditions_old_analyzer_bug.sql | 37 +++++-------------- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference index 38f7434e427..9ba9848ee2d 100644 --- a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference +++ b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference @@ -1,4 +1,4 @@ this query used to be broken in old analyser: -c1 c2 yes true 1 +c1 c2 yes true this query worked: -c1 c2 yes true 1 +yes true diff --git a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql index 8c82c5c3b53..d5e7f10e395 100644 --- a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql +++ b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql @@ -6,37 +6,18 @@ AS SELECT 'c2' as column_b; select 'this query used to be broken in old analyser:'; -SELECT - *, - multiIf(column_b IN ( - SELECT 'c2' as someproduct - ), 'yes', 'no') AS condition_1, - multiIf(column_b = 'c2', 'true', 'false') AS condition_2, - --condition in WHERE is true - (condition_1 IN ('yes')) AND (condition_2 in ('true')) as cond -FROM -( - SELECT column_a, column_b - FROM bugcheck1 -) +SELECT *, + multiIf(column_b IN (SELECT 'c2' as someproduct), 'yes', 'no') AS condition_1, + multiIf(column_b = 'c2', 'true', 'false') AS condition_2 +FROM (SELECT column_a, column_b FROM bugcheck1) WHERE (condition_1 IN ('yes')) AND (condition_2 in ('true')) settings allow_experimental_analyzer=0; select 'this query worked:'; + SELECT - *, - multiIf(column_b IN ( - SELECT 'c2' as someproduct - ), 'yes', 'no') AS condition_1, - multiIf(column_b = 'c2', 'true', 'false') AS condition_2, - --condition in WHERE is true - (condition_1 IN ('yes')) AND (condition_2 in ('true')) as cond -FROM -( - SELECT column_a, column_b - FROM bugcheck1 -) ---the next line is the only difference: --- WHERE (condition_1 IN ('yes')) AND (condition_2 in ('true')) -WHERE (condition_2 in ('true')) + multiIf(column_b IN (SELECT 'c2' as someproduct), 'yes', 'no') AS condition_1, + multiIf(column_b = 'c2', 'true', 'false') AS condition_2 +FROM (SELECT column_a, column_b FROM bugcheck1) +WHERE (condition_1 IN ('yes')) AND (condition_2 in ('true')) settings allow_experimental_analyzer=0; From 8c2e4900e301b7b54c8c266f2c2c8ef4a4ef5e2c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 12 Jul 2024 10:48:34 +0000 Subject: [PATCH 093/161] Introduce isServerConstant() flag to functions. --- src/Analyzer/Resolve/QueryAnalyzer.cpp | 6 +- src/Functions/IFunction.h | 12 ++++ src/Functions/IFunctionAdaptors.h | 2 + src/Functions/getScalar.cpp | 4 ++ src/Functions/serverConstants.cpp | 56 +++++++++++-------- src/Planner/PlannerExpressionAnalysis.cpp | 6 +- .../02992_analyzer_group_by_const.sql | 43 ++++++++++++++ 7 files changed, 99 insertions(+), 30 deletions(-) diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 576c4943ccb..818e75f9594 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -3410,14 +3410,14 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi function_base = function->build(argument_columns); /// Do not constant fold get scalar functions - bool disable_constant_folding = function_name == "__getScalar" || function_name == "shardNum" || - function_name == "shardCount" || function_name == "hostName" || function_name == "tcpPort"; + // bool disable_constant_folding = function_name == "__getScalar" || function_name == "shardNum" || + // function_name == "shardCount" || function_name == "hostName" || function_name == "tcpPort"; /** If function is suitable for constant folding try to convert it to constant. * Example: SELECT plus(1, 1); * Result: SELECT 2; */ - if (function_base->isSuitableForConstantFolding() && !disable_constant_folding) + if (function_base->isSuitableForConstantFolding()) // && !disable_constant_folding) { auto result_type = function_base->getResultType(); auto executable_function = function_base->prepare(argument_columns); diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index a66456cabee..23aab420338 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -230,6 +230,17 @@ public: virtual bool isDeterministicInScopeOfQuery() const { return true; } + /** This is a special flags for functions which return constant value for the server, + * but the result could be different for different servers in distributed query. + * + * This fuctions can't support constant folding on the initiator, but can on the follower. + * We can't apply some optimizations as well (e.g. can't remove constant result from GROUP BY key). + * So, it is convenient to have a special flag for them. + * + * Examples are: "__getScalar" and every function from serverConstants.cpp + */ + virtual bool isServerConstant() const { return false; } + /** Lets you know if the function is monotonic in a range of values. * This is used to work with the index in a sorted chunk of data. * And allows to use the index not only when it is written, for example `date >= const`, but also, for example, `toMonth(date) >= 11`. @@ -488,6 +499,7 @@ public: virtual bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const { return false; } virtual bool isDeterministic() const { return true; } virtual bool isDeterministicInScopeOfQuery() const { return true; } + virtual bool isServerConstant() const { return false; } virtual bool isStateful() const { return false; } using ShortCircuitSettings = IFunctionBase::ShortCircuitSettings; diff --git a/src/Functions/IFunctionAdaptors.h b/src/Functions/IFunctionAdaptors.h index 04bd03a776e..c9929a083c1 100644 --- a/src/Functions/IFunctionAdaptors.h +++ b/src/Functions/IFunctionAdaptors.h @@ -86,6 +86,8 @@ public: bool isDeterministicInScopeOfQuery() const override { return function->isDeterministicInScopeOfQuery(); } + bool isServerConstant() const override { return function->isServerConstant(); } + bool isShortCircuit(ShortCircuitSettings & settings, size_t number_of_arguments) const override { return function->isShortCircuit(settings, number_of_arguments); } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & args) const override { return function->isSuitableForShortCircuitArgumentsExecution(args); } diff --git a/src/Functions/getScalar.cpp b/src/Functions/getScalar.cpp index 7196cbc0a36..5131dca962e 100644 --- a/src/Functions/getScalar.cpp +++ b/src/Functions/getScalar.cpp @@ -49,6 +49,8 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + bool isServerConstant() const override { return true; } + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { if (arguments.size() != 1 || !isString(arguments[0].type) || !arguments[0].column || !isColumnConst(*arguments[0].column)) @@ -105,6 +107,8 @@ public: bool isDeterministic() const override { return false; } + bool isServerConstant() const override { return true; } + bool isSuitableForConstantFolding() const override { return !is_distributed; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } diff --git a/src/Functions/serverConstants.cpp b/src/Functions/serverConstants.cpp index e7e423058f1..1e2c8c48bab 100644 --- a/src/Functions/serverConstants.cpp +++ b/src/Functions/serverConstants.cpp @@ -20,117 +20,125 @@ namespace DB namespace { + template + class FunctionServerConstantBase : public FunctionConstantBase + { + public: + using FunctionConstantBase::FunctionConstantBase; + bool isServerConstant() const override { return true; } + }; + #if defined(__ELF__) && !defined(OS_FREEBSD) /// buildId() - returns the compiler build id of the running binary. - class FunctionBuildId : public FunctionConstantBase + class FunctionBuildId : public FunctionServerConstantBase { public: static constexpr auto name = "buildId"; static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - explicit FunctionBuildId(ContextPtr context) : FunctionConstantBase(SymbolIndex::instance().getBuildIDHex(), context->isDistributed()) {} + explicit FunctionBuildId(ContextPtr context) : FunctionServerConstantBase(SymbolIndex::instance().getBuildIDHex(), context->isDistributed()) {} }; #endif /// Get the host name. It is constant on single server, but is not constant in distributed queries. - class FunctionHostName : public FunctionConstantBase + class FunctionHostName : public FunctionServerConstantBase { public: static constexpr auto name = "hostName"; static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - explicit FunctionHostName(ContextPtr context) : FunctionConstantBase(DNSResolver::instance().getHostName(), context->isDistributed()) {} + explicit FunctionHostName(ContextPtr context) : FunctionServerConstantBase(DNSResolver::instance().getHostName(), context->isDistributed()) {} }; - class FunctionServerUUID : public FunctionConstantBase + class FunctionServerUUID : public FunctionServerConstantBase { public: static constexpr auto name = "serverUUID"; static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - explicit FunctionServerUUID(ContextPtr context) : FunctionConstantBase(ServerUUID::get(), context->isDistributed()) {} + explicit FunctionServerUUID(ContextPtr context) : FunctionServerConstantBase(ServerUUID::get(), context->isDistributed()) {} }; - class FunctionTCPPort : public FunctionConstantBase + class FunctionTCPPort : public FunctionServerConstantBase { public: static constexpr auto name = "tcpPort"; static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - explicit FunctionTCPPort(ContextPtr context) : FunctionConstantBase(context->getTCPPort(), context->isDistributed()) {} + explicit FunctionTCPPort(ContextPtr context) : FunctionServerConstantBase(context->getTCPPort(), context->isDistributed()) {} }; /// Returns timezone for current session. - class FunctionTimezone : public FunctionConstantBase + class FunctionTimezone : public FunctionServerConstantBase { public: static constexpr auto name = "timezone"; static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - explicit FunctionTimezone(ContextPtr context) : FunctionConstantBase(DateLUT::instance().getTimeZone(), context->isDistributed()) {} + explicit FunctionTimezone(ContextPtr context) : FunctionServerConstantBase(DateLUT::instance().getTimeZone(), context->isDistributed()) {} }; /// Returns the server time zone (timezone in which server runs). - class FunctionServerTimezone : public FunctionConstantBase + class FunctionServerTimezone : public FunctionServerConstantBase { public: static constexpr auto name = "serverTimezone"; static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - explicit FunctionServerTimezone(ContextPtr context) : FunctionConstantBase(DateLUT::serverTimezoneInstance().getTimeZone(), context->isDistributed()) {} + explicit FunctionServerTimezone(ContextPtr context) : FunctionServerConstantBase(DateLUT::serverTimezoneInstance().getTimeZone(), context->isDistributed()) {} }; /// Returns server uptime in seconds. - class FunctionUptime : public FunctionConstantBase + class FunctionUptime : public FunctionServerConstantBase { public: static constexpr auto name = "uptime"; static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - explicit FunctionUptime(ContextPtr context) : FunctionConstantBase(context->getUptimeSeconds(), context->isDistributed()) {} + explicit FunctionUptime(ContextPtr context) : FunctionServerConstantBase(context->getUptimeSeconds(), context->isDistributed()) {} }; /// version() - returns the current version as a string. - class FunctionVersion : public FunctionConstantBase + class FunctionVersion : public FunctionServerConstantBase { public: static constexpr auto name = "version"; static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - explicit FunctionVersion(ContextPtr context) : FunctionConstantBase(VERSION_STRING, context->isDistributed()) {} + explicit FunctionVersion(ContextPtr context) : FunctionServerConstantBase(VERSION_STRING, context->isDistributed()) {} }; /// revision() - returns the current revision. - class FunctionRevision : public FunctionConstantBase + class FunctionRevision : public FunctionServerConstantBase { public: static constexpr auto name = "revision"; static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - explicit FunctionRevision(ContextPtr context) : FunctionConstantBase(ClickHouseRevision::getVersionRevision(), context->isDistributed()) {} + explicit FunctionRevision(ContextPtr context) : FunctionServerConstantBase(ClickHouseRevision::getVersionRevision(), context->isDistributed()) {} }; - class FunctionZooKeeperSessionUptime : public FunctionConstantBase + class FunctionZooKeeperSessionUptime : public FunctionServerConstantBase { public: static constexpr auto name = "zookeeperSessionUptime"; explicit FunctionZooKeeperSessionUptime(ContextPtr context) - : FunctionConstantBase(context->getZooKeeperSessionUptime(), context->isDistributed()) + : FunctionServerConstantBase(context->getZooKeeperSessionUptime(), context->isDistributed()) { } static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } }; - class FunctionGetOSKernelVersion : public FunctionConstantBase + class FunctionGetOSKernelVersion : public FunctionServerConstantBase { public: static constexpr auto name = "getOSKernelVersion"; - explicit FunctionGetOSKernelVersion(ContextPtr context) : FunctionConstantBase(Poco::Environment::osName() + " " + Poco::Environment::osVersion(), context->isDistributed()) {} + explicit FunctionGetOSKernelVersion(ContextPtr context) : FunctionServerConstantBase(Poco::Environment::osName() + " " + Poco::Environment::osVersion(), context->isDistributed()) {} static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } }; - class FunctionDisplayName : public FunctionConstantBase + class FunctionDisplayName : public FunctionServerConstantBase { public: static constexpr auto name = "displayName"; - explicit FunctionDisplayName(ContextPtr context) : FunctionConstantBase(context->getConfigRef().getString("display_name", getFQDNOrHostName()), context->isDistributed()) {} + explicit FunctionDisplayName(ContextPtr context) : FunctionServerConstantBase(context->getConfigRef().getString("display_name", getFQDNOrHostName()), context->isDistributed()) {} static FunctionPtr create(ContextPtr context) {return std::make_shared(context); } }; } diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index ceb506d1bbb..f0de3e69113 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -85,14 +85,14 @@ bool canRemoveConstantFromGroupByKey(const ConstantNode & root) else if (function_node) { /// Do not allow removing constants like `hostName()` - if (!function_node->getFunctionOrThrow()->isDeterministic()) + if (function_node->getFunctionOrThrow()->isServerConstant()) return false; for (const auto & child : function_node->getArguments()) nodes.push(child.get()); } - else - return false; + // else + // return false; } return true; diff --git a/tests/queries/0_stateless/02992_analyzer_group_by_const.sql b/tests/queries/0_stateless/02992_analyzer_group_by_const.sql index ede6e0deed9..89049515a37 100644 --- a/tests/queries/0_stateless/02992_analyzer_group_by_const.sql +++ b/tests/queries/0_stateless/02992_analyzer_group_by_const.sql @@ -30,3 +30,46 @@ SELECT min(dummy) FROM remote('127.0.0.{2,3}', system.one) GROUP BY y; + +SELECT + count(), + now() AS c1 +FROM remote('127.0.0.{1,2}', default, ttt) +GROUP BY c1 FORMAT Null; + +SELECT + count(), + now() AS c1 +FROM remote('127.0.0.{3,2}', default, ttt) +GROUP BY c1 FORMAT Null; + +SELECT + count(), + now() AS c1 +FROM remote('127.0.0.{1,2}', default, ttt) +GROUP BY c1 + 1 FORMAT Null; + +SELECT + count(), + now() AS c1 +FROM remote('127.0.0.{3,2}', default, ttt) +GROUP BY c1 + 1 FORMAT Null; + +CREATE TABLE ttt (hr DateTime, ts DateTime) ENGINE=Memory +as select '2000-01-01' d, d; + +SELECT + count(), + tuple(nullIf(toDateTime(formatDateTime(hr, '%F %T', 'America/Los_Angeles'), 'America/Los_Angeles'), toDateTime(0))) as c1, + defaultValueOfArgumentType(toTimeZone(ts, 'America/Los_Angeles')) as c2, + formatDateTime(hr, '%F %T', 'America/Los_Angeles') as c3 +FROM remote('127.0.0.{1,2}', default, ttt) +GROUP BY c1, c2, c3 FORMAT Null; + +SELECT + count(), + tuple(nullIf(toDateTime(formatDateTime(hr, '%F %T', 'America/Los_Angeles'), 'America/Los_Angeles'), toDateTime(0))) as c1, + defaultValueOfArgumentType(toTimeZone(ts, 'America/Los_Angeles')) as c2, + formatDateTime(hr, '%F %T', 'America/Los_Angeles') as c3 +FROM remote('127.0.0.{3,2}', default, ttt) +GROUP BY c1, c2, c3 FORMAT Null; From 64764a531f1625777397145a6a67ced07a192278 Mon Sep 17 00:00:00 2001 From: maxvostrikov Date: Fri, 12 Jul 2024 12:49:38 +0200 Subject: [PATCH 094/161] squash! added a stateless test to check if an analyser bug described in https://github.com/ClickHouse/ClickHouse/issues/66333 is still happening --- ...where_2_conditions_old_analyzer_bug.reference | 2 ++ ...f_and_where_2_conditions_old_analyzer_bug.sql | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference index 9ba9848ee2d..5512e59cc73 100644 --- a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference +++ b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.reference @@ -2,3 +2,5 @@ this query used to be broken in old analyser: c1 c2 yes true this query worked: yes true +experimental analyzer: +c1 c2 yes true diff --git a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql index d5e7f10e395..39d772324bd 100644 --- a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql +++ b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql @@ -7,8 +7,8 @@ AS SELECT select 'this query used to be broken in old analyser:'; SELECT *, - multiIf(column_b IN (SELECT 'c2' as someproduct), 'yes', 'no') AS condition_1, - multiIf(column_b = 'c2', 'true', 'false') AS condition_2 + multiIf(column_b IN (SELECT 'c2' as someproduct), 'yes', 'no') AS condition_1, + multiIf(column_b = 'c2', 'true', 'false') AS condition_2 FROM (SELECT column_a, column_b FROM bugcheck1) WHERE (condition_1 IN ('yes')) AND (condition_2 in ('true')) settings allow_experimental_analyzer=0; @@ -16,8 +16,16 @@ settings allow_experimental_analyzer=0; select 'this query worked:'; SELECT - multiIf(column_b IN (SELECT 'c2' as someproduct), 'yes', 'no') AS condition_1, - multiIf(column_b = 'c2', 'true', 'false') AS condition_2 + multiIf(column_b IN (SELECT 'c2' as someproduct), 'yes', 'no') AS condition_1, + multiIf(column_b = 'c2', 'true', 'false') AS condition_2 FROM (SELECT column_a, column_b FROM bugcheck1) WHERE (condition_1 IN ('yes')) AND (condition_2 in ('true')) settings allow_experimental_analyzer=0; + +select 'experimental analyzer:'; +SELECT *, + multiIf(column_b IN (SELECT 'c2' as someproduct), 'yes', 'no') AS condition_1, + multiIf(column_b = 'c2', 'true', 'false') AS condition_2 +FROM (SELECT column_a, column_b FROM bugcheck1) +WHERE (condition_1 IN ('yes')) AND (condition_2 in ('true')) +settings allow_experimental_analyzer=1; From dbfc9fc71fb32a8bc14d50ee6f15db224157fd22 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 12 Jul 2024 13:04:41 +0200 Subject: [PATCH 095/161] Update 03036_dynamic_read_subcolumns.sh --- tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh index a1cb35e4609..65517061b99 100755 --- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh +++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh @@ -7,7 +7,7 @@ CLICKHOUSE_LOG_COMMENT= # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --optimize_functions_to_subcolumns=0" +CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1" function test() From 1e9f47707fc7b47703c0e2957da09192bd044e81 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 12 Jul 2024 13:05:40 +0200 Subject: [PATCH 096/161] Fix optimizing array size to subcolumns in dynamic subcolumn --- src/Storages/ColumnsDescription.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index c07583cd39d..da749812167 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -701,15 +701,15 @@ std::optional ColumnsDescription::tryGetColumn(const GetColumns auto jt = subcolumns.get<0>().find(column_name); if (jt != subcolumns.get<0>().end()) return *jt; - } - /// Check for dynamic subcolumns. - auto [ordinary_column_name, dynamic_subcolumn_name] = Nested::splitName(column_name); - it = columns.get<1>().find(ordinary_column_name); - if (it != columns.get<1>().end() && it->type->hasDynamicSubcolumns()) - { - if (auto dynamic_subcolumn_type = it->type->tryGetSubcolumnType(dynamic_subcolumn_name)) - return NameAndTypePair(ordinary_column_name, dynamic_subcolumn_name, it->type, dynamic_subcolumn_type); + /// Check for dynamic subcolumns. + auto [ordinary_column_name, dynamic_subcolumn_name] = Nested::splitName(column_name); + it = columns.get<1>().find(ordinary_column_name); + if (it != columns.get<1>().end() && it->type->hasDynamicSubcolumns()) + { + if (auto dynamic_subcolumn_type = it->type->tryGetSubcolumnType(dynamic_subcolumn_name)) + return NameAndTypePair(ordinary_column_name, dynamic_subcolumn_name, it->type, dynamic_subcolumn_type); + } } return {}; From c7180e67bde8fa7924c77b999a0dafdc2b4b283d Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 12 Jul 2024 13:08:53 +0100 Subject: [PATCH 097/161] impl --- cmake/limit_jobs.cmake | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 8e48fc9b9d8..3a759b90fe3 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -42,9 +42,14 @@ endif () # But use 2 parallel jobs, since: # - this is what llvm does # - and I've verfied that lld-11 does not use all available CPU time (in peak) while linking one binary -if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 2) - message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.") - set (PARALLEL_LINK_JOBS 2) +if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO) + if (ARCH_AARCH64) + message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 1.") + set (PARALLEL_LINK_JOBS 1) + elseif (PARALLEL_LINK_JOBS GREATER 2) + message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.") + set (PARALLEL_LINK_JOBS 2) + endif () endif() message(STATUS "Building sub-tree with ${PARALLEL_COMPILE_JOBS} compile jobs and ${PARALLEL_LINK_JOBS} linker jobs (system: ${NUMBER_OF_LOGICAL_CORES} cores, ${TOTAL_PHYSICAL_MEMORY} MB RAM, 'OFF' means the native core count).") From 6e67edfeaa8f1ec3e71275e2cc7c8ada4c2c80ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 12 Jul 2024 16:45:42 +0200 Subject: [PATCH 098/161] Conflicts --- src/Functions/FunctionsJSON.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index db1602b1939..47040545677 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include From 64d2b01d4a1a56f7cab978142cb15cf71ff0b4a1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 12 Jul 2024 15:51:16 +0000 Subject: [PATCH 099/161] Fixing style --- src/Functions/IFunction.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index 23aab420338..12931b51df2 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -233,7 +233,7 @@ public: /** This is a special flags for functions which return constant value for the server, * but the result could be different for different servers in distributed query. * - * This fuctions can't support constant folding on the initiator, but can on the follower. + * This functions can't support constant folding on the initiator, but can on the follower. * We can't apply some optimizations as well (e.g. can't remove constant result from GROUP BY key). * So, it is convenient to have a special flag for them. * From 00fa457a5d14a96b9f3e7ed3b937506f07476789 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 12 Jul 2024 17:06:07 +0000 Subject: [PATCH 100/161] Fixing test. --- src/Functions/getMacro.cpp | 2 + .../02992_analyzer_group_by_const.sql | 52 +++++++++---------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/Functions/getMacro.cpp b/src/Functions/getMacro.cpp index 8172fc8ba2e..b7f0b34d652 100644 --- a/src/Functions/getMacro.cpp +++ b/src/Functions/getMacro.cpp @@ -53,6 +53,8 @@ public: /// getMacro may return different values on different shards/replicas, so it's not constant for distributed query bool isSuitableForConstantFolding() const override { return !is_distributed; } + bool isServerConstant() const override { return true; } + size_t getNumberOfArguments() const override { return 1; diff --git a/tests/queries/0_stateless/02992_analyzer_group_by_const.sql b/tests/queries/0_stateless/02992_analyzer_group_by_const.sql index 89049515a37..2a9e673d7bc 100644 --- a/tests/queries/0_stateless/02992_analyzer_group_by_const.sql +++ b/tests/queries/0_stateless/02992_analyzer_group_by_const.sql @@ -31,39 +31,39 @@ SELECT FROM remote('127.0.0.{2,3}', system.one) GROUP BY y; -SELECT - count(), - now() AS c1 -FROM remote('127.0.0.{1,2}', default, ttt) -GROUP BY c1 FORMAT Null; - -SELECT - count(), - now() AS c1 -FROM remote('127.0.0.{3,2}', default, ttt) -GROUP BY c1 FORMAT Null; - -SELECT - count(), - now() AS c1 -FROM remote('127.0.0.{1,2}', default, ttt) -GROUP BY c1 + 1 FORMAT Null; - -SELECT - count(), - now() AS c1 -FROM remote('127.0.0.{3,2}', default, ttt) -GROUP BY c1 + 1 FORMAT Null; - CREATE TABLE ttt (hr DateTime, ts DateTime) ENGINE=Memory as select '2000-01-01' d, d; +SELECT + count(), + now() AS c1 +FROM remote('127.0.0.{1,2}', currentDatabase(), ttt) +GROUP BY c1 FORMAT Null; + +SELECT + count(), + now() AS c1 +FROM remote('127.0.0.{3,2}', currentDatabase(), ttt) +GROUP BY c1 FORMAT Null; + +SELECT + count(), + now() AS c1 +FROM remote('127.0.0.{1,2}', currentDatabase(), ttt) +GROUP BY c1 + 1 FORMAT Null; + +SELECT + count(), + now() AS c1 +FROM remote('127.0.0.{3,2}', currentDatabase(), ttt) +GROUP BY c1 + 1 FORMAT Null; + SELECT count(), tuple(nullIf(toDateTime(formatDateTime(hr, '%F %T', 'America/Los_Angeles'), 'America/Los_Angeles'), toDateTime(0))) as c1, defaultValueOfArgumentType(toTimeZone(ts, 'America/Los_Angeles')) as c2, formatDateTime(hr, '%F %T', 'America/Los_Angeles') as c3 -FROM remote('127.0.0.{1,2}', default, ttt) +FROM remote('127.0.0.{1,2}', currentDatabase(), ttt) GROUP BY c1, c2, c3 FORMAT Null; SELECT @@ -71,5 +71,5 @@ SELECT tuple(nullIf(toDateTime(formatDateTime(hr, '%F %T', 'America/Los_Angeles'), 'America/Los_Angeles'), toDateTime(0))) as c1, defaultValueOfArgumentType(toTimeZone(ts, 'America/Los_Angeles')) as c2, formatDateTime(hr, '%F %T', 'America/Los_Angeles') as c3 -FROM remote('127.0.0.{3,2}', default, ttt) +FROM remote('127.0.0.{3,2}', currentDatabase(), ttt) GROUP BY c1, c2, c3 FORMAT Null; From b38928b09f75bc88c722c10ba401623a880a7b70 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 12 Jul 2024 19:25:32 +0200 Subject: [PATCH 101/161] Update 00992_system_parts_race_condition_zookeeper_long.sh --- .../00992_system_parts_race_condition_zookeeper_long.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh index d45cc3a6871..4887c409844 100755 --- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh +++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh @@ -41,7 +41,7 @@ function thread3() function thread4() { - while true; do $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table0 FINAL"; done + while true; do $CLICKHOUSE_CLIENT --receive_timeout=3 -q "OPTIMIZE TABLE alter_table0 FINAL" | grep -Fv "Timeout exceeded while receiving data from server"; done } function thread5() From 8d07c522f155b3b6ec9760b9ecef28c90598dd1f Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 12 Jul 2024 17:13:33 +0100 Subject: [PATCH 102/161] reduce amount of parallel linker jobs further --- cmake/limit_jobs.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 3a759b90fe3..17d8dd42a2c 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -44,8 +44,13 @@ endif () # - and I've verfied that lld-11 does not use all available CPU time (in peak) while linking one binary if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO) if (ARCH_AARCH64) + # aarch64 builds start to often fail with OOMs (reason not yet clear), for now let's limit the concurrency message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 1.") set (PARALLEL_LINK_JOBS 1) + if (LINKER_NAME MATCHES "lld") + math(EXPR LTO_JOBS ${NUMBER_OF_LOGICAL_CORES}/4) + set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--thinlto-jobs=${LTO_JOBS}") + endif() elseif (PARALLEL_LINK_JOBS GREATER 2) message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.") set (PARALLEL_LINK_JOBS 2) From 0bb3d07e8ed0f5e7005186e47446d57f8fea0e9f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 12 Jul 2024 20:12:39 +0200 Subject: [PATCH 103/161] fix --- src/Processors/Formats/Impl/NpyRowInputFormat.cpp | 3 +++ tests/queries/0_stateless/02895_npy_format.reference | 12 ++++++------ tests/queries/0_stateless/02895_npy_format.sh | 12 ++++++------ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/Processors/Formats/Impl/NpyRowInputFormat.cpp b/src/Processors/Formats/Impl/NpyRowInputFormat.cpp index 65e0f9dd192..773cbc9268e 100644 --- a/src/Processors/Formats/Impl/NpyRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/NpyRowInputFormat.cpp @@ -445,6 +445,9 @@ bool NpyRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & /* elements_in_current_column *= header.shape[i]; } + if (typeid_cast(current_column)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected nesting level of column '{}', expected {}", column->getName(), header.shape.size() - 1); + for (size_t i = 0; i != elements_in_current_column; ++i) readValue(current_column); diff --git a/tests/queries/0_stateless/02895_npy_format.reference b/tests/queries/0_stateless/02895_npy_format.reference index f9e77644a35..52972f0acbd 100644 --- a/tests/queries/0_stateless/02895_npy_format.reference +++ b/tests/queries/0_stateless/02895_npy_format.reference @@ -85,12 +85,12 @@ c [4,5,6] [[1,2],[3,4]] [[5,6],[7,8]] -0 -0 -0 -0 -0 -0 +1 +1 +1 +1 +1 +1 1 [2.199219,1.099609,3.300781] [4.25,3.34961,6.628906] diff --git a/tests/queries/0_stateless/02895_npy_format.sh b/tests/queries/0_stateless/02895_npy_format.sh index 9d05303a091..194b2bc1fe4 100755 --- a/tests/queries/0_stateless/02895_npy_format.sh +++ b/tests/queries/0_stateless/02895_npy_format.sh @@ -52,14 +52,14 @@ $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/two_dim.npy', Npy, 'v $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/three_dim.npy', Npy, 'value Array(Array(Int8))')" $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Array(Float32)')" 2>&1 | grep -c "BAD_ARGUMENTS" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value UUID')" 2>&1 | grep -c "BAD_ARGUMENTS" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Tuple(UInt8)')" 2>&1 | grep -c "BAD_ARGUMENTS" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value UUID')" 2>&1 | grep -c "UNKNOWN_TYPE" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Tuple(UInt8)')" 2>&1 | grep -c "UNKNOWN_TYPE" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Int8')" 2>&1 | grep -c "BAD_ARGUMENTS" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_str.npy', Npy, 'value Int8')" 2>&1 | grep -c "BAD_ARGUMENTS" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_unicode.npy', Npy, 'value Float32')" 2>&1 | grep -c "BAD_ARGUMENTS" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Int8')" 2>&1 | grep -c "ILLEGAL_COLUMN" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_str.npy', Npy, 'value Int8')" 2>&1 | grep -c "ILLEGAL_COLUMN" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_unicode.npy', Npy, 'value Float32')" 2>&1 | grep -c "ILLEGAL_COLUMN" -$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/complex.npy')" 2>&1 | grep -c "BAD_ARGUMENTS" +$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/complex.npy')" 2>&1 | grep -c "CANNOT_EXTRACT_TABLE_STRUCTURE" $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/float_16.npy')" From 96d3c311900fcb089cfccef965847634bd23de4a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jul 2024 18:51:00 +0200 Subject: [PATCH 104/161] Update play.html --- programs/server/play.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/play.html b/programs/server/play.html index b5bcc687c27..9590a65524c 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -639,7 +639,7 @@ renderChart(json); } else { renderUnparsedResult(response); - stats.innerText = `Elapsed: ${(elapsed_msec/1000).toFixed(3)} sec.`; + stats.innerText = `Elapsed (client-side): ${(elapsed_msec / 1000).toFixed(3)} sec.`; } document.getElementById('check-mark').style.display = 'inline'; } else { From a1229c1ebbb1a783cf75914cc4dd7e1eb002bb6f Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Sat, 13 Jul 2024 20:37:13 +0200 Subject: [PATCH 105/161] Update install.md Production builds will not run on older ARM CPUs --- docs/en/getting-started/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 98e73dec451..6209ef3c8ee 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -15,7 +15,7 @@ You have four options for getting up and running with ClickHouse: - **[ClickHouse Cloud](https://clickhouse.com/cloud/):** The official ClickHouse as a service, - built by, maintained and supported by the creators of ClickHouse - **[Quick Install](#quick-install):** an easy-to-download binary for testing and developing with ClickHouse -- **[Production Deployments](#available-installation-options):** ClickHouse can run on any Linux, FreeBSD, or macOS with x86-64, ARM, or PowerPC64LE CPU architecture +- **[Production Deployments](#available-installation-options):** ClickHouse can run on any Linux, FreeBSD, or macOS with x86-64, modern ARM (ARMv8.2-A up), or PowerPC64LE CPU architecture - **[Docker Image](https://hub.docker.com/r/clickhouse/clickhouse-server/):** use the official Docker image in Docker Hub ## ClickHouse Cloud From 09b753ecc59e4e8bec78f63faac97e89df4c6c80 Mon Sep 17 00:00:00 2001 From: Max K Date: Sat, 13 Jul 2024 20:15:47 +0200 Subject: [PATCH 106/161] CI: Check job's exit status and report if killed --- tests/ci/ci.py | 52 +++++++++++++++++++++++--------------------- tests/ci/ci_utils.py | 12 ++++++++++ tests/ci/report.py | 26 ++++++++++++++++++++++ 3 files changed, 65 insertions(+), 25 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 5e13def95f1..f99a5dad92f 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -15,7 +15,7 @@ import upload_result_helper from build_check import get_release_or_pr from ci_config import CI from ci_metadata import CiMetadata -from ci_utils import GHActions, normalize_string, Shell +from ci_utils import GHActions, normalize_string, Utils from clickhouse_helper import ( CiLogsCredentials, ClickHouseHelper, @@ -264,7 +264,7 @@ def check_missing_images_on_dockerhub( def _pre_action(s3, indata, pr_info): print("Clear dmesg") - Shell.run("sudo dmesg --clear ||:") + Utils.clear_dmesg() CommitStatusData.cleanup() JobReport.cleanup() BuildResult.cleanup() @@ -1035,6 +1035,7 @@ def main() -> int: elif args.pre: assert indata, "Run config must be provided via --infile" _pre_action(s3, indata, pr_info) + JobReport.create_pre_report().dump() ### RUN action: start elif args.run: @@ -1131,22 +1132,22 @@ def main() -> int: exit_code = 1 else: exit_code = _run_test(check_name, args.run_command) + job_report = JobReport.load() if JobReport.exist() else None + assert ( + job_report + ), "BUG. There must be job report either real report, or pre-report if job was killed" + job_report.exit_code = exit_code + job_report.dump() ### RUN action: end ### POST action: start elif args.post: - has_oom_error = False - if Shell.check( - "sudo dmesg -T | grep -q -e 'Out of memory: Killed process' -e 'oom_reaper: reaped process' -e 'oom-kill:constraint=CONSTRAINT_NONE'" - ): - print("WARNING: OOM while job execution") - CIBuddy(dry_run=not pr_info.is_release).post_error( - "Out Of Memory", job_name=_get_ext_check_name(args.job_name) - ) - has_oom_error = True - job_report = JobReport.load() if JobReport.exist() else None - if job_report and not job_report.job_skipped: + assert ( + job_report + ), "BUG. There must be job report either real report, or pre-report if job was killed" + if not job_report.job_skipped and not job_report.pre_report: + # it's a real job report ch_helper = ClickHouseHelper() check_url = "" @@ -1248,29 +1249,30 @@ def main() -> int: ) elif job_report.job_skipped: print(f"Skipped after rerun check {[args.job_name]} - do nothing") - else: + elif job_report.job_skipped: + print(f"Job was skipped {[args.job_name]} - do nothing") + elif job_report.pre_report: + print(f"ERROR: Job was killed - generate evidence") + job_report.update_duration() + # Job was killed! + if Utils.is_killed_with_oom(): + print("WARNING: OOM while job execution") + error = f"Out Of Memory, exit_code {job_report.exit_code}, after {job_report.duration}s" + else: + error = f"Unknown, exit_code {job_report.exit_code}, after {job_report.duration}s" + CIBuddy().post_error(error, job_name=_get_ext_check_name(args.job_name)) if CI.is_test_job(args.job_name): - if has_oom_error: - description = "ERROR: Out Of Memory" - else: - description = "ERROR: Unknown job status" - print( - f"No job report for {[args.job_name]} - post status [{description}]" - ) gh = GitHub(get_best_robot_token(), per_page=100) commit = get_commit(gh, pr_info.sha) post_commit_status( commit, ERROR, "", - description, + "Error: " + error, _get_ext_check_name(args.job_name), pr_info, dump_to_file=True, ) - else: - # no job report - print(f"No job report for {[args.job_name]} - do nothing") ### POST action: end ### MARK SUCCESS action: start diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py index abc4a88989d..44bd37fe260 100644 --- a/tests/ci/ci_utils.py +++ b/tests/ci/ci_utils.py @@ -96,3 +96,15 @@ class Utils: if match: return int(match.group(1)) return None + + @staticmethod + def is_killed_with_oom(): + if Shell.check( + "sudo dmesg -T | grep -q -e 'Out of memory: Killed process' -e 'oom_reaper: reaped process' -e 'oom-kill:constraint=CONSTRAINT_NONE'" + ): + return True + return False + + @staticmethod + def clear_dmesg(): + Shell.run("sudo dmesg --clear ||:") diff --git a/tests/ci/report.py b/tests/ci/report.py index 5ca911f638f..4be7b438f4f 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -297,7 +297,33 @@ class JobReport: build_dir_for_upload: Union[Path, str] = "" # if False no GH commit status will be created by CI need_commit_status: bool = True + # indicates that this is not real job report but report for the job that was skipped by rerun check job_skipped: bool = False + # indicates that report generated by CI script in order to check later if job was killed before real report is generated + pre_report: bool = False + exit_code: int = -1 + + @staticmethod + def create_pre_report() -> "JobReport": + return JobReport( + status=ERROR, + description="", + test_results=[], + start_time=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), + duration=0.0, + additional_files=[], + pre_report=True, + ) + + def update_duration(self): + if not self.start_time: + self.duration = 0.0 + else: + start_time = datetime.datetime.strptime( + self.start_time, "%Y-%m-%d %H:%M:%S" + ) + current_time = datetime.datetime.utcnow() + self.duration = (current_time - start_time).total_seconds() def __post_init__(self): assert self.status in (SUCCESS, ERROR, FAILURE, PENDING) From 707994b876145b93bcd57e204f1b2d449496e998 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jul 2024 21:48:53 +0200 Subject: [PATCH 107/161] Add a test for #37557 --- .../03204_distributed_with_scalar_subquery.reference | 0 .../03204_distributed_with_scalar_subquery.sql | 10 ++++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/03204_distributed_with_scalar_subquery.reference create mode 100644 tests/queries/0_stateless/03204_distributed_with_scalar_subquery.sql diff --git a/tests/queries/0_stateless/03204_distributed_with_scalar_subquery.reference b/tests/queries/0_stateless/03204_distributed_with_scalar_subquery.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03204_distributed_with_scalar_subquery.sql b/tests/queries/0_stateless/03204_distributed_with_scalar_subquery.sql new file mode 100644 index 00000000000..0a07ce48268 --- /dev/null +++ b/tests/queries/0_stateless/03204_distributed_with_scalar_subquery.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS t_c3oollc8r; +CREATE TABLE t_c3oollc8r (c_k37 Int32, c_y String, c_bou Int32, c_g1 Int32, c_lfntfzg Int32, c_kntw50q Int32) ENGINE = MergeTree ORDER BY (); + +SELECT ( + SELECT c_k37 + FROM t_c3oollc8r + ) > c_lfntfzg +FROM remote('127.0.0.{1,2}', currentDatabase(), t_c3oollc8r); + +DROP TABLE t_c3oollc8r; From 1e03ca16b20e2b185d0b96de0f47f88e7293cf6a Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Sat, 13 Jul 2024 21:52:01 +0200 Subject: [PATCH 108/161] Update developer-instruction.md --- docs/en/development/developer-instruction.md | 54 ++++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index 0a1fe58b16f..4b11fb71230 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -226,15 +226,59 @@ Other IDEs you can use are [Sublime Text](https://www.sublimetext.com/), [Visual ## Writing Code {#writing-code} -The description of ClickHouse architecture can be found here: https://clickhouse.com/docs/en/development/architecture/ +Below you can find some quick links which may be useful when writing code for ClickHouse: -The Code Style Guide: https://clickhouse.com/docs/en/development/style/ +- [ClickHouse architecture description](https://clickhouse.com/docs/en/development/architecture/). +- [The code style guide](https://clickhouse.com/docs/en/development/style/). +- [Adding third-party libraries](https://clickhouse.com/docs/en/development/contrib/#adding-third-party-libraries) +- [Writing tests](https://clickhouse.com/docs/en/development/tests/) +- [List of open issues](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3Ahacktoberfest) -Adding third-party libraries: https://clickhouse.com/docs/en/development/contrib/#adding-third-party-libraries +## Writing Documentation {#writing-documentation} -Writing tests: https://clickhouse.com/docs/en/development/tests/ +As part of every pull request which adds a new feature, it is necessary to write documentation for it. If you'd like to preview your documentation changes the instructions for how to build the documentation page locally are available in the README.md file [here](https://github.com/ClickHouse/clickhouse-docs). When adding a new function to ClickHouse you can use the template below as a guide: -List of tasks: https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3Ahacktoberfest +```markdown +# newFunctionName + +A short description of the function goes here. It should describe briefly what it does and a typical usage case. + +**Syntax** + +\```sql +newFunctionName(arg1, arg2[, arg3]) +\``` + +**Arguments** + +- `arg1` — Description of the argument. [DataType](../data-types/float.md) +- `arg2` — Description of the argument. [DataType](../data-types/float.md) +- `arg3` — Description of optional argument (optional). [DataType](../data-types/float.md) + +**Implementation Details** + +A description of implementation details if relevant. + +**Returned value** + +- Returns {insert what the function returns here}. [DataType](../data-types/float.md) + +**Example** + +Query: + +\```sql +SELECT 'write your example query here'; +\``` + +Response: + +\```response +┌───────────────────────────────────┐ +│ the result of the query │ +└───────────────────────────────────┘ +\``` +``` ## Test Data {#test-data} From 7e0daae303eda27229743eaa9d5f9371c8bcb6a4 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 13 Jul 2024 20:05:39 +0000 Subject: [PATCH 109/161] Revert "Fix style" This reverts commit 0eb081f899bd0fa9c185add84c9c59e212dfd6ca. --- tests/integration/test_lost_part/test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/test_lost_part/test.py b/tests/integration/test_lost_part/test.py index e88b84775b4..5070b3c5b68 100644 --- a/tests/integration/test_lost_part/test.py +++ b/tests/integration/test_lost_part/test.py @@ -266,9 +266,7 @@ def test_lost_last_part(start_cluster): "ALTER TABLE mt3 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"} ) - partition_id = node1.query( - "select distinct _partition_id from mt3 where p = 'x'" - ).strip() + partition_id = node1.query("select distinct _partition_id from mt3 where p = 'x'").strip() remove_part_from_disk(node1, "mt3", f"{partition_id}_0_0_0") # other way to detect broken parts From 9b87a1469b910d643e603f49228ef883116b3905 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 13 Jul 2024 20:06:12 +0000 Subject: [PATCH 110/161] Revert "Fix integration tests" This reverts commit 85f5fe832a23fa25e9784728b9f26d712057148a. --- tests/integration/test_backward_compatibility/test_functions.py | 2 -- tests/integration/test_lost_part/test.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/test_backward_compatibility/test_functions.py b/tests/integration/test_backward_compatibility/test_functions.py index 551a0550c0c..fc03a77030e 100644 --- a/tests/integration/test_backward_compatibility/test_functions.py +++ b/tests/integration/test_backward_compatibility/test_functions.py @@ -168,8 +168,6 @@ def test_string_functions(start_cluster): "filesystemAvailable", # Exclude it for now. Looks like the result depends on the build type. "farmHash64", - # Removed in 24.7 - "partitionId", ] functions = filter(lambda x: x not in excludes, functions) diff --git a/tests/integration/test_lost_part/test.py b/tests/integration/test_lost_part/test.py index 5070b3c5b68..5d0ec383a39 100644 --- a/tests/integration/test_lost_part/test.py +++ b/tests/integration/test_lost_part/test.py @@ -266,7 +266,7 @@ def test_lost_last_part(start_cluster): "ALTER TABLE mt3 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"} ) - partition_id = node1.query("select distinct _partition_id from mt3 where p = 'x'").strip() + partition_id = node1.query("select _partition_id").strip() remove_part_from_disk(node1, "mt3", f"{partition_id}_0_0_0") # other way to detect broken parts From cef13cfa723508e40d203eff4b185a8ae716c911 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 13 Jul 2024 20:06:19 +0000 Subject: [PATCH 111/161] Revert "Update 02415_all_new_functions_must_be_documented.reference" This reverts commit ee4162f5ec3c34e2e35eec5fca012d75aee84401. --- .../02415_all_new_functions_must_be_documented.reference | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index eb5ae5b4f44..a152066a460 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -534,6 +534,7 @@ parseDateTimeInJodaSyntaxOrZero parseDateTimeOrNull parseDateTimeOrZero parseTimeDelta +partitionId path pathFull pi From eb9a629868899c144f30a3b5fe19b9832aafc43e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 13 Jul 2024 20:06:37 +0000 Subject: [PATCH 112/161] Revert "remove partitionID function, update tests and remove from documentation" This reverts commit c3b72386a2f26ad40431eca0c1985b14663b43ca. --- .../functions/other-functions.md | 60 ++++++++++++++++ src/Functions/partitionId.cpp | 71 +++++++++++++++++++ tests/integration/test_lost_part/test.py | 2 +- .../01748_partition_id_pruning.sql | 6 +- 4 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 src/Functions/partitionId.cpp diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 8f9fd9abb0d..68d2c12f25c 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2984,6 +2984,66 @@ Result: └─────────┘ ``` +## partitionId + +Returns computed [partition](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) IDs of its arguments. + +:::note +This function is slow and should not be called for large amount of rows. +::: + +**Syntax** + +```sql +partitionId(x[, y, ...]); +``` + +**Arguments** + +- `x` — Column for which to return the partition ID. +- `y, ...` — Remaining N columns for which to return the partition ID (optional). + +**Return Type** + +- Partition ID that the row belongs to. [String](../data-types/string.md). + +**Example** + +Query: + +```sql +DROP TABLE IF EXISTS mt; +CREATE TABLE mt +( + `i` int, + `j` int +) +ENGINE = MergeTree +PARTITION BY i +ORDER BY j +SETTINGS index_granularity = 1; + +INSERT INTO mt VALUES (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); + +SELECT * FROM mt WHERE _partition_id = partitionId(1); +SELECT * FROM mt WHERE _partition_id = partitionId(2); +``` +Result: + +```response + ┌─i─┬─j─┐ +1. │ 1 │ 1 │ +2. │ 1 │ 2 │ +3. │ 1 │ 3 │ + └───┴───┘ + ┌─i─┬─j─┐ +1. │ 2 │ 4 │ +2. │ 2 │ 5 │ +3. │ 2 │ 6 │ + └───┴───┘ +``` + + ## shardNum Returns the index of a shard which processes a part of data in a distributed query. Indices are started from `1`. diff --git a/src/Functions/partitionId.cpp b/src/Functions/partitionId.cpp new file mode 100644 index 00000000000..e2e9038cd8b --- /dev/null +++ b/src/Functions/partitionId.cpp @@ -0,0 +1,71 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + + +/** partitionId(x, y, ...) is a function that computes partition ids of arguments. + * The function is slow and should not be called for large amount of rows. + */ +class FunctionPartitionId : public IFunction +{ +public: + static constexpr auto name = "partitionId"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + + size_t getNumberOfArguments() const override { return 0; } + + bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const override { return true; } + + bool useDefaultImplementationForNulls() const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.empty()) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires at least one argument.", getName()); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + Block sample_block(arguments); + size_t size = arguments.size(); + + auto result_column = ColumnString::create(); + for (size_t j = 0; j < input_rows_count; ++j) + { + Row row(size); + for (size_t i = 0; i < size; ++i) + arguments[i].column->get(j, row[i]); + MergeTreePartition partition(std::move(row)); + result_column->insert(partition.getID(sample_block)); + } + return result_column; + } +}; + +REGISTER_FUNCTION(PartitionId) +{ + factory.registerFunction(); +} + +} diff --git a/tests/integration/test_lost_part/test.py b/tests/integration/test_lost_part/test.py index 5d0ec383a39..b8e67551d79 100644 --- a/tests/integration/test_lost_part/test.py +++ b/tests/integration/test_lost_part/test.py @@ -266,7 +266,7 @@ def test_lost_last_part(start_cluster): "ALTER TABLE mt3 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"} ) - partition_id = node1.query("select _partition_id").strip() + partition_id = node1.query("select partitionId('x')").strip() remove_part_from_disk(node1, "mt3", f"{partition_id}_0_0_0") # other way to detect broken parts diff --git a/tests/queries/0_stateless/01748_partition_id_pruning.sql b/tests/queries/0_stateless/01748_partition_id_pruning.sql index 0e942573652..b637528bc6c 100644 --- a/tests/queries/0_stateless/01748_partition_id_pruning.sql +++ b/tests/queries/0_stateless/01748_partition_id_pruning.sql @@ -8,17 +8,17 @@ insert into x values (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); set max_rows_to_read = 3; -select * from x where _partition_id = '1'; +select * from x where _partition_id = partitionId(1); set max_rows_to_read = 5; -- one row for subquery + subquery -select * from x where _partition_id in (select number + 1 from numbers(1)); +select * from x where _partition_id in (select partitionId(number + 1) from numbers(1)); -- trivial count optimization test set max_rows_to_read = 2; -- one row for subquery + subquery itself -- TODO: Relax the limits because we might build prepared set twice with _minmax_count_projection set max_rows_to_read = 3; -select count() from x where _partition_id in (select number + 1 from numbers(1)); +select count() from x where _partition_id in (select partitionId(number + 1) from numbers(1)); drop table x; From 2f913f193ca8a08b10800d5b4b9d0b9473db91d1 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 13 Jul 2024 20:25:58 +0000 Subject: [PATCH 113/161] Fixup docs --- .../functions/other-functions.md | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 68d2c12f25c..cb31d21e1b3 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2986,7 +2986,7 @@ Result: ## partitionId -Returns computed [partition](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) IDs of its arguments. +Computes the [partition ID](../../engines/table-engines/mergetree-family/custom-partitioning-key.md). :::note This function is slow and should not be called for large amount of rows. @@ -3003,44 +3003,44 @@ partitionId(x[, y, ...]); - `x` — Column for which to return the partition ID. - `y, ...` — Remaining N columns for which to return the partition ID (optional). -**Return Type** +**Returned Value** -- Partition ID that the row belongs to. [String](../data-types/string.md). +- Partition ID that the row would belong to. [String](../data-types/string.md). **Example** Query: ```sql -DROP TABLE IF EXISTS mt; -CREATE TABLE mt +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab ( - `i` int, - `j` int + i int, + j int ) ENGINE = MergeTree PARTITION BY i -ORDER BY j -SETTINGS index_granularity = 1; +ORDER BY tuple(); -INSERT INTO mt VALUES (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); +INSERT INTO tab VALUES (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); -SELECT * FROM mt WHERE _partition_id = partitionId(1); -SELECT * FROM mt WHERE _partition_id = partitionId(2); +SELECT i, j, partitionId(i), _partition_id FROM tab ORDER BY i, j; ``` + Result: ```response - ┌─i─┬─j─┐ -1. │ 1 │ 1 │ -2. │ 1 │ 2 │ -3. │ 1 │ 3 │ - └───┴───┘ - ┌─i─┬─j─┐ -1. │ 2 │ 4 │ -2. │ 2 │ 5 │ -3. │ 2 │ 6 │ - └───┴───┘ +┌─i─┬─j─┬─partitionId(i)─┬─_partition_id─┐ +│ 1 │ 1 │ 1 │ 1 │ +│ 1 │ 2 │ 1 │ 1 │ +│ 1 │ 3 │ 1 │ 1 │ +└───┴───┴────────────────┴───────────────┘ +┌─i─┬─j─┬─partitionId(i)─┬─_partition_id─┐ +│ 2 │ 4 │ 2 │ 2 │ +│ 2 │ 5 │ 2 │ 2 │ +│ 2 │ 6 │ 2 │ 2 │ +└───┴───┴────────────────┴───────────────┘ ``` From 6ba7765b8114828b97ac6ca9191de1bb98b8d53d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jul 2024 23:33:29 +0200 Subject: [PATCH 114/161] Update 03203_multiif_and_where_2_conditions_old_analyzer_bug.sql --- ...3203_multiif_and_where_2_conditions_old_analyzer_bug.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql index 39d772324bd..c97333f7df4 100644 --- a/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql +++ b/tests/queries/0_stateless/03203_multiif_and_where_2_conditions_old_analyzer_bug.sql @@ -1,4 +1,6 @@ -CREATE OR REPLACE TABLE bugcheck1 +DROP TABLE IF EXISTS bugcheck1; + +CREATE TABLE bugcheck1 ENGINE = MergeTree ORDER BY tuple() AS SELECT @@ -29,3 +31,5 @@ SELECT *, FROM (SELECT column_a, column_b FROM bugcheck1) WHERE (condition_1 IN ('yes')) AND (condition_2 in ('true')) settings allow_experimental_analyzer=1; + +DROP TABLE bugcheck1; From dfb831f86119588f63a48f12807c44d568995ab4 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 13 Jul 2024 22:07:46 +0000 Subject: [PATCH 115/161] Try to fix an msan issue --- src/Functions/changeDate.cpp | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/Functions/changeDate.cpp b/src/Functions/changeDate.cpp index 5965f3d1d00..19e4c165ee3 100644 --- a/src/Functions/changeDate.cpp +++ b/src/Functions/changeDate.cpp @@ -1,6 +1,5 @@ #include "Common/DateLUTImpl.h" #include "Common/Exception.h" -#include #include #include #include @@ -101,19 +100,16 @@ public: template ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & input_type, const DataTypePtr & result_type, size_t input_rows_count) const { - bool is_const = (isColumnConst(*arguments[0].column) && isColumnConst(*arguments[1].column)); - size_t result_rows_count = (is_const ? 1 : input_rows_count); - typename ResultDataType::ColumnType::MutablePtr result_col; if constexpr (std::is_same_v) { auto scale = DataTypeDateTime64::default_scale; if constexpr (std::is_same_v) scale = typeid_cast(*result_type).getScale(); - result_col = ResultDataType::ColumnType::create(result_rows_count, scale); + result_col = ResultDataType::ColumnType::create(input_rows_count, scale); } else - result_col = ResultDataType::ColumnType::create(result_rows_count); + result_col = ResultDataType::ColumnType::create(input_rows_count); auto date_time_col = arguments[0].column->convertToFullIfNeeded(); const auto & date_time_col_data = typeid_cast(*date_time_col).getData(); @@ -133,7 +129,7 @@ public: for (size_t j = 0; j < scale; ++j) deg *= 10; - for (size_t i = 0; i < result_rows_count; ++i) + for (size_t i = 0; i < input_rows_count; ++i) { Int64 time = date_lut.toNumYYYYMMDDhhmmss(date_time_col_data[i] / deg); Int64 fraction = date_time_col_data[i] % deg; @@ -144,7 +140,7 @@ public: else if constexpr (std::is_same_v && std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - for (size_t i = 0; i < result_rows_count; ++i) + for (size_t i = 0; i < input_rows_count; ++i) { Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(date_time_col_data[i]))) * 1'000'000; result_col_data[i] = getChangedDate(time, value_col_data[i], result_type, date_lut, 3, 0); @@ -153,7 +149,7 @@ public: else if constexpr (std::is_same_v && std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - for (size_t i = 0; i < result_rows_count; ++i) + for (size_t i = 0; i < input_rows_count; ++i) { Int64 time = static_cast(date_lut.toNumYYYYMMDD(ExtendedDayNum(date_time_col_data[i]))) * 1'000'000; result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); @@ -162,7 +158,7 @@ public: else if constexpr (std::is_same_v) { const auto & date_lut = typeid_cast(*result_type).getTimeZone(); - for (size_t i = 0; i < result_rows_count; ++i) + for (size_t i = 0; i < input_rows_count; ++i) { Int64 time = date_lut.toNumYYYYMMDDhhmmss(date_time_col_data[i]); result_col_data[i] = static_cast(getChangedDate(time, value_col_data[i], result_type, date_lut)); @@ -171,7 +167,7 @@ public: else { const auto & date_lut = DateLUT::instance(); - for (size_t i = 0; i < result_rows_count; ++i) + for (size_t i = 0; i < input_rows_count; ++i) { Int64 time; if (isDate(input_type)) @@ -186,9 +182,6 @@ public: } } - if (is_const) - return ColumnConst::create(std::move(result_col), input_rows_count); - return result_col; } From a90e10c438e58e896482f958ca905e7dc21a9993 Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Sat, 13 Jul 2024 22:38:28 +0000 Subject: [PATCH 116/161] add alias partitionI, update tests, and docs --- docs/en/sql-reference/functions/other-functions.md | 10 +++++----- src/Functions/partitionId.cpp | 1 + tests/integration/test_lost_part/test.py | 2 +- .../queries/0_stateless/01748_partition_id_pruning.sql | 6 +++--- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index cb31d21e1b3..260457b3be1 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2984,7 +2984,7 @@ Result: └─────────┘ ``` -## partitionId +## partitionID Computes the [partition ID](../../engines/table-engines/mergetree-family/custom-partitioning-key.md). @@ -2995,7 +2995,7 @@ This function is slow and should not be called for large amount of rows. **Syntax** ```sql -partitionId(x[, y, ...]); +partitionID(x[, y, ...]); ``` **Arguments** @@ -3025,18 +3025,18 @@ ORDER BY tuple(); INSERT INTO tab VALUES (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); -SELECT i, j, partitionId(i), _partition_id FROM tab ORDER BY i, j; +SELECT i, j, partitionID(i), _partition_id FROM tab ORDER BY i, j; ``` Result: ```response -┌─i─┬─j─┬─partitionId(i)─┬─_partition_id─┐ +┌─i─┬─j─┬─partitionID(i)─┬─_partition_id─┐ │ 1 │ 1 │ 1 │ 1 │ │ 1 │ 2 │ 1 │ 1 │ │ 1 │ 3 │ 1 │ 1 │ └───┴───┴────────────────┴───────────────┘ -┌─i─┬─j─┬─partitionId(i)─┬─_partition_id─┐ +┌─i─┬─j─┬─partitionID(i)─┬─_partition_id─┐ │ 2 │ 4 │ 2 │ 2 │ │ 2 │ 5 │ 2 │ 2 │ │ 2 │ 6 │ 2 │ 2 │ diff --git a/src/Functions/partitionId.cpp b/src/Functions/partitionId.cpp index e2e9038cd8b..00ff3981c88 100644 --- a/src/Functions/partitionId.cpp +++ b/src/Functions/partitionId.cpp @@ -66,6 +66,7 @@ public: REGISTER_FUNCTION(PartitionId) { factory.registerFunction(); + factory.registerAlias("partitionID", "partitionId", FunctionFactory::CaseInsensitive); } } diff --git a/tests/integration/test_lost_part/test.py b/tests/integration/test_lost_part/test.py index b8e67551d79..c2469d6c524 100644 --- a/tests/integration/test_lost_part/test.py +++ b/tests/integration/test_lost_part/test.py @@ -266,7 +266,7 @@ def test_lost_last_part(start_cluster): "ALTER TABLE mt3 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"} ) - partition_id = node1.query("select partitionId('x')").strip() + partition_id = node1.query("select partitionID('x')").strip() remove_part_from_disk(node1, "mt3", f"{partition_id}_0_0_0") # other way to detect broken parts diff --git a/tests/queries/0_stateless/01748_partition_id_pruning.sql b/tests/queries/0_stateless/01748_partition_id_pruning.sql index b637528bc6c..f492a4a8871 100644 --- a/tests/queries/0_stateless/01748_partition_id_pruning.sql +++ b/tests/queries/0_stateless/01748_partition_id_pruning.sql @@ -8,17 +8,17 @@ insert into x values (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); set max_rows_to_read = 3; -select * from x where _partition_id = partitionId(1); +select * from x where _partition_id = partitionID(1); set max_rows_to_read = 5; -- one row for subquery + subquery -select * from x where _partition_id in (select partitionId(number + 1) from numbers(1)); +select * from x where _partition_id in (select partitionID(number + 1) from numbers(1)); -- trivial count optimization test set max_rows_to_read = 2; -- one row for subquery + subquery itself -- TODO: Relax the limits because we might build prepared set twice with _minmax_count_projection set max_rows_to_read = 3; -select count() from x where _partition_id in (select partitionId(number + 1) from numbers(1)); +select count() from x where _partition_id in (select partitionID(number + 1) from numbers(1)); drop table x; diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 935b08c1b88..e5f06b468a0 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -2193,6 +2193,7 @@ parseTimeDelta parseable parsers partitionId +partitionID pathFull pclmulqdq pcre From 22611781b0944670b4b33aa65e6c11733b4e1f3e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 12 Jul 2024 14:40:40 +0200 Subject: [PATCH 117/161] Fix jemalloc assertion due to non-monotonic CLOCK_MONOTONIC_COARSE Recently one tricky assertion of jemalloc had been discovered [1]: Failed assertion: "nstime_compare(&decay->epoch, new_time) <= 0" [1]: https://github.com/ClickHouse/ClickHouse/issues/66193 And as it turns out it is really possible for CLOCK_MONOTONIC_COARSE to go backwards, in a nutshell it can be done with ADJ_FREQUENCY, you can find example here [2]. And I can't trigger this issue for non-coarse clocks. [2]: https://gist.github.com/azat/7ea7f50ed75591b1af2d675a240ea94c?permalink_comment_id=5119222#gistcomment-5119222 But, jemalloc do not call clock_gettime() that frequently (I've verified it), so it can use non-coarse version - CLOCK_MONOTONIC I've also measured the latency of CLOCK_MONOTONIC and CLOCK_MONOTONIC_COARSE, and it is 20ns vs 4ns per call [3], so make this change affect performance you need really frequently calls of clock_gettime. [3]: https://gist.github.com/azat/622fa1f9a5d8e7d546ee9d294501961d?permalink_comment_id=5119245#gistcomment-5119245 Interesting, that this bug started to appears only after jemalloc heap profiler had been enabled by default [4], no clue why (I would believe more in a more frequent calls to clock_adjtime(ADJ_FREQUENCY), but I can't verify this) [4]: https://github.com/ClickHouse/ClickHouse/pull/65702 To be continued... Fixes: https://github.com/ClickHouse/ClickHouse/issues/66193 Signed-off-by: Azat Khuzhin --- contrib/jemalloc-cmake/README | 11 +++++++++++ .../jemalloc/internal/jemalloc_internal_defs.h.in | 2 +- .../jemalloc/internal/jemalloc_internal_defs.h.in | 2 +- .../jemalloc/internal/jemalloc_internal_defs.h.in | 2 +- .../jemalloc/internal/jemalloc_internal_defs.h.in | 2 +- .../jemalloc/internal/jemalloc_internal_defs.h.in | 2 +- .../jemalloc/internal/jemalloc_internal_defs.h.in | 2 +- .../jemalloc/internal/jemalloc_internal_defs.h.in | 2 +- 8 files changed, 18 insertions(+), 7 deletions(-) diff --git a/contrib/jemalloc-cmake/README b/contrib/jemalloc-cmake/README index 91b58448c1f..89d7c818445 100644 --- a/contrib/jemalloc-cmake/README +++ b/contrib/jemalloc-cmake/README @@ -4,3 +4,14 @@ It allows to integrate JEMalloc into CMake project. - Added JEMALLOC_CONFIG_MALLOC_CONF substitution - Add musl support (USE_MUSL) - Also note, that darwin build requires JEMALLOC_PREFIX, while others do not +- JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE should be disabled + + CLOCK_MONOTONIC_COARSE can go backwards after clock_adjtime(ADJ_FREQUENCY) + Let's disable it for now, and this menas that CLOCK_MONOTONIC will be used, + and this, should not be a problem, since: + - jemalloc do not call clock_gettime() that frequently + - the difference is CLOCK_MONOTONIC 20ns and CLOCK_MONOTONIC_COARSE 4ns + + This can be done with the following command: + + gg JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE | cut -d: -f1 | xargs sed -i 's@#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE@/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */@' diff --git a/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in index e027ee97d1a..b58d5c1eb8e 100644 --- a/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in @@ -96,7 +96,7 @@ /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE +/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */ /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 597f2d59933..8bfc4b60f3d 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -98,7 +98,7 @@ /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE +/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */ /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. diff --git a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in index 0a72e71496b..d5197c6127f 100644 --- a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in @@ -98,7 +98,7 @@ /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE +/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */ /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. diff --git a/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in index 597f2d59933..8bfc4b60f3d 100644 --- a/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -98,7 +98,7 @@ /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE +/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */ /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. diff --git a/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in index 531f2bca0c2..b3f3a489a83 100644 --- a/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in @@ -96,7 +96,7 @@ /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE +/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */ /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index d21098a4dcc..cf9acdb0b80 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -98,7 +98,7 @@ /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE +/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */ /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. diff --git a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in index e08a2bed2ec..729bd82c2fa 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in @@ -99,7 +99,7 @@ /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE +/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */ /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. From 669ce7cd40881a075fe473027e6668d13683c840 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 12 Jul 2024 15:01:40 +0200 Subject: [PATCH 118/161] Enable jemalloc heap profiler by default for debug builds as well Since the issue with jemalloc assertion is clear, let's revert that workaround Signed-off-by: Azat Khuzhin --- contrib/jemalloc-cmake/CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 1fbfd29a3bd..191886981c3 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -34,11 +34,7 @@ if (OS_LINUX) # avoid spurious latencies and additional work associated with # MADV_DONTNEED. See # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000") - else() - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true") - endif() + set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true") else() set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000") endif() From 80c8511004008e5caaec08ad7f869baa578cad6c Mon Sep 17 00:00:00 2001 From: Max K Date: Sun, 14 Jul 2024 11:39:31 +0200 Subject: [PATCH 119/161] CI: Add retry for GH set_comment_status call --- tests/ci/ci.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index f99a5dad92f..979d108378d 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -6,6 +6,7 @@ import os import re import subprocess import sys +import time from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, List, Optional @@ -550,7 +551,17 @@ def _update_gh_statuses_action(indata: Dict, s3: S3Helper) -> None: except Exception as e: raise e print("Going to update overall CI report") - set_status_comment(commit, pr_info) + for retry in range(2): + try: + set_status_comment(commit, pr_info) + break + except Exception as e: + print( + f"WARNING: Failed to update CI Running status, attempt [{retry + 1}], exception [{e}]" + ) + time.sleep(1) + else: + print("ERROR: All retry attempts failed.") print("... CI report update - done") From 379706d2d5118dcd53c87f661b7a06add00bee18 Mon Sep 17 00:00:00 2001 From: Max K Date: Sun, 14 Jul 2024 11:47:18 +0200 Subject: [PATCH 120/161] fix for job filtering --- tests/ci/ci.py | 4 ++-- tests/ci/ci_cache.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 979d108378d..8dcf3fc4c69 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1007,10 +1007,10 @@ def main() -> int: args.skip_jobs, ) + ci_cache.print_status() if IS_CI and pr_info.is_pr and not ci_settings.no_ci_cache: ci_cache.filter_out_not_affected_jobs() - - ci_cache.print_status() + ci_cache.print_status() if IS_CI and not pr_info.is_merge_queue: # wait for pending jobs to be finished, await_jobs is a long blocking call diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index ae3b8f9e9a4..7552b10b873 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -714,7 +714,7 @@ class CiCache: if CI.is_test_job(job_name) and job_name != CI.JobNames.BUILD_CHECK: if job_config.reference_job_name: reference_name = job_config.reference_job_name - reference_config = self.jobs_to_do[reference_name] + reference_config = CI.JOB_CONFIGS[reference_name] else: reference_name = job_name reference_config = job_config @@ -745,7 +745,8 @@ class CiCache: del self.jobs_to_do[job] if job in self.jobs_to_wait: del self.jobs_to_wait[job] - self.jobs_to_skip.append(job) + if job in self.jobs_to_skip: + self.jobs_to_skip.remove(job) def await_pending_jobs(self, is_release: bool, dry_run: bool = False) -> None: """ From ad86e3060a907bd3b81c56724cea6df30823716e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 14 Jul 2024 11:48:13 +0000 Subject: [PATCH 121/161] OpenSSL: Follow-up to #66064 More commits were added to the upstream fix after #66064 has been merged in ClickHouse. This PR adds these commits to keep things in-sync. --- contrib/openssl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/openssl b/contrib/openssl index ee2bb8513b2..66deddc1e53 160000 --- a/contrib/openssl +++ b/contrib/openssl @@ -1 +1 @@ -Subproject commit ee2bb8513b28bf86b35404dd17a0e29305ca9e08 +Subproject commit 66deddc1e53cda8706604a019777259372d1bd62 From f374cdfe69a8d2bc1f7360ead578a5323331650d Mon Sep 17 00:00:00 2001 From: Max K Date: Sun, 14 Jul 2024 13:53:20 +0200 Subject: [PATCH 122/161] CI: Fix for job filtering in PRs --- tests/ci/ci_cache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index 7552b10b873..5c6d3b05021 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -533,19 +533,19 @@ class CiCache: job=job, batch=0, num_batches=job_config.num_batches, - release_branch=not job_config.pr_only, + release_branch=True, ) or self.is_pending( job=job, batch=0, num_batches=job_config.num_batches, - release_branch=not job_config.pr_only, + release_branch=True, ) or self.is_failed( job=job, batch=0, num_batches=job_config.num_batches, - release_branch=not job_config.pr_only, + release_branch=True, ) ) From 9dabf205e623680807bffe022d8b8b6b856779d8 Mon Sep 17 00:00:00 2001 From: Max K Date: Sun, 14 Jul 2024 14:01:10 +0200 Subject: [PATCH 123/161] CI: Fix for await to always await for builds --- tests/ci/ci_cache.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index 5c6d3b05021..5229c754d70 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -765,14 +765,19 @@ class CiCache: MAX_JOB_NUM_TO_WAIT = 3 round_cnt = 0 - # FIXME: temporary experiment: lets enable await for PR' workflows but for a shorter time + def _has_build_job(): + for job in self.jobs_to_wait: + if CI.is_build_job(job): + return True + return False + if not is_release: - MAX_ROUNDS_TO_WAIT = 3 + # in PRs we can wait only for builds, TIMEOUT*MAX_ROUNDS_TO_WAIT=100min is enough + MAX_ROUNDS_TO_WAIT = 2 while ( - len(self.jobs_to_wait) > MAX_JOB_NUM_TO_WAIT - and round_cnt < MAX_ROUNDS_TO_WAIT - ): + len(self.jobs_to_wait) > MAX_JOB_NUM_TO_WAIT or _has_build_job() + ) and round_cnt < MAX_ROUNDS_TO_WAIT: round_cnt += 1 GHActions.print_in_group( f"Wait pending jobs, round [{round_cnt}/{MAX_ROUNDS_TO_WAIT}]:", From 5bce8086a86716f03ceb36f22412ed21852c610e Mon Sep 17 00:00:00 2001 From: Max K Date: Sun, 14 Jul 2024 19:02:20 +0200 Subject: [PATCH 124/161] CI: Create release workflow updates --- tests/ci/create_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py index 7f4cf8c787a..cd01b646ebe 100755 --- a/tests/ci/create_release.py +++ b/tests/ci/create_release.py @@ -120,7 +120,7 @@ class ReleaseInfo: expected_version = copy(version) expected_version.bump() expected_tag_prefix = ( - f"v{expected_version.major}.{expected_version.minor}-" + f"v{expected_version.major}.{expected_version.minor}." ) expected_tag_suffix = "-new" else: From fd44c533cfefbb15e9c9aa6663973c4533ebf643 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jul 2024 20:57:02 +0200 Subject: [PATCH 125/161] Add one more revision to ignore --- .git-blame-ignore-revs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 06e893fabb3..5088bf90ede 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -13,3 +13,6 @@ # dbms/ → src/ # (though it is unlikely that you will see it in blame) 06446b4f08a142d6f1bc30664c47ded88ab51782 + +# Applied Black formatter for Python code +e6f5a3f98b21ba99cf274a9833797889e020a2b3 From 2ddd0078dcf5f33004cfb558328303c29306a32a Mon Sep 17 00:00:00 2001 From: Max K Date: Sun, 14 Jul 2024 21:07:41 +0200 Subject: [PATCH 126/161] rename reusable docker workflow --- .github/workflows/backport_branches.yml | 2 +- .github/workflows/create_release.yml | 2 +- .../workflows/{reusable_docker.yml => docker_test_images.yml} | 0 .github/workflows/master.yml | 2 +- .github/workflows/merge_queue.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/pull_request.yml | 2 +- .github/workflows/release_branches.yml | 2 +- tests/ci/create_release.py | 2 ++ 9 files changed, 9 insertions(+), 7 deletions(-) rename .github/workflows/{reusable_docker.yml => docker_test_images.yml} (100%) diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 64c3d2f8342..60bd79560eb 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -62,7 +62,7 @@ jobs: BuildDockers: needs: [RunConfig] if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/reusable_docker.yml + uses: ./.github/workflows/docker_test_images.yml with: data: ${{ needs.RunConfig.outputs.data }} CompatibilityCheckX86: diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 972aff90195..d8d27531f28 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -94,7 +94,7 @@ jobs: echo "Generate Security" python3 ./utils/security-generator/generate_security.py > SECURITY.md git diff HEAD - - name: Generate ChangeLog + - name: Create ChangeLog PR if: ${{ inputs.type == 'patch' && ! inputs.dry-run }} uses: peter-evans/create-pull-request@v6 with: diff --git a/.github/workflows/reusable_docker.yml b/.github/workflows/docker_test_images.yml similarity index 100% rename from .github/workflows/reusable_docker.yml rename to .github/workflows/docker_test_images.yml diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 2a7e6f737ab..d27b1987532 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -58,7 +58,7 @@ jobs: # BuildDockers: # needs: [RunConfig] # if: ${{ !failure() && !cancelled() }} -# uses: ./.github/workflows/reusable_docker.yml +# uses: ./.github/workflows/docker_test_images.yml # with: # data: ${{ needs.RunConfig.outputs.data }} # StyleCheck: diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml index 01685ee1f5a..c08c3fb30ac 100644 --- a/.github/workflows/merge_queue.yml +++ b/.github/workflows/merge_queue.yml @@ -51,7 +51,7 @@ jobs: BuildDockers: needs: [RunConfig] if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }} - uses: ./.github/workflows/reusable_docker.yml + uses: ./.github/workflows/docker_test_images.yml with: data: ${{ needs.RunConfig.outputs.data }} StyleCheck: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3e1c5576e7d..bffe5b4c1bf 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -40,7 +40,7 @@ jobs: } >> "$GITHUB_OUTPUT" BuildDockers: needs: [RunConfig] - uses: ./.github/workflows/reusable_docker.yml + uses: ./.github/workflows/docker_test_images.yml with: data: "${{ needs.RunConfig.outputs.data }}" set_latest: true diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c9f4f858825..04ce4d29ce9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -72,7 +72,7 @@ jobs: BuildDockers: needs: [RunConfig] if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }} - uses: ./.github/workflows/reusable_docker.yml + uses: ./.github/workflows/docker_test_images.yml with: data: ${{ needs.RunConfig.outputs.data }} StyleCheck: diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 6bf846d7535..e1a97b91016 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -57,7 +57,7 @@ jobs: BuildDockers: needs: [RunConfig] if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/reusable_docker.yml + uses: ./.github/workflows/docker_test_images.yml with: data: ${{ needs.RunConfig.outputs.data }} CompatibilityCheckX86: diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py index cd01b646ebe..f377e12d11c 100755 --- a/tests/ci/create_release.py +++ b/tests/ci/create_release.py @@ -618,6 +618,8 @@ sudo apt install --yes --no-install-recommends python3-dev python3-pip gh unzip sudo apt install --yes python3-boto3 sudo apt install --yes python3-github sudo apt install --yes python3-unidiff +sudo apt install --yes python3-tqdm # cloud changelog +sudo apt install --yes python3-thefuzz # cloud changelog sudo apt install --yes s3fs ### INSTALL AWS CLI From 20faa70389a2a7e3d4c30d3d647e96129e1eb8e2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jul 2024 03:30:03 +0200 Subject: [PATCH 127/161] Fix bad test `02530_dictionaries_update_field` --- .../02530_dictionaries_update_field.reference | 60 ------------------- .../02530_dictionaries_update_field.sh | 35 ++++++----- 2 files changed, 21 insertions(+), 74 deletions(-) diff --git a/tests/queries/0_stateless/02530_dictionaries_update_field.reference b/tests/queries/0_stateless/02530_dictionaries_update_field.reference index 4d5a7447a49..ada13443a10 100644 --- a/tests/queries/0_stateless/02530_dictionaries_update_field.reference +++ b/tests/queries/0_stateless/02530_dictionaries_update_field.reference @@ -1,102 +1,42 @@ flat --- { echoOn } - INSERT INTO table_for_update_field_dictionary VALUES (1, 'First', now()); -SELECT key, value FROM dict_flat ORDER BY key ASC; 1 First -INSERT INTO table_for_update_field_dictionary VALUES (2, 'Second', now()); -SELECT sleepEachRow(1) FROM numbers(10) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -SELECT key, value FROM dict_flat ORDER BY key ASC; 1 First 2 Second -INSERT INTO table_for_update_field_dictionary VALUES (2, 'SecondUpdated', now()); -INSERT INTO table_for_update_field_dictionary VALUES (3, 'Third', now()); -SELECT sleepEachRow(1) FROM numbers(20) SETTINGS function_sleep_max_microseconds_per_block = 20000000 FORMAT Null; -SELECT key, value FROM dict_flat ORDER BY key ASC; 1 First 2 SecondUpdated 3 Third flat/custom --- { echoOn } - INSERT INTO table_for_update_field_dictionary VALUES (1, 'First', now()); -SELECT key, value FROM dict_flat_custom ORDER BY key ASC; 1 First -INSERT INTO table_for_update_field_dictionary VALUES (2, 'Second', now()); -SELECT sleepEachRow(1) FROM numbers(10) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -SELECT key, value FROM dict_flat_custom ORDER BY key ASC; 1 First 2 Second -INSERT INTO table_for_update_field_dictionary VALUES (2, 'SecondUpdated', now()); -INSERT INTO table_for_update_field_dictionary VALUES (3, 'Third', now()); -SELECT sleepEachRow(1) FROM numbers(20) SETTINGS function_sleep_max_microseconds_per_block = 20000000 FORMAT Null; -SELECT key, value FROM dict_flat_custom ORDER BY key ASC; 1 First 2 SecondUpdated 3 Third hashed --- { echoOn } - INSERT INTO table_for_update_field_dictionary VALUES (1, 'First', now()); -SELECT key, value FROM dict_hashed ORDER BY key ASC; 1 First -INSERT INTO table_for_update_field_dictionary VALUES (2, 'Second', now()); -SELECT sleepEachRow(1) FROM numbers(10) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -SELECT key, value FROM dict_hashed ORDER BY key ASC; 1 First 2 Second -INSERT INTO table_for_update_field_dictionary VALUES (2, 'SecondUpdated', now()); -INSERT INTO table_for_update_field_dictionary VALUES (3, 'Third', now()); -SELECT sleepEachRow(1) FROM numbers(20) SETTINGS function_sleep_max_microseconds_per_block = 20000000 FORMAT Null; -SELECT key, value FROM dict_hashed ORDER BY key ASC; 1 First 2 SecondUpdated 3 Third hashed/custom --- { echoOn } - INSERT INTO table_for_update_field_dictionary VALUES (1, 'First', now()); -SELECT key, value FROM dict_hashed_custom ORDER BY key ASC; 1 First -INSERT INTO table_for_update_field_dictionary VALUES (2, 'Second', now()); -SELECT sleepEachRow(1) FROM numbers(10) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -SELECT key, value FROM dict_hashed_custom ORDER BY key ASC; 1 First 2 Second -INSERT INTO table_for_update_field_dictionary VALUES (2, 'SecondUpdated', now()); -INSERT INTO table_for_update_field_dictionary VALUES (3, 'Third', now()); -SELECT sleepEachRow(1) FROM numbers(20) SETTINGS function_sleep_max_microseconds_per_block = 20000000 FORMAT Null; -SELECT key, value FROM dict_hashed_custom ORDER BY key ASC; 1 First 2 SecondUpdated 3 Third complex_key_hashed --- { echoOn } - INSERT INTO table_for_update_field_dictionary VALUES (1, 'First', now()); -SELECT key, value FROM dict_complex_key_hashed ORDER BY key ASC; 1 First -INSERT INTO table_for_update_field_dictionary VALUES (2, 'Second', now()); -SELECT sleepEachRow(1) FROM numbers(10) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -SELECT key, value FROM dict_complex_key_hashed ORDER BY key ASC; 1 First 2 Second -INSERT INTO table_for_update_field_dictionary VALUES (2, 'SecondUpdated', now()); -INSERT INTO table_for_update_field_dictionary VALUES (3, 'Third', now()); -SELECT sleepEachRow(1) FROM numbers(20) SETTINGS function_sleep_max_microseconds_per_block = 20000000 FORMAT Null; -SELECT key, value FROM dict_complex_key_hashed ORDER BY key ASC; 1 First 2 SecondUpdated 3 Third complex_key_hashed/custom --- { echoOn } - INSERT INTO table_for_update_field_dictionary VALUES (1, 'First', now()); -SELECT key, value FROM dict_complex_key_hashed_custom ORDER BY key ASC; 1 First -INSERT INTO table_for_update_field_dictionary VALUES (2, 'Second', now()); -SELECT sleepEachRow(1) FROM numbers(10) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -SELECT key, value FROM dict_complex_key_hashed_custom ORDER BY key ASC; 1 First 2 Second -INSERT INTO table_for_update_field_dictionary VALUES (2, 'SecondUpdated', now()); -INSERT INTO table_for_update_field_dictionary VALUES (3, 'Third', now()); -SELECT sleepEachRow(1) FROM numbers(20) SETTINGS function_sleep_max_microseconds_per_block = 20000000 FORMAT Null; -SELECT key, value FROM dict_complex_key_hashed_custom ORDER BY key ASC; 1 First 2 SecondUpdated 3 Third diff --git a/tests/queries/0_stateless/02530_dictionaries_update_field.sh b/tests/queries/0_stateless/02530_dictionaries_update_field.sh index 9ced78a1196..a8e5804fba4 100755 --- a/tests/queries/0_stateless/02530_dictionaries_update_field.sh +++ b/tests/queries/0_stateless/02530_dictionaries_update_field.sh @@ -35,7 +35,7 @@ for layout in "${layouts[@]}"; do echo "$layout" fi - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT --multiquery " TRUNCATE TABLE table_for_update_field_dictionary; CREATE DICTIONARY $dictionary_name @@ -49,24 +49,31 @@ for layout in "${layouts[@]}"; do LAYOUT($layout()) LIFETIME(1); - -- { echoOn } - INSERT INTO table_for_update_field_dictionary VALUES (1, 'First', now()); - SELECT key, value FROM $dictionary_name ORDER BY key ASC; + INSERT INTO table_for_update_field_dictionary VALUES (1, 'First', now());" - INSERT INTO table_for_update_field_dictionary VALUES (2, 'Second', now()); - SELECT sleepEachRow(1) FROM numbers(10) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; + while true + do + $CLICKHOUSE_CLIENT --query "SELECT key, value FROM $dictionary_name ORDER BY key ASC" | grep -A10 -B10 'First' && break; + sleep .1; + done - SELECT key, value FROM $dictionary_name ORDER BY key ASC; + $CLICKHOUSE_CLIENT --query "INSERT INTO table_for_update_field_dictionary VALUES (2, 'Second', now());" - INSERT INTO table_for_update_field_dictionary VALUES (2, 'SecondUpdated', now()); - INSERT INTO table_for_update_field_dictionary VALUES (3, 'Third', now()); - SELECT sleepEachRow(1) FROM numbers(20) SETTINGS function_sleep_max_microseconds_per_block = 20000000 FORMAT Null; + while true + do + $CLICKHOUSE_CLIENT --query "SELECT key, value FROM $dictionary_name ORDER BY key ASC" | grep -A10 -B10 'Second' && break; + sleep .1; + done - SELECT key, value FROM $dictionary_name ORDER BY key ASC; - -- { echoOff } + $CLICKHOUSE_CLIENT --query "INSERT INTO table_for_update_field_dictionary VALUES (2, 'SecondUpdated', now()), (3, 'Third', now())" - DROP DICTIONARY $dictionary_name; - " + while true + do + $CLICKHOUSE_CLIENT --query "SELECT key, value FROM $dictionary_name ORDER BY key ASC" | grep -A10 -B10 'SecondUpdated' && break; + sleep .1; + done + + $CLICKHOUSE_CLIENT --query "DROP DICTIONARY $dictionary_name" done done From bc1b19591d1c78ba9b782722a27c1d29ce1d8d7d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jul 2024 05:13:38 +0200 Subject: [PATCH 128/161] Disable suspend on Ctrl+Z --- src/Client/ClientBase.cpp | 5 +++-- src/Client/LineReader.h | 3 +-- src/Client/ReplxxLineReader.cpp | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 5d472ba99b9..c6a070219a3 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -80,6 +80,7 @@ #include #include "config.h" + namespace fs = std::filesystem; using namespace std::literals; @@ -2565,12 +2566,12 @@ void ClientBase::runInteractive() word_break_characters, highlight_callback); #else + (void)word_break_characters; LineReader lr( history_file, getClientConfiguration().has("multiline"), query_extenders, - query_delimiters, - word_break_characters); + query_delimiters); #endif static const std::initializer_list> backslash_aliases = diff --git a/src/Client/LineReader.h b/src/Client/LineReader.h index 0172bd7ec22..8c101401190 100644 --- a/src/Client/LineReader.h +++ b/src/Client/LineReader.h @@ -46,8 +46,7 @@ public: Patterns delimiters, std::istream & input_stream_ = std::cin, std::ostream & output_stream_ = std::cout, - int in_fd_ = STDIN_FILENO - ); + int in_fd_ = STDIN_FILENO); virtual ~LineReader() = default; diff --git a/src/Client/ReplxxLineReader.cpp b/src/Client/ReplxxLineReader.cpp index 46600168695..3b3508d1a58 100644 --- a/src/Client/ReplxxLineReader.cpp +++ b/src/Client/ReplxxLineReader.cpp @@ -362,6 +362,9 @@ ReplxxLineReader::ReplxxLineReader( rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); }); rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); }); + /// We don't want the default, "suspend" behavior, it confuses people. + rx.bind_key_internal(replxx::Replxx::KEY::control('Z'), "insert_character"); + auto commit_action = [this](char32_t code) { /// If we allow multiline and there is already something in the input, start a newline. From 89f8c577e472d0c3709220427c2da1e880eea3ad Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jul 2024 05:34:14 +0200 Subject: [PATCH 129/161] Do not fix anything --- tests/queries/0_stateless/00719_parallel_ddl_db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00719_parallel_ddl_db.sh b/tests/queries/0_stateless/00719_parallel_ddl_db.sh index ceba24df7e4..ac4e4eae4f2 100755 --- a/tests/queries/0_stateless/00719_parallel_ddl_db.sh +++ b/tests/queries/0_stateless/00719_parallel_ddl_db.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -DB_SUFFIX=$RANDOM +DB_SUFFIX=${RANDOM}${RANDOM}${RANDOM}${RANDOM} ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS parallel_ddl_${DB_SUFFIX}" function query() From cd719bf8fc76370821c6b1d46a226e6784dce055 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 15 Jul 2024 08:57:11 +0200 Subject: [PATCH 130/161] Make the alias case-sensitive --- src/Functions/partitionId.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/partitionId.cpp b/src/Functions/partitionId.cpp index 00ff3981c88..dab56f39adc 100644 --- a/src/Functions/partitionId.cpp +++ b/src/Functions/partitionId.cpp @@ -66,7 +66,7 @@ public: REGISTER_FUNCTION(PartitionId) { factory.registerFunction(); - factory.registerAlias("partitionID", "partitionId", FunctionFactory::CaseInsensitive); + factory.registerAlias("partitionID", "partitionId"); } } From 7ef89f169ba53a73c4645d46d48a2744dfd5a4da Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 15 Jul 2024 08:59:39 +0200 Subject: [PATCH 131/161] CI: Fix for skipping Builds_2 in PRs' CI --- tests/ci/ci_cache.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index 5229c754d70..44cdb19c3a1 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -710,6 +710,7 @@ class CiCache: """ remove_from_to_do = [] required_builds = [] + has_test_jobs_to_skip = False for job_name, job_config in self.jobs_to_do.items(): if CI.is_test_job(job_name) and job_name != CI.JobNames.BUILD_CHECK: if job_config.reference_job_name: @@ -723,30 +724,30 @@ class CiCache: job_config=reference_config, ): remove_from_to_do.append(job_name) + has_test_jobs_to_skip = True else: required_builds += ( job_config.required_builds if job_config.required_builds else [] ) - has_builds_to_do = False for job_name, job_config in self.jobs_to_do.items(): if CI.is_build_job(job_name): if job_name not in required_builds: remove_from_to_do.append(job_name) - else: - has_builds_to_do = True - if not has_builds_to_do: + if not required_builds: remove_from_to_do.append(CI.JobNames.BUILD_CHECK) - for job in remove_from_to_do: - print(f"Filter job [{job}] - not affected by the change") - if job in self.jobs_to_do: - del self.jobs_to_do[job] - if job in self.jobs_to_wait: - del self.jobs_to_wait[job] - if job in self.jobs_to_skip: - self.jobs_to_skip.remove(job) + if has_test_jobs_to_skip: + # if there are no test jobs to skip, then we must not skip anything - it's a new CI run with new build digest + for job in remove_from_to_do: + print(f"Filter job [{job}] - not affected by the change") + if job in self.jobs_to_do: + del self.jobs_to_do[job] + if job in self.jobs_to_wait: + del self.jobs_to_wait[job] + if job in self.jobs_to_skip: + self.jobs_to_skip.remove(job) def await_pending_jobs(self, is_release: bool, dry_run: bool = False) -> None: """ From 4fd832fe482eebd909e642fe2ae2e7468cccebe2 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 15 Jul 2024 09:57:32 +0200 Subject: [PATCH 132/161] Fix 02918_parallel_replicas_custom_key_unavailable_replica --- ...02918_parallel_replicas_custom_key_unavailable_replica.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.sql b/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.sql index b9bc6974c47..c36b5bebd58 100644 --- a/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.sql +++ b/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.sql @@ -6,6 +6,9 @@ INSERT INTO 02918_parallel_replicas SELECT toString(number), number % 4 FROM num SET prefer_localhost_replica=0; +--- if we try to query unavaialble replica, connection will be retried +--- but a warning log message will be printed out +SET send_logs_level='error'; -- { echoOn } SELECT y, count() FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas) @@ -26,5 +29,6 @@ GROUP BY y ORDER BY y SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='default'; -- { echoOff } +SET send_logs_level='warning'; DROP TABLE 02918_parallel_replicas; From f311612e91aa195a5074cb104d2bb9845babbc6b Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 15 Jul 2024 09:59:23 +0200 Subject: [PATCH 133/161] check filter_out_not_affected_jobs in unit test --- tests/ci/ci_cache.py | 30 +++++++++++++++--------------- tests/ci/test_ci_config.py | 1 + 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index 44cdb19c3a1..417d5dbc262 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -729,25 +729,25 @@ class CiCache: required_builds += ( job_config.required_builds if job_config.required_builds else [] ) - - for job_name, job_config in self.jobs_to_do.items(): - if CI.is_build_job(job_name): - if job_name not in required_builds: - remove_from_to_do.append(job_name) + if has_test_jobs_to_skip: + # If there are tests to skip, it means build digest has not been changed. + # No need to test builds. Let's keep all builds required for test jobs and skip the others + for job_name, job_config in self.jobs_to_do.items(): + if CI.is_build_job(job_name): + if job_name not in required_builds: + remove_from_to_do.append(job_name) if not required_builds: remove_from_to_do.append(CI.JobNames.BUILD_CHECK) - if has_test_jobs_to_skip: - # if there are no test jobs to skip, then we must not skip anything - it's a new CI run with new build digest - for job in remove_from_to_do: - print(f"Filter job [{job}] - not affected by the change") - if job in self.jobs_to_do: - del self.jobs_to_do[job] - if job in self.jobs_to_wait: - del self.jobs_to_wait[job] - if job in self.jobs_to_skip: - self.jobs_to_skip.remove(job) + for job in remove_from_to_do: + print(f"Filter job [{job}] - not affected by the change") + if job in self.jobs_to_do: + del self.jobs_to_do[job] + if job in self.jobs_to_wait: + del self.jobs_to_wait[job] + if job in self.jobs_to_skip: + self.jobs_to_skip.remove(job) def await_pending_jobs(self, is_release: bool, dry_run: bool = False) -> None: """ diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py index e12a67bfc92..c901994affa 100644 --- a/tests/ci/test_ci_config.py +++ b/tests/ci/test_ci_config.py @@ -283,6 +283,7 @@ class TestCIConfig(unittest.TestCase): ci_cache = CIPY._configure_jobs( S3Helper(), pr_info, settings, skip_jobs=False, dry_run=True ) + ci_cache.filter_out_not_affected_jobs() actual_jobs_to_do = list(ci_cache.jobs_to_do) expected_jobs_to_do = [] for set_ in settings.ci_sets: From 2af0edd9e9a509e1bffa15e8da8454a4feb7f0ed Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 15 Jul 2024 08:47:08 +0000 Subject: [PATCH 134/161] Some minor fixups --- base/base/cgroupsv2.cpp | 14 +++++++------- base/base/cgroupsv2.h | 7 +++---- base/base/getMemoryAmount.cpp | 3 +-- src/Common/CgroupsMemoryUsageObserver.cpp | 12 ++++++------ src/Common/getNumberOfPhysicalCPUCores.cpp | 2 +- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/base/base/cgroupsv2.cpp b/base/base/cgroupsv2.cpp index 466ebbc3ffb..87f62bf377d 100644 --- a/base/base/cgroupsv2.cpp +++ b/base/base/cgroupsv2.cpp @@ -3,8 +3,9 @@ #include #include -#include +#include +namespace fs = std::filesystem; bool cgroupsV2Enabled() { @@ -13,11 +14,11 @@ bool cgroupsV2Enabled() { /// This file exists iff the host has cgroups v2 enabled. auto controllers_file = default_cgroups_mount / "cgroup.controllers"; - if (!std::filesystem::exists(controllers_file)) + if (!fs::exists(controllers_file)) return false; return true; } - catch (const std::filesystem::filesystem_error &) /// all "underlying OS API errors", typically: permission denied + catch (const fs::filesystem_error &) /// all "underlying OS API errors", typically: permission denied { return false; /// not logging the exception as most callers fall back to cgroups v1 } @@ -33,7 +34,7 @@ bool cgroupsV2MemoryControllerEnabled() /// According to https://docs.kernel.org/admin-guide/cgroup-v2.html, file "cgroup.controllers" defines which controllers are available /// for the current + child cgroups. The set of available controllers can be restricted from level to level using file /// "cgroups.subtree_control". It is therefore sufficient to check the bottom-most nested "cgroup.controllers" file. - auto cgroup_dir = currentCGroupV2Path(); + fs::path cgroup_dir = cgroupV2PathOfProcess(); if (cgroup_dir.empty()) return false; std::ifstream controllers_file(cgroup_dir / "cgroup.controllers"); @@ -47,7 +48,7 @@ bool cgroupsV2MemoryControllerEnabled() #endif } -std::filesystem::path currentCGroupV2Path() +fs::path cgroupV2PathOfProcess() { #if defined(OS_LINUX) chassert(cgroupsV2Enabled()); @@ -63,9 +64,8 @@ std::filesystem::path currentCGroupV2Path() static const std::string v2_prefix = "0::/"; if (!cgroup.starts_with(v2_prefix)) return {}; - - // the 'root' cgroup can have empty path, which is valid cgroup = cgroup.substr(v2_prefix.length()); + /// Note: The 'root' cgroup can have an empty cgroup name, this is valid return default_cgroups_mount / cgroup; #else return {}; diff --git a/base/base/cgroupsv2.h b/base/base/cgroupsv2.h index 2c58682ce31..cfb916ff358 100644 --- a/base/base/cgroupsv2.h +++ b/base/base/cgroupsv2.h @@ -1,7 +1,6 @@ #pragma once #include -#include #if defined(OS_LINUX) /// I think it is possible to mount the cgroups hierarchy somewhere else (e.g. when in containers). @@ -16,7 +15,7 @@ bool cgroupsV2Enabled(); /// Assumes that cgroupsV2Enabled() is enabled. bool cgroupsV2MemoryControllerEnabled(); -/// Detects which cgroup the process belong and returns the path to it in sysfs (for cgroups v2). -/// Returns an empty path if the cgroup cannot be determined. +/// Detects which cgroup v2 the process belongs to and returns the filesystem path to the cgroup. +/// Returns an empty path the cgroup cannot be determined. /// Assumes that cgroupsV2Enabled() is enabled. -std::filesystem::path currentCGroupV2Path(); +std::filesystem::path cgroupV2PathOfProcess(); diff --git a/base/base/getMemoryAmount.cpp b/base/base/getMemoryAmount.cpp index 9bd5ad75445..afdb6ba068a 100644 --- a/base/base/getMemoryAmount.cpp +++ b/base/base/getMemoryAmount.cpp @@ -23,8 +23,7 @@ std::optional getCgroupsV2MemoryLimit() if (!cgroupsV2MemoryControllerEnabled()) return {}; - auto current_cgroup = currentCGroupV2Path(); - + std::filesystem::path current_cgroup = cgroupV2PathOfProcess(); if (current_cgroup.empty()) return {}; diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp index e034319b21f..02bde0d80b7 100644 --- a/src/Common/CgroupsMemoryUsageObserver.cpp +++ b/src/Common/CgroupsMemoryUsageObserver.cpp @@ -25,6 +25,7 @@ #endif using namespace DB; +namespace fs = std::filesystem; namespace DB { @@ -69,7 +70,7 @@ uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, const std::string & ke struct CgroupsV1Reader : ICgroupsReader { - explicit CgroupsV1Reader(const std::filesystem::path & stat_file_dir) : buf(stat_file_dir / "memory.stat") { } + explicit CgroupsV1Reader(const fs::path & stat_file_dir) : buf(stat_file_dir / "memory.stat") { } uint64_t readMemoryUsage() override { @@ -85,7 +86,7 @@ private: struct CgroupsV2Reader : ICgroupsReader { - explicit CgroupsV2Reader(const std::filesystem::path & stat_file_dir) + explicit CgroupsV2Reader(const fs::path & stat_file_dir) : current_buf(stat_file_dir / "memory.current"), stat_buf(stat_file_dir / "memory.stat") { } @@ -129,8 +130,7 @@ std::optional getCgroupsV2Path() if (!cgroupsV2MemoryControllerEnabled()) return {}; - auto current_cgroup = currentCGroupV2Path(); - + fs::path current_cgroup = cgroupV2PathOfProcess(); if (current_cgroup.empty()) return {}; @@ -140,7 +140,7 @@ std::optional getCgroupsV2Path() { const auto current_path = current_cgroup / "memory.current"; const auto stat_path = current_cgroup / "memory.stat"; - if (std::filesystem::exists(current_path) && std::filesystem::exists(stat_path)) + if (fs::exists(current_path) && fs::exists(stat_path)) return {current_cgroup}; current_cgroup = current_cgroup.parent_path(); } @@ -150,7 +150,7 @@ std::optional getCgroupsV2Path() std::optional getCgroupsV1Path() { auto path = default_cgroups_mount / "memory/memory.stat"; - if (!std::filesystem::exists(path)) + if (!fs::exists(path)) return {}; return {default_cgroups_mount / "memory"}; } diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index b16c635f23e..34a1add2f0e 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -38,7 +38,7 @@ uint32_t getCGroupLimitedCPUCores(unsigned default_cpu_count) if (cgroupsV2Enabled()) { /// First, we identify the path of the cgroup the process belongs - auto cgroup_path = currentCGroupV2Path(); + std::filesystem::path cgroup_path = cgroupV2PathOfProcess(); if (cgroup_path.empty()) return default_cpu_count; From 2a0253d1e27ea3e55a54eb42da7b8aca9987edf9 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Mon, 15 Jul 2024 13:14:29 +0200 Subject: [PATCH 135/161] Tests: rename bad log names --- docker/test/stateful/run.sh | 2 +- docker/test/stateless/run.sh | 2 +- docker/test/stateless/setup_hdfs_minicluster.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 1f28d1ac6ea..35ffeee5438 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -25,7 +25,7 @@ source /utils.lib azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence & ./setup_minio.sh stateful -./mc admin trace clickminio > /test_output/rubbish.log & +./mc admin trace clickminio > /test_output/minio.log & MC_ADMIN_PID=$! config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index ae7ed1bcd20..c0bfc12bc75 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -54,7 +54,7 @@ source /utils.lib /usr/share/clickhouse-test/config/install.sh ./setup_minio.sh stateless -m./c admin trace clickminio > /test_output/rubbish.log & +./mc admin trace clickminio > /test_output/minio.log & MC_ADMIN_PID=$! ./setup_hdfs_minicluster.sh diff --git a/docker/test/stateless/setup_hdfs_minicluster.sh b/docker/test/stateless/setup_hdfs_minicluster.sh index 6671e73562a..15a54f59096 100755 --- a/docker/test/stateless/setup_hdfs_minicluster.sh +++ b/docker/test/stateless/setup_hdfs_minicluster.sh @@ -10,7 +10,7 @@ cd hadoop-3.3.1 export JAVA_HOME=/usr mkdir -p target/test/data chown clickhouse ./target/test/data -sudo -E -u clickhouse bin/mapred minicluster -format -nomr -nnport 12222 >> /test_output/garbage.log 2>&1 & +sudo -E -u clickhouse bin/mapred minicluster -format -nomr -nnport 12222 >> /test_output/hdfs_minicluster.log 2>&1 & while ! nc -z localhost 12222; do sleep 1 From de029e278ad4461c7ce981146fdc1dcf18ff0b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 15 Jul 2024 13:30:48 +0200 Subject: [PATCH 136/161] Add additional log masking --- tests/ci/functional_test_check.py | 14 ++++++++++++++ tests/ci/stress_check.py | 15 +++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index ef9f4dc016e..3531a55f8ae 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -24,6 +24,18 @@ from tee_popen import TeePopen NO_CHANGES_MSG = "Nothing to run" +class SensitiveFormatter(logging.Formatter): + @staticmethod + def _filter(s): + return re.sub( + r"(.*)(AZURE_CONNECTION_STRING.*\')(.*)", r"\1AZURE_CONNECTION_STRING\3", s + ) + + def format(self, record): + original = logging.Formatter.format(self, record) + return self._filter(original) + + def get_additional_envs( check_name: str, run_by_hash_num: int, run_by_hash_total: int ) -> List[str]: @@ -213,6 +225,8 @@ def parse_args(): def main(): logging.basicConfig(level=logging.INFO) + for handler in logging.root.handlers: + handler.setFormatter(SensitiveFormatter(handler.formatter._fmt)) stopwatch = Stopwatch() diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 486bfc25e22..83a410551a1 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -3,6 +3,7 @@ import csv import logging import os +import re import subprocess import sys from pathlib import Path @@ -19,6 +20,18 @@ from stopwatch import Stopwatch from tee_popen import TeePopen +class SensitiveFormatter(logging.Formatter): + @staticmethod + def _filter(s): + return re.sub( + r"(.*)(AZURE_CONNECTION_STRING.*\')(.*)", r"\1AZURE_CONNECTION_STRING\3", s + ) + + def format(self, record): + original = logging.Formatter.format(self, record) + return self._filter(original) + + def get_additional_envs(check_name: str) -> List[str]: result = [] azure_connection_string = get_parameter_from_ssm("azure_connection_string") @@ -117,6 +130,8 @@ def process_results( def run_stress_test(docker_image_name: str) -> None: logging.basicConfig(level=logging.INFO) + for handler in logging.root.handlers: + handler.setFormatter(SensitiveFormatter(handler.formatter._fmt)) stopwatch = Stopwatch() temp_path = Path(TEMP_PATH) From 6f036fad598bd0fb6903802c3df709f15b97c4cd Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 15 Jul 2024 13:35:20 +0200 Subject: [PATCH 137/161] CI: Fix ci await function --- tests/ci/ci_cache.py | 17 ++--- tests/ci/test_ci_config.py | 128 +++++++++++++++++++++++++------------ 2 files changed, 93 insertions(+), 52 deletions(-) diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py index 417d5dbc262..9486a286a8d 100644 --- a/tests/ci/ci_cache.py +++ b/tests/ci/ci_cache.py @@ -763,22 +763,13 @@ class CiCache: # TIMEOUT * MAX_ROUNDS_TO_WAIT must be less than 6h (GH job timeout) with a room for rest RunConfig work TIMEOUT = 3000 # 50 min MAX_ROUNDS_TO_WAIT = 6 - MAX_JOB_NUM_TO_WAIT = 3 round_cnt = 0 - def _has_build_job(): - for job in self.jobs_to_wait: - if CI.is_build_job(job): - return True - return False - if not is_release: # in PRs we can wait only for builds, TIMEOUT*MAX_ROUNDS_TO_WAIT=100min is enough MAX_ROUNDS_TO_WAIT = 2 - while ( - len(self.jobs_to_wait) > MAX_JOB_NUM_TO_WAIT or _has_build_job() - ) and round_cnt < MAX_ROUNDS_TO_WAIT: + while round_cnt < MAX_ROUNDS_TO_WAIT: round_cnt += 1 GHActions.print_in_group( f"Wait pending jobs, round [{round_cnt}/{MAX_ROUNDS_TO_WAIT}]:", @@ -820,6 +811,10 @@ class CiCache: f"Job [{job_name}_[{batch}/{num_batches}]] is not pending anymore" ) job_config.batches.remove(batch) + if not job_config.batches: + print(f"Remove job [{job_name}] from jobs_to_do") + self.jobs_to_skip.append(job_name) + del self.jobs_to_do[job_name] else: print( f"NOTE: Job [{job_name}:{batch}] finished failed - do not add to ready" @@ -830,9 +825,7 @@ class CiCache: await_finished.add(job_name) for job in await_finished: - self.jobs_to_skip.append(job) del self.jobs_to_wait[job] - del self.jobs_to_do[job] if not dry_run: expired_sec = int(time.time()) - start_at diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py index c901994affa..4336783e0d5 100644 --- a/tests/ci/test_ci_config.py +++ b/tests/ci/test_ci_config.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 - +import copy import unittest import random @@ -416,6 +416,30 @@ class TestCIConfig(unittest.TestCase): """ checks ci.py job configuration """ + + def _reset_ci_cache_to_wait_all_jobs(ci_cache): + # pretend there are pending jobs that we need to wait + ci_cache.jobs_to_wait = dict(ci_cache.jobs_to_do) + for job, config in ci_cache.jobs_to_wait.items(): + assert config.batches + config.pending_batches = list(config.batches) + + for batch in range(config.num_batches): + record = CiCache.Record( + record_type=CiCache.RecordType.PENDING, + job_name=job, + job_digest=ci_cache.job_digests[job], + batch=batch, + num_batches=config.num_batches, + release_branch=True, + ) + for record_t_, records_ in ci_cache.records.items(): + if record_t_.value == CiCache.RecordType.PENDING.value: + records_[record.to_str_key()] = record + assert not ci_cache.jobs_to_skip + assert ci_cache.jobs_to_wait + ci_cache.jobs_to_skip = [] + settings = CiSettings() settings.no_ci_cache = True pr_info = PRInfo(github_event=_TEST_EVENT_JSON) @@ -432,26 +456,6 @@ class TestCIConfig(unittest.TestCase): assert not ci_cache.jobs_to_skip assert not ci_cache.jobs_to_wait - # pretend there are pending jobs that we need to wait - ci_cache.jobs_to_wait = dict(ci_cache.jobs_to_do) - for job, config in ci_cache.jobs_to_wait.items(): - assert not config.pending_batches - assert config.batches - config.pending_batches = list(config.batches) - for job, config in ci_cache.jobs_to_wait.items(): - for batch in range(config.num_batches): - record = CiCache.Record( - record_type=CiCache.RecordType.PENDING, - job_name=job, - job_digest=ci_cache.job_digests[job], - batch=batch, - num_batches=config.num_batches, - release_branch=True, - ) - for record_t_, records_ in ci_cache.records.items(): - if record_t_.value == CiCache.RecordType.PENDING.value: - records_[record.to_str_key()] = record - def _test_await_for_batch( ci_cache: CiCache, record_type: CiCache.RecordType, batch: int ) -> None: @@ -477,32 +481,76 @@ class TestCIConfig(unittest.TestCase): and batch < config_.num_batches ): assert batch not in config_.pending_batches - else: - assert batch in config_.pending_batches for _, config_ in ci_cache.jobs_to_do.items(): # jobs to do must have batches to run before/after await # if it's an empty list after await - apparently job has not been removed after await assert config_.batches - _test_await_for_batch(ci_cache, CiCache.RecordType.SUCCESSFUL, 0) - # check all one-batch jobs are in jobs_to_skip - for job in all_jobs_in_wf: - config = CI.JOB_CONFIGS[job] - if config.num_batches == 1: - self.assertTrue(job in ci_cache.jobs_to_skip) - self.assertTrue(job not in ci_cache.jobs_to_do) - else: - self.assertTrue(job not in ci_cache.jobs_to_skip) - self.assertTrue(job in ci_cache.jobs_to_do) - - _test_await_for_batch(ci_cache, CiCache.RecordType.FAILED, 1) - _test_await_for_batch(ci_cache, CiCache.RecordType.SUCCESSFUL, 2) - - self.assertTrue(len(ci_cache.jobs_to_skip) > 0) - self.assertTrue(len(ci_cache.jobs_to_do) > 0) + _reset_ci_cache_to_wait_all_jobs(ci_cache) + _test_await_for_batch(ci_cache, CiCache.RecordType.FAILED, 0) + tested = False + for job, config in ci_cache.jobs_to_do.items(): + if config.batches == [0]: + tested = True + self.assertTrue( + job not in ci_cache.jobs_to_wait, + "Job must be removed from @jobs_to_wait, because its only batch has FAILED cache record", + ) self.assertCountEqual( - list(ci_cache.jobs_to_do) + ci_cache.jobs_to_skip, all_jobs_in_wf + ci_cache.jobs_to_skip, + [], + "No jobs must be skipped, since all cache records are of type FAILED", + ) + assert tested + + # reset jobs_to_wait after previous test + _reset_ci_cache_to_wait_all_jobs(ci_cache) + assert not ci_cache.jobs_to_skip + + # set batch 0 as SUCCESSFUL in ci cache + jobs_to_do_prev = list(ci_cache.jobs_to_do) + jobs_to_skip_prev = [] + jobs_to_wait_prev = list(ci_cache.jobs_to_wait) + _test_await_for_batch(ci_cache, CiCache.RecordType.SUCCESSFUL, 0) + self.assertTrue(len(jobs_to_skip_prev) != len(ci_cache.jobs_to_skip)) + self.assertTrue(len(jobs_to_wait_prev) > len(ci_cache.jobs_to_wait)) + self.assertCountEqual( + list(ci_cache.jobs_to_do) + ci_cache.jobs_to_skip, + jobs_to_do_prev + jobs_to_skip_prev, + ) + + # set batch 1 as SUCCESSFUL in ci cache + jobs_to_do_prev = list(ci_cache.jobs_to_do) + jobs_to_skip_prev = list(ci_cache.jobs_to_skip) + jobs_to_wait_prev = list(ci_cache.jobs_to_wait) + _test_await_for_batch(ci_cache, CiCache.RecordType.SUCCESSFUL, 1) + self.assertTrue(len(jobs_to_skip_prev) != len(ci_cache.jobs_to_skip)) + self.assertTrue(len(jobs_to_wait_prev) > len(ci_cache.jobs_to_wait)) + self.assertCountEqual( + list(ci_cache.jobs_to_do) + ci_cache.jobs_to_skip, + jobs_to_do_prev + jobs_to_skip_prev, + ) + + # set batch 3, 4, 5, 6 as SUCCESSFUL in ci cache + jobs_to_do_prev = list(ci_cache.jobs_to_do) + jobs_to_skip_prev = list(ci_cache.jobs_to_skip) + jobs_to_wait_prev = list(ci_cache.jobs_to_wait) + _test_await_for_batch(ci_cache, CiCache.RecordType.SUCCESSFUL, 2) + self.assertTrue(ci_cache.jobs_to_do) + _test_await_for_batch(ci_cache, CiCache.RecordType.SUCCESSFUL, 3) + self.assertTrue(ci_cache.jobs_to_do) + _test_await_for_batch(ci_cache, CiCache.RecordType.SUCCESSFUL, 4) + self.assertTrue(ci_cache.jobs_to_do) + _test_await_for_batch(ci_cache, CiCache.RecordType.SUCCESSFUL, 5) + self.assertTrue( + not ci_cache.jobs_to_do + ) # by this moment there must be no jobs left as batch 5 is currently the maximum + self.assertTrue(len(jobs_to_skip_prev) != len(ci_cache.jobs_to_skip)) + self.assertTrue(len(jobs_to_wait_prev) > len(ci_cache.jobs_to_wait)) + self.assertCountEqual( + list(ci_cache.jobs_to_do) + ci_cache.jobs_to_skip, + jobs_to_do_prev + jobs_to_skip_prev, ) def test_ci_py_filters_not_affected_jobs_in_prs(self): From b6b4b046a39994242abbd7d8637ddfb17b0c60a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 15 Jul 2024 13:42:00 +0200 Subject: [PATCH 138/161] Style --- tests/ci/functional_test_check.py | 3 ++- tests/ci/stress_check.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 3531a55f8ae..41c7ed963c9 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -226,7 +226,8 @@ def parse_args(): def main(): logging.basicConfig(level=logging.INFO) for handler in logging.root.handlers: - handler.setFormatter(SensitiveFormatter(handler.formatter._fmt)) + # pylint: disable=protected-access + handler.setFormatter(SensitiveFormatter(handler.formatter._fmt)) # type: ignore stopwatch = Stopwatch() diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 83a410551a1..85da601e379 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -131,7 +131,8 @@ def process_results( def run_stress_test(docker_image_name: str) -> None: logging.basicConfig(level=logging.INFO) for handler in logging.root.handlers: - handler.setFormatter(SensitiveFormatter(handler.formatter._fmt)) + # pylint: disable=protected-access + handler.setFormatter(SensitiveFormatter(handler.formatter._fmt)) # type: ignore stopwatch = Stopwatch() temp_path = Path(TEMP_PATH) From d768251f99840ca468af835d07347c9870ca7a34 Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 15 Jul 2024 13:43:07 +0200 Subject: [PATCH 139/161] fix handling of skipped build --- tests/ci/ci.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 8dcf3fc4c69..b17d4a0325e 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1068,6 +1068,15 @@ def main() -> int: if build_result: if build_result.status == SUCCESS: previous_status = build_result.status + JobReport( + status=SUCCESS, + description="", + test_results=[], + start_time="", + duration=0.0, + additional_files=[], + job_skipped=True, + ).dump() else: # FIXME: Consider reusing failures for build jobs. # Just remove this if/else - that makes build job starting and failing immediately From 31fdfbc737093797905ac0fe3b7fbfba9e09854d Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 15 Jul 2024 13:59:57 +0200 Subject: [PATCH 140/161] add fetching exit code if job was killed --- .github/workflows/reusable_test.yml | 3 +++ tests/ci/ci.py | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index c01dd8ca9d4..3e88ba9a508 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -102,6 +102,9 @@ jobs: --job-name '${{inputs.test_name}}' \ --run \ --run-command '''${{inputs.run_command}}''' + if [ "$?" != 0 ]; then + echo JOB_EXIT_CODE="$?" >> "$GITHUB_ENV" + fi - name: Post run if: ${{ !cancelled() }} run: | diff --git a/tests/ci/ci.py b/tests/ci/ci.py index b17d4a0325e..5fc3e2978ca 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1274,10 +1274,15 @@ def main() -> int: elif job_report.pre_report: print(f"ERROR: Job was killed - generate evidence") job_report.update_duration() - # Job was killed! + ret_code = os.getenv("JOB_EXIT_CODE", "") + if ret_code: + try: + job_report.exit_code = int(ret_code) + except ValueError: + pass if Utils.is_killed_with_oom(): print("WARNING: OOM while job execution") - error = f"Out Of Memory, exit_code {job_report.exit_code}, after {job_report.duration}s" + error = f"Out Of Memory, exit_code {job_report.exit_code}, after {int(job_report.duration)}s" else: error = f"Unknown, exit_code {job_report.exit_code}, after {job_report.duration}s" CIBuddy().post_error(error, job_name=_get_ext_check_name(args.job_name)) From 1e6edbd4aeaa2602c66f3dbff66cfac78aa05a9d Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 15 Jul 2024 14:14:44 +0200 Subject: [PATCH 141/161] ci buddy message format updates --- tests/ci/ci_buddy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py index ea690bb602c..363da121840 100644 --- a/tests/ci/ci_buddy.py +++ b/tests/ci/ci_buddy.py @@ -26,6 +26,7 @@ class CIBuddy: self.pr_number = pr_info.number self.head_ref = pr_info.head_ref self.commit_url = pr_info.commit_html_url + self.sha = pr_info.sha[:10] @staticmethod def _get_webhooks(): @@ -69,8 +70,8 @@ class CIBuddy: line_err = f":red_circle: *Error: {error_description}*\n\n" line_ghr = f" *Runner:* `{instance_type}`, `{instance_id}`\n" line_job = f" *Job:* `{job_name}`\n" - line_pr_ = f" *PR:* \n" - line_br_ = f" *Branch:* `{self.head_ref}`, <{self.commit_url}|commit>\n" + line_pr_ = f" *PR:* , <{self.commit_url}|{self.sha}>\n" + line_br_ = f" *Branch:* `{self.head_ref}`, <{self.commit_url}|{self.sha}>\n" message = line_err message += line_job if with_instance_info: From f7dfa21bfff8f775c7af722b037be58c57caa76a Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 15 Jul 2024 14:39:29 +0200 Subject: [PATCH 142/161] style fixes --- .github/workflows/reusable_test.yml | 3 ++- tests/ci/ci.py | 2 +- tests/ci/ci_buddy.py | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index 3e88ba9a508..05b0aa88ba4 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -102,7 +102,8 @@ jobs: --job-name '${{inputs.test_name}}' \ --run \ --run-command '''${{inputs.run_command}}''' - if [ "$?" != 0 ]; then + exit_code = "$?" + if [ "$exit_code" != 0 ]; then echo JOB_EXIT_CODE="$?" >> "$GITHUB_ENV" fi - name: Post run diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 5fc3e2978ca..cf285f4b97d 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1284,7 +1284,7 @@ def main() -> int: print("WARNING: OOM while job execution") error = f"Out Of Memory, exit_code {job_report.exit_code}, after {int(job_report.duration)}s" else: - error = f"Unknown, exit_code {job_report.exit_code}, after {job_report.duration}s" + error = f"Unknown, exit_code {job_report.exit_code}, after {int(job_report.duration)}s" CIBuddy().post_error(error, job_name=_get_ext_check_name(args.job_name)) if CI.is_test_job(args.job_name): gh = GitHub(get_best_robot_token(), per_page=100) diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py index 363da121840..c650b876610 100644 --- a/tests/ci/ci_buddy.py +++ b/tests/ci/ci_buddy.py @@ -71,7 +71,9 @@ class CIBuddy: line_ghr = f" *Runner:* `{instance_type}`, `{instance_id}`\n" line_job = f" *Job:* `{job_name}`\n" line_pr_ = f" *PR:* , <{self.commit_url}|{self.sha}>\n" - line_br_ = f" *Branch:* `{self.head_ref}`, <{self.commit_url}|{self.sha}>\n" + line_br_ = ( + f" *Branch:* `{self.head_ref}`, <{self.commit_url}|{self.sha}>\n" + ) message = line_err message += line_job if with_instance_info: @@ -86,4 +88,4 @@ class CIBuddy: if __name__ == "__main__": # test buddy = CIBuddy(dry_run=True) - buddy.post_error("Out of memory") + buddy.post_error("TEst") From bba45958c5b3e62d4f99902610a46c4779bc8f40 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 15 Jul 2024 13:07:38 +0000 Subject: [PATCH 143/161] Review fixes. --- .../Resolve/IdentifierResolveScope.cpp | 3 +-- src/Analyzer/Resolve/IdentifierResolveScope.h | 4 +-- src/Analyzer/Resolve/QueryAnalyzer.cpp | 26 +++++++++---------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/Analyzer/Resolve/IdentifierResolveScope.cpp b/src/Analyzer/Resolve/IdentifierResolveScope.cpp index 1600efacf4a..eb3e2179440 100644 --- a/src/Analyzer/Resolve/IdentifierResolveScope.cpp +++ b/src/Analyzer/Resolve/IdentifierResolveScope.cpp @@ -12,10 +12,9 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -IdentifierResolveScope::IdentifierResolveScope(QueryTreeNodePtr scope_node_, IdentifierResolveScope * parent_scope_, bool is_query_) +IdentifierResolveScope::IdentifierResolveScope(QueryTreeNodePtr scope_node_, IdentifierResolveScope * parent_scope_) : scope_node(std::move(scope_node_)) , parent_scope(parent_scope_) - , is_query(is_query_) { if (parent_scope) { diff --git a/src/Analyzer/Resolve/IdentifierResolveScope.h b/src/Analyzer/Resolve/IdentifierResolveScope.h index 917e032321d..ab2e27cc14d 100644 --- a/src/Analyzer/Resolve/IdentifierResolveScope.h +++ b/src/Analyzer/Resolve/IdentifierResolveScope.h @@ -128,7 +128,7 @@ constexpr auto PROJECTION_NAME_PLACEHOLDER = "__projection_name_placeholder"; struct IdentifierResolveScope { /// Construct identifier resolve scope using scope node, and parent scope - IdentifierResolveScope(QueryTreeNodePtr scope_node_, IdentifierResolveScope * parent_scope_, bool is_query_); + IdentifierResolveScope(QueryTreeNodePtr scope_node_, IdentifierResolveScope * parent_scope_); QueryTreeNodePtr scope_node; @@ -188,8 +188,6 @@ struct IdentifierResolveScope /// Join retutns NULLs instead of default values bool join_use_nulls = false; - bool is_query; - /// JOINs count size_t joins_count = 0; diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 6965036bb0c..a0043c8c04f 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -119,7 +119,7 @@ QueryAnalyzer::~QueryAnalyzer() = default; void QueryAnalyzer::resolve(QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression, ContextPtr context) { - IdentifierResolveScope scope(node, nullptr /*parent_scope*/, true /*is_query*/); + IdentifierResolveScope scope(node, nullptr /*parent_scope*/); if (!scope.context) scope.context = context; @@ -2169,7 +2169,7 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I if (apply_transformer->getApplyTransformerType() == ApplyColumnTransformerType::LAMBDA) { auto lambda_expression_to_resolve = expression_node->clone(); - IdentifierResolveScope lambda_scope(expression_node, &scope /*parent_scope*/, false /*is_query*/); + IdentifierResolveScope lambda_scope(expression_node, &scope /*parent_scope*/); node_projection_names = resolveLambda(expression_node, lambda_expression_to_resolve, {node}, lambda_scope); auto & lambda_expression_to_resolve_typed = lambda_expression_to_resolve->as(); node = lambda_expression_to_resolve_typed.getExpression(); @@ -3042,7 +3042,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi auto lambda_expression_clone = lambda_expression_untyped->clone(); - IdentifierResolveScope lambda_scope(lambda_expression_clone, &scope /*parent_scope*/, false /*is_query*/); + IdentifierResolveScope lambda_scope(lambda_expression_clone, &scope /*parent_scope*/); ProjectionNames lambda_projection_names = resolveLambda(lambda_expression_untyped, lambda_expression_clone, function_arguments, lambda_scope); auto & resolved_lambda = lambda_expression_clone->as(); @@ -3297,7 +3297,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi lambda_arguments.push_back(std::make_shared(std::move(column_name_and_type), lambda_to_resolve)); } - IdentifierResolveScope lambda_scope(lambda_to_resolve, &scope /*parent_scope*/, false /*is_query*/); + IdentifierResolveScope lambda_scope(lambda_to_resolve, &scope /*parent_scope*/); lambda_projection_names = resolveLambda(lambda_argument, lambda_to_resolve, lambda_arguments, lambda_scope); if (auto * lambda_list_node_result = lambda_to_resolve_typed.getExpression()->as()) @@ -3518,7 +3518,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( auto node_type = node->getNodeType(); if (!allow_table_expression && (node_type == QueryTreeNodeType::QUERY || node_type == QueryTreeNodeType::UNION)) { - IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/, false /*is_query*/); + IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/); subquery_scope.subquery_depth = scope.subquery_depth + 1; evaluateScalarSubqueryIfNeeded(node, subquery_scope); @@ -3625,7 +3625,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( else union_node->setIsCTE(false); - IdentifierResolveScope subquery_scope(resolved_identifier_node, &scope /*parent_scope*/, true /*is_query*/); + IdentifierResolveScope subquery_scope(resolved_identifier_node, &scope /*parent_scope*/); subquery_scope.subquery_depth = scope.subquery_depth + 1; /// CTE is being resolved, it's required to forbid to resolve to it again @@ -3758,7 +3758,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( [[fallthrough]]; case QueryTreeNodeType::UNION: { - IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/, true /*is_query*/); + IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/); subquery_scope.subquery_depth = scope.subquery_depth + 1; std::string projection_name = "_subquery_" + std::to_string(subquery_counter); @@ -3834,7 +3834,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( } /// Check parent scopes until find current query scope. - if (scope_ptr->is_query) + if (scope_ptr->scope_node->getNodeType() == QueryTreeNodeType::QUERY) break; } } @@ -4122,7 +4122,7 @@ void QueryAnalyzer::resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpo bool is_column_constant = interpolate_node_typed.getExpression()->getNodeType() == QueryTreeNodeType::CONSTANT; auto & interpolation_to_resolve = interpolate_node_typed.getInterpolateExpression(); - IdentifierResolveScope interpolate_scope(interpolation_to_resolve, &scope /*parent_scope*/, false /*is_query*/); + IdentifierResolveScope interpolate_scope(interpolation_to_resolve, &scope /*parent_scope*/); auto fake_column_node = std::make_shared(NameAndTypePair(column_to_interpolate_name, interpolate_node_typed.getExpression()->getResultType()), interpolate_node_typed.getExpression()); if (is_column_constant) @@ -4420,7 +4420,7 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table */ alias_column_to_resolve = column_name_to_column_node[alias_column_to_resolve_name]; - IdentifierResolveScope alias_column_resolve_scope(alias_column_to_resolve, nullptr /*parent_scope*/, false /*is_query*/); + IdentifierResolveScope alias_column_resolve_scope(alias_column_to_resolve, nullptr /*parent_scope*/); alias_column_resolve_scope.column_name_to_column_node = std::move(column_name_to_column_node); alias_column_resolve_scope.context = scope.context; @@ -5012,7 +5012,7 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS left_subquery->getProjection().getNodes().push_back(projection_node->clone()); left_subquery->getJoinTree() = left_table_expression; - IdentifierResolveScope left_subquery_scope(left_subquery, nullptr /*parent_scope*/, true /*is_query*/); + IdentifierResolveScope left_subquery_scope(left_subquery, nullptr /*parent_scope*/); resolveQuery(left_subquery, left_subquery_scope); const auto & resolved_nodes = left_subquery->getProjection().getNodes(); @@ -5621,7 +5621,7 @@ void QueryAnalyzer::resolveUnion(const QueryTreeNodePtr & union_node, Identifier auto & non_recursive_query_mutable_context = non_recursive_query_is_query_node ? non_recursive_query->as().getMutableContext() : non_recursive_query->as().getMutableContext(); - IdentifierResolveScope non_recursive_subquery_scope(non_recursive_query, &scope /*parent_scope*/, true /*is_query*/); + IdentifierResolveScope non_recursive_subquery_scope(non_recursive_query, &scope /*parent_scope*/); non_recursive_subquery_scope.subquery_depth = scope.subquery_depth + 1; if (non_recursive_query_is_query_node) @@ -5652,7 +5652,7 @@ void QueryAnalyzer::resolveUnion(const QueryTreeNodePtr & union_node, Identifier { auto & query_node = queries_nodes[i]; - IdentifierResolveScope subquery_scope(query_node, &scope /*parent_scope*/, true /*is_subquery*/); + IdentifierResolveScope subquery_scope(query_node, &scope /*parent_scope*/); if (recursive_cte_table_node) subquery_scope.expression_argument_name_to_node[union_node_typed.getCTEName()] = recursive_cte_table_node; From 7d42a44944b9876f14749c1e6ebeb309a0750008 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 15 Jul 2024 13:08:31 +0000 Subject: [PATCH 144/161] Review fixes. --- src/Analyzer/Resolve/QueryAnalyzer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index a0043c8c04f..a0be2a53ef6 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -511,7 +511,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden /// exception with this settings enabled(https://github.com/ClickHouse/ClickHouse/issues/52494). subquery_context->setSetting("use_structure_from_insertion_table_in_table_functions", false); - auto options = SelectQueryOptions(QueryProcessingStage::Complete, scope.subquery_depth, true /*is_query*/); + auto options = SelectQueryOptions(QueryProcessingStage::Complete, scope.subquery_depth, true /*is_subquery*/); options.only_analyze = only_analyze; auto interpreter = std::make_unique(node->toAST(), subquery_context, subquery_context->getViewSource(), options); From e3d28f92688d63fd7d417553ad77944a380d3d30 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 15 Jul 2024 15:12:23 +0200 Subject: [PATCH 145/161] Update 02443_detach_attach_partition.sh --- tests/queries/0_stateless/02443_detach_attach_partition.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02443_detach_attach_partition.sh b/tests/queries/0_stateless/02443_detach_attach_partition.sh index d72d771a150..6a47b7d8d61 100755 --- a/tests/queries/0_stateless/02443_detach_attach_partition.sh +++ b/tests/queries/0_stateless/02443_detach_attach_partition.sh @@ -73,7 +73,7 @@ kill -TERM $PID_1 && kill -TERM $PID_2 && kill -TERM $PID_3 && kill -TERM $PID_4 wait $CLICKHOUSE_CLIENT -q "SELECT '$CLICKHOUSE_DATABASE', 'threads finished'" -wait_for_queries_to_finish +wait_for_queries_to_finish 60 $CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA alter_table0" $CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA alter_table1" From 43b3a133e9c908e1850ef643b22a2331181a6cdb Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 15 Jul 2024 14:57:40 +0200 Subject: [PATCH 146/161] add prints to check_mergeable_state --- .github/workflows/reusable_test.yml | 6 ++---- tests/ci/merge_pr.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index 05b0aa88ba4..ca6df5df14e 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -102,10 +102,8 @@ jobs: --job-name '${{inputs.test_name}}' \ --run \ --run-command '''${{inputs.run_command}}''' - exit_code = "$?" - if [ "$exit_code" != 0 ]; then - echo JOB_EXIT_CODE="$?" >> "$GITHUB_ENV" - fi + # shellcheck disable=SC2319 + echo "JOB_EXIT_CODE=$?" >> "$GITHUB_ENV" - name: Post run if: ${{ !cancelled() }} run: | diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py index 59749abb4fa..94456506879 100644 --- a/tests/ci/merge_pr.py +++ b/tests/ci/merge_pr.py @@ -260,18 +260,29 @@ def main(): failed_to_get_info = False has_failed_statuses = False for status in statuses: - if not CI.is_required(status.context): + if not CI.is_required(status.context) or status.context in ( + CI.StatusNames.SYNC, + CI.StatusNames.PR_CHECK, + ): + # CI.StatusNames.SYNC or CI.StatusNames.PR_CHECK should not be checked continue + print(f"Check status [{status.context}], [{status.state}]") if status.state == FAILURE: has_failed_statuses = True failed_cnt = Utils.get_failed_tests_number(status.description) if failed_cnt is None: failed_to_get_info = True + print( + f"WARNING: failed to get number of failed tests from [{status.description}]" + ) else: if failed_cnt > max_failed_tests_per_job: job_name_with_max_failures = status.context max_failed_tests_per_job = failed_cnt total_failed_tests += failed_cnt + print( + f"Failed test cases in [{status.context}] is [{failed_cnt}], total failures [{total_failed_tests}]" + ) elif status.state != SUCCESS and status.context not in ( CI.StatusNames.SYNC, CI.StatusNames.PR_CHECK, From cfb2183d118b783d142a721c89c04b1835dfa3ed Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 15 Jul 2024 17:25:43 +0200 Subject: [PATCH 147/161] Update 01396_inactive_replica_cleanup_nodes_zookeeper.sh --- .../01396_inactive_replica_cleanup_nodes_zookeeper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh index b81bb75891d..bff85b3e29f 100755 --- a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh +++ b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh @@ -30,7 +30,7 @@ $CLICKHOUSE_CLIENT --max_block_size 1 --min_insert_block_size_rows 1 --min_inser for _ in {1..60}; do $CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS" - [[ $($CLICKHOUSE_CLIENT --query "SELECT sum(toUInt32(extract(message, 'Removed (\d+) old log entries'))) FROM system.text_log WHERE event_date >= yesterday() AND logger_name LIKE '%' || '$CLICKHOUSE_DATABASE' || '%r1%(ReplicatedMergeTreeCleanupThread)%' AND message LIKE '%Removed % old log entries%'") -gt $((SCALE - 100)) ]] && break; + [[ $($CLICKHOUSE_CLIENT --query "SELECT sum(toUInt32(extract(message, 'Removed (\d+) old log entries'))) FROM system.text_log WHERE event_date >= yesterday() AND logger_name LIKE '%' || '$CLICKHOUSE_DATABASE' || '%r1%(ReplicatedMergeTreeCleanupThread)%' AND message LIKE '%Removed % old log entries%'") -gt $((SCALE - 10)) ]] && break; sleep 1 done From e06f1ed108f12b24fff716dc380bb5adce27a921 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Mon, 15 Jul 2024 17:59:00 +0200 Subject: [PATCH 148/161] Update the MySQL interface doc with compatibility notes --- docs/en/interfaces/mysql.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/en/interfaces/mysql.md b/docs/en/interfaces/mysql.md index 42820505406..9ad13ae4f05 100644 --- a/docs/en/interfaces/mysql.md +++ b/docs/en/interfaces/mysql.md @@ -6,7 +6,18 @@ sidebar_label: MySQL Interface # MySQL Interface -ClickHouse supports the MySQL wire protocol. This allow tools that are MySQL-compatible to interact with ClickHouse seamlessly (e.g. [Looker Studio](../integrations/data-visualization/looker-studio-and-clickhouse.md)). +ClickHouse supports the MySQL wire protocol. This allows certain clients that do not have native ClickHouse connectors leverage the MySQL protocol instead, and it has been validated with the following BI tools: + +- [Looker Studio](../integrations/data-visualization/looker-studio-and-clickhouse.md) +- [Tableau Online](../integrations/tableau-online) +- [QuickSight](../integrations/quicksight) + +If you are trying other untested clients or integrations, keep in mind that there could be the following limitations: + +- SSL implementation might not be fully compatible; there could be potential [TLS SNI](https://www.cloudflare.com/learning/ssl/what-is-sni/) issues. +- A particular tool might require dialect features (e.g., MySQL-specific functions or settings) that are not implemented yet. + +If there is a native driver available (e.g., [DBeaver](../integrations/dbeaver)), it is always preferred to use it instead of the MySQL interface. Additionally, while most of the MySQL language clients should work fine, MySQL interface is not guaranteed to be a drop-in replacement for a codebase with existing MySQL queries. ## Enabling the MySQL Interface On ClickHouse Cloud From 293b5c81d088223f9428e4b9e73f036bc8165d7d Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Mon, 15 Jul 2024 18:10:46 +0200 Subject: [PATCH 149/161] Add an entry about MySQL interface issues reporting --- docs/en/interfaces/mysql.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/interfaces/mysql.md b/docs/en/interfaces/mysql.md index 9ad13ae4f05..30a8d9569a9 100644 --- a/docs/en/interfaces/mysql.md +++ b/docs/en/interfaces/mysql.md @@ -19,6 +19,8 @@ If you are trying other untested clients or integrations, keep in mind that ther If there is a native driver available (e.g., [DBeaver](../integrations/dbeaver)), it is always preferred to use it instead of the MySQL interface. Additionally, while most of the MySQL language clients should work fine, MySQL interface is not guaranteed to be a drop-in replacement for a codebase with existing MySQL queries. +If your use case involves a particular tool that does not have a native ClickHouse driver, and you would like to use it via the MySQL interface and you found certain incompatibilities - please [create an issue](https://github.com/ClickHouse/ClickHouse/issues) in the ClickHouse repository. + ## Enabling the MySQL Interface On ClickHouse Cloud 1. After creating your ClickHouse Cloud Service, on the credentials screen, select the MySQL tab From 50f068efe930e264de938da46040a1e056200032 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Mon, 15 Jul 2024 17:00:53 +0000 Subject: [PATCH 150/161] add dedicated runner to libfuzzer, update docker --- docker/test/libfuzzer/run_libfuzzer.py | 70 +--------------------- tests/ci/build_download_helper.py | 2 +- tests/fuzz/build.sh | 3 + utils/libfuzzer/runner.py | 80 ++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 69 deletions(-) create mode 100644 utils/libfuzzer/runner.py diff --git a/docker/test/libfuzzer/run_libfuzzer.py b/docker/test/libfuzzer/run_libfuzzer.py index fa67805dfa5..25d5ede028d 100755 --- a/docker/test/libfuzzer/run_libfuzzer.py +++ b/docker/test/libfuzzer/run_libfuzzer.py @@ -1,77 +1,11 @@ #!/usr/bin/env python3 -import configparser -import logging -import os -from pathlib import Path -import subprocess - -DEBUGGER = os.getenv("DEBUGGER", "") -FUZZER_ARGS = os.getenv("FUZZER_ARGS", "") - - -def run_fuzzer(fuzzer: str): - logging.info(f"Running fuzzer {fuzzer}...") - - corpus_dir = f"{fuzzer}.in" - with Path(corpus_dir) as path: - if not path.exists() or not path.is_dir(): - corpus_dir = "" - - options_file = f"{fuzzer}.options" - custom_libfuzzer_options = "" - - with Path(options_file) as path: - if path.exists() and path.is_file(): - parser = configparser.ConfigParser() - parser.read(path) - - if parser.has_section("asan"): - os.environ["ASAN_OPTIONS"] = ( - f"{os.environ['ASAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['asan'].items())}" - ) - - if parser.has_section("msan"): - os.environ["MSAN_OPTIONS"] = ( - f"{os.environ['MSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['msan'].items())}" - ) - - if parser.has_section("ubsan"): - os.environ["UBSAN_OPTIONS"] = ( - f"{os.environ['UBSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['ubsan'].items())}" - ) - - if parser.has_section("libfuzzer"): - custom_libfuzzer_options = " ".join( - "-%s=%s" % (key, value) - for key, value in parser["libfuzzer"].items() - ) - - cmd_line = f"{DEBUGGER} ./{fuzzer} {FUZZER_ARGS} {corpus_dir}" - if custom_libfuzzer_options: - cmd_line += f" {custom_libfuzzer_options}" - - if not "-dict=" in cmd_line and Path(f"{fuzzer}.dict").exists(): - cmd_line += f" -dict={fuzzer}.dict" - - cmd_line += " < /dev/null" - - logging.info(f"...will execute: {cmd_line}") - subprocess.check_call(cmd_line, shell=True) +import runner def main(): - logging.basicConfig(level=logging.INFO) - - subprocess.check_call("ls -al", shell=True) - - with Path() as current: - for fuzzer in current.iterdir(): - if (current / fuzzer).is_file() and os.access(current / fuzzer, os.X_OK): - run_fuzzer(fuzzer) - + runner.run() exit(0) - if __name__ == "__main__": main() diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index 8482abb26e0..0b7dd6dd0c9 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -275,5 +275,5 @@ def download_fuzzers( check_name, reports_path, result_path, - lambda x: x.endswith(("_fuzzer", ".dict", ".options", "_seed_corpus.zip")), + lambda x: x.endswith(("_fuzzer", ".dict", ".options", "_seed_corpus.zip", "runner.py")), ) diff --git a/tests/fuzz/build.sh b/tests/fuzz/build.sh index 12f41f6e079..cf871d2de58 100755 --- a/tests/fuzz/build.sh +++ b/tests/fuzz/build.sh @@ -1,5 +1,8 @@ #!/bin/bash -eu +# copy runner +cp $SRC/utils/libfuzzer/runner.py $OUT/ + # copy fuzzer options and dictionaries cp $SRC/tests/fuzz/*.dict $OUT/ cp $SRC/tests/fuzz/*.options $OUT/ diff --git a/utils/libfuzzer/runner.py b/utils/libfuzzer/runner.py new file mode 100644 index 00000000000..435eb49383b --- /dev/null +++ b/utils/libfuzzer/runner.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 + +import configparser +import logging +import os +from pathlib import Path +import subprocess + +DEBUGGER = os.getenv("DEBUGGER", "") +FUZZER_ARGS = os.getenv("FUZZER_ARGS", "") + + +def run_fuzzer(fuzzer: str): + logging.info(f"Running fuzzer {fuzzer}...") + + corpus_dir = f"{fuzzer}.in" + with Path(corpus_dir) as path: + if not path.exists() or not path.is_dir(): + corpus_dir = "" + + options_file = f"{fuzzer}.options" + custom_libfuzzer_options = "" + fuzzer_arguments = "" + + with Path(options_file) as path: + if path.exists() and path.is_file(): + parser = configparser.ConfigParser() + parser.read(path) + + if parser.has_section("asan"): + os.environ["ASAN_OPTIONS"] = ( + f"{os.environ['ASAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['asan'].items())}" + ) + + if parser.has_section("msan"): + os.environ["MSAN_OPTIONS"] = ( + f"{os.environ['MSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['msan'].items())}" + ) + + if parser.has_section("ubsan"): + os.environ["UBSAN_OPTIONS"] = ( + f"{os.environ['UBSAN_OPTIONS']}:{':'.join('%s=%s' % (key, value) for key, value in parser['ubsan'].items())}" + ) + + if parser.has_section("libfuzzer"): + custom_libfuzzer_options = " ".join( + "-%s=%s" % (key, value) + for key, value in parser["libfuzzer"].items() + ) + + if parser.has_section("fuzzer_arguments"): + fuzzer_arguments = " ".join( + ("%s" % key) if value == "" else ("%s=%s" % (key, value)) + for key, value in parser["fuzzer_arguments"].items() + ) + + cmd_line = f"{DEBUGGER} ./{fuzzer} {FUZZER_ARGS} {corpus_dir}" + if custom_libfuzzer_options: + cmd_line += f" {custom_libfuzzer_options}" + if fuzzer_arguments: + cmd_line += f" {fuzzer_arguments}" + + if not "-dict=" in cmd_line and Path(f"{fuzzer}.dict").exists(): + cmd_line += f" -dict={fuzzer}.dict" + + cmd_line += " < /dev/null" + + logging.info(f"...will execute: {cmd_line}") + subprocess.check_call(cmd_line, shell=True) + + +def run(): + logging.basicConfig(level=logging.INFO) + + subprocess.check_call("ls -al", shell=True) + + with Path() as current: + for fuzzer in current.iterdir(): + if (current / fuzzer).is_file() and os.access(current / fuzzer, os.X_OK): + run_fuzzer(fuzzer) From 874b4d1af09496be6cc6601186379e8ad925b05a Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Mon, 15 Jul 2024 19:15:45 +0200 Subject: [PATCH 151/161] fix tidy build --- src/Client/ConnectionPool.h | 2 +- src/Interpreters/Context.cpp | 1 - utils/keeper-bench/Runner.cpp | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Client/ConnectionPool.h b/src/Client/ConnectionPool.h index d83ecdd75f9..0fcb3c4e7e1 100644 --- a/src/Client/ConnectionPool.h +++ b/src/Client/ConnectionPool.h @@ -103,7 +103,7 @@ public: Entry get(const ConnectionTimeouts & timeouts, /// NOLINT const Settings & settings, - bool force_connected = true) override; + bool force_connected) override; std::string getDescription() const { diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index fc1e87e7b7e..2602afd8b78 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -51,7 +51,6 @@ #include #include #include -#include #include #include #include diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 587e015b340..cd7f1437cf7 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include From 97c9739abcb6b6e5da113ac12a35078fe694aff6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 15 Jul 2024 17:21:50 +0000 Subject: [PATCH 152/161] Automatic style fix --- docker/test/libfuzzer/run_libfuzzer.py | 1 + tests/ci/build_download_helper.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/test/libfuzzer/run_libfuzzer.py b/docker/test/libfuzzer/run_libfuzzer.py index 25d5ede028d..06c816c8943 100755 --- a/docker/test/libfuzzer/run_libfuzzer.py +++ b/docker/test/libfuzzer/run_libfuzzer.py @@ -7,5 +7,6 @@ def main(): runner.run() exit(0) + if __name__ == "__main__": main() diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index 0b7dd6dd0c9..c3e516f836d 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -275,5 +275,7 @@ def download_fuzzers( check_name, reports_path, result_path, - lambda x: x.endswith(("_fuzzer", ".dict", ".options", "_seed_corpus.zip", "runner.py")), + lambda x: x.endswith( + ("_fuzzer", ".dict", ".options", "_seed_corpus.zip", "runner.py") + ), ) From 5fc4fada610b9d7a62528eaecb600bd5d16d48ae Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 15 Jul 2024 18:18:15 +0200 Subject: [PATCH 153/161] add info about previous release --- tests/ci/ci_utils.py | 5 +++-- tests/ci/create_release.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py index 44bd37fe260..25875e55df6 100644 --- a/tests/ci/ci_utils.py +++ b/tests/ci/ci_utils.py @@ -49,14 +49,15 @@ class GHActions: class Shell: @classmethod def run_strict(cls, command): - subprocess.run( - command + " 2>&1", + res = subprocess.run( + command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True, ) + return res.stdout.strip() @classmethod def run(cls, command): diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py index f377e12d11c..e5723e00e2a 100755 --- a/tests/ci/create_release.py +++ b/tests/ci/create_release.py @@ -14,6 +14,7 @@ from ssh import SSHAgent from env_helper import GITHUB_REPOSITORY, S3_BUILDS_BUCKET from s3_helper import S3Helper from autoscale_runners_lambda.lambda_shared.pr import Labels +from ci_utils import Shell from version_helper import ( FILE_WITH_VERSION_PATH, GENERATED_CONTRIBUTORS, @@ -65,6 +66,8 @@ class ReleaseInfo: commit_sha: str # lts or stable codename: str + previous_release_tag: str + previous_release_sha: str @staticmethod def from_file(file_path: str) -> "ReleaseInfo": @@ -79,6 +82,8 @@ class ReleaseInfo: version = None release_branch = None release_tag = None + previous_release_tag = None + previous_release_sha = None codename = None assert release_type in ("patch", "new") if release_type == "new": @@ -101,6 +106,11 @@ class ReleaseInfo: codename = ( VersionType.STABLE ) # dummy value (artifactory won't be updated for new release) + previous_release_tag = expected_prev_tag + previous_release_sha = Shell.run_strict( + f"git rev-parse {previous_release_tag}" + ) + assert previous_release_sha if release_type == "patch": with checkout(commit_ref): _, commit_sha = ShellRunner.run(f"git rev-parse {commit_ref}") @@ -118,6 +128,7 @@ class ReleaseInfo: ) if version.patch == 1: expected_version = copy(version) + previous_release_tag = f"v{version.major}.{version.minor}.1.1-new" expected_version.bump() expected_tag_prefix = ( f"v{expected_version.major}.{expected_version.minor}." @@ -128,6 +139,7 @@ class ReleaseInfo: f"v{version.major}.{version.minor}.{version.patch-1}." ) expected_tag_suffix = f"-{version.get_stable_release_type()}" + previous_release_tag = git.latest_tag if git.latest_tag.startswith( expected_tag_prefix ) and git.latest_tag.endswith(expected_tag_suffix): @@ -137,8 +149,15 @@ class ReleaseInfo: False ), f"BUG: Unexpected latest tag [{git.latest_tag}] expected [{expected_tag_prefix}*{expected_tag_suffix}]" + previous_release_sha = Shell.run_strict( + f"git rev-parse {previous_release_tag}" + ) + assert previous_release_sha + assert ( release_branch + and previous_release_tag + and previous_release_sha and commit_sha and release_tag and version @@ -150,6 +169,8 @@ class ReleaseInfo: release_tag=release_tag, version=version.string, codename=codename, + previous_release_tag=previous_release_tag, + previous_release_sha=previous_release_sha, ) with open(outfile, "w", encoding="utf-8") as f: print(json.dumps(dataclasses.asdict(res), indent=2), file=f) From 5efbb42a2a0efbb5a32267fec60ab73eddaea4cf Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Mon, 15 Jul 2024 19:31:41 +0200 Subject: [PATCH 154/161] Add QuickSight to spell check ignore --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index a2a2fcd967a..987c4e8cac1 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -760,6 +760,7 @@ QueryCacheMisses QueryPreempted QueryThread QuickAssist +QuickSight QuoteMeta RBAC RClickHouse From 5361ade8e709f3ebe8916aa80669f4bec8b5c726 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Mon, 15 Jul 2024 19:10:07 +0000 Subject: [PATCH 155/161] optimize --- docker/test/libfuzzer/Dockerfile | 2 +- docker/test/libfuzzer/run_libfuzzer.py | 12 ------------ tests/ci/build_download_helper.py | 4 +--- tests/fuzz/build.sh | 3 --- utils/libfuzzer/runner.py | 6 +++++- 5 files changed, 7 insertions(+), 20 deletions(-) delete mode 100755 docker/test/libfuzzer/run_libfuzzer.py diff --git a/docker/test/libfuzzer/Dockerfile b/docker/test/libfuzzer/Dockerfile index e6eb2ae336e..9ded2ca4e3a 100644 --- a/docker/test/libfuzzer/Dockerfile +++ b/docker/test/libfuzzer/Dockerfile @@ -39,7 +39,7 @@ ENV FUZZER_ARGS="-max_total_time=60" SHELL ["/bin/bash", "-c"] CMD set -o pipefail \ - && timeout -s 9 1h /run_libfuzzer.py 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log + && timeout -s 9 1h ./utils/libfuzzer/runner.py 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log # docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer diff --git a/docker/test/libfuzzer/run_libfuzzer.py b/docker/test/libfuzzer/run_libfuzzer.py deleted file mode 100755 index 06c816c8943..00000000000 --- a/docker/test/libfuzzer/run_libfuzzer.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python3 - -import runner - - -def main(): - runner.run() - exit(0) - - -if __name__ == "__main__": - main() diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index c3e516f836d..8482abb26e0 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -275,7 +275,5 @@ def download_fuzzers( check_name, reports_path, result_path, - lambda x: x.endswith( - ("_fuzzer", ".dict", ".options", "_seed_corpus.zip", "runner.py") - ), + lambda x: x.endswith(("_fuzzer", ".dict", ".options", "_seed_corpus.zip")), ) diff --git a/tests/fuzz/build.sh b/tests/fuzz/build.sh index cf871d2de58..12f41f6e079 100755 --- a/tests/fuzz/build.sh +++ b/tests/fuzz/build.sh @@ -1,8 +1,5 @@ #!/bin/bash -eu -# copy runner -cp $SRC/utils/libfuzzer/runner.py $OUT/ - # copy fuzzer options and dictionaries cp $SRC/tests/fuzz/*.dict $OUT/ cp $SRC/tests/fuzz/*.options $OUT/ diff --git a/utils/libfuzzer/runner.py b/utils/libfuzzer/runner.py index 435eb49383b..bbe648dbbc2 100644 --- a/utils/libfuzzer/runner.py +++ b/utils/libfuzzer/runner.py @@ -69,7 +69,7 @@ def run_fuzzer(fuzzer: str): subprocess.check_call(cmd_line, shell=True) -def run(): +def main(): logging.basicConfig(level=logging.INFO) subprocess.check_call("ls -al", shell=True) @@ -78,3 +78,7 @@ def run(): for fuzzer in current.iterdir(): if (current / fuzzer).is_file() and os.access(current / fuzzer, os.X_OK): run_fuzzer(fuzzer) + + +if __name__ == "__main__": + main() From 0176e0b2ae534fee35a6e2bcab8a74a0783e8137 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Mon, 15 Jul 2024 19:27:54 +0000 Subject: [PATCH 156/161] fix --- docker/test/libfuzzer/Dockerfile | 4 ---- tests/ci/libfuzzer_test_check.py | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docker/test/libfuzzer/Dockerfile b/docker/test/libfuzzer/Dockerfile index 9ded2ca4e3a..3ffae0cd921 100644 --- a/docker/test/libfuzzer/Dockerfile +++ b/docker/test/libfuzzer/Dockerfile @@ -33,13 +33,9 @@ RUN apt-get update \ COPY requirements.txt / RUN pip3 install --no-cache-dir -r /requirements.txt -COPY * / - ENV FUZZER_ARGS="-max_total_time=60" SHELL ["/bin/bash", "-c"] -CMD set -o pipefail \ - && timeout -s 9 1h ./utils/libfuzzer/runner.py 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log # docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index 1f5936c3fec..d9e33229932 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -74,7 +74,8 @@ def get_run_command( f"--volume={repo_path}/tests:/usr/share/clickhouse-test " f"--volume={result_path}:/test_output " "--security-opt seccomp=unconfined " # required to issue io_uring sys-calls - f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" + f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image} " + "python3 ./utils/runner.py" ) From c05b2bfd39bdb12290f2c698bfdbdec41021a45e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jul 2024 05:33:33 +0200 Subject: [PATCH 157/161] More clarity in the test `03001_consider_lwd_when_merge` --- tests/queries/0_stateless/03001_consider_lwd_when_merge.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03001_consider_lwd_when_merge.sql b/tests/queries/0_stateless/03001_consider_lwd_when_merge.sql index 988d7058f21..2b10c72ae1b 100644 --- a/tests/queries/0_stateless/03001_consider_lwd_when_merge.sql +++ b/tests/queries/0_stateless/03001_consider_lwd_when_merge.sql @@ -7,12 +7,14 @@ SETTINGS max_bytes_to_merge_at_max_space_in_pool = 80000, exclude_deleted_rows_f INSERT INTO lwd_merge SELECT number FROM numbers(10000); INSERT INTO lwd_merge SELECT number FROM numbers(10000, 10000); -OPTIMIZE TABLE lwd_merge; +SET optimize_throw_if_noop = 1; + +OPTIMIZE TABLE lwd_merge; -- { serverError CANNOT_ASSIGN_OPTIMIZE } SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 'lwd_merge' AND active = 1; DELETE FROM lwd_merge WHERE id % 10 > 0; -OPTIMIZE TABLE lwd_merge; +OPTIMIZE TABLE lwd_merge; -- { serverError CANNOT_ASSIGN_OPTIMIZE } SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 'lwd_merge' AND active = 1; ALTER TABLE lwd_merge MODIFY SETTING exclude_deleted_rows_for_part_size_in_merge = 1; From f3047cc78dc310e72b913964429f925876f27fd1 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Fri, 12 Jul 2024 10:35:16 +0800 Subject: [PATCH 158/161] fixed type mismatch in cross join --- src/Interpreters/HashJoin/HashJoin.cpp | 2 +- tests/queries/0_stateless/00202_cross_join.reference | 1 + tests/queries/0_stateless/00202_cross_join.sql | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/HashJoin/HashJoin.cpp b/src/Interpreters/HashJoin/HashJoin.cpp index fa8ebd2c0f0..0c7cad4360d 100644 --- a/src/Interpreters/HashJoin/HashJoin.cpp +++ b/src/Interpreters/HashJoin/HashJoin.cpp @@ -125,7 +125,7 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s if (isCrossOrComma(kind)) { data->type = Type::CROSS; - sample_block_with_columns_to_add = right_sample_block; + sample_block_with_columns_to_add = materializeBlock(right_sample_block); } else if (table_join->getClauses().empty()) { diff --git a/tests/queries/0_stateless/00202_cross_join.reference b/tests/queries/0_stateless/00202_cross_join.reference index a8db281730a..e134631383d 100644 --- a/tests/queries/0_stateless/00202_cross_join.reference +++ b/tests/queries/0_stateless/00202_cross_join.reference @@ -43,3 +43,4 @@ 2 2 2 3 2 4 +1 1 1 1 1 diff --git a/tests/queries/0_stateless/00202_cross_join.sql b/tests/queries/0_stateless/00202_cross_join.sql index 8d62c56b3f1..e4929d038c3 100644 --- a/tests/queries/0_stateless/00202_cross_join.sql +++ b/tests/queries/0_stateless/00202_cross_join.sql @@ -5,3 +5,5 @@ SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN SET allow_experimental_analyzer = 1; SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN (SELECT number AS y FROM system.numbers LIMIT 5) js2; + +SELECT * FROM ( SELECT 1 AS a, toLowCardinality(1), 1) AS t1 CROSS JOIN (SELECT toLowCardinality(1 AS a), 1 AS b) AS t2; From 9c1532e02f5f1dfeb2f3833afe585e26d08ccbf0 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Mon, 15 Jul 2024 09:46:51 +0800 Subject: [PATCH 159/161] add tests --- tests/queries/0_stateless/00202_cross_join.reference | 1 - tests/queries/0_stateless/00202_cross_join.sql | 2 -- .../queries/0_stateless/03205_column_type_check.reference | 2 ++ tests/queries/0_stateless/03205_column_type_check.sql | 7 +++++++ 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/03205_column_type_check.reference create mode 100644 tests/queries/0_stateless/03205_column_type_check.sql diff --git a/tests/queries/0_stateless/00202_cross_join.reference b/tests/queries/0_stateless/00202_cross_join.reference index e134631383d..a8db281730a 100644 --- a/tests/queries/0_stateless/00202_cross_join.reference +++ b/tests/queries/0_stateless/00202_cross_join.reference @@ -43,4 +43,3 @@ 2 2 2 3 2 4 -1 1 1 1 1 diff --git a/tests/queries/0_stateless/00202_cross_join.sql b/tests/queries/0_stateless/00202_cross_join.sql index e4929d038c3..8d62c56b3f1 100644 --- a/tests/queries/0_stateless/00202_cross_join.sql +++ b/tests/queries/0_stateless/00202_cross_join.sql @@ -5,5 +5,3 @@ SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN SET allow_experimental_analyzer = 1; SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN (SELECT number AS y FROM system.numbers LIMIT 5) js2; - -SELECT * FROM ( SELECT 1 AS a, toLowCardinality(1), 1) AS t1 CROSS JOIN (SELECT toLowCardinality(1 AS a), 1 AS b) AS t2; diff --git a/tests/queries/0_stateless/03205_column_type_check.reference b/tests/queries/0_stateless/03205_column_type_check.reference new file mode 100644 index 00000000000..3b6c93a0610 --- /dev/null +++ b/tests/queries/0_stateless/03205_column_type_check.reference @@ -0,0 +1,2 @@ +1 nan 1048575 2 +1 1 1 1 1 diff --git a/tests/queries/0_stateless/03205_column_type_check.sql b/tests/queries/0_stateless/03205_column_type_check.sql new file mode 100644 index 00000000000..ab122821eb0 --- /dev/null +++ b/tests/queries/0_stateless/03205_column_type_check.sql @@ -0,0 +1,7 @@ +SELECT * FROM (SELECT toUInt256(1)) AS t, (SELECT greatCircleAngle(toLowCardinality(toNullable(toUInt256(1048575))), 257, -9223372036854775808, 1048576), 1048575, materialize(2)) AS u; + + +SET join_algorithm='hash'; +SET allow_experimental_join_condition=1; +SELECT * FROM ( SELECT 1 AS a, toLowCardinality(1), 1) AS t1 CROSS JOIN (SELECT toLowCardinality(1 AS a), 1 AS b) AS t2; + From 77c4e6ae8ca03022c99c1f3f328ef1c8d282ed7b Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 16 Jul 2024 12:21:08 +0200 Subject: [PATCH 160/161] Stateless tests: disable sequential tests in parallel tests --- docker/test/stateless/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index c0bfc12bc75..2b535f8dd23 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -12,7 +12,7 @@ MAX_RUN_TIME=$((MAX_RUN_TIME == 0 ? 7200 : MAX_RUN_TIME)) USE_DATABASE_REPLICATED=${USE_DATABASE_REPLICATED:=0} USE_SHARED_CATALOG=${USE_SHARED_CATALOG:=0} -RUN_SEQUENTIAL_TESTS_IN_PARALLEL=1 +RUN_SEQUENTIAL_TESTS_IN_PARALLEL=0 if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] || [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then RUN_SEQUENTIAL_TESTS_IN_PARALLEL=0 From e8627601b52e28a4d4203cd365cd5e4693e2af57 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:08:25 +0200 Subject: [PATCH 161/161] Add a note about reading subcolumns from Array type in documentation --- docs/en/sql-reference/data-types/array.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md index e5a8ce5d18b..1737fdd88b2 100644 --- a/docs/en/sql-reference/data-types/array.md +++ b/docs/en/sql-reference/data-types/array.md @@ -96,3 +96,22 @@ Result: │ 1 │ [2] │ [[4,1]] │ └───────────┴───────────┴───────────┘ ``` + +## Reading nested subcolumns from Array + +If nested type `T` inside `Array` has subcolumns (for example, if it's a [named tuple](./tuple.md)), you can read its subcolumns from an `Array(T)` type with the same subcolumn names. The type of a subcolumn will be `Array` of the type of original subcolumn. + +**Example** + +```sql +CREATE TABLE t_arr (arr Array(Tuple(field1 UInt32, field2 String))) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_arr VALUES ([(1, 'Hello'), (2, 'World')]), ([(3, 'This'), (4, 'is'), (5, 'subcolumn')]); +SELECT arr.field1, toTypeName(arr.field1), arr.field2, toTypeName(arr.field2) from t_arr; +``` + +```test +┌─arr.field1─┬─toTypeName(arr.field1)─┬─arr.field2────────────────┬─toTypeName(arr.field2)─┐ +│ [1,2] │ Array(UInt32) │ ['Hello','World'] │ Array(String) │ +│ [3,4,5] │ Array(UInt32) │ ['This','is','subcolumn'] │ Array(String) │ +└────────────┴────────────────────────┴───────────────────────────┴────────────────────────┘ +```