From 699b9d40263078285a8fce6a031bc74ce72c16d3 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Wed, 6 Nov 2024 20:20:44 +0800 Subject: [PATCH 01/20] fix comments --- src/Functions/parseDateTime.cpp | 277 ++++++++++++++++++++++---------- 1 file changed, 194 insertions(+), 83 deletions(-) diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 65bc65fb45c..976be53a21e 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -57,8 +57,15 @@ namespace Null }; + enum class ReturnType: uint8_t + { + DateTime, + DateTime64 + }; + constexpr Int32 minYear = 1970; constexpr Int32 maxYear = 2106; + constexpr Int32 maxPrecisionOfDateTime64 = 6; const std::unordered_map> dayOfWeekMap{ {"mon", {"day", 1}}, @@ -570,8 +577,8 @@ namespace } }; - /// _FUNC_(str[, format, timezone]) - template + /// _FUNC_(str[scale, format, timezone]) + template class FunctionParseDateTimeImpl : public IFunction { public: @@ -602,79 +609,112 @@ namespace {"time", static_cast(&isString), nullptr, "String"} }; - FunctionArgumentDescriptors optional_args{ - {"format", static_cast(&isString), nullptr, "String"}, - {"timezone", static_cast(&isString), &isColumnConst, "const String"} - }; - + FunctionArgumentDescriptors optional_args; + if constexpr (return_type == ReturnType::DateTime64) + { + optional_args = { + {"precision or format", static_cast([](const IDataType & data_type) -> bool { + return isUInt(data_type) || isString(data_type); + }), nullptr, "Number or String"}, + {"format", static_cast(&isString), nullptr, "String"}, + {"timezone", static_cast(&isString), &isColumnConst, "const String"} + }; + } + else + optional_args = { + {"format", static_cast(&isString), nullptr, "String"}, + {"timezone", static_cast(&isString), &isColumnConst, "const String"} + }; validateFunctionArguments(*this, arguments, mandatory_args, optional_args); String time_zone_name = getTimeZone(arguments).getTimeZone(); - DataTypePtr date_type = nullptr; - if constexpr (parseDateTime64) + DataTypePtr data_type; + if constexpr (return_type == ReturnType::DateTime64) { - String format = getFormat(arguments); - std::vector instructions = parseFormat(format); - UInt32 scale = 0; - if (!instructions.empty()) + if (arguments.size() == 1) + return std::make_shared(0, time_zone_name); + else { - for (const auto & ins : instructions) + UInt32 precision = 0; + if (isUInt(arguments[1].type)) { - if (scale > 0) - break; - const String fragment = ins.getFragment(); + const auto * col_precision = checkAndGetColumnConst(arguments[1].column.get()); + if (col_precision) + precision = col_precision->getValue(); + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "The input precision value may exceed the max value of `DateTime64`: {}.", + maxPrecisionOfDateTime64); + } + /// Construct the return type `DataTypDateTime64` with precision and time zone name. The precision value can be specified or be extracted + /// from the format string by computing how many 'S' characters are contained in the format's micorsceond fragment. + String format = getFormat(arguments, precision); + std::vector instructions = parseFormat(format); + for (const auto & instruction : instructions) + { + const String & fragment = instruction.getFragment(); + UInt32 val = 0; for (char ch : fragment) { if (ch != 'S') { - scale = 0; + val = 0; break; } else - scale++; + val++; } + /// If the precision is already specified by the second parameter, but it not equals the value that extract from the format string, + /// then we should throw an exception; If the precision is not specified, then we set its value as the extracted one. + if (val != 0 && precision != 0 && val != precision) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "The precision of input format string {} not equals the given precision value {}.", + format, + precision); + else if (precision == 0 && val != 0) + precision = val; } + if (precision > maxPrecisionOfDateTime64) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "The precision of the input format string {} exceed the max precision value {}.", + format, + maxPrecisionOfDateTime64); + data_type = std::make_shared(precision, time_zone_name); } - date_type = std::make_shared(scale, time_zone_name); } else - date_type = std::make_shared(time_zone_name); + data_type = std::make_shared(time_zone_name); if (error_handling == ErrorHandling::Null) - return std::make_shared(date_type); - return date_type; + return std::make_shared(data_type); + return data_type; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { - ColumnUInt8::MutablePtr col_null_map; + DataTypePtr non_null_result_type; if constexpr (error_handling == ErrorHandling::Null) - col_null_map = ColumnUInt8::create(input_rows_count, 0); - if constexpr (parseDateTime64) + non_null_result_type = removeNullable(result_type); + else + non_null_result_type = result_type; + + if constexpr (return_type == ReturnType::DateTime64) { - const DataTypeDateTime64 * datatime64_type = checkAndGetDataType(removeNullable(result_type).get()); - auto col_res = ColumnDateTime64::create(input_rows_count, datatime64_type->getScale()); - PaddedPODArray & res_data = col_res->getData(); - executeImpl2(arguments, result_type, input_rows_count, res_data, col_null_map); - if constexpr (error_handling == ErrorHandling::Null) - return ColumnNullable::create(std::move(col_res), std::move(col_null_map)); - else - return col_res; + const auto * datatime64_type = checkAndGetDataType(non_null_result_type.get()); + MutableColumnPtr col_res = ColumnDateTime64::create(input_rows_count, datatime64_type->getScale()); + ColumnDateTime64 * col_datetime64 = assert_cast(col_res.get()); + return executeImpl2(arguments, result_type, input_rows_count, col_res, col_datetime64->getData()); } else { - auto col_res = ColumnDateTime::create(input_rows_count); - PaddedPODArray & res_data = col_res->getData(); - executeImpl2(arguments, result_type, input_rows_count, res_data, col_null_map); - if constexpr (error_handling == ErrorHandling::Null) - return ColumnNullable::create(std::move(col_res), std::move(col_null_map)); - else - return col_res; + MutableColumnPtr col_res = ColumnDateTime::create(input_rows_count); + ColumnDateTime * col_datetime = assert_cast(col_res.get()); + return executeImpl2(arguments, result_type, input_rows_count, col_res, col_datetime->getData()); } } template - void executeImpl2(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, - PaddedPODArray & res_data, ColumnUInt8::MutablePtr & col_null_map) const + ColumnPtr executeImpl2(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, + MutableColumnPtr & col_res, PaddedPODArray & res_data) const { const auto * col_str = checkAndGetColumn(arguments[0].column.get()); if (!col_str) @@ -683,8 +723,21 @@ namespace "Illegal column {} of first ('str') argument of function {}. Must be string.", arguments[0].column->getName(), getName()); + + ColumnUInt8::MutablePtr col_null_map; + if constexpr (error_handling == ErrorHandling::Null) + col_null_map = ColumnUInt8::create(input_rows_count, 0); - String format = getFormat(arguments); + Int64 multiplier = 0; + UInt32 precision = 0; + if constexpr (return_type == ReturnType::DateTime64) + { + const DataTypeDateTime64 * datatime64_type = checkAndGetDataType(removeNullable(result_type).get()); + precision = datatime64_type->getScale(); + multiplier = DecimalUtils::scaleMultiplier(precision); + } + + String format = getFormat(arguments, precision); const auto & time_zone = getTimeZone(arguments); std::vector instructions = parseFormat(format); @@ -733,8 +786,8 @@ namespace Int64OrError result = 0; - /// Ensure all input was consumed - if (!parseDateTime64 && cur < end) + /// Ensure all input was consumed when the return type is `DateTime`. + if (return_type == ReturnType::DateTime && cur < end) { result = tl::unexpected(ErrorCodeAndMessage( ErrorCodes::CANNOT_PARSE_DATETIME, @@ -747,12 +800,8 @@ namespace { if (result = datetime.buildDateTime(time_zone); result.has_value()) { - if constexpr (parseDateTime64) - { - const DataTypeDateTime64 * datatime64_type = checkAndGetDataType(removeNullable(result_type).get()); - Int64 multiplier = DecimalUtils::scaleMultiplier(datatime64_type->getScale()); + if constexpr (return_type == ReturnType::DateTime64) res_data[i] = static_cast(*result) * multiplier + datetime.microsecond; - } else res_data[i] = static_cast(*result); } @@ -777,6 +826,10 @@ namespace } } } + if constexpr (error_handling == ErrorHandling::Null) + return ColumnNullable::create(std::move(col_res), std::move(col_null_map)); + else + return std::move(col_res); } @@ -808,7 +861,7 @@ namespace explicit Instruction(const String & literal_) : literal(literal_), fragment("LITERAL") { } explicit Instruction(String && literal_) : literal(std::move(literal_)), fragment("LITERAL") { } - String getFragment() const { return fragment; } + const String & getFragment() const { return fragment; } /// For debug [[maybe_unused]] String toString() const @@ -1695,7 +1748,7 @@ namespace } [[nodiscard]] - static PosOrError jodaMicroSecondOfSecond(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static PosOrError jodaMicrosecondOfSecond(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 microsecond; ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, microsecond))) @@ -1704,25 +1757,25 @@ namespace } [[nodiscard]] - static PosOrError jodaTimezoneId(size_t, Pos cur, Pos end, const String &, DateTime & date) + static PosOrError jodaTimezone(size_t, Pos cur, Pos end, const String &, DateTime & date) { - String dateTimeZone; + String read_time_zone; while (cur <= end) { - dateTimeZone += *cur; + read_time_zone += *cur; ++cur; } - const DateLUTImpl & date_time_zone = DateLUT::instance(dateTimeZone); + const DateLUTImpl & date_time_zone = DateLUT::instance(read_time_zone); const auto result = date.buildDateTime(date_time_zone); if (result.has_value()) { - const auto timezoneOffset = date_time_zone.timezoneOffset(*result); + const DateLUTImpl::Time timezone_offset = date_time_zone.timezoneOffset(*result); date.has_time_zone_offset = true; - date.time_zone_offset = timezoneOffset; + date.time_zone_offset = timezone_offset; return cur; } else - RETURN_ERROR(ErrorCodes::CANNOT_PARSE_DATETIME, "Unable to build date time from timezone {}", dateTimeZone) + RETURN_ERROR(ErrorCodes::CANNOT_PARSE_DATETIME, "Unable to build date time from timezone {}", read_time_zone) } [[nodiscard]] @@ -1745,8 +1798,22 @@ namespace Int32 hour; ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, hour))) + if (hour < 0 || hour > 23) + RETURN_ERROR( + ErrorCodes::CANNOT_PARSE_DATETIME, + "Unable to parse fragment {} from {} because of the hour of datetime not in range [0, 23]: {}", + fragment, + std::string_view(cur, end - cur), + std::string_view(cur, 1)) Int32 minute; ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, minute))) + if (minute < 0 || minute > 59) + RETURN_ERROR( + ErrorCodes::CANNOT_PARSE_DATETIME, + "Unable to parse fragment {} from {} because of the minute of datetime not in range [0, 59]: {}", + fragment, + std::string_view(cur, end - cur), + std::string_view(cur, 1)) date.has_time_zone_offset = true; date.time_zone_offset = sign * (hour * 3600 + minute * 60); return cur; @@ -2133,10 +2200,10 @@ namespace instructions.emplace_back(ACTION_ARGS_WITH_BIND(Instruction::jodaSecondOfMinute, repetitions)); break; case 'S': - instructions.emplace_back(ACTION_ARGS_WITH_BIND(Instruction::jodaMicroSecondOfSecond, repetitions)); + instructions.emplace_back(ACTION_ARGS_WITH_BIND(Instruction::jodaMicrosecondOfSecond, repetitions)); break; case 'z': - instructions.emplace_back(ACTION_ARGS_WITH_BIND(Instruction::jodaTimezoneId, repetitions)); + instructions.emplace_back(ACTION_ARGS_WITH_BIND(Instruction::jodaTimezone, repetitions)); break; case 'Z': instructions.emplace_back(ACTION_ARGS_WITH_BIND(Instruction::jodaTimezoneOffset, repetitions)); @@ -2156,26 +2223,45 @@ namespace } - String getFormat(const ColumnsWithTypeAndName & arguments) const + String getFormat(const ColumnsWithTypeAndName & arguments, UInt32 precision) const { - if (arguments.size() == 1) + size_t format_arg_index = 1; + if constexpr (return_type == ReturnType::DateTime64) { - if constexpr (parse_syntax == ParseSyntax::MySQL) - return "%Y-%m-%d %H:%i:%s"; + /// When parse `DateTime64` like `parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-05 12:22.22.123', 3), then the format is treated + /// as default value `yyyy-MM-dd HH:mm:ss`. + /// When parse `DateTime64` like `parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-05 12:22:22.123', 'yyyy-MM-dd HH:mm:ss.SSS')`, + /// then the second argument is the format. + /// When parse `DateTime64` like `parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-05 12:22:22.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS')`, + /// then the third argument is the format. + if (arguments.size() > 1 && isString(removeNullable(arguments[1].type))) + format_arg_index = 1; else - return "yyyy-MM-dd HH:mm:ss"; + format_arg_index = 2; + } + + if (arguments.size() <= format_arg_index) + { + String format; + if constexpr (parse_syntax == ParseSyntax::MySQL) + format = "%Y-%m-%d %H:%i:%s"; + else + format = "yyyy-MM-dd HH:mm:ss"; + if (precision > 0) + format += "." + String(precision, 'S'); + return format; } else { - if (!arguments[1].column || !isColumnConst(*arguments[1].column)) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Argument at index {} for function {} must be constant", 1, getName()); + if (!arguments[format_arg_index].column || !isColumnConst(*arguments[format_arg_index].column)) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Argument at index {} for function {} must be constant", format_arg_index, getName()); - const auto * col_format = checkAndGetColumnConst(arguments[1].column.get()); + const auto * col_format = checkAndGetColumnConst(arguments[format_arg_index].column.get()); if (!col_format) throw Exception( ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of second ('format') argument of function {}. Must be constant string.", - arguments[1].column->getName(), + arguments[format_arg_index].column->getName(), getName()); return col_format->getValue(); } @@ -2183,15 +2269,19 @@ namespace const DateLUTImpl & getTimeZone(const ColumnsWithTypeAndName & arguments) const { - if (arguments.size() < 3) + size_t timezone_arg_index = 2; + if constexpr (return_type == ReturnType::DateTime64) + timezone_arg_index = 3; + + if (arguments.size() <= timezone_arg_index) return DateLUT::instance(); - const auto * col = checkAndGetColumnConst(arguments[2].column.get()); + const auto * col = checkAndGetColumnConst(arguments[timezone_arg_index].column.get()); if (!col) throw Exception( ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of third ('timezone') argument of function {}. Must be constant String.", - arguments[2].column->getName(), + arguments[timezone_arg_index].column->getName(), getName()); String time_zone = col->getValue(); @@ -2229,6 +2319,21 @@ namespace static constexpr auto name = "parseDateTimeInJodaSyntaxOrNull"; }; + struct NameParseDateTime64 + { + static constexpr auto name = "parseDateTime64"; + }; + + struct NameParseDateTime64OrZero + { + static constexpr auto name = "parseDateTime64OrZero"; + }; + + struct NameParseDateTime64OrNull + { + static constexpr auto name = "parseDateTime64OrNull"; + }; + struct NameParseDateTime64InJodaSyntax { static constexpr auto name = "parseDateTime64InJodaSyntax"; @@ -2244,15 +2349,18 @@ namespace static constexpr auto name = "parseDateTime64InJodaSyntaxOrNull"; }; - using FunctionParseDateTime = FunctionParseDateTimeImpl; - using FunctionParseDateTimeOrZero = FunctionParseDateTimeImpl; - using FunctionParseDateTimeOrNull = FunctionParseDateTimeImpl; - using FunctionParseDateTimeInJodaSyntax = FunctionParseDateTimeImpl; - using FunctionParseDateTimeInJodaSyntaxOrZero = FunctionParseDateTimeImpl; - using FunctionParseDateTimeInJodaSyntaxOrNull = FunctionParseDateTimeImpl; - using FunctionParseDateTime64InJodaSyntax = FunctionParseDateTimeImpl; - using FunctionParseDateTime64InJodaSyntaxOrZero = FunctionParseDateTimeImpl; - using FunctionParseDateTime64InJodaSyntaxOrNull = FunctionParseDateTimeImpl; + using FunctionParseDateTime = FunctionParseDateTimeImpl; + using FunctionParseDateTimeOrZero = FunctionParseDateTimeImpl; + using FunctionParseDateTimeOrNull = FunctionParseDateTimeImpl; + using FunctionParseDateTime64 = FunctionParseDateTimeImpl; + using FunctionParseDateTime64OrZero = FunctionParseDateTimeImpl; + using FunctionParseDateTime64OrNull = FunctionParseDateTimeImpl; + using FunctionParseDateTimeInJodaSyntax = FunctionParseDateTimeImpl; + using FunctionParseDateTimeInJodaSyntaxOrZero = FunctionParseDateTimeImpl; + using FunctionParseDateTimeInJodaSyntaxOrNull = FunctionParseDateTimeImpl; + using FunctionParseDateTime64InJodaSyntax = FunctionParseDateTimeImpl; + using FunctionParseDateTime64InJodaSyntaxOrZero = FunctionParseDateTimeImpl; + using FunctionParseDateTime64InJodaSyntaxOrNull = FunctionParseDateTimeImpl; } REGISTER_FUNCTION(ParseDateTime) @@ -2262,6 +2370,9 @@ REGISTER_FUNCTION(ParseDateTime) factory.registerFunction(); factory.registerFunction(); factory.registerAlias("str_to_date", FunctionParseDateTimeOrNull::name, FunctionFactory::Case::Insensitive); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); From 1c74206bf2fddd2aad8f96699769e20dd122979a Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Thu, 7 Nov 2024 18:00:14 +0800 Subject: [PATCH 02/20] add parseDateTime64 functions --- src/Functions/parseDateTime.cpp | 145 ++++++++++++------ .../03252_parse_datetime64.reference | 17 ++ .../0_stateless/03252_parse_datetime64.sql | 32 ++++ ..._parse_datetime64_in_joda_syntax.reference | 32 ++-- .../03252_parse_datetime64_in_joda_syntax.sql | 60 ++++++-- 5 files changed, 214 insertions(+), 72 deletions(-) create mode 100644 tests/queries/0_stateless/03252_parse_datetime64.reference create mode 100644 tests/queries/0_stateless/03252_parse_datetime64.sql diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 976be53a21e..9f7f78dcbe2 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -193,6 +193,7 @@ namespace Int32 minute = 0; /// range [0, 59] Int32 second = 0; /// range [0, 59] Int32 microsecond = 0; /// range [0, 999999] + UInt32 scale = 0; /// The microsecond scale of DateTime64. bool is_am = true; /// If is_hour_of_half_day = true and is_am = false (i.e. pm) then add 12 hours to the result DateTime bool hour_starts_at_1 = false; /// Whether the hour is clockhour @@ -221,6 +222,7 @@ namespace minute = 0; second = 0; microsecond = 0; + scale = 0; is_am = true; hour_starts_at_1 = false; @@ -599,7 +601,7 @@ namespace bool useDefaultImplementationForConstants() const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2, 3}; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } @@ -613,9 +615,9 @@ namespace if constexpr (return_type == ReturnType::DateTime64) { optional_args = { - {"precision or format", static_cast([](const IDataType & data_type) -> bool { + {"scale/format", static_cast([](const IDataType & data_type) -> bool { return isUInt(data_type) || isString(data_type); - }), nullptr, "Number or String"}, + }), nullptr, "UInt or String"}, {"format", static_cast(&isString), nullptr, "String"}, {"timezone", static_cast(&isString), &isColumnConst, "const String"} }; @@ -631,24 +633,34 @@ namespace DataTypePtr data_type; if constexpr (return_type == ReturnType::DateTime64) { + UInt32 scale = 0; if (arguments.size() == 1) - return std::make_shared(0, time_zone_name); + { + /// In MySQL parse syntax, the scale of microseond is 6. + if constexpr (parse_syntax == ParseSyntax::MySQL) + scale = 6; + } else { - UInt32 precision = 0; if (isUInt(arguments[1].type)) { - const auto * col_precision = checkAndGetColumnConst(arguments[1].column.get()); - if (col_precision) - precision = col_precision->getValue(); + const auto * col_scale = checkAndGetColumnConst(arguments[1].column.get()); + if (col_scale) + scale = col_scale->getValue(); else throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The input precision value may exceed the max value of `DateTime64`: {}.", + "The input scale value may exceed the max value of `DateTime64`: {}.", maxPrecisionOfDateTime64); } - /// Construct the return type `DataTypDateTime64` with precision and time zone name. The precision value can be specified or be extracted - /// from the format string by computing how many 'S' characters are contained in the format's micorsceond fragment. - String format = getFormat(arguments, precision); + else + { + if constexpr (parse_syntax == ParseSyntax::MySQL) + scale = 6; + } + + /// Construct the return type `DataTypDateTime64` with scale and time zone name. The scale value can be specified or be extracted + /// from the format string by c how many 'S' characters are contained in the format's micorsceond fragment. + String format = getFormat(arguments, scale); std::vector instructions = parseFormat(format); for (const auto & instruction : instructions) { @@ -664,26 +676,27 @@ namespace else val++; } - /// If the precision is already specified by the second parameter, but it not equals the value that extract from the format string, - /// then we should throw an exception; If the precision is not specified, then we set its value as the extracted one. - if (val != 0 && precision != 0 && val != precision) + /// If the scale is already specified by the second argument, but it not equals the value that extract from the format string, + /// then we should throw an exception; If the scale is not specified, then we should set its value as the extracted one. + if (val != 0 && scale != 0 && val != scale) throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The precision of input format string {} not equals the given precision value {}.", + "The scale of input format string {} not equals the given scale value {}.", format, - precision); - else if (precision == 0 && val != 0) - precision = val; + scale); + else if (scale == 0 && val != 0) + scale = val; } - if (precision > maxPrecisionOfDateTime64) + if (scale > maxPrecisionOfDateTime64) throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The precision of the input format string {} exceed the max precision value {}.", + "The scale of the input format string {} exceed the max scale value {}.", format, maxPrecisionOfDateTime64); - data_type = std::make_shared(precision, time_zone_name); } + data_type = std::make_shared(scale, time_zone_name); } else data_type = std::make_shared(time_zone_name); + if (error_handling == ErrorHandling::Null) return std::make_shared(data_type); return data_type; @@ -729,15 +742,15 @@ namespace col_null_map = ColumnUInt8::create(input_rows_count, 0); Int64 multiplier = 0; - UInt32 precision = 0; + UInt32 scale = 0; if constexpr (return_type == ReturnType::DateTime64) { const DataTypeDateTime64 * datatime64_type = checkAndGetDataType(removeNullable(result_type).get()); - precision = datatime64_type->getScale(); - multiplier = DecimalUtils::scaleMultiplier(precision); + scale = datatime64_type->getScale(); + multiplier = DecimalUtils::scaleMultiplier(scale); } - String format = getFormat(arguments, precision); + const String format = getFormat(arguments, scale); const auto & time_zone = getTimeZone(arguments); std::vector instructions = parseFormat(format); @@ -746,6 +759,9 @@ namespace for (size_t i = 0; i < input_rows_count; ++i) { datetime.reset(); + if constexpr (return_type == ReturnType::DateTime64) + datetime.scale = scale; + StringRef str_ref = col_str->getDataAt(i); Pos cur = str_ref.data; Pos end = str_ref.data + str_ref.size; @@ -787,7 +803,7 @@ namespace Int64OrError result = 0; /// Ensure all input was consumed when the return type is `DateTime`. - if (return_type == ReturnType::DateTime && cur < end) + if (cur < end) { result = tl::unexpected(ErrorCodeAndMessage( ErrorCodes::CANNOT_PARSE_DATETIME, @@ -938,6 +954,28 @@ namespace return cur; } + template + [[nodiscard]] + static PosOrError readNumber6(Pos cur, Pos end, [[maybe_unused]] const String & fragment, T & res) + { + if constexpr (need_check_space == NeedCheckSpace::Yes) + RETURN_ERROR_IF_FAILED(checkSpace(cur, end, 6, "readNumber6 requires size >= 6", fragment)) + + res = (*cur - '0'); + ++cur; + res = res * 10 + (*cur - '0'); + ++cur; + res = res * 10 + (*cur - '0'); + ++cur; + res = res * 10 + (*cur - '0'); + ++cur; + res = res * 10 + (*cur - '0'); + ++cur; + res = res * 10 + (*cur - '0'); + ++cur; + return cur; + } + [[nodiscard]] static VoidOrError checkSpace(Pos cur, Pos end, size_t len, const String & msg, const String & fragment) { @@ -1358,13 +1396,18 @@ namespace } [[nodiscard]] - static PosOrError mysqlMicrosecond(Pos cur, Pos end, const String & fragment, DateTime & /*date*/) + static PosOrError mysqlMicrosecond(Pos cur, Pos end, const String & fragment, DateTime & date) { - RETURN_ERROR_IF_FAILED(checkSpace(cur, end, 6, "mysqlMicrosecond requires size >= 6", fragment)) - - for (size_t i = 0; i < 6; ++i) - ASSIGN_RESULT_OR_RETURN_ERROR(cur, (assertNumber(cur, end, fragment))) - + if (date.scale != 6) + RETURN_ERROR( + ErrorCodes::CANNOT_PARSE_DATETIME, + "Unable to parse fragment {} from {} because of the microsecond's scale {} is not 6", + fragment, + std::string_view(cur, end - cur), + std::to_string(date.scale)) + Int32 microsecond = 0; + ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumber6(cur, end, fragment, microsecond))) + RETURN_ERROR_IF_FAILED(date.setMicrosecond(microsecond)) return cur; } @@ -1775,7 +1818,7 @@ namespace return cur; } else - RETURN_ERROR(ErrorCodes::CANNOT_PARSE_DATETIME, "Unable to build date time from timezone {}", read_time_zone) + RETURN_ERROR(ErrorCodes::CANNOT_PARSE_DATETIME, "Unable to parse date time from timezone {}", read_time_zone) } [[nodiscard]] @@ -2223,7 +2266,7 @@ namespace } - String getFormat(const ColumnsWithTypeAndName & arguments, UInt32 precision) const + String getFormat(const ColumnsWithTypeAndName & arguments, UInt32 scale) const { size_t format_arg_index = 1; if constexpr (return_type == ReturnType::DateTime64) @@ -2247,15 +2290,17 @@ namespace format = "%Y-%m-%d %H:%i:%s"; else format = "yyyy-MM-dd HH:mm:ss"; - if (precision > 0) - format += "." + String(precision, 'S'); + if (scale > 0) + { + if constexpr (parse_syntax == ParseSyntax::MySQL) + format += ".%f"; + else + format += "." + String(scale, 'S'); + } return format; } else { - if (!arguments[format_arg_index].column || !isColumnConst(*arguments[format_arg_index].column)) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Argument at index {} for function {} must be constant", format_arg_index, getName()); - const auto * col_format = checkAndGetColumnConst(arguments[format_arg_index].column.get()); if (!col_format) throw Exception( @@ -2269,18 +2314,24 @@ namespace const DateLUTImpl & getTimeZone(const ColumnsWithTypeAndName & arguments) const { - size_t timezone_arg_index = 2; - if constexpr (return_type == ReturnType::DateTime64) - timezone_arg_index = 3; - - if (arguments.size() <= timezone_arg_index) + if (arguments.size() < 3) return DateLUT::instance(); - + else if constexpr (return_type == ReturnType::DateTime64) + { + /// If the return type is DateTime64, and the second argument is UInt type for scale, then it has 2 reasonable situations: + /// the first like parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-07 17:27.30.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT+8') + /// the second like parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-07 17:27.30.123456', 6, '%Y-%m-%d %H:%i:%s.%f'). And for the + /// first one, we should return the last argument as its timezone, and for the second one, we should return the default time zone as + /// `DateLUT::instance()`. + if (isUInt(arguments[1].type) && arguments.size() < 4) + return DateLUT::instance(); + } + size_t timezone_arg_index = arguments.size() - 1; const auto * col = checkAndGetColumnConst(arguments[timezone_arg_index].column.get()); if (!col) throw Exception( ErrorCodes::ILLEGAL_COLUMN, - "Illegal column {} of third ('timezone') argument of function {}. Must be constant String.", + "Illegal column {} of ('timezone') argument of function {}. Must be constant String.", arguments[timezone_arg_index].column->getName(), getName()); diff --git a/tests/queries/0_stateless/03252_parse_datetime64.reference b/tests/queries/0_stateless/03252_parse_datetime64.reference new file mode 100644 index 00000000000..27dcef6bf68 --- /dev/null +++ b/tests/queries/0_stateless/03252_parse_datetime64.reference @@ -0,0 +1,17 @@ +2024-10-09 10:30:10.123456 +2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 +2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 +2024-10-09 10:30:10.123456 +1970-01-01 08:00:00.000000 +1970-01-01 08:00:00.000 +1970-01-01 08:00:00.000 +2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 +2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 +1970-01-01 08:00:00.000 +2024-10-09 10:30:10.123456 +\N +\N +\N +2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 +2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 +\N diff --git a/tests/queries/0_stateless/03252_parse_datetime64.sql b/tests/queries/0_stateless/03252_parse_datetime64.sql new file mode 100644 index 00000000000..d28b6e586f7 --- /dev/null +++ b/tests/queries/0_stateless/03252_parse_datetime64.sql @@ -0,0 +1,32 @@ +set session_timezone = 'Asia/Shanghai'; + +select parseDateTime64('2024-10-09 10:30:10.123456'); +select parseDateTime64('2024-10-09 10:30:10.123'); -- { serverError NOT_ENOUGH_SPACE } +select parseDateTime64('2024-10-09 10:30:10', 3); -- { serverError NOT_ENOUGH_SPACE } +select parseDateTime64('2024-10-09 10:30:10.', 3); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} +select parseDateTime64('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } +select parseDateTime64('2024-10-09 10:30:10.123456', 6), parseDateTime64('2024-10-09 10:30:10.123456', '%Y-%m-%d %H:%i:%s.%f'); +select parseDateTime64('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f'), parseDateTime64('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7'); +select parseDateTime64('2024-10-09 10:30:10.123', 3, '%Y-%m-%d %H:%i:%s.%f'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64('2024-10-09 10:30:10.123', 6, '%Y-%m-%d %H:%i:%s.%f'); -- { serverError NOT_ENOUGH_SPACE } + +select parseDateTime64OrZero('2024-10-09 10:30:10.123456'); +select parseDateTime64OrZero('2024-10-09 10:30:10.123'); +select parseDateTime64OrZero('2024-10-09 10:30:10', 3); +select parseDateTime64OrZero('2024-10-09 10:30:10.', 3); +select parseDateTime64OrZero('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} +select parseDateTime64OrZero('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } +select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', '%Y-%m-%d %H:%i:%s.%f'); +select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f'), parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7'); +select parseDateTime64OrZero('2024-10-09 10:30:10.123', 3, '%Y-%m-%d %H:%i:%s.%f'); + +select parseDateTime64OrNull('2024-10-09 10:30:10.123456'); +select parseDateTime64OrNull('2024-10-09 10:30:10.123'); +select parseDateTime64OrNull('2024-10-09 10:30:10', 3); +select parseDateTime64OrNull('2024-10-09 10:30:10.', 3); +select parseDateTime64OrNull('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} +select parseDateTime64OrNull('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } +select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', '%Y-%m-%d %H:%i:%s.%f'); +select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f'), parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7');; +select parseDateTime64OrNull('2024-10-09 10:30:10.123', 3, '%Y-%m-%d %H:%i:%s.%f'); \ No newline at end of file diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference index 063b76b152c..0b4a28c4b38 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference @@ -1,14 +1,26 @@ -2024-10-09 10:30:10.123 -2024-10-09 10:30:10.123456 -2024-10-10 02:30:10.123456 +2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 +2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 +2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 +2024-10-10 02:42:10.123456 2024-10-10 01:30:10.123456 -2024-10-09 10:30:10.123 -2024-10-09 10:30:10.123456 -1970-01-01 08:00:00.000000000 -2024-10-10 02:30:10.123456 2024-10-10 01:30:10.123456 -2024-10-09 10:30:10.123 -2024-10-09 10:30:10.123456 +1970-01-01 08:00:00.000 +1970-01-01 08:00:00.000 +2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 +2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 +2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 +2024-10-10 02:42:10.123456 +1970-01-01 08:00:00.000000 +2024-10-10 01:30:10.123456 +2024-10-10 01:30:10.123456 +1970-01-01 08:00:00.000000 +\N +\N +2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 +2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 +2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 +2024-10-10 02:42:10.123456 \N -2024-10-10 02:30:10.123456 2024-10-10 01:30:10.123456 +2024-10-10 01:30:10.123456 +\N diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql index 9ea854bc324..8482677e9c9 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql @@ -1,19 +1,49 @@ set session_timezone = 'Asia/Shanghai'; -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -- { serverError CANNOT_PARSE_DATETIME } -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10', 3); -- { serverError NOT_ENOUGH_SPACE } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.', 3); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 6); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); -- {serverError CANNOT_PARSE_DATETIME} +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); +-- incorrect timezone offset and timezone +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-8000', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456ABCD', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10', 3); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.', 3); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 6); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); +-- incorrect timezone offset and timezone +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-8000', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456ABCD', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10', 3); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.', 3); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 6); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); +-- incorrect timezone offset and timezone +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-8000', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456ABCD', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } \ No newline at end of file From a6b08187b31b3d3d6a5432bf8f39fe81ab5d81a7 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Thu, 7 Nov 2024 20:03:44 +0800 Subject: [PATCH 03/20] checkstyle and doc --- .../functions/type-conversion-functions.md | 46 ++++++++++++++++++- src/Functions/parseDateTime.cpp | 27 +++++------ 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 91bae2fe9da..c44d9ddb12b 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -6867,9 +6867,53 @@ Same as for [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax) except that Same as for [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax) except that it returns `NULL` when it encounters a date format that cannot be processed. +## parseDateTime64 + +Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datetime64.md) according to a [MySQL format string](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_date-format). + +**Syntax** + +``` sql +parseDateTime64(str[, [scale, [format[, timezone]]]]) +``` + +**Arguments** + +- `str` — The String to be parsed +- `scale` - The precision for [DateTime64](../data-types/datetime64.md). Optional, default is 6 if not specified. +- `format` — The format string. Optional. `%Y-%m-%d %H:%i:%s` if not specified. +- `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional. + +**Returned value(s)** + +Returns [DateTime64](../data-types/datetime64.md) type values parsed from input string according to a MySQL style format string. + +## parseDateTime64OrZero +Same as for [parseDateTime64](#parsedatetime64) except that it returns zero date when it encounters a date format that cannot be processed. + +## parseDateTime64OrNull +Same as for [parseDateTime64](#parsedatetime64) except that it returns zero date when it encounters a date format that cannot be processed. + ## parseDateTime64InJodaSyntax -Similar to [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax). Differently, it returns a value of type [DateTime64](../data-types/datetime64.md). +Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datetime64.md) according to a [Joda format string](https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html). + +**Syntax** + +``` sql +parseDateTime64InJodaSyntax(str[, [scale, [format[, timezone]]]]) +``` + +**Arguments** + +- `str` — The String to be parsed +- `scale` - The precision for [DateTime64](../data-types/datetime64.md). Optional, default is 6 if not specified. +- `format` — The format string. Optional. `yyyy-MM-dd HH:mm:ss` if not specified. +- `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional. + +**Returned value(s)** + +Returns [DateTime64](../data-types/datetime64.md) type values parsed from input string according to a joda style format string. ## parseDateTime64InJodaSyntaxOrZero diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 9f7f78dcbe2..5743278e104 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -613,15 +613,12 @@ namespace FunctionArgumentDescriptors optional_args; if constexpr (return_type == ReturnType::DateTime64) - { - optional_args = { - {"scale/format", static_cast([](const IDataType & data_type) -> bool { - return isUInt(data_type) || isString(data_type); - }), nullptr, "UInt or String"}, + optional_args = {{"scale/format", static_cast( + [](const IDataType & data_type) -> bool { return isUInt(data_type) || isString(data_type); } + ), nullptr, "UInt or String"}, {"format", static_cast(&isString), nullptr, "String"}, {"timezone", static_cast(&isString), &isColumnConst, "const String"} }; - } else optional_args = { {"format", static_cast(&isString), nullptr, "String"}, @@ -659,7 +656,7 @@ namespace } /// Construct the return type `DataTypDateTime64` with scale and time zone name. The scale value can be specified or be extracted - /// from the format string by c how many 'S' characters are contained in the format's micorsceond fragment. + /// from the format string by counting how many 'S' characters are contained in the format's micorsceond fragment. String format = getFormat(arguments, scale); std::vector instructions = parseFormat(format); for (const auto & instruction : instructions) @@ -676,7 +673,7 @@ namespace else val++; } - /// If the scale is already specified by the second argument, but it not equals the value that extract from the format string, + /// If the scale is already specified by the second argument, but it not equals the value that extract from the format string, /// then we should throw an exception; If the scale is not specified, then we should set its value as the extracted one. if (val != 0 && scale != 0 && val != scale) throw Exception(ErrorCodes::BAD_ARGUMENTS, @@ -687,7 +684,7 @@ namespace scale = val; } if (scale > maxPrecisionOfDateTime64) - throw Exception(ErrorCodes::BAD_ARGUMENTS, + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale of the input format string {} exceed the max scale value {}.", format, maxPrecisionOfDateTime64); @@ -709,7 +706,7 @@ namespace non_null_result_type = removeNullable(result_type); else non_null_result_type = result_type; - + if constexpr (return_type == ReturnType::DateTime64) { const auto * datatime64_type = checkAndGetDataType(non_null_result_type.get()); @@ -726,7 +723,7 @@ namespace } template - ColumnPtr executeImpl2(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, + ColumnPtr executeImpl2(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, MutableColumnPtr & col_res, PaddedPODArray & res_data) const { const auto * col_str = checkAndGetColumn(arguments[0].column.get()); @@ -736,7 +733,7 @@ namespace "Illegal column {} of first ('str') argument of function {}. Must be string.", arguments[0].column->getName(), getName()); - + ColumnUInt8::MutablePtr col_null_map; if constexpr (error_handling == ErrorHandling::Null) col_null_map = ColumnUInt8::create(input_rows_count, 0); @@ -802,7 +799,7 @@ namespace Int64OrError result = 0; - /// Ensure all input was consumed when the return type is `DateTime`. + /// Ensure all input was consumed. if (cur < end) { result = tl::unexpected(ErrorCodeAndMessage( @@ -2273,7 +2270,7 @@ namespace { /// When parse `DateTime64` like `parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-05 12:22.22.123', 3), then the format is treated /// as default value `yyyy-MM-dd HH:mm:ss`. - /// When parse `DateTime64` like `parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-05 12:22:22.123', 'yyyy-MM-dd HH:mm:ss.SSS')`, + /// When parse `DateTime64` like `parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-05 12:22:22.123', 'yyyy-MM-dd HH:mm:ss.SSS')`, /// then the second argument is the format. /// When parse `DateTime64` like `parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-05 12:22:22.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS')`, /// then the third argument is the format. @@ -2321,7 +2318,7 @@ namespace /// If the return type is DateTime64, and the second argument is UInt type for scale, then it has 2 reasonable situations: /// the first like parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-07 17:27.30.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT+8') /// the second like parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-07 17:27.30.123456', 6, '%Y-%m-%d %H:%i:%s.%f'). And for the - /// first one, we should return the last argument as its timezone, and for the second one, we should return the default time zone as + /// first one, we should return the last argument as its timezone, and for the second one, we should return the default time zone as /// `DateLUT::instance()`. if (isUInt(arguments[1].type) && arguments.size() < 4) return DateLUT::instance(); From 3332bce1dc94e7fddccc1865df01eb029e5f7e52 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Thu, 7 Nov 2024 20:38:44 +0800 Subject: [PATCH 04/20] fix doc and comments --- .../functions/type-conversion-functions.md | 6 +++--- src/Functions/parseDateTime.cpp | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index c44d9ddb12b..8043b21744a 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -6880,7 +6880,7 @@ parseDateTime64(str[, [scale, [format[, timezone]]]]) **Arguments** - `str` — The String to be parsed -- `scale` - The precision for [DateTime64](../data-types/datetime64.md). Optional, default is 6 if not specified. +- `scale` - The precision for [DateTime64](../data-types/datetime64.md). Optional, default 6 if not specified. - `format` — The format string. Optional. `%Y-%m-%d %H:%i:%s` if not specified. - `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional. @@ -6892,7 +6892,7 @@ Returns [DateTime64](../data-types/datetime64.md) type values parsed from input Same as for [parseDateTime64](#parsedatetime64) except that it returns zero date when it encounters a date format that cannot be processed. ## parseDateTime64OrNull -Same as for [parseDateTime64](#parsedatetime64) except that it returns zero date when it encounters a date format that cannot be processed. +Same as for [parseDateTime64](#parsedatetime64) except that it returns `NULL` when it encounters a date format that cannot be processed. ## parseDateTime64InJodaSyntax @@ -6907,7 +6907,7 @@ parseDateTime64InJodaSyntax(str[, [scale, [format[, timezone]]]]) **Arguments** - `str` — The String to be parsed -- `scale` - The precision for [DateTime64](../data-types/datetime64.md). Optional, default is 6 if not specified. +- `scale` - The precision for [DateTime64](../data-types/datetime64.md). Optional, default 0 if not specified. - `format` — The format string. Optional. `yyyy-MM-dd HH:mm:ss` if not specified. - `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional. diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 5743278e104..7190c1ad6f8 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -65,7 +65,7 @@ namespace constexpr Int32 minYear = 1970; constexpr Int32 maxYear = 2106; - constexpr Int32 maxPrecisionOfDateTime64 = 6; + constexpr Int32 maxScaleOfDateTime64 = 6; const std::unordered_map> dayOfWeekMap{ {"mon", {"day", 1}}, @@ -193,7 +193,7 @@ namespace Int32 minute = 0; /// range [0, 59] Int32 second = 0; /// range [0, 59] Int32 microsecond = 0; /// range [0, 999999] - UInt32 scale = 0; /// The microsecond scale of DateTime64. + UInt32 scale = 0; /// The scale of DateTime64, range [0, 6]. bool is_am = true; /// If is_hour_of_half_day = true and is_am = false (i.e. pm) then add 12 hours to the result DateTime bool hour_starts_at_1 = false; /// Whether the hour is clockhour @@ -646,8 +646,8 @@ namespace scale = col_scale->getValue(); else throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The input scale value may exceed the max value of `DateTime64`: {}.", - maxPrecisionOfDateTime64); + "The input scale value may exceed the max scale value of `DateTime64`: {}.", + maxScaleOfDateTime64); } else { @@ -656,7 +656,7 @@ namespace } /// Construct the return type `DataTypDateTime64` with scale and time zone name. The scale value can be specified or be extracted - /// from the format string by counting how many 'S' characters are contained in the format's micorsceond fragment. + /// from the format string by counting how many 'S' characters are contained in the format's microsceond fragment. String format = getFormat(arguments, scale); std::vector instructions = parseFormat(format); for (const auto & instruction : instructions) @@ -683,11 +683,11 @@ namespace else if (scale == 0 && val != 0) scale = val; } - if (scale > maxPrecisionOfDateTime64) + if (scale > maxScaleOfDateTime64) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale of the input format string {} exceed the max scale value {}.", format, - maxPrecisionOfDateTime64); + maxScaleOfDateTime64); } data_type = std::make_shared(scale, time_zone_name); } @@ -1398,7 +1398,7 @@ namespace if (date.scale != 6) RETURN_ERROR( ErrorCodes::CANNOT_PARSE_DATETIME, - "Unable to parse fragment {} from {} because of the microsecond's scale {} is not 6", + "Unable to parse fragment {} from {} because of the datetime scale {} is not 6", fragment, std::string_view(cur, end - cur), std::to_string(date.scale)) From ab79efe40f8785a7bd947cd5919feafedfb88259 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Mon, 11 Nov 2024 14:07:19 +0800 Subject: [PATCH 05/20] make scale argument not optional --- .../functions/type-conversion-functions.md | 4 +- src/Functions/parseDateTime.cpp | 130 +++++++----------- .../03252_parse_datetime64.reference | 13 +- .../0_stateless/03252_parse_datetime64.sql | 38 ++--- ..._parse_datetime64_in_joda_syntax.reference | 5 - .../03252_parse_datetime64_in_joda_syntax.sql | 12 +- 6 files changed, 69 insertions(+), 133 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 8043b21744a..72e6fda03f7 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -6880,7 +6880,7 @@ parseDateTime64(str[, [scale, [format[, timezone]]]]) **Arguments** - `str` — The String to be parsed -- `scale` - The precision for [DateTime64](../data-types/datetime64.md). Optional, default 6 if not specified. +- `scale` - The scale of [DateTime64](../data-types/datetime64.md). - `format` — The format string. Optional. `%Y-%m-%d %H:%i:%s` if not specified. - `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional. @@ -6907,7 +6907,7 @@ parseDateTime64InJodaSyntax(str[, [scale, [format[, timezone]]]]) **Arguments** - `str` — The String to be parsed -- `scale` - The precision for [DateTime64](../data-types/datetime64.md). Optional, default 0 if not specified. +- `scale` - The scale of [DateTime64](../data-types/datetime64.md). - `format` — The format string. Optional. `yyyy-MM-dd HH:mm:ss` if not specified. - `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional. diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 7190c1ad6f8..72e3ba661ca 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -607,87 +607,71 @@ namespace DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - FunctionArgumentDescriptors mandatory_args{ - {"time", static_cast(&isString), nullptr, "String"} - }; - + FunctionArgumentDescriptors mandatory_args; FunctionArgumentDescriptors optional_args; if constexpr (return_type == ReturnType::DateTime64) - optional_args = {{"scale/format", static_cast( - [](const IDataType & data_type) -> bool { return isUInt(data_type) || isString(data_type); } - ), nullptr, "UInt or String"}, - {"format", static_cast(&isString), nullptr, "String"}, - {"timezone", static_cast(&isString), &isColumnConst, "const String"} + { + mandatory_args = { + {"time", static_cast(&isString), nullptr, "String"}, + {"scale", static_cast(&isUInt8), nullptr, "UInt8"} }; - else optional_args = { {"format", static_cast(&isString), nullptr, "String"}, {"timezone", static_cast(&isString), &isColumnConst, "const String"} }; + } + else + { + mandatory_args = {{"time", static_cast(&isString), nullptr, "String"}}; + optional_args = { + {"format", static_cast(&isString), nullptr, "String"}, + {"timezone", static_cast(&isString), &isColumnConst, "const String"} + }; + } validateFunctionArguments(*this, arguments, mandatory_args, optional_args); String time_zone_name = getTimeZone(arguments).getTimeZone(); DataTypePtr data_type; if constexpr (return_type == ReturnType::DateTime64) { - UInt32 scale = 0; - if (arguments.size() == 1) + UInt8 scale = 0; + if (isUInt8(arguments[1].type)) { - /// In MySQL parse syntax, the scale of microseond is 6. - if constexpr (parse_syntax == ParseSyntax::MySQL) - scale = 6; - } - else - { - if (isUInt(arguments[1].type)) - { - const auto * col_scale = checkAndGetColumnConst(arguments[1].column.get()); - if (col_scale) - scale = col_scale->getValue(); - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The input scale value may exceed the max scale value of `DateTime64`: {}.", - maxScaleOfDateTime64); - } + const auto * col_scale = checkAndGetColumnConst(arguments[1].column.get()); + if (col_scale) + scale = col_scale->getValue(); else - { - if constexpr (parse_syntax == ParseSyntax::MySQL) - scale = 6; - } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale argument is not Const(UInt8) type."); + } + if (parse_syntax == ParseSyntax::MySQL && scale != 6) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale value {} of MySQL parse syntax is not 6.", std::to_string(scale)); + if (scale > maxScaleOfDateTime64) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "The scale argument's value {} exceed the max scale value {}.", std::to_string(scale), std::to_string(maxScaleOfDateTime64)); - /// Construct the return type `DataTypDateTime64` with scale and time zone name. The scale value can be specified or be extracted - /// from the format string by counting how many 'S' characters are contained in the format's microsceond fragment. - String format = getFormat(arguments, scale); - std::vector instructions = parseFormat(format); - for (const auto & instruction : instructions) + String format = getFormat(arguments, scale); + std::vector instructions = parseFormat(format); + for (const auto & instruction : instructions) + { + /// Check scale by counting how may 'S' characters exists in the format string. + const String & fragment = instruction.getFragment(); + UInt32 s_cnt = 0; + for (char ch : fragment) { - const String & fragment = instruction.getFragment(); - UInt32 val = 0; - for (char ch : fragment) + if (ch != 'S') { - if (ch != 'S') - { - val = 0; - break; - } - else - val++; + s_cnt = 0; + break; } - /// If the scale is already specified by the second argument, but it not equals the value that extract from the format string, - /// then we should throw an exception; If the scale is not specified, then we should set its value as the extracted one. - if (val != 0 && scale != 0 && val != scale) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The scale of input format string {} not equals the given scale value {}.", - format, - scale); - else if (scale == 0 && val != 0) - scale = val; + else + s_cnt++; } - if (scale > maxScaleOfDateTime64) + /// If the number of 'S' character in format string not euqals the scale, then throw an exception to report error. + if (s_cnt != 0 && s_cnt != scale) throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The scale of the input format string {} exceed the max scale value {}.", + "The scale of input format string {} not equals the given scale value {}.", format, - maxScaleOfDateTime64); + std::to_string(scale)); } data_type = std::make_shared(scale, time_zone_name); } @@ -2267,18 +2251,7 @@ namespace { size_t format_arg_index = 1; if constexpr (return_type == ReturnType::DateTime64) - { - /// When parse `DateTime64` like `parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-05 12:22.22.123', 3), then the format is treated - /// as default value `yyyy-MM-dd HH:mm:ss`. - /// When parse `DateTime64` like `parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-05 12:22:22.123', 'yyyy-MM-dd HH:mm:ss.SSS')`, - /// then the second argument is the format. - /// When parse `DateTime64` like `parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-05 12:22:22.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS')`, - /// then the third argument is the format. - if (arguments.size() > 1 && isString(removeNullable(arguments[1].type))) - format_arg_index = 1; - else - format_arg_index = 2; - } + format_arg_index = 2; if (arguments.size() <= format_arg_index) { @@ -2311,18 +2284,11 @@ namespace const DateLUTImpl & getTimeZone(const ColumnsWithTypeAndName & arguments) const { - if (arguments.size() < 3) + if (return_type == ReturnType::DateTime && arguments.size() < 3) return DateLUT::instance(); - else if constexpr (return_type == ReturnType::DateTime64) - { - /// If the return type is DateTime64, and the second argument is UInt type for scale, then it has 2 reasonable situations: - /// the first like parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-07 17:27.30.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT+8') - /// the second like parseDateTime64[InJodaSyntax][OrZero/OrNull]('2024-11-07 17:27.30.123456', 6, '%Y-%m-%d %H:%i:%s.%f'). And for the - /// first one, we should return the last argument as its timezone, and for the second one, we should return the default time zone as - /// `DateLUT::instance()`. - if (isUInt(arguments[1].type) && arguments.size() < 4) - return DateLUT::instance(); - } + else if (return_type == ReturnType::DateTime64 && arguments.size() < 4) + return DateLUT::instance(); + size_t timezone_arg_index = arguments.size() - 1; const auto * col = checkAndGetColumnConst(arguments[timezone_arg_index].column.get()); if (!col) diff --git a/tests/queries/0_stateless/03252_parse_datetime64.reference b/tests/queries/0_stateless/03252_parse_datetime64.reference index 27dcef6bf68..263c9b5d8ea 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64.reference +++ b/tests/queries/0_stateless/03252_parse_datetime64.reference @@ -1,17 +1,8 @@ -2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 +2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 1970-01-01 08:00:00.000000 -1970-01-01 08:00:00.000 -1970-01-01 08:00:00.000 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 -2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 -1970-01-01 08:00:00.000 2024-10-09 10:30:10.123456 -\N -\N -\N -2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 -2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 -\N +1970-01-01 08:00:00.000000 diff --git a/tests/queries/0_stateless/03252_parse_datetime64.sql b/tests/queries/0_stateless/03252_parse_datetime64.sql index d28b6e586f7..2a6ef254887 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64.sql +++ b/tests/queries/0_stateless/03252_parse_datetime64.sql @@ -1,32 +1,22 @@ set session_timezone = 'Asia/Shanghai'; -select parseDateTime64('2024-10-09 10:30:10.123456'); -select parseDateTime64('2024-10-09 10:30:10.123'); -- { serverError NOT_ENOUGH_SPACE } -select parseDateTime64('2024-10-09 10:30:10', 3); -- { serverError NOT_ENOUGH_SPACE } -select parseDateTime64('2024-10-09 10:30:10.', 3); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64('2024-10-09 10:30:10.123456'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} +select parseDateTime64('2024-10-09 10:30:10', 3); -- { serverError BAD_ARGUMENTS } select parseDateTime64('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } -select parseDateTime64('2024-10-09 10:30:10.123456', 6), parseDateTime64('2024-10-09 10:30:10.123456', '%Y-%m-%d %H:%i:%s.%f'); -select parseDateTime64('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f'), parseDateTime64('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7'); -select parseDateTime64('2024-10-09 10:30:10.123', 3, '%Y-%m-%d %H:%i:%s.%f'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64('2024-10-09 10:30:10.123456', 6), parseDateTime64('2024-10-09 10:30:10.123456', 6,'%Y-%m-%d %H:%i:%s.%f'); +select parseDateTime64('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7'); select parseDateTime64('2024-10-09 10:30:10.123', 6, '%Y-%m-%d %H:%i:%s.%f'); -- { serverError NOT_ENOUGH_SPACE } -select parseDateTime64OrZero('2024-10-09 10:30:10.123456'); -select parseDateTime64OrZero('2024-10-09 10:30:10.123'); -select parseDateTime64OrZero('2024-10-09 10:30:10', 3); -select parseDateTime64OrZero('2024-10-09 10:30:10.', 3); +select parseDateTime64OrZero('2024-10-09 10:30:10.123456'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} +select parseDateTime64OrZero('2024-10-09 10:30:10', 3); -- { serverError BAD_ARGUMENTS } select parseDateTime64OrZero('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64OrZero('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } -select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', '%Y-%m-%d %H:%i:%s.%f'); -select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f'), parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7'); -select parseDateTime64OrZero('2024-10-09 10:30:10.123', 3, '%Y-%m-%d %H:%i:%s.%f'); +select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f'); +select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7'); +select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%fffff'); -select parseDateTime64OrNull('2024-10-09 10:30:10.123456'); -select parseDateTime64OrNull('2024-10-09 10:30:10.123'); -select parseDateTime64OrNull('2024-10-09 10:30:10', 3); -select parseDateTime64OrNull('2024-10-09 10:30:10.', 3); +select parseDateTime64OrNull('2024-10-09 10:30:10.123456'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} +select parseDateTime64OrNull('2024-10-09 10:30:10', 3); -- { serverError BAD_ARGUMENTS } select parseDateTime64OrNull('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64OrNull('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } -select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', '%Y-%m-%d %H:%i:%s.%f'); -select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f'), parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7');; -select parseDateTime64OrNull('2024-10-09 10:30:10.123', 3, '%Y-%m-%d %H:%i:%s.%f'); \ No newline at end of file +select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6,'%Y-%m-%d %H:%i:%s.%f'); +select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7');; +select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%fffff'); \ No newline at end of file diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference index 0b4a28c4b38..99252ce55ca 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference @@ -1,12 +1,9 @@ 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 -2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-10 02:42:10.123456 2024-10-10 01:30:10.123456 2024-10-10 01:30:10.123456 1970-01-01 08:00:00.000 -1970-01-01 08:00:00.000 -2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-10 02:42:10.123456 @@ -15,8 +12,6 @@ 2024-10-10 01:30:10.123456 1970-01-01 08:00:00.000000 \N -\N -2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-10 02:42:10.123456 diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql index 8482677e9c9..bcb0fb5a362 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql @@ -1,11 +1,9 @@ set session_timezone = 'Asia/Shanghai'; select parseDateTime64InJodaSyntax('2024-10-09 10:30:10', 3); -- { serverError NOT_ENOUGH_SPACE } -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.', 3); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64InJodaSyntax('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 6); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); @@ -17,11 +15,9 @@ select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-8000', 6, 'yyyy-M select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456ABCD', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10', 3); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.', 3); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 6); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); @@ -33,11 +29,9 @@ select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-8000', 6, ' select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456ABCD', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10', 3); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.', 3); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10', 9); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 6); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); From 0768e0b265dc3a7d83d3a4c3ea9ba8625fe70994 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Mon, 11 Nov 2024 14:26:58 +0800 Subject: [PATCH 06/20] update doc & comments --- .../functions/type-conversion-functions.md | 10 ++++----- src/Functions/parseDateTime.cpp | 22 ++++++------------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 72e6fda03f7..a92d7055fd5 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -6874,14 +6874,14 @@ Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datet **Syntax** ``` sql -parseDateTime64(str[, [scale, [format[, timezone]]]]) +parseDateTime64(str, scale, [format[, timezone]]) ``` **Arguments** -- `str` — The String to be parsed +- `str` — The String to be parsed. - `scale` - The scale of [DateTime64](../data-types/datetime64.md). -- `format` — The format string. Optional. `%Y-%m-%d %H:%i:%s` if not specified. +- `format` — The format string. Optional. `%Y-%m-%d %H:%i:%s.%f` if not specified. - `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional. **Returned value(s)** @@ -6901,12 +6901,12 @@ Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datet **Syntax** ``` sql -parseDateTime64InJodaSyntax(str[, [scale, [format[, timezone]]]]) +parseDateTime64InJodaSyntax(str, scale, [format[, timezone]]) ``` **Arguments** -- `str` — The String to be parsed +- `str` — The String to be parsed. - `scale` - The scale of [DateTime64](../data-types/datetime64.md). - `format` — The format string. Optional. `yyyy-MM-dd HH:mm:ss` if not specified. - `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional. diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 72e3ba661ca..9fea8a4f130 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -608,26 +608,18 @@ namespace DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { FunctionArgumentDescriptors mandatory_args; - FunctionArgumentDescriptors optional_args; if constexpr (return_type == ReturnType::DateTime64) - { mandatory_args = { {"time", static_cast(&isString), nullptr, "String"}, - {"scale", static_cast(&isUInt8), nullptr, "UInt8"} + {"scale", static_cast(&isUInt8), &isColumnConst, "UInt8"} }; - optional_args = { - {"format", static_cast(&isString), nullptr, "String"}, - {"timezone", static_cast(&isString), &isColumnConst, "const String"} - }; - } else - { mandatory_args = {{"time", static_cast(&isString), nullptr, "String"}}; - optional_args = { - {"format", static_cast(&isString), nullptr, "String"}, - {"timezone", static_cast(&isString), &isColumnConst, "const String"} - }; - } + + FunctionArgumentDescriptors optional_args{ + {"format", static_cast(&isString), nullptr, "String"}, + {"timezone", static_cast(&isString), &isColumnConst, "const String"} + }; validateFunctionArguments(*this, arguments, mandatory_args, optional_args); String time_zone_name = getTimeZone(arguments).getTimeZone(); @@ -644,7 +636,7 @@ namespace throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale argument is not Const(UInt8) type."); } if (parse_syntax == ParseSyntax::MySQL && scale != 6) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale value {} of MySQL parse syntax is not 6.", std::to_string(scale)); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale argument's value {} of MySQL parse syntax is not 6.", std::to_string(scale)); if (scale > maxScaleOfDateTime64) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale argument's value {} exceed the max scale value {}.", std::to_string(scale), std::to_string(maxScaleOfDateTime64)); From 373f00c127d84cd70147bd63962f94b85d1a60b6 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 12 Nov 2024 11:25:14 +0800 Subject: [PATCH 07/20] fix review & ci --- .../functions/date-time-functions.md | 6 +- .../functions/type-conversion-functions.md | 2 + src/Functions/parseDateTime.cpp | 145 ++++++++++-------- ..._parse_datetime64_in_joda_syntax.reference | 12 +- .../03252_parse_datetime64_in_joda_syntax.sql | 12 +- .../aspell-ignore/en/aspell-dict.txt | 11 +- 6 files changed, 105 insertions(+), 83 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 2357b5b2fdd..9643e4aba8a 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -4489,9 +4489,9 @@ Using replacement fields, you can define a pattern for the resulting string. | k | clockhour of day (1~24) | number | 24 | | m | minute of hour | number | 30 | | s | second of minute | number | 55 | -| S | fraction of second (not supported yet) | number | 978 | -| z | time zone (short name not supported yet) | text | Pacific Standard Time; PST | -| Z | time zone offset/id (not supported yet) | zone | -0800; -08:00; America/Los_Angeles | +| S | fraction of second | number | 978 | +| z | time zone | text | Eastern Standard Time; EST | +| Z | time zone offset | zone | -0800; -0812 | | ' | escape for text | delimiter | | | '' | single quote | literal | ' | diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index a92d7055fd5..02f796e8336 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -6889,9 +6889,11 @@ parseDateTime64(str, scale, [format[, timezone]]) Returns [DateTime64](../data-types/datetime64.md) type values parsed from input string according to a MySQL style format string. ## parseDateTime64OrZero + Same as for [parseDateTime64](#parsedatetime64) except that it returns zero date when it encounters a date format that cannot be processed. ## parseDateTime64OrNull + Same as for [parseDateTime64](#parsedatetime64) except that it returns `NULL` when it encounters a date format that cannot be processed. ## parseDateTime64InJodaSyntax diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 9fea8a4f130..41e2a8ee21e 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -38,6 +38,7 @@ namespace ErrorCodes extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; extern const int CANNOT_PARSE_DATETIME; extern const int NOT_ENOUGH_SPACE; + extern const int LOGICAL_ERROR; } namespace @@ -193,7 +194,7 @@ namespace Int32 minute = 0; /// range [0, 59] Int32 second = 0; /// range [0, 59] Int32 microsecond = 0; /// range [0, 999999] - UInt32 scale = 0; /// The scale of DateTime64, range [0, 6]. + UInt32 scale = 0; /// Scale of DateTime64. When parse syntax is `Joda`, it range [0, 6]; When parse syntax is `MySQL`, it should be 6 bool is_am = true; /// If is_hour_of_half_day = true and is_am = false (i.e. pm) then add 12 hours to the result DateTime bool hour_starts_at_1 = false; /// Whether the hour is clockhour @@ -458,6 +459,18 @@ namespace return {}; } + [[nodiscard]] + VoidOrError setScale(UInt8 scale_, ParseSyntax parse_syntax_) + { + if (parse_syntax_ == ParseSyntax::MySQL && scale_ != 6) + RETURN_ERROR(ErrorCodes::CANNOT_PARSE_DATETIME, "Value {} for scale must be 6 for MySQL parse syntax", std::to_string(scale_)) + else if (parse_syntax_ == ParseSyntax::Joda && scale_ > 6) + RETURN_ERROR(ErrorCodes::CANNOT_PARSE_DATETIME, "Value {} for scale must be in the range [0, 6] for joda syntax", std::to_string(scale_)) + + scale = scale_; + return {}; + } + /// For debug [[maybe_unused]] String toString() const { @@ -579,7 +592,8 @@ namespace } }; - /// _FUNC_(str[scale, format, timezone]) + /// _FUNC_(str[, format, timezone]) /// for ReturnType::DateTime + /// _FUNC_(str, scale[, format, timezone]). /// for ReturnType::DateTime64 template class FunctionParseDateTimeImpl : public IFunction { @@ -608,13 +622,13 @@ namespace DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { FunctionArgumentDescriptors mandatory_args; - if constexpr (return_type == ReturnType::DateTime64) + if constexpr (return_type == ReturnType::DateTime) + mandatory_args = {{"time", static_cast(&isString), nullptr, "String"}}; + else mandatory_args = { {"time", static_cast(&isString), nullptr, "String"}, {"scale", static_cast(&isUInt8), &isColumnConst, "UInt8"} }; - else - mandatory_args = {{"time", static_cast(&isString), nullptr, "String"}}; FunctionArgumentDescriptors optional_args{ {"format", static_cast(&isString), nullptr, "String"}, @@ -624,22 +638,21 @@ namespace String time_zone_name = getTimeZone(arguments).getTimeZone(); DataTypePtr data_type; - if constexpr (return_type == ReturnType::DateTime64) + if constexpr (return_type == ReturnType::DateTime) + data_type = std::make_shared(time_zone_name); + else { UInt8 scale = 0; - if (isUInt8(arguments[1].type)) - { - const auto * col_scale = checkAndGetColumnConst(arguments[1].column.get()); - if (col_scale) - scale = col_scale->getValue(); - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale argument is not Const(UInt8) type."); - } + const auto * col_scale = checkAndGetColumnConst(arguments[1].column.get()); + if (col_scale) + scale = col_scale->getValue(); + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "The scale argument is not Const(UInt8) type."); + if (parse_syntax == ParseSyntax::MySQL && scale != 6) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale argument's value {} of MySQL parse syntax is not 6.", std::to_string(scale)); if (scale > maxScaleOfDateTime64) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The scale argument's value {} exceed the max scale value {}.", std::to_string(scale), std::to_string(maxScaleOfDateTime64)); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale argument's value {} exceed the max scale value {}.", std::to_string(scale), std::to_string(maxScaleOfDateTime64)); String format = getFormat(arguments, scale); std::vector instructions = parseFormat(format); @@ -647,28 +660,26 @@ namespace { /// Check scale by counting how may 'S' characters exists in the format string. const String & fragment = instruction.getFragment(); - UInt32 s_cnt = 0; + UInt32 s_count = 0; for (char ch : fragment) { if (ch != 'S') { - s_cnt = 0; + s_count = 0; break; } else - s_cnt++; + s_count++; } - /// If the number of 'S' character in format string not euqals the scale, then throw an exception to report error. - if (s_cnt != 0 && s_cnt != scale) + /// If the number of 'S' characters in format string not euqals the scale, then throw an exception to report error. + if (s_count != 0 && s_count != scale) throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The scale of input format string {} not equals the given scale value {}.", + "The number of 'S' characters in the input format string {} does not equal the given scale value {}.", format, std::to_string(scale)); } data_type = std::make_shared(scale, time_zone_name); } - else - data_type = std::make_shared(time_zone_name); if (error_handling == ErrorHandling::Null) return std::make_shared(data_type); @@ -679,23 +690,24 @@ namespace { DataTypePtr non_null_result_type; if constexpr (error_handling == ErrorHandling::Null) + /// Remove Nullable wrapper. It will be added back later. non_null_result_type = removeNullable(result_type); else non_null_result_type = result_type; - if constexpr (return_type == ReturnType::DateTime64) + if constexpr (return_type == ReturnType::DateTime) + { + MutableColumnPtr col_res = ColumnDateTime::create(input_rows_count); + ColumnDateTime * col_datetime = assert_cast(col_res.get()); + return executeImpl2(arguments, result_type, input_rows_count, col_res, col_datetime->getData()); + } + else { const auto * datatime64_type = checkAndGetDataType(non_null_result_type.get()); MutableColumnPtr col_res = ColumnDateTime64::create(input_rows_count, datatime64_type->getScale()); ColumnDateTime64 * col_datetime64 = assert_cast(col_res.get()); return executeImpl2(arguments, result_type, input_rows_count, col_res, col_datetime64->getData()); } - else - { - MutableColumnPtr col_res = ColumnDateTime::create(input_rows_count); - ColumnDateTime * col_datetime = assert_cast(col_res.get()); - return executeImpl2(arguments, result_type, input_rows_count, col_res, col_datetime->getData()); - } } template @@ -732,41 +744,52 @@ namespace for (size_t i = 0; i < input_rows_count; ++i) { datetime.reset(); - if constexpr (return_type == ReturnType::DateTime64) - datetime.scale = scale; StringRef str_ref = col_str->getDataAt(i); Pos cur = str_ref.data; Pos end = str_ref.data + str_ref.size; bool error = false; + auto handleError = [&](const auto & result) -> void + { + if constexpr (error_handling == ErrorHandling::Zero) + { + res_data[i] = 0; + error = true; + } + else if constexpr (error_handling == ErrorHandling::Null) + { + res_data[i] = 0; + col_null_map->getData()[i] = 1; + error = true; + } + else + { + static_assert(error_handling == ErrorHandling::Exception); + const ErrorCodeAndMessage & err = result.error(); + throw Exception(err.error_code, "{}", err.error_message); + } + }; + + if constexpr (return_type == ReturnType::DateTime64) + { + if (auto result = datetime.setScale(static_cast(scale), parse_syntax); !result.has_value()) + { + handleError(result); + continue; + } + } + for (const auto & instruction : instructions) { - if (auto result = instruction.perform(cur, end, datetime); result.has_value()) + if (auto result = instruction.perform(cur, end, datetime) ; result.has_value()) { cur = *result; } else { - if constexpr (error_handling == ErrorHandling::Zero) - { - res_data[i] = 0; - error = true; - break; - } - else if constexpr (error_handling == ErrorHandling::Null) - { - res_data[i] = 0; - col_null_map->getData()[i] = 1; - error = true; - break; - } - else - { - static_assert(error_handling == ErrorHandling::Exception); - const ErrorCodeAndMessage & err = result.error(); - throw Exception(err.error_code, "{}", err.error_message); - } + handleError(result); + break; } } @@ -789,10 +812,10 @@ namespace { if (result = datetime.buildDateTime(time_zone); result.has_value()) { - if constexpr (return_type == ReturnType::DateTime64) - res_data[i] = static_cast(*result) * multiplier + datetime.microsecond; - else + if constexpr (return_type == ReturnType::DateTime) res_data[i] = static_cast(*result); + else + res_data[i] = static_cast(*result) * multiplier + datetime.microsecond; } } @@ -2241,11 +2264,9 @@ namespace String getFormat(const ColumnsWithTypeAndName & arguments, UInt32 scale) const { - size_t format_arg_index = 1; - if constexpr (return_type == ReturnType::DateTime64) - format_arg_index = 2; + size_t index_of_format_string_arg = (return_type == ReturnType::DateTime) ? 1 : 2; - if (arguments.size() <= format_arg_index) + if (arguments.size() <= index_of_format_string_arg) { String format; if constexpr (parse_syntax == ParseSyntax::MySQL) @@ -2263,12 +2284,12 @@ namespace } else { - const auto * col_format = checkAndGetColumnConst(arguments[format_arg_index].column.get()); + const auto * col_format = checkAndGetColumnConst(arguments[index_of_format_string_arg].column.get()); if (!col_format) throw Exception( ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of second ('format') argument of function {}. Must be constant string.", - arguments[format_arg_index].column->getName(), + arguments[index_of_format_string_arg].column->getName(), getName()); return col_format->getValue(); } diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference index 99252ce55ca..2c5f7191c60 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference @@ -1,21 +1,21 @@ 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-10 02:42:10.123456 -2024-10-10 01:30:10.123456 -2024-10-10 01:30:10.123456 +2024-10-09 23:30:10.123456 +2024-10-09 23:30:10.123456 1970-01-01 08:00:00.000 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-10 02:42:10.123456 1970-01-01 08:00:00.000000 -2024-10-10 01:30:10.123456 -2024-10-10 01:30:10.123456 +2024-10-09 23:30:10.123456 +2024-10-09 23:30:10.123456 1970-01-01 08:00:00.000000 \N 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-10 02:42:10.123456 \N -2024-10-10 01:30:10.123456 -2024-10-10 01:30:10.123456 +2024-10-09 23:30:10.123456 +2024-10-09 23:30:10.123456 \N diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql index bcb0fb5a362..cb9d802dab0 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql @@ -8,8 +8,8 @@ select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH: select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); -- {serverError CANNOT_PARSE_DATETIME} -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); -- incorrect timezone offset and timezone select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-8000', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456ABCD', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } @@ -22,8 +22,8 @@ select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM- select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); -- incorrect timezone offset and timezone select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-8000', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456ABCD', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } @@ -36,8 +36,8 @@ select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM- select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456America/Los_Angeles', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); -- incorrect timezone offset and timezone select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-8000', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456ABCD', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } \ No newline at end of file diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index a08143467cd..21fe37d0526 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1,4 +1,4 @@ -personal_ws-1.1 en 2984 +personal_ws-1.1 en 3154 AArch ACLs ALTERs @@ -186,7 +186,6 @@ ComplexKeyCache ComplexKeyDirect ComplexKeyHashed Composable -composable ConcurrencyControlAcquired ConcurrencyControlSoftLimit Config @@ -405,12 +404,12 @@ ITION Identifiant IdentifierQuotingRule IdentifierQuotingStyle -Incrementing -IndexesAreNeighbors -InfluxDB InJodaSyntax InJodaSyntaxOrNull InJodaSyntaxOrZero +Incrementing +IndexesAreNeighbors +InfluxDB Instana IntN Integrations @@ -1099,7 +1098,6 @@ URLHash URLHierarchy URLPathHierarchy USearch -USearch UTCTimestamp UUIDNumToString UUIDStringToNum @@ -1995,6 +1993,7 @@ jbod jdbc jemalloc jeprof +joda joinGet joinGetOrNull json From eea644dc45e7b83f80a839827bf76c903e3d859d Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 12 Nov 2024 11:35:50 +0800 Subject: [PATCH 08/20] remove unused changes --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 21fe37d0526..b5962d44cb2 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1,4 +1,4 @@ -personal_ws-1.1 en 3154 +personal_ws-1.1 en 2984 AArch ACLs ALTERs @@ -186,6 +186,7 @@ ComplexKeyCache ComplexKeyDirect ComplexKeyHashed Composable +composable ConcurrencyControlAcquired ConcurrencyControlSoftLimit Config @@ -404,12 +405,12 @@ ITION Identifiant IdentifierQuotingRule IdentifierQuotingStyle -InJodaSyntax -InJodaSyntaxOrNull -InJodaSyntaxOrZero Incrementing IndexesAreNeighbors InfluxDB +InJodaSyntax +InJodaSyntaxOrNull +InJodaSyntaxOrZero Instana IntN Integrations @@ -1098,6 +1099,7 @@ URLHash URLHierarchy URLPathHierarchy USearch +USearch UTCTimestamp UUIDNumToString UUIDStringToNum From 9fb1cf20c7c9ea76db1508f249b29922d5539821 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 12 Nov 2024 12:25:38 +0800 Subject: [PATCH 09/20] fix ci test --- src/Functions/parseDateTime.cpp | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 41e2a8ee21e..eee91ed74ec 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -1394,16 +1394,26 @@ namespace [[nodiscard]] static PosOrError mysqlMicrosecond(Pos cur, Pos end, const String & fragment, DateTime & date) { - if (date.scale != 6) - RETURN_ERROR( - ErrorCodes::CANNOT_PARSE_DATETIME, - "Unable to parse fragment {} from {} because of the datetime scale {} is not 6", - fragment, - std::string_view(cur, end - cur), - std::to_string(date.scale)) - Int32 microsecond = 0; - ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumber6(cur, end, fragment, microsecond))) - RETURN_ERROR_IF_FAILED(date.setMicrosecond(microsecond)) + if constexpr (return_type == ReturnType::DateTime) + { + RETURN_ERROR_IF_FAILED(checkSpace(cur, end, 6, "mysqlMicrosecond requires size >= 6", fragment)) + + for (size_t i = 0; i < 6; ++i) + ASSIGN_RESULT_OR_RETURN_ERROR(cur, (assertNumber(cur, end, fragment))) + } + else + { + if (date.scale != 6) + RETURN_ERROR( + ErrorCodes::CANNOT_PARSE_DATETIME, + "Unable to parse fragment {} from {} because of the datetime scale {} is not 6", + fragment, + std::string_view(cur, end - cur), + std::to_string(date.scale)) + Int32 microsecond = 0; + ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumber6(cur, end, fragment, microsecond))) + RETURN_ERROR_IF_FAILED(date.setMicrosecond(microsecond)) + } return cur; } From 701fb76edb3593aba595285f0fe124a184fa4764 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 12 Nov 2024 13:03:38 +0800 Subject: [PATCH 10/20] fix ci test --- .../02415_all_new_functions_must_be_documented.reference | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index dea41174c65..b97394742ea 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -513,6 +513,7 @@ parseDateTime parseDateTime32BestEffort parseDateTime32BestEffortOrNull parseDateTime32BestEffortOrZero +parseDateTime64 parseDateTime64BestEffort parseDateTime64BestEffortOrNull parseDateTime64BestEffortOrZero @@ -522,6 +523,8 @@ parseDateTime64BestEffortUSOrZero parseDateTime64InJodaSyntax parseDateTime64InJodaSyntaxOrNull parseDateTime64InJodaSyntaxOrZero +parseDateTime64OrNull +parseDateTime64OrZero parseDateTimeBestEffort parseDateTimeBestEffortOrNull parseDateTimeBestEffortOrZero From 2507daa3b1934579c0720055a0fc1287ff26e738 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 12 Nov 2024 14:57:15 +0800 Subject: [PATCH 11/20] fix build error --- src/Functions/parseDateTime.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index eee91ed74ec..9dfa3701863 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -750,7 +750,7 @@ namespace Pos end = str_ref.data + str_ref.size; bool error = false; - auto handleError = [&](const auto & result) -> void + auto handleError = [&](const auto & result [[maybe_unused]]) -> void { if constexpr (error_handling == ErrorHandling::Zero) { From 178bf896525d54f425f214073611d1b7390a8912 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 12 Nov 2024 19:30:51 +0800 Subject: [PATCH 12/20] fix ci ast error --- src/Functions/parseDateTime.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 9dfa3701863..ac8087136cf 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -615,7 +615,13 @@ namespace bool useDefaultImplementationForConstants() const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2, 3}; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override + { + if constexpr (return_type == ReturnType::DateTime) + return {1, 2}; + else + return {1, 2, 3}; + } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } @@ -2299,7 +2305,7 @@ namespace throw Exception( ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of second ('format') argument of function {}. Must be constant string.", - arguments[index_of_format_string_arg].column->getName(), + arguments[index_of_format_string_arg].name, getName()); return col_format->getValue(); } From 7634e646e8ac50f821457dc2811e58322448987f Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 12 Nov 2024 21:23:09 +0800 Subject: [PATCH 13/20] fix review --- .../functions/type-conversion-functions.md | 4 +- src/Functions/parseDateTime.cpp | 110 +++++++++--------- .../03252_parse_datetime64.reference | 2 + .../0_stateless/03252_parse_datetime64.sql | 6 +- ..._parse_datetime64_in_joda_syntax.reference | 2 + .../03252_parse_datetime64_in_joda_syntax.sql | 4 +- 6 files changed, 65 insertions(+), 63 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 02f796e8336..7578ac33d8d 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -6874,7 +6874,7 @@ Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datet **Syntax** ``` sql -parseDateTime64(str, scale, [format[, timezone]]) +parseDateTime64(str, scale[, format[, timezone]]) ``` **Arguments** @@ -6903,7 +6903,7 @@ Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datet **Syntax** ``` sql -parseDateTime64InJodaSyntax(str, scale, [format[, timezone]]) +parseDateTime64InJodaSyntax(str, scale[, format[, timezone]]) ``` **Arguments** diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index ac8087136cf..4b65500ccba 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -32,13 +32,13 @@ namespace Setting namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; - extern const int NOT_IMPLEMENTED; extern const int BAD_ARGUMENTS; - extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; extern const int CANNOT_PARSE_DATETIME; - extern const int NOT_ENOUGH_SPACE; + extern const int ILLEGAL_COLUMN; extern const int LOGICAL_ERROR; + extern const int NOT_ENOUGH_SPACE; + extern const int NOT_IMPLEMENTED; + extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; } namespace @@ -66,7 +66,6 @@ namespace constexpr Int32 minYear = 1970; constexpr Int32 maxYear = 2106; - constexpr Int32 maxScaleOfDateTime64 = 6; const std::unordered_map> dayOfWeekMap{ {"mon", {"day", 1}}, @@ -465,7 +464,7 @@ namespace if (parse_syntax_ == ParseSyntax::MySQL && scale_ != 6) RETURN_ERROR(ErrorCodes::CANNOT_PARSE_DATETIME, "Value {} for scale must be 6 for MySQL parse syntax", std::to_string(scale_)) else if (parse_syntax_ == ParseSyntax::Joda && scale_ > 6) - RETURN_ERROR(ErrorCodes::CANNOT_PARSE_DATETIME, "Value {} for scale must be in the range [0, 6] for joda syntax", std::to_string(scale_)) + RETURN_ERROR(ErrorCodes::CANNOT_PARSE_DATETIME, "Value {} for scale must be in the range [0, 6] for Joda syntax", std::to_string(scale_)) scale = scale_; return {}; @@ -627,14 +626,11 @@ namespace DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - FunctionArgumentDescriptors mandatory_args; - if constexpr (return_type == ReturnType::DateTime) - mandatory_args = {{"time", static_cast(&isString), nullptr, "String"}}; - else - mandatory_args = { - {"time", static_cast(&isString), nullptr, "String"}, - {"scale", static_cast(&isUInt8), &isColumnConst, "UInt8"} - }; + FunctionArgumentDescriptors mandatory_args = { + {"time", static_cast(&isString), nullptr, "String"}}; + if constexpr (return_type == ReturnType::DateTime64) + mandatory_args.push_back( + {"scale", static_cast(&isUInt8), &isColumnConst, "UInt8"}); FunctionArgumentDescriptors optional_args{ {"format", static_cast(&isString), nullptr, "String"}, @@ -649,41 +645,11 @@ namespace else { UInt8 scale = 0; - const auto * col_scale = checkAndGetColumnConst(arguments[1].column.get()); - if (col_scale) + if (const auto * col_scale = checkAndGetColumnConst(arguments[1].column.get()); col_scale != nullptr) scale = col_scale->getValue(); else throw Exception(ErrorCodes::LOGICAL_ERROR, "The scale argument is not Const(UInt8) type."); - if (parse_syntax == ParseSyntax::MySQL && scale != 6) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale argument's value {} of MySQL parse syntax is not 6.", std::to_string(scale)); - if (scale > maxScaleOfDateTime64) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "The scale argument's value {} exceed the max scale value {}.", std::to_string(scale), std::to_string(maxScaleOfDateTime64)); - - String format = getFormat(arguments, scale); - std::vector instructions = parseFormat(format); - for (const auto & instruction : instructions) - { - /// Check scale by counting how may 'S' characters exists in the format string. - const String & fragment = instruction.getFragment(); - UInt32 s_count = 0; - for (char ch : fragment) - { - if (ch != 'S') - { - s_count = 0; - break; - } - else - s_count++; - } - /// If the number of 'S' characters in format string not euqals the scale, then throw an exception to report error. - if (s_count != 0 && s_count != scale) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The number of 'S' characters in the input format string {} does not equal the given scale value {}.", - format, - std::to_string(scale)); - } data_type = std::make_shared(scale, time_zone_name); } @@ -742,9 +708,37 @@ namespace } const String format = getFormat(arguments, scale); - const auto & time_zone = getTimeZone(arguments); std::vector instructions = parseFormat(format); + Int64OrError result = 0; + /// Check scale by counting how many 'S' characters exist in the format string when parse syntax is Joda and + /// return type is DateTime64. + if constexpr (return_type == ReturnType::DateTime64 && parse_syntax == ParseSyntax::Joda) + { + UInt32 s_count = 0; + for (const auto & instruction : instructions) + { + const String & fragment = instruction.getFragment(); + for (char ch : fragment) + { + if (ch != 'S') + { + s_count = 0; + break; + } + else + ++s_count; + } + } + /// If the number of 'S' characters in format string not euqals the scale, then throw an exception to report error. + if (s_count != 0 && s_count != scale) + result = tl::unexpected(ErrorCodeAndMessage(ErrorCodes::BAD_ARGUMENTS, + "The number of 'S' characters in the input format string {} does not equal the given scale value {}.", + format, + std::to_string(scale))); + } + + const auto & time_zone = getTimeZone(arguments); /// Make datetime fit in a cache line. alignas(64) DateTime datetime; for (size_t i = 0; i < input_rows_count; ++i) @@ -756,7 +750,7 @@ namespace Pos end = str_ref.data + str_ref.size; bool error = false; - auto handleError = [&](const auto & result [[maybe_unused]]) -> void + auto handle_error = [&]([[maybe_unused]] const auto & res) -> void { if constexpr (error_handling == ErrorHandling::Zero) { @@ -772,29 +766,35 @@ namespace else { static_assert(error_handling == ErrorHandling::Exception); - const ErrorCodeAndMessage & err = result.error(); + const ErrorCodeAndMessage & err = res.error(); throw Exception(err.error_code, "{}", err.error_message); } }; + if (!result.has_value()) + { + handle_error(result); + continue; + } + if constexpr (return_type == ReturnType::DateTime64) { - if (auto result = datetime.setScale(static_cast(scale), parse_syntax); !result.has_value()) + if (auto res = datetime.setScale(static_cast(scale), parse_syntax); !res.has_value()) { - handleError(result); + handle_error(res); continue; } } for (const auto & instruction : instructions) { - if (auto result = instruction.perform(cur, end, datetime) ; result.has_value()) + if (auto res = instruction.perform(cur, end, datetime) ; res.has_value()) { - cur = *result; + cur = *res; } else { - handleError(result); + handle_error(res); break; } } @@ -802,8 +802,6 @@ namespace if (error) continue; - Int64OrError result = 0; - /// Ensure all input was consumed. if (cur < end) { @@ -2305,7 +2303,7 @@ namespace throw Exception( ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of second ('format') argument of function {}. Must be constant string.", - arguments[index_of_format_string_arg].name, + arguments[index_of_format_string_arg].column->getName(), getName()); return col_format->getValue(); } diff --git a/tests/queries/0_stateless/03252_parse_datetime64.reference b/tests/queries/0_stateless/03252_parse_datetime64.reference index 263c9b5d8ea..0113948b366 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64.reference +++ b/tests/queries/0_stateless/03252_parse_datetime64.reference @@ -1,8 +1,10 @@ 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 +1970-01-01 08:00:00.000 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 1970-01-01 08:00:00.000000 +\N 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 1970-01-01 08:00:00.000000 diff --git a/tests/queries/0_stateless/03252_parse_datetime64.sql b/tests/queries/0_stateless/03252_parse_datetime64.sql index 2a6ef254887..e43440e062f 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64.sql +++ b/tests/queries/0_stateless/03252_parse_datetime64.sql @@ -1,21 +1,21 @@ set session_timezone = 'Asia/Shanghai'; select parseDateTime64('2024-10-09 10:30:10.123456'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} -select parseDateTime64('2024-10-09 10:30:10', 3); -- { serverError BAD_ARGUMENTS } +select parseDateTime64('2024-10-09 10:30:10', 3); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64('2024-10-09 10:30:10.123456', 6), parseDateTime64('2024-10-09 10:30:10.123456', 6,'%Y-%m-%d %H:%i:%s.%f'); select parseDateTime64('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7'); select parseDateTime64('2024-10-09 10:30:10.123', 6, '%Y-%m-%d %H:%i:%s.%f'); -- { serverError NOT_ENOUGH_SPACE } select parseDateTime64OrZero('2024-10-09 10:30:10.123456'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} -select parseDateTime64OrZero('2024-10-09 10:30:10', 3); -- { serverError BAD_ARGUMENTS } +select parseDateTime64OrZero('2024-10-09 10:30:10', 3); select parseDateTime64OrZero('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f'); select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7'); select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%fffff'); select parseDateTime64OrNull('2024-10-09 10:30:10.123456'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} -select parseDateTime64OrNull('2024-10-09 10:30:10', 3); -- { serverError BAD_ARGUMENTS } +select parseDateTime64OrNull('2024-10-09 10:30:10', 3); select parseDateTime64OrNull('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6,'%Y-%m-%d %H:%i:%s.%f'); select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7');; diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference index 2c5f7191c60..efda25ee21d 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference @@ -6,6 +6,7 @@ 1970-01-01 08:00:00.000 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 +1970-01-01 08:00:00.000 2024-10-10 02:42:10.123456 1970-01-01 08:00:00.000000 2024-10-09 23:30:10.123456 @@ -14,6 +15,7 @@ \N 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 +\N 2024-10-10 02:42:10.123456 \N 2024-10-09 23:30:10.123456 diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql index cb9d802dab0..b432eb97536 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql @@ -19,7 +19,7 @@ select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10', -3); -- {serverE select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 6); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); @@ -33,7 +33,7 @@ select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10', -3); -- {serverE select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 6); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); From 4ea106712f481794f94d39c9564eaf66cbf8f219 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Wed, 13 Nov 2024 11:49:31 +0800 Subject: [PATCH 14/20] fix comments --- src/Functions/parseDateTime.cpp | 147 ++++++++---------- .../03252_parse_datetime64.reference | 2 - .../0_stateless/03252_parse_datetime64.sql | 4 +- ..._parse_datetime64_in_joda_syntax.reference | 2 - .../03252_parse_datetime64_in_joda_syntax.sql | 4 +- 5 files changed, 71 insertions(+), 88 deletions(-) diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 4b65500ccba..956ede4295e 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -35,7 +35,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int CANNOT_PARSE_DATETIME; extern const int ILLEGAL_COLUMN; - extern const int LOGICAL_ERROR; extern const int NOT_ENOUGH_SPACE; extern const int NOT_IMPLEMENTED; extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; @@ -193,13 +192,13 @@ namespace Int32 minute = 0; /// range [0, 59] Int32 second = 0; /// range [0, 59] Int32 microsecond = 0; /// range [0, 999999] - UInt32 scale = 0; /// Scale of DateTime64. When parse syntax is `Joda`, it range [0, 6]; When parse syntax is `MySQL`, it should be 6 + UInt32 scale = 0; /// scale of the result DateTime64. Always 6 for ParseSytax == MySQL, [0, 6] for ParseSyntax == Joda. bool is_am = true; /// If is_hour_of_half_day = true and is_am = false (i.e. pm) then add 12 hours to the result DateTime bool hour_starts_at_1 = false; /// Whether the hour is clockhour bool is_hour_of_half_day = false; /// Whether the hour is of half day - bool has_time_zone_offset = false; /// If true, time zone offset is explicitly specified. + bool has_time_zone_offset = false; /// If true, timezone offset is explicitly specified. Int64 time_zone_offset = 0; /// Offset in seconds between current timezone to UTC. void reset() @@ -630,10 +629,10 @@ namespace {"time", static_cast(&isString), nullptr, "String"}}; if constexpr (return_type == ReturnType::DateTime64) mandatory_args.push_back( - {"scale", static_cast(&isUInt8), &isColumnConst, "UInt8"}); + {"scale", static_cast(&isUInt8), &isColumnConst, "const UInt8"}); FunctionArgumentDescriptors optional_args{ - {"format", static_cast(&isString), nullptr, "String"}, + {"format", static_cast(&isString), nullptr, "const String"}, {"timezone", static_cast(&isString), &isColumnConst, "const String"} }; validateFunctionArguments(*this, arguments, mandatory_args, optional_args); @@ -648,7 +647,10 @@ namespace if (const auto * col_scale = checkAndGetColumnConst(arguments[1].column.get()); col_scale != nullptr) scale = col_scale->getValue(); else - throw Exception(ErrorCodes::LOGICAL_ERROR, "The scale argument is not Const(UInt8) type."); + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type in 2nd ('scale') argument of function {}. Must be constant UInt8.", + getName()); data_type = std::make_shared(scale, time_zone_name); } @@ -660,12 +662,11 @@ namespace ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { - DataTypePtr non_null_result_type; + DataTypePtr result_type_without_nullable; if constexpr (error_handling == ErrorHandling::Null) - /// Remove Nullable wrapper. It will be added back later. - non_null_result_type = removeNullable(result_type); + result_type_without_nullable = removeNullable(result_type); /// Remove Nullable wrapper. It will be added back later. else - non_null_result_type = result_type; + result_type_without_nullable = result_type; if constexpr (return_type == ReturnType::DateTime) { @@ -675,46 +676,44 @@ namespace } else { - const auto * datatime64_type = checkAndGetDataType(non_null_result_type.get()); - MutableColumnPtr col_res = ColumnDateTime64::create(input_rows_count, datatime64_type->getScale()); + const auto * result_type_without_nullable_casted = checkAndGetDataType(result_type_without_nullable.get()); + MutableColumnPtr col_res = ColumnDateTime64::create(input_rows_count, result_type_without_nullable_casted->getScale()); ColumnDateTime64 * col_datetime64 = assert_cast(col_res.get()); return executeImpl2(arguments, result_type, input_rows_count, col_res, col_datetime64->getData()); } } template - ColumnPtr executeImpl2(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, + ColumnPtr executeImpl2( + const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, MutableColumnPtr & col_res, PaddedPODArray & res_data) const { const auto * col_str = checkAndGetColumn(arguments[0].column.get()); if (!col_str) throw Exception( ErrorCodes::ILLEGAL_COLUMN, - "Illegal column {} of first ('str') argument of function {}. Must be string.", - arguments[0].column->getName(), + "Illegal type in 1st ('time') argument of function {}. Must be String.", getName()); - ColumnUInt8::MutablePtr col_null_map; - if constexpr (error_handling == ErrorHandling::Null) - col_null_map = ColumnUInt8::create(input_rows_count, 0); - Int64 multiplier = 0; UInt32 scale = 0; if constexpr (return_type == ReturnType::DateTime64) { - const DataTypeDateTime64 * datatime64_type = checkAndGetDataType(removeNullable(result_type).get()); - scale = datatime64_type->getScale(); + const DataTypeDateTime64 * result_type_without_nullable_casted = checkAndGetDataType(removeNullable(result_type).get()); + scale = result_type_without_nullable_casted->getScale(); multiplier = DecimalUtils::scaleMultiplier(scale); } - const String format = getFormat(arguments, scale); - std::vector instructions = parseFormat(format); - Int64OrError result = 0; + ColumnUInt8::MutablePtr col_null_map; + if constexpr (error_handling == ErrorHandling::Null) + col_null_map = ColumnUInt8::create(input_rows_count, 0); + + const String format = getFormat(arguments, scale); + const std::vector instructions = parseFormat(format); - /// Check scale by counting how many 'S' characters exist in the format string when parse syntax is Joda and - /// return type is DateTime64. if constexpr (return_type == ReturnType::DateTime64 && parse_syntax == ParseSyntax::Joda) { + /// How many 'S' characters does the format string contain? UInt32 s_count = 0; for (const auto & instruction : instructions) { @@ -730,14 +729,13 @@ namespace ++s_count; } } - /// If the number of 'S' characters in format string not euqals the scale, then throw an exception to report error. + /// The number of 'S' characters in the format string must be equal to the scale. if (s_count != 0 && s_count != scale) - result = tl::unexpected(ErrorCodeAndMessage(ErrorCodes::BAD_ARGUMENTS, - "The number of 'S' characters in the input format string {} does not equal the given scale value {}.", + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "The number of 'S' characters in the input format string {} is different than the scale {}.", format, - std::to_string(scale))); + std::to_string(scale)); } - const auto & time_zone = getTimeZone(arguments); /// Make datetime fit in a cache line. alignas(64) DateTime datetime; @@ -750,59 +748,48 @@ namespace Pos end = str_ref.data + str_ref.size; bool error = false; - auto handle_error = [&]([[maybe_unused]] const auto & res) -> void - { - if constexpr (error_handling == ErrorHandling::Zero) - { - res_data[i] = 0; - error = true; - } - else if constexpr (error_handling == ErrorHandling::Null) - { - res_data[i] = 0; - col_null_map->getData()[i] = 1; - error = true; - } - else - { - static_assert(error_handling == ErrorHandling::Exception); - const ErrorCodeAndMessage & err = res.error(); - throw Exception(err.error_code, "{}", err.error_message); - } - }; - - if (!result.has_value()) - { - handle_error(result); - continue; - } - if constexpr (return_type == ReturnType::DateTime64) { - if (auto res = datetime.setScale(static_cast(scale), parse_syntax); !res.has_value()) + if (auto result = datetime.setScale(static_cast(scale), parse_syntax); !result.has_value()) { - handle_error(res); - continue; + const ErrorCodeAndMessage & err = result.error(); + throw Exception(err.error_code, "Invalid scale value: {}, {}", std::to_string(scale), err.error_message); } } for (const auto & instruction : instructions) { - if (auto res = instruction.perform(cur, end, datetime) ; res.has_value()) + if (auto result = instruction.perform(cur, end, datetime); result.has_value()) { - cur = *res; + cur = *result; } else { - handle_error(res); - break; + if constexpr (error_handling == ErrorHandling::Zero) + { + res_data[i] = 0; + error = true; + } + else if constexpr (error_handling == ErrorHandling::Null) + { + res_data[i] = 0; + col_null_map->getData()[i] = 1; + error = true; + } + else + { + static_assert(error_handling == ErrorHandling::Exception); + const ErrorCodeAndMessage & err = result.error(); + throw Exception(err.error_code, "{}", err.error_message); + } } } if (error) continue; - /// Ensure all input was consumed. + Int64OrError result = 0; + /// Ensure all input was consumed if (cur < end) { result = tl::unexpected(ErrorCodeAndMessage( @@ -1410,7 +1397,7 @@ namespace if (date.scale != 6) RETURN_ERROR( ErrorCodes::CANNOT_PARSE_DATETIME, - "Unable to parse fragment {} from {} because of the datetime scale {} is not 6", + "Unable to parse fragment {} from {} because the datetime scale {} is not 6", fragment, std::string_view(cur, end - cur), std::to_string(date.scale)) @@ -1835,6 +1822,7 @@ namespace static PosOrError jodaTimezoneOffset(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { RETURN_ERROR_IF_FAILED(checkSpace(cur, end, 5, "jodaTimezoneOffset requires size >= 5", fragment)) + Int32 sign; if (*cur == '-') sign = -1; @@ -1843,7 +1831,7 @@ namespace else RETURN_ERROR( ErrorCodes::CANNOT_PARSE_DATETIME, - "Unable to parse fragment {} from {} because of unknown sign time zone offset: {}", + "Unable to parse fragment {} from {} because of unknown sign in time zone offset: {}", fragment, std::string_view(cur, end - cur), std::string_view(cur, 1)) @@ -1854,7 +1842,7 @@ namespace if (hour < 0 || hour > 23) RETURN_ERROR( ErrorCodes::CANNOT_PARSE_DATETIME, - "Unable to parse fragment {} from {} because of the hour of datetime not in range [0, 23]: {}", + "Unable to parse fragment {} from {} because the hour of datetime not in range [0, 23]: {}", fragment, std::string_view(cur, end - cur), std::string_view(cur, 1)) @@ -1863,7 +1851,7 @@ namespace if (minute < 0 || minute > 59) RETURN_ERROR( ErrorCodes::CANNOT_PARSE_DATETIME, - "Unable to parse fragment {} from {} because of the minute of datetime not in range [0, 59]: {}", + "Unable to parse fragment {} from {} because the minute of datetime not in range [0, 59]: {}", fragment, std::string_view(cur, end - cur), std::string_view(cur, 1)) @@ -2282,6 +2270,7 @@ namespace if (arguments.size() <= index_of_format_string_arg) { + /// No format string given. Use default format string. String format; if constexpr (parse_syntax == ParseSyntax::MySQL) format = "%Y-%m-%d %H:%i:%s"; @@ -2302,8 +2291,7 @@ namespace if (!col_format) throw Exception( ErrorCodes::ILLEGAL_COLUMN, - "Illegal column {} of second ('format') argument of function {}. Must be constant string.", - arguments[index_of_format_string_arg].column->getName(), + "Illegal type in 'format' argument of function {}. Must be constant String.", getName()); return col_format->getValue(); } @@ -2316,13 +2304,12 @@ namespace else if (return_type == ReturnType::DateTime64 && arguments.size() < 4) return DateLUT::instance(); - size_t timezone_arg_index = arguments.size() - 1; - const auto * col = checkAndGetColumnConst(arguments[timezone_arg_index].column.get()); + size_t index_of_timezone_arg = arguments.size() - 1; + const auto * col = checkAndGetColumnConst(arguments[index_of_timezone_arg].column.get()); if (!col) throw Exception( ErrorCodes::ILLEGAL_COLUMN, - "Illegal column {} of ('timezone') argument of function {}. Must be constant String.", - arguments[timezone_arg_index].column->getName(), + "Illegal type in 'timezone' argument of function {}. Must be constant String.", getName()); String time_zone = col->getValue(); @@ -2411,16 +2398,16 @@ REGISTER_FUNCTION(ParseDateTime) factory.registerFunction(); factory.registerFunction(); factory.registerAlias("str_to_date", FunctionParseDateTimeOrNull::name, FunctionFactory::Case::Insensitive); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); } diff --git a/tests/queries/0_stateless/03252_parse_datetime64.reference b/tests/queries/0_stateless/03252_parse_datetime64.reference index 0113948b366..263c9b5d8ea 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64.reference +++ b/tests/queries/0_stateless/03252_parse_datetime64.reference @@ -1,10 +1,8 @@ 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 -1970-01-01 08:00:00.000 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 1970-01-01 08:00:00.000000 -\N 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 1970-01-01 08:00:00.000000 diff --git a/tests/queries/0_stateless/03252_parse_datetime64.sql b/tests/queries/0_stateless/03252_parse_datetime64.sql index e43440e062f..247c015e501 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64.sql +++ b/tests/queries/0_stateless/03252_parse_datetime64.sql @@ -8,14 +8,14 @@ select parseDateTime64('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', select parseDateTime64('2024-10-09 10:30:10.123', 6, '%Y-%m-%d %H:%i:%s.%f'); -- { serverError NOT_ENOUGH_SPACE } select parseDateTime64OrZero('2024-10-09 10:30:10.123456'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} -select parseDateTime64OrZero('2024-10-09 10:30:10', 3); +select parseDateTime64OrZero('2024-10-09 10:30:10', 3); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64OrZero('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f'); select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7'); select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%fffff'); select parseDateTime64OrNull('2024-10-09 10:30:10.123456'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} -select parseDateTime64OrNull('2024-10-09 10:30:10', 3); +select parseDateTime64OrNull('2024-10-09 10:30:10', 3); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64OrNull('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6,'%Y-%m-%d %H:%i:%s.%f'); select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7');; diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference index efda25ee21d..2c5f7191c60 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference @@ -6,7 +6,6 @@ 1970-01-01 08:00:00.000 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 -1970-01-01 08:00:00.000 2024-10-10 02:42:10.123456 1970-01-01 08:00:00.000000 2024-10-09 23:30:10.123456 @@ -15,7 +14,6 @@ \N 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 -\N 2024-10-10 02:42:10.123456 \N 2024-10-09 23:30:10.123456 diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql index b432eb97536..559f1fbb3f2 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql @@ -19,7 +19,7 @@ select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10', -3); -- {serverE select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 6); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); @@ -33,7 +33,7 @@ select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10', -3); -- {serverE select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 6); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); From 5e4dd5d32afeb2b6f6215909fb8edeb7021f1299 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Wed, 13 Nov 2024 12:16:22 +0800 Subject: [PATCH 15/20] fix comments --- src/Functions/parseDateTime.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 956ede4295e..4ea90a091b8 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -742,7 +742,6 @@ namespace for (size_t i = 0; i < input_rows_count; ++i) { datetime.reset(); - StringRef str_ref = col_str->getDataAt(i); Pos cur = str_ref.data; Pos end = str_ref.data + str_ref.size; @@ -769,12 +768,14 @@ namespace { res_data[i] = 0; error = true; + break; } else if constexpr (error_handling == ErrorHandling::Null) { res_data[i] = 0; col_null_map->getData()[i] = 1; error = true; + break; } else { From 66a216f63cd531fa7ff538641d9c0e8410ca8a23 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Thu, 14 Nov 2024 16:25:41 +0800 Subject: [PATCH 16/20] fix comments --- .../functions/type-conversion-functions.md | 6 +- src/Functions/parseDateTime.cpp | 109 +++++++----------- .../02668_parse_datetime.reference | 25 ++++ .../0_stateless/02668_parse_datetime.sql | 19 +++ ...68_parse_datetime_in_joda_syntax.reference | 44 +++++++ .../02668_parse_datetime_in_joda_syntax.sql | 31 +++++ .../03252_parse_datetime64.reference | 8 -- .../0_stateless/03252_parse_datetime64.sql | 22 ---- ..._parse_datetime64_in_joda_syntax.reference | 21 ---- .../03252_parse_datetime64_in_joda_syntax.sql | 43 ------- 10 files changed, 160 insertions(+), 168 deletions(-) delete mode 100644 tests/queries/0_stateless/03252_parse_datetime64.reference delete mode 100644 tests/queries/0_stateless/03252_parse_datetime64.sql delete mode 100644 tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference delete mode 100644 tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 7578ac33d8d..1c92a459e13 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -6874,13 +6874,12 @@ Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datet **Syntax** ``` sql -parseDateTime64(str, scale[, format[, timezone]]) +parseDateTime64(str[, format[, timezone]]) ``` **Arguments** - `str` — The String to be parsed. -- `scale` - The scale of [DateTime64](../data-types/datetime64.md). - `format` — The format string. Optional. `%Y-%m-%d %H:%i:%s.%f` if not specified. - `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional. @@ -6903,13 +6902,12 @@ Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datet **Syntax** ``` sql -parseDateTime64InJodaSyntax(str, scale[, format[, timezone]]) +parseDateTime64InJodaSyntax(str[, format[, timezone]]) ``` **Arguments** - `str` — The String to be parsed. -- `scale` - The scale of [DateTime64](../data-types/datetime64.md). - `format` — The format string. Optional. `yyyy-MM-dd HH:mm:ss` if not specified. - `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional. diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 4ea90a091b8..c8e800202f0 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -613,26 +613,17 @@ namespace bool useDefaultImplementationForConstants() const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override - { - if constexpr (return_type == ReturnType::DateTime) - return {1, 2}; - else - return {1, 2, 3}; - } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - FunctionArgumentDescriptors mandatory_args = { - {"time", static_cast(&isString), nullptr, "String"}}; - if constexpr (return_type == ReturnType::DateTime64) - mandatory_args.push_back( - {"scale", static_cast(&isUInt8), &isColumnConst, "const UInt8"}); - + FunctionArgumentDescriptors mandatory_args{ + {"time", static_cast(&isString), nullptr, "String"} + }; FunctionArgumentDescriptors optional_args{ - {"format", static_cast(&isString), nullptr, "const String"}, + {"format", static_cast(&isString), &isColumnConst, "const String"}, {"timezone", static_cast(&isString), &isColumnConst, "const String"} }; validateFunctionArguments(*this, arguments, mandatory_args, optional_args); @@ -643,16 +634,30 @@ namespace data_type = std::make_shared(time_zone_name); else { - UInt8 scale = 0; - if (const auto * col_scale = checkAndGetColumnConst(arguments[1].column.get()); col_scale != nullptr) - scale = col_scale->getValue(); + if constexpr (parse_syntax == ParseSyntax::MySQL) + data_type = std::make_shared(6, time_zone_name); else - throw Exception( - ErrorCodes::ILLEGAL_COLUMN, - "Illegal type in 2nd ('scale') argument of function {}. Must be constant UInt8.", - getName()); - - data_type = std::make_shared(scale, time_zone_name); + { + String format = getFormat(arguments); + std::vector instructions = parseFormat(format); + /// How many 'S' characters does the format string contain? + UInt32 s_count = 0; + for (const auto & instruction : instructions) + { + const String fragment = instruction.getFragment(); + for (char c : fragment) + { + if (c == 'S') + ++s_count; + else + break; + } + if (s_count > 0) + break; + } + /// Use s_count as DateTime64's scale. + data_type = std::make_shared(s_count, time_zone_name); + } } if (error_handling == ErrorHandling::Null) @@ -708,34 +713,8 @@ namespace if constexpr (error_handling == ErrorHandling::Null) col_null_map = ColumnUInt8::create(input_rows_count, 0); - const String format = getFormat(arguments, scale); + const String format = getFormat(arguments); const std::vector instructions = parseFormat(format); - - if constexpr (return_type == ReturnType::DateTime64 && parse_syntax == ParseSyntax::Joda) - { - /// How many 'S' characters does the format string contain? - UInt32 s_count = 0; - for (const auto & instruction : instructions) - { - const String & fragment = instruction.getFragment(); - for (char ch : fragment) - { - if (ch != 'S') - { - s_count = 0; - break; - } - else - ++s_count; - } - } - /// The number of 'S' characters in the format string must be equal to the scale. - if (s_count != 0 && s_count != scale) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "The number of 'S' characters in the input format string {} is different than the scale {}.", - format, - std::to_string(scale)); - } const auto & time_zone = getTimeZone(arguments); /// Make datetime fit in a cache line. alignas(64) DateTime datetime; @@ -2265,30 +2244,23 @@ namespace } - String getFormat(const ColumnsWithTypeAndName & arguments, UInt32 scale) const + String getFormat(const ColumnsWithTypeAndName & arguments) const { - size_t index_of_format_string_arg = (return_type == ReturnType::DateTime) ? 1 : 2; - - if (arguments.size() <= index_of_format_string_arg) + if (arguments.size() == 1) { - /// No format string given. Use default format string. - String format; if constexpr (parse_syntax == ParseSyntax::MySQL) - format = "%Y-%m-%d %H:%i:%s"; - else - format = "yyyy-MM-dd HH:mm:ss"; - if (scale > 0) { - if constexpr (parse_syntax == ParseSyntax::MySQL) - format += ".%f"; + if constexpr (return_type == ReturnType::DateTime) + return "%Y-%m-%d %H:%i:%s"; else - format += "." + String(scale, 'S'); + return "%Y-%m-%d %H:%i:%s.%f"; } - return format; + else + return "yyyy-MM-dd HH:mm:ss"; } else { - const auto * col_format = checkAndGetColumnConst(arguments[index_of_format_string_arg].column.get()); + const auto * col_format = checkAndGetColumnConst(arguments[1].column.get()); if (!col_format) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -2300,13 +2272,10 @@ namespace const DateLUTImpl & getTimeZone(const ColumnsWithTypeAndName & arguments) const { - if (return_type == ReturnType::DateTime && arguments.size() < 3) - return DateLUT::instance(); - else if (return_type == ReturnType::DateTime64 && arguments.size() < 4) + if (arguments.size() < 3) return DateLUT::instance(); - size_t index_of_timezone_arg = arguments.size() - 1; - const auto * col = checkAndGetColumnConst(arguments[index_of_timezone_arg].column.get()); + const auto * col = checkAndGetColumnConst(arguments[2].column.get()); if (!col) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/tests/queries/0_stateless/02668_parse_datetime.reference b/tests/queries/0_stateless/02668_parse_datetime.reference index b67ca2d8b76..bd78da75656 100644 --- a/tests/queries/0_stateless/02668_parse_datetime.reference +++ b/tests/queries/0_stateless/02668_parse_datetime.reference @@ -274,3 +274,28 @@ select parseDateTime('08 13, 2022, 07:58:32', '%c %e, %G, %k:%i:%s', 'UTC'); set session_timezone = 'UTC'; -- don't randomize the session timezone select parseDateTime('2021-01-04 23:12:34') = toDateTime('2021-01-04 23:12:34'); 1 +-- The following is test of parseDateTime64[OrNull/OrZero] +select parseDateTime64('2021-01-04 23:12:34.118112') = toDateTime64('2021-01-04 23:12:34.118112', 6); +1 +select parseDateTime64('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('2021-01-04 23:12:34.118112', 6); +1 +select parseDateTime64('2021-01-04 23:12:34.118'); -- { serverError NOT_ENOUGH_SPACE } +select parseDateTime64('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError NOT_ENOUGH_SPACE } +select parseDateTime64('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64OrNull('2021-01-04 23:12:34.118') IS NULL; +1 +select parseDateTime64OrNull('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f') IS NULL; +1 +select parseDateTime64OrNull('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s') IS NULL; +1 +select parseDateTime64OrNull('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f') IS NULL; +1 +select parseDateTime64OrZero('2021-01-04 23:12:34.118') = toDateTime64('1970-01-01 00:00:00', 6); +1 +select parseDateTime64OrZero('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('1970-01-01 00:00:00', 6); +1 +select parseDateTime64OrZero('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s') = toDateTime64('1970-01-01 00:00:00', 6); +1 +select parseDateTime64OrZero('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('1970-01-01 00:00:00', 6); +1 diff --git a/tests/queries/0_stateless/02668_parse_datetime.sql b/tests/queries/0_stateless/02668_parse_datetime.sql index 7b3aed60a4a..4a3c31dd3bf 100644 --- a/tests/queries/0_stateless/02668_parse_datetime.sql +++ b/tests/queries/0_stateless/02668_parse_datetime.sql @@ -191,5 +191,24 @@ select parseDateTime('08 13, 2022, 07:58:32', '%c %e, %G, %k:%i:%s', 'UTC'); set session_timezone = 'UTC'; -- don't randomize the session timezone select parseDateTime('2021-01-04 23:12:34') = toDateTime('2021-01-04 23:12:34'); +-- The following is test of parseDateTime64 +-- %f +select parseDateTime64('2021-01-04 23:12:34.118112') = toDateTime64('2021-01-04 23:12:34.118112', 6); +select parseDateTime64('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('2021-01-04 23:12:34.118112', 6); +select parseDateTime64('2021-01-04 23:12:34.118'); -- { serverError NOT_ENOUGH_SPACE } +select parseDateTime64('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError NOT_ENOUGH_SPACE } +select parseDateTime64('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s'); -- { serverError CANNOT_PARSE_DATETIME } +-- Test of parseDateTime64OrNull +select parseDateTime64OrNull('2021-01-04 23:12:34.118') IS NULL; +select parseDateTime64OrNull('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f') IS NULL; +select parseDateTime64OrNull('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s') IS NULL; +select parseDateTime64OrNull('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f') IS NULL; +-- Test of parseDateTime64OrZero +select parseDateTime64OrZero('2021-01-04 23:12:34.118') = toDateTime64('1970-01-01 00:00:00', 6); +select parseDateTime64OrZero('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('1970-01-01 00:00:00', 6); +select parseDateTime64OrZero('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s') = toDateTime64('1970-01-01 00:00:00', 6); +select parseDateTime64OrZero('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('1970-01-01 00:00:00', 6); + -- { echoOff } diff --git a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference index 6f560577ab5..585ed1e2718 100644 --- a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference +++ b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference @@ -360,3 +360,47 @@ select parseDateTimeInJodaSyntax('12 AM', 'h a', 'UTC', 'a fourth argument'); -- set session_timezone = 'UTC'; -- don't randomize the session timezone select parseDateTimeInJodaSyntax('2021-01-04 23:12:34') = toDateTime('2021-01-04 23:12:34'); 1 +-- Test timezone and timezone offset for parseDateTimeInJodaSyntax +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10-0812'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10-0812', 'yyyy-MM-dd HH:mm:ssZ') = toDateTime64('2024-10-09 18:42:10', 6); +1 +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10-08123', 'yyyy-MM-dd HH:mm:ssZZZ'); -- {serverError CANNOT_PARSE_DATETIME} +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10EST', 'yyyy-MM-dd HH:mm:ssz') = toDateTime64('2024-10-09 15:30:10', 6); +1 +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10EST', 'yyyy-MM-dd HH:mm:sszzz') = toDateTime64('2024-10-09 15:30:10', 6); +1 +-- incorrect timezone offset and timezone +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10-8000', 'yyyy-MM-dd HH:mm:ssZ'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10ABCD', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError BAD_ARGUMENTS } +-- The following is test of parseDateTime64InJodaSyntax[OrNull/OrZero] +select parseDateTime64InJodaSyntax('2021-01-04 23:12:34') = toDateTime64('2021-01-04 23:12:34', 0); +1 +select parseDateTime64InJodaSyntax('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SSS') = toDateTime64('2021-01-04 23:12:34.331', 3); +1 +select parseDateTime64InJodaSyntax('2021/01/04 23:12:34.331', 'yyyy/MM/dd HH:mm:ss.SSS') = toDateTime64('2021-01-04 23:12:34.331', 3); +1 +select parseDateTime64InJodaSyntax('2021-01-04 23:12:34.331'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SSSS') = toDateTime64('2021-01-04 23:12:34.0331', 4); +1 +select parseDateTime64InJodaSyntax('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SS'); -- { serverError CANNOT_PARSE_DATETIME } +-- Test timezone and timezone offset for paseDatetTime64InJodaSyntax +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10-0812'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0812', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2024-10-09 18:42:10.123456', 6); +1 +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-08123', 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); -- {serverError CANNOT_PARSE_DATETIME} +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 15:30:10.123456', 6); +1 +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz') = toDateTime64('2024-10-09 15:30:10.123456', 6); +1 +-- incorrect timezone offset and timezone +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-8000', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456ABCD', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } +-- Test for parseDateTime64InJodaSyntaxOrNull / parseDateTime64InJodaSyntaxOrZero +select parseDateTime64InJodaSyntaxOrNull('2021-01-04 23:12:34.331') IS NULL; +1 +select parseDateTime64InJodaSyntaxOrNull('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SS') IS NULL; +1 +select parseDateTime64InJodaSyntaxOrZero('2021-01-04 23:12:34.331') = toDateTime64('1970-01-01 00:00:00', 0); +1 +select parseDateTime64InJodaSyntaxOrZero('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SS') = toDateTime64('1970-01-01 00:00:00', 0); +1 diff --git a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql index 28d14607ba6..ab22d56ea21 100644 --- a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql +++ b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql @@ -245,5 +245,36 @@ select parseDateTimeInJodaSyntax('12 AM', 'h a', 'UTC', 'a fourth argument'); -- -- The format string argument is optional set session_timezone = 'UTC'; -- don't randomize the session timezone select parseDateTimeInJodaSyntax('2021-01-04 23:12:34') = toDateTime('2021-01-04 23:12:34'); +-- Test timezone and timezone offset for parseDateTimeInJodaSyntax +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10-0812'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10-0812', 'yyyy-MM-dd HH:mm:ssZ') = toDateTime64('2024-10-09 18:42:10', 6); +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10-08123', 'yyyy-MM-dd HH:mm:ssZZZ'); -- {serverError CANNOT_PARSE_DATETIME} +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10EST', 'yyyy-MM-dd HH:mm:ssz') = toDateTime64('2024-10-09 15:30:10', 6); +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10EST', 'yyyy-MM-dd HH:mm:sszzz') = toDateTime64('2024-10-09 15:30:10', 6); +-- incorrect timezone offset and timezone +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10-8000', 'yyyy-MM-dd HH:mm:ssZ'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTimeInJodaSyntax('2024-10-09 10:30:10ABCD', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError BAD_ARGUMENTS } + +-- The following is test of parseDateTime64InJodaSyntax[OrNull/OrZero] +select parseDateTime64InJodaSyntax('2021-01-04 23:12:34') = toDateTime64('2021-01-04 23:12:34', 0); +select parseDateTime64InJodaSyntax('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SSS') = toDateTime64('2021-01-04 23:12:34.331', 3); +select parseDateTime64InJodaSyntax('2021/01/04 23:12:34.331', 'yyyy/MM/dd HH:mm:ss.SSS') = toDateTime64('2021-01-04 23:12:34.331', 3); +select parseDateTime64InJodaSyntax('2021-01-04 23:12:34.331'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SSSS') = toDateTime64('2021-01-04 23:12:34.0331', 4); +select parseDateTime64InJodaSyntax('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SS'); -- { serverError CANNOT_PARSE_DATETIME } +-- Test timezone and timezone offset for paseDatetTime64InJodaSyntax +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10-0812'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0812', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2024-10-09 18:42:10.123456', 6); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-08123', 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); -- {serverError CANNOT_PARSE_DATETIME} +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 15:30:10.123456', 6); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz') = toDateTime64('2024-10-09 15:30:10.123456', 6); +-- incorrect timezone offset and timezone +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-8000', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456ABCD', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } +-- Test for parseDateTime64InJodaSyntaxOrNull / parseDateTime64InJodaSyntaxOrZero +select parseDateTime64InJodaSyntaxOrNull('2021-01-04 23:12:34.331') IS NULL; +select parseDateTime64InJodaSyntaxOrNull('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SS') IS NULL; +select parseDateTime64InJodaSyntaxOrZero('2021-01-04 23:12:34.331') = toDateTime64('1970-01-01 00:00:00', 0); +select parseDateTime64InJodaSyntaxOrZero('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SS') = toDateTime64('1970-01-01 00:00:00', 0); -- { echoOff } diff --git a/tests/queries/0_stateless/03252_parse_datetime64.reference b/tests/queries/0_stateless/03252_parse_datetime64.reference deleted file mode 100644 index 263c9b5d8ea..00000000000 --- a/tests/queries/0_stateless/03252_parse_datetime64.reference +++ /dev/null @@ -1,8 +0,0 @@ -2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 -2024-10-09 10:30:10.123456 -2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 -2024-10-09 10:30:10.123456 -1970-01-01 08:00:00.000000 -2024-10-09 10:30:10.123456 2024-10-09 10:30:10.123456 -2024-10-09 10:30:10.123456 -1970-01-01 08:00:00.000000 diff --git a/tests/queries/0_stateless/03252_parse_datetime64.sql b/tests/queries/0_stateless/03252_parse_datetime64.sql deleted file mode 100644 index 247c015e501..00000000000 --- a/tests/queries/0_stateless/03252_parse_datetime64.sql +++ /dev/null @@ -1,22 +0,0 @@ -set session_timezone = 'Asia/Shanghai'; - -select parseDateTime64('2024-10-09 10:30:10.123456'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} -select parseDateTime64('2024-10-09 10:30:10', 3); -- { serverError CANNOT_PARSE_DATETIME } -select parseDateTime64('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64('2024-10-09 10:30:10.123456', 6), parseDateTime64('2024-10-09 10:30:10.123456', 6,'%Y-%m-%d %H:%i:%s.%f'); -select parseDateTime64('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7'); -select parseDateTime64('2024-10-09 10:30:10.123', 6, '%Y-%m-%d %H:%i:%s.%f'); -- { serverError NOT_ENOUGH_SPACE } - -select parseDateTime64OrZero('2024-10-09 10:30:10.123456'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} -select parseDateTime64OrZero('2024-10-09 10:30:10', 3); -- { serverError CANNOT_PARSE_DATETIME } -select parseDateTime64OrZero('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f'); -select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7'); -select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%fffff'); - -select parseDateTime64OrNull('2024-10-09 10:30:10.123456'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} -select parseDateTime64OrNull('2024-10-09 10:30:10', 3); -- { serverError CANNOT_PARSE_DATETIME } -select parseDateTime64OrNull('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6), parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6,'%Y-%m-%d %H:%i:%s.%f'); -select parseDateTime64OrNull('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%f', 'Etc/GMT-7');; -select parseDateTime64OrZero('2024-10-09 10:30:10.123456', 6, '%Y-%m-%d %H:%i:%s.%fffff'); \ No newline at end of file diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference deleted file mode 100644 index 2c5f7191c60..00000000000 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference +++ /dev/null @@ -1,21 +0,0 @@ -2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 -2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 -2024-10-10 02:42:10.123456 -2024-10-09 23:30:10.123456 -2024-10-09 23:30:10.123456 -1970-01-01 08:00:00.000 -2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 -2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 -2024-10-10 02:42:10.123456 -1970-01-01 08:00:00.000000 -2024-10-09 23:30:10.123456 -2024-10-09 23:30:10.123456 -1970-01-01 08:00:00.000000 -\N -2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 -2024-10-09 10:30:10.123 2024-10-09 10:30:10.000123 -2024-10-10 02:42:10.123456 -\N -2024-10-09 23:30:10.123456 -2024-10-09 23:30:10.123456 -\N diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql deleted file mode 100644 index 559f1fbb3f2..00000000000 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql +++ /dev/null @@ -1,43 +0,0 @@ -set session_timezone = 'Asia/Shanghai'; - -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10', 3); -- { serverError NOT_ENOUGH_SPACE } -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 6); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); -- {serverError CANNOT_PARSE_DATETIME} -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); --- incorrect timezone offset and timezone -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-8000', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -- { serverError CANNOT_PARSE_DATETIME } -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456ABCD', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } - -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10', 3); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 6); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); --- incorrect timezone offset and timezone -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-8000', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456ABCD', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } - -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10', 3); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10', -3); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT} -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 6); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSS'), parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 3, 'yyyy-MM-dd HH:mm:ss.SSSS'); -- { serverError BAD_ARGUMENTS } -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0812', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-08123', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456EST', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz'); --- incorrect timezone offset and timezone -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-8000', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456ABCD', 6, 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } \ No newline at end of file From 1dc969c5602c81cc7f33322d763c1d7471e863b5 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Thu, 14 Nov 2024 16:48:17 +0800 Subject: [PATCH 17/20] fix comments --- src/Functions/parseDateTime.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index c8e800202f0..9a9a8fd93b4 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -590,8 +590,7 @@ namespace } }; - /// _FUNC_(str[, format, timezone]) /// for ReturnType::DateTime - /// _FUNC_(str, scale[, format, timezone]). /// for ReturnType::DateTime64 + /// _FUNC_(str[, format, timezone]) template class FunctionParseDateTimeImpl : public IFunction { From 61d0440f858267d3f1c46c6250835b0367199e11 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Thu, 14 Nov 2024 19:14:15 +0800 Subject: [PATCH 18/20] modify test --- .../0_stateless/02668_parse_datetime.sql | 14 +++++- .../02668_parse_datetime_in_joda_syntax.sql | 48 +++++++++++++++++-- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/02668_parse_datetime.sql b/tests/queries/0_stateless/02668_parse_datetime.sql index 4a3c31dd3bf..e56418ae5ed 100644 --- a/tests/queries/0_stateless/02668_parse_datetime.sql +++ b/tests/queries/0_stateless/02668_parse_datetime.sql @@ -192,19 +192,31 @@ set session_timezone = 'UTC'; -- don't randomize the session timezone select parseDateTime('2021-01-04 23:12:34') = toDateTime('2021-01-04 23:12:34'); -- The following is test of parseDateTime64 --- %f +select 'parseDateTime64'; +select parseDateTime64(''); +select parseDateTime64('2177-10-09 10:30:10.123'); +select parseDateTime64('08:01', 'HH:ss'); +select parseDateTime64('2024-01-02', 'yyyy-MM-dd'); +select parseDateTime64('10:30:50', 'HH:mm:ss'); select parseDateTime64('2021-01-04 23:12:34.118112') = toDateTime64('2021-01-04 23:12:34.118112', 6); select parseDateTime64('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('2021-01-04 23:12:34.118112', 6); select parseDateTime64('2021-01-04 23:12:34.118'); -- { serverError NOT_ENOUGH_SPACE } select parseDateTime64('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError NOT_ENOUGH_SPACE } select parseDateTime64('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s'); -- { serverError CANNOT_PARSE_DATETIME } +--leap years and non-leap years +select parseDateTime64('2024-02-29 11:23:34.123433', '%Y-%m-%d %H:%i:%s.%f'); +select parseDateTime64('2023-02-29 11:22:33.123433', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64('2024-02-28 23:22:33.123433', '%Y-%m-%d %H:%i:%s.%f'); +select parseDateTime64('2023-02-28 23:22:33.123433', '%Y-%m-%d %H:%i:%s.%f'); -- Test of parseDateTime64OrNull +select 'parseDateTime64OrNull'; select parseDateTime64OrNull('2021-01-04 23:12:34.118') IS NULL; select parseDateTime64OrNull('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f') IS NULL; select parseDateTime64OrNull('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s') IS NULL; select parseDateTime64OrNull('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f') IS NULL; -- Test of parseDateTime64OrZero +select 'parseDateTime64OrZero'; select parseDateTime64OrZero('2021-01-04 23:12:34.118') = toDateTime64('1970-01-01 00:00:00', 6); select parseDateTime64OrZero('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('1970-01-01 00:00:00', 6); select parseDateTime64OrZero('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s') = toDateTime64('1970-01-01 00:00:00', 6); diff --git a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql index ab22d56ea21..a4d5738d91d 100644 --- a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql +++ b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql @@ -256,6 +256,15 @@ select parseDateTimeInJodaSyntax('2024-10-09 10:30:10-8000', 'yyyy-MM-dd HH:mm:s select parseDateTimeInJodaSyntax('2024-10-09 10:30:10ABCD', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError BAD_ARGUMENTS } -- The following is test of parseDateTime64InJodaSyntax[OrNull/OrZero] +select 'parseDateTime64InJodaSyntax'; +select parseDateTime64InJodaSyntax('', ''); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +select parseDateTime64InJodaSyntax('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('+0000', 'Z'); +select parseDateTime64InJodaSyntax('08:01', 'HH:ss'); +select parseDateTime64InJodaSyntax('2024-01-02', 'yyyy-MM-dd'); +select parseDateTime64InJodaSyntax('10:30:50', 'HH:mm:ss'); +select parseDateTime64InJodaSyntax('2024-12-31 23:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntax('2024-01-01 00:00:01.123456+0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntax('2021-01-04 23:12:34') = toDateTime64('2021-01-04 23:12:34', 0); select parseDateTime64InJodaSyntax('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SSS') = toDateTime64('2021-01-04 23:12:34.331', 3); select parseDateTime64InJodaSyntax('2021/01/04 23:12:34.331', 'yyyy/MM/dd HH:mm:ss.SSS') = toDateTime64('2021-01-04 23:12:34.331', 3); @@ -268,13 +277,44 @@ select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0812', 'yyyy-MM-d select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-08123', 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); -- {serverError CANNOT_PARSE_DATETIME} select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 15:30:10.123456', 6); select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz') = toDateTime64('2024-10-09 15:30:10.123456', 6); +select parseDateTime64InJodaSyntax('2024-11-05-0800 01:02:03.123456', 'yyyy-MM-ddZ HH:mm:ss.SSSSSS'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456Australia/Adelaide', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntax('999999 10-09-202410:30:10', 'SSSSSSSSS dd-MM-yyyyHH:mm:ss'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0845', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -- incorrect timezone offset and timezone select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-8000', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456ABCD', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } +select parseDateTime64InJodaSyntax('2023-02-29 11:22:33Not/Timezone', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError BAD_ARGUMENTS } +--leap years and non-leap years +select parseDateTime64InJodaSyntax('2024-02-29 11:23:34America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); +select parseDateTime64InJodaSyntax('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2024-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); +select parseDateTime64InJodaSyntax('2023-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); +select parseDateTime64InJodaSyntax('2024-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); +select parseDateTime64InJodaSyntax('2023-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); -- Test for parseDateTime64InJodaSyntaxOrNull / parseDateTime64InJodaSyntaxOrZero -select parseDateTime64InJodaSyntaxOrNull('2021-01-04 23:12:34.331') IS NULL; -select parseDateTime64InJodaSyntaxOrNull('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SS') IS NULL; -select parseDateTime64InJodaSyntaxOrZero('2021-01-04 23:12:34.331') = toDateTime64('1970-01-01 00:00:00', 0); -select parseDateTime64InJodaSyntaxOrZero('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SS') = toDateTime64('1970-01-01 00:00:00', 0); +select 'parseDateTime64InJodaSyntaxOrNull'; +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntaxOrNull('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); +select parseDateTime64InJodaSyntaxOrNull('', ''); +select parseDateTime64InJodaSyntaxOrNull('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); +select 'parseDateTime64InJodaSyntaxOrZero'; +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntaxOrZero('wrong value', 'yyyy-dd-MM HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntaxOrZero('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); +select parseDateTime64InJodaSyntaxOrZero('', ''); +select parseDateTime64InJodaSyntaxOrZero('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- { echoOff } From 3fb4bf41bc928944ec92e914fbd2b4f048a15f0d Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Thu, 14 Nov 2024 20:02:48 +0800 Subject: [PATCH 19/20] reslove conflict --- .../02668_parse_datetime.reference | 16 +++- .../0_stateless/02668_parse_datetime.sql | 16 ++-- ...68_parse_datetime_in_joda_syntax.reference | 76 ++++++++++++++++-- .../02668_parse_datetime_in_joda_syntax.sql | 80 +++++++++---------- 4 files changed, 129 insertions(+), 59 deletions(-) diff --git a/tests/queries/0_stateless/02668_parse_datetime.reference b/tests/queries/0_stateless/02668_parse_datetime.reference index bd78da75656..1733a9dd4df 100644 --- a/tests/queries/0_stateless/02668_parse_datetime.reference +++ b/tests/queries/0_stateless/02668_parse_datetime.reference @@ -274,15 +274,26 @@ select parseDateTime('08 13, 2022, 07:58:32', '%c %e, %G, %k:%i:%s', 'UTC'); set session_timezone = 'UTC'; -- don't randomize the session timezone select parseDateTime('2021-01-04 23:12:34') = toDateTime('2021-01-04 23:12:34'); 1 --- The following is test of parseDateTime64[OrNull/OrZero] +-- The following is test of parseDateTime64 +select parseDateTime64(''); -- { serverError NOT_ENOUGH_SPACE } +select parseDateTime64('2177-10-09 10:30:10.123'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64('2021-01-04 23:12:34.118112') = toDateTime64('2021-01-04 23:12:34.118112', 6); 1 select parseDateTime64('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('2021-01-04 23:12:34.118112', 6); 1 select parseDateTime64('2021-01-04 23:12:34.118'); -- { serverError NOT_ENOUGH_SPACE } select parseDateTime64('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError NOT_ENOUGH_SPACE } -select parseDateTime64('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s'); -- { serverError CANNOT_PARSE_DATETIME } +--leap years and non-leap years +select parseDateTime64('2024-02-29 11:23:34.123433', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('2024-02-29 11:23:34.123433', 6); +1 +select parseDateTime64('2023-02-29 11:22:33.123433', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64('2024-02-28 23:22:33.123433', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('2024-02-28 23:22:33.123433', 6); +1 +select parseDateTime64('2023-02-28 23:22:33.123433', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('2023-02-28 23:22:33.123433', 6); +1 +-- Test of parseDateTime64OrNull select parseDateTime64OrNull('2021-01-04 23:12:34.118') IS NULL; 1 select parseDateTime64OrNull('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f') IS NULL; @@ -291,6 +302,7 @@ select parseDateTime64OrNull('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s') 1 select parseDateTime64OrNull('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f') IS NULL; 1 +-- Test of parseDateTime64OrZero select parseDateTime64OrZero('2021-01-04 23:12:34.118') = toDateTime64('1970-01-01 00:00:00', 6); 1 select parseDateTime64OrZero('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('1970-01-01 00:00:00', 6); diff --git a/tests/queries/0_stateless/02668_parse_datetime.sql b/tests/queries/0_stateless/02668_parse_datetime.sql index e56418ae5ed..1f80fb4d1d8 100644 --- a/tests/queries/0_stateless/02668_parse_datetime.sql +++ b/tests/queries/0_stateless/02668_parse_datetime.sql @@ -192,12 +192,8 @@ set session_timezone = 'UTC'; -- don't randomize the session timezone select parseDateTime('2021-01-04 23:12:34') = toDateTime('2021-01-04 23:12:34'); -- The following is test of parseDateTime64 -select 'parseDateTime64'; -select parseDateTime64(''); -select parseDateTime64('2177-10-09 10:30:10.123'); -select parseDateTime64('08:01', 'HH:ss'); -select parseDateTime64('2024-01-02', 'yyyy-MM-dd'); -select parseDateTime64('10:30:50', 'HH:mm:ss'); +select parseDateTime64(''); -- { serverError NOT_ENOUGH_SPACE } +select parseDateTime64('2177-10-09 10:30:10.123'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64('2021-01-04 23:12:34.118112') = toDateTime64('2021-01-04 23:12:34.118112', 6); select parseDateTime64('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('2021-01-04 23:12:34.118112', 6); select parseDateTime64('2021-01-04 23:12:34.118'); -- { serverError NOT_ENOUGH_SPACE } @@ -205,18 +201,16 @@ select parseDateTime64('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f'); -- { select parseDateTime64('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s'); -- { serverError CANNOT_PARSE_DATETIME } --leap years and non-leap years -select parseDateTime64('2024-02-29 11:23:34.123433', '%Y-%m-%d %H:%i:%s.%f'); +select parseDateTime64('2024-02-29 11:23:34.123433', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('2024-02-29 11:23:34.123433', 6); select parseDateTime64('2023-02-29 11:22:33.123433', '%Y-%m-%d %H:%i:%s.%f'); -- { serverError CANNOT_PARSE_DATETIME } -select parseDateTime64('2024-02-28 23:22:33.123433', '%Y-%m-%d %H:%i:%s.%f'); -select parseDateTime64('2023-02-28 23:22:33.123433', '%Y-%m-%d %H:%i:%s.%f'); +select parseDateTime64('2024-02-28 23:22:33.123433', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('2024-02-28 23:22:33.123433', 6); +select parseDateTime64('2023-02-28 23:22:33.123433', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('2023-02-28 23:22:33.123433', 6); -- Test of parseDateTime64OrNull -select 'parseDateTime64OrNull'; select parseDateTime64OrNull('2021-01-04 23:12:34.118') IS NULL; select parseDateTime64OrNull('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f') IS NULL; select parseDateTime64OrNull('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s') IS NULL; select parseDateTime64OrNull('2021-01-04 23:12:34.11811235', '%Y-%m-%d %H:%i:%s.%f') IS NULL; -- Test of parseDateTime64OrZero -select 'parseDateTime64OrZero'; select parseDateTime64OrZero('2021-01-04 23:12:34.118') = toDateTime64('1970-01-01 00:00:00', 6); select parseDateTime64OrZero('2021-01-04 23:12:34.118', '%Y-%m-%d %H:%i:%s.%f') = toDateTime64('1970-01-01 00:00:00', 6); select parseDateTime64OrZero('2021-01-04 23:12:34.118112', '%Y-%m-%d %H:%i:%s') = toDateTime64('1970-01-01 00:00:00', 6); diff --git a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference index 585ed1e2718..d2b940562ff 100644 --- a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference +++ b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference @@ -373,6 +373,21 @@ select parseDateTimeInJodaSyntax('2024-10-09 10:30:10EST', 'yyyy-MM-dd HH:mm:ssz select parseDateTimeInJodaSyntax('2024-10-09 10:30:10-8000', 'yyyy-MM-dd HH:mm:ssZ'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTimeInJodaSyntax('2024-10-09 10:30:10ABCD', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError BAD_ARGUMENTS } -- The following is test of parseDateTime64InJodaSyntax[OrNull/OrZero] +select parseDateTime64InJodaSyntax('', '') = toDateTime64('1970-01-01 00:00:00', 0); +1 +select parseDateTime64InJodaSyntax('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('+0000', 'Z') = toDateTime64('1970-01-01 00:00:00', 0); +1 +select parseDateTime64InJodaSyntax('08:01', 'HH:ss') = toDateTime64('1970-01-01 08:00:01', 0); +1 +select parseDateTime64InJodaSyntax('2024-01-02', 'yyyy-MM-dd') = toDateTime64('2024-01-02 00:00:00', 0); +1 +select parseDateTime64InJodaSyntax('10:30:50', 'HH:mm:ss') = toDateTime64('1970-01-01 10:30:50', 0); +1 +select parseDateTime64InJodaSyntax('2024-12-31 23:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2025-01-01 07:30:10.123456', 6); +1 +select parseDateTime64InJodaSyntax('2024-01-01 00:00:01.123456+0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2023-12-31 16:00:01.123456', 6); +1 select parseDateTime64InJodaSyntax('2021-01-04 23:12:34') = toDateTime64('2021-01-04 23:12:34', 0); 1 select parseDateTime64InJodaSyntax('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SSS') = toDateTime64('2021-01-04 23:12:34.331', 3); @@ -392,15 +407,66 @@ select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 'yyyy-MM-dd 1 select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz') = toDateTime64('2024-10-09 15:30:10.123456', 6); 1 +select parseDateTime64InJodaSyntax('2024-11-05-0800 01:02:03.123456', 'yyyy-MM-ddZ HH:mm:ss.SSSSSS') = toDateTime64('2024-11-05 09:02:03.123456', 6); +1 +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 17:30:10.123456', 6); +1 +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456Australia/Adelaide', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 00:00:10.123456', 6); +1 +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS') = toDateTime64('2024-09-10 10:30:10.123', 3); +1 +select parseDateTime64InJodaSyntax('999999 10-09-202410:30:10', 'SSSSSSSSS dd-MM-yyyyHH:mm:ss'); -- {serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0845', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2024-10-09 19:15:10.123456', 6); +1 -- incorrect timezone offset and timezone select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-8000', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456ABCD', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } --- Test for parseDateTime64InJodaSyntaxOrNull / parseDateTime64InJodaSyntaxOrZero -select parseDateTime64InJodaSyntaxOrNull('2021-01-04 23:12:34.331') IS NULL; +select parseDateTime64InJodaSyntax('2023-02-29 11:22:33Not/Timezone', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError BAD_ARGUMENTS } +--leap years and non-leap years +select parseDateTime64InJodaSyntax('2024-02-29 11:23:34America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz') = toDateTime64('2024-02-29 19:23:34', 0); 1 -select parseDateTime64InJodaSyntaxOrNull('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SS') IS NULL; +select parseDateTime64InJodaSyntax('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2024-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz') = toDateTime64('2024-02-29 07:22:33', 0); 1 -select parseDateTime64InJodaSyntaxOrZero('2021-01-04 23:12:34.331') = toDateTime64('1970-01-01 00:00:00', 0); +select parseDateTime64InJodaSyntax('2023-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz') = toDateTime64('2023-03-01 07:22:33', 0); 1 -select parseDateTime64InJodaSyntaxOrZero('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SS') = toDateTime64('1970-01-01 00:00:00', 0); +select parseDateTime64InJodaSyntax('2024-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2023-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); -- { serverError CANNOT_PARSE_DATETIME } +-- Test for parseDateTime64InJodaSyntaxOrNull +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS') = toDateTime64('2024-10-09 10:30:10.123', 3); +1 +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS') = toDateTime64('2024-10-09 10:30:10.123456', 6); +1 +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2024-10-09 18:30:10.123456', 6); +1 +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 17:30:10.123456', 6); +1 +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS') = toDateTime64('2024-09-10 10:30:10.123', 3); +1 +select parseDateTime64InJodaSyntaxOrNull('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz') is NULL; +1 +select parseDateTime64InJodaSyntaxOrNull('', '') = toDateTime64('1970-01-01 00:00:00', 0); +1 +select parseDateTime64InJodaSyntaxOrNull('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS') is NULL; +1 +-- Test for parseDateTime64InJodaSyntaxOrZero +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS') = toDateTime64('2024-10-09 10:30:10.123', 3); +1 +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS') = toDateTime64('2024-10-09 10:30:10.123456', 6); +1 +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2024-10-09 18:30:10.123456', 6); +1 +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 17:30:10.123456', 6); +1 +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS') = toDateTime64('2024-09-10 10:30:10.123', 3); +1 +select parseDateTime64InJodaSyntaxOrZero('wrong value', 'yyyy-dd-MM HH:mm:ss.SSS') = toDateTime64('1970-01-01 00:00:00.000', 3); +1 +select parseDateTime64InJodaSyntaxOrZero('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz') = toDateTime64('1970-01-01 00:00:00', 0); +1 +select parseDateTime64InJodaSyntaxOrZero('', '') = toDateTime64('1970-01-01 00:00:00', 0); +1 +select parseDateTime64InJodaSyntaxOrZero('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS') = toDateTime64('1970-01-01 00:00:00.000', 3); 1 diff --git a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql index a4d5738d91d..a041403139f 100644 --- a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql +++ b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql @@ -256,15 +256,14 @@ select parseDateTimeInJodaSyntax('2024-10-09 10:30:10-8000', 'yyyy-MM-dd HH:mm:s select parseDateTimeInJodaSyntax('2024-10-09 10:30:10ABCD', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError BAD_ARGUMENTS } -- The following is test of parseDateTime64InJodaSyntax[OrNull/OrZero] -select 'parseDateTime64InJodaSyntax'; -select parseDateTime64InJodaSyntax('', ''); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +select parseDateTime64InJodaSyntax('', '') = toDateTime64('1970-01-01 00:00:00', 0); select parseDateTime64InJodaSyntax('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- { serverError CANNOT_PARSE_DATETIME } -select parseDateTime64InJodaSyntax('+0000', 'Z'); -select parseDateTime64InJodaSyntax('08:01', 'HH:ss'); -select parseDateTime64InJodaSyntax('2024-01-02', 'yyyy-MM-dd'); -select parseDateTime64InJodaSyntax('10:30:50', 'HH:mm:ss'); -select parseDateTime64InJodaSyntax('2024-12-31 23:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntax('2024-01-01 00:00:01.123456+0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntax('+0000', 'Z') = toDateTime64('1970-01-01 00:00:00', 0); +select parseDateTime64InJodaSyntax('08:01', 'HH:ss') = toDateTime64('1970-01-01 08:00:01', 0); +select parseDateTime64InJodaSyntax('2024-01-02', 'yyyy-MM-dd') = toDateTime64('2024-01-02 00:00:00', 0); +select parseDateTime64InJodaSyntax('10:30:50', 'HH:mm:ss') = toDateTime64('1970-01-01 10:30:50', 0); +select parseDateTime64InJodaSyntax('2024-12-31 23:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2025-01-01 07:30:10.123456', 6); +select parseDateTime64InJodaSyntax('2024-01-01 00:00:01.123456+0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2023-12-31 16:00:01.123456', 6); select parseDateTime64InJodaSyntax('2021-01-04 23:12:34') = toDateTime64('2021-01-04 23:12:34', 0); select parseDateTime64InJodaSyntax('2021-01-04 23:12:34.331', 'yyyy-MM-dd HH:mm:ss.SSS') = toDateTime64('2021-01-04 23:12:34.331', 3); select parseDateTime64InJodaSyntax('2021/01/04 23:12:34.331', 'yyyy/MM/dd HH:mm:ss.SSS') = toDateTime64('2021-01-04 23:12:34.331', 3); @@ -277,44 +276,43 @@ select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0812', 'yyyy-MM-d select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-08123', 'yyyy-MM-dd HH:mm:ss.SSSSSSZZZ'); -- {serverError CANNOT_PARSE_DATETIME} select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 15:30:10.123456', 6); select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456EST', 'yyyy-MM-dd HH:mm:ss.SSSSSSzzz') = toDateTime64('2024-10-09 15:30:10.123456', 6); -select parseDateTime64InJodaSyntax('2024-11-05-0800 01:02:03.123456', 'yyyy-MM-ddZ HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456Australia/Adelaide', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntax('999999 10-09-202410:30:10', 'SSSSSSSSS dd-MM-yyyyHH:mm:ss'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0845', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntax('2024-11-05-0800 01:02:03.123456', 'yyyy-MM-ddZ HH:mm:ss.SSSSSS') = toDateTime64('2024-11-05 09:02:03.123456', 6); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 17:30:10.123456', 6); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456Australia/Adelaide', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 00:00:10.123456', 6); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS') = toDateTime64('2024-09-10 10:30:10.123', 3); +select parseDateTime64InJodaSyntax('999999 10-09-202410:30:10', 'SSSSSSSSS dd-MM-yyyyHH:mm:ss'); -- {serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0845', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2024-10-09 19:15:10.123456', 6); -- incorrect timezone offset and timezone select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-8000', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456ABCD', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -- { serverError BAD_ARGUMENTS } select parseDateTime64InJodaSyntax('2023-02-29 11:22:33Not/Timezone', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError BAD_ARGUMENTS } --leap years and non-leap years -select parseDateTime64InJodaSyntax('2024-02-29 11:23:34America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); +select parseDateTime64InJodaSyntax('2024-02-29 11:23:34America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz') = toDateTime64('2024-02-29 19:23:34', 0); select parseDateTime64InJodaSyntax('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError CANNOT_PARSE_DATETIME } -select parseDateTime64InJodaSyntax('2024-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -select parseDateTime64InJodaSyntax('2023-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -select parseDateTime64InJodaSyntax('2024-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); -select parseDateTime64InJodaSyntax('2023-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); --- Test for parseDateTime64InJodaSyntaxOrNull / parseDateTime64InJodaSyntaxOrZero -select 'parseDateTime64InJodaSyntaxOrNull'; -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrNull('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -select parseDateTime64InJodaSyntaxOrNull('', ''); -select parseDateTime64InJodaSyntaxOrNull('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -select 'parseDateTime64InJodaSyntaxOrZero'; -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrZero('wrong value', 'yyyy-dd-MM HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrZero('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -select parseDateTime64InJodaSyntaxOrZero('', ''); -select parseDateTime64InJodaSyntaxOrZero('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntax('2024-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz') = toDateTime64('2024-02-29 07:22:33', 0); +select parseDateTime64InJodaSyntax('2023-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz') = toDateTime64('2023-03-01 07:22:33', 0); +select parseDateTime64InJodaSyntax('2024-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2023-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); -- { serverError CANNOT_PARSE_DATETIME } +-- Test for parseDateTime64InJodaSyntaxOrNull +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS') = toDateTime64('2024-10-09 10:30:10.123', 3); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS') = toDateTime64('2024-10-09 10:30:10.123456', 6); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2024-10-09 18:30:10.123456', 6); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 17:30:10.123456', 6); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS') = toDateTime64('2024-09-10 10:30:10.123', 3); +select parseDateTime64InJodaSyntaxOrNull('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz') is NULL; +select parseDateTime64InJodaSyntaxOrNull('', '') = toDateTime64('1970-01-01 00:00:00', 0); +select parseDateTime64InJodaSyntaxOrNull('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS') is NULL; +-- Test for parseDateTime64InJodaSyntaxOrZero +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS') = toDateTime64('2024-10-09 10:30:10.123', 3); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS') = toDateTime64('2024-10-09 10:30:10.123456', 6); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ') = toDateTime64('2024-10-09 18:30:10.123456', 6); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz') = toDateTime64('2024-10-09 17:30:10.123456', 6); +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS') = toDateTime64('2024-09-10 10:30:10.123', 3); +select parseDateTime64InJodaSyntaxOrZero('wrong value', 'yyyy-dd-MM HH:mm:ss.SSS') = toDateTime64('1970-01-01 00:00:00.000', 3); +select parseDateTime64InJodaSyntaxOrZero('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz') = toDateTime64('1970-01-01 00:00:00', 0); +select parseDateTime64InJodaSyntaxOrZero('', '') = toDateTime64('1970-01-01 00:00:00', 0); +select parseDateTime64InJodaSyntaxOrZero('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS') = toDateTime64('1970-01-01 00:00:00.000', 3); -- { echoOff } From 0305f044940f75b2837ec55bc5591dd95a78fa64 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Thu, 14 Nov 2024 20:20:54 +0800 Subject: [PATCH 20/20] delete 03252 parse datetime64 test --- ..._parse_datetime64_in_joda_syntax.reference | 44 --------------- .../03252_parse_datetime64_in_joda_syntax.sql | 54 ------------------- 2 files changed, 98 deletions(-) delete mode 100644 tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference delete mode 100644 tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference deleted file mode 100644 index e4be64155d1..00000000000 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference +++ /dev/null @@ -1,44 +0,0 @@ -parseDateTime64InJodaSyntax -2077-10-09 10:30:10.123 -1970-01-01 08:00:00 -1970-01-01 08:00:01 -2024-01-02 00:00:00 -1970-01-01 10:30:50 -2025-01-01 15:30:10.123456 -2024-01-01 00:00:01.123456 -2024-10-09 10:30:10.123 -2024-10-09 10:30:10.123456 -2024-10-10 02:30:10.123456 -2024-11-05 17:02:03.123456 -2024-10-10 01:30:10.123456 -2024-10-09 08:00:10.123456 -2024-09-10 10:30:10.123 -2024-09-10 10:30:10.000999999 -2024-10-10 03:15:10.123456 -2024-03-01 03:23:34 -2024-02-29 15:22:33 -2023-03-01 15:22:33 -2024-03-04 16:22:33 -2023-03-04 16:22:33 -parseDateTime64InJodaSyntaxOrZero -2024-10-09 10:30:10.123 -2024-10-09 10:30:10.123456 -1970-01-01 08:00:00.000000000 -2024-10-10 02:30:10.123456 -2024-10-10 01:30:10.123456 -2024-09-10 10:30:10.123 -1970-01-01 08:00:00.000 -1970-01-01 08:00:00 -1970-01-01 08:00:00 -1970-01-01 08:00:00.000 -parseDateTime64InJodaSyntaxOrNull -2024-10-09 10:30:10.123 -2024-10-09 10:30:10.123456 -\N -2024-10-10 02:30:10.123456 -2024-10-10 01:30:10.123456 -2024-09-10 10:30:10.123 -\N -\N -\N -1970-01-01 00:00:00 diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql deleted file mode 100644 index 7e024288f1c..00000000000 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql +++ /dev/null @@ -1,54 +0,0 @@ -set session_timezone = 'Asia/Shanghai'; - -select 'parseDateTime64InJodaSyntax'; -select parseDateTime64InJodaSyntax('', ''); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } -select parseDateTime64InJodaSyntax('2077-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntax('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- { serverError CANNOT_PARSE_DATETIME } -select parseDateTime64InJodaSyntax('+0000', 'Z'); -select parseDateTime64InJodaSyntax('08:01', 'HH:ss'); -select parseDateTime64InJodaSyntax('2024-01-02', 'yyyy-MM-dd'); -select parseDateTime64InJodaSyntax('10:30:50', 'HH:mm:ss'); -select parseDateTime64InJodaSyntax('2024-12-31 23:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntax('2024-01-01 00:00:01.123456+0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -- { serverError CANNOT_PARSE_DATETIME } -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntax('2024-11-05-0800 01:02:03.123456', 'yyyy-MM-ddZ HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456Australia/Adelaide', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntax('999999 10-09-202410:30:10', 'SSSSSSSSS dd-MM-yyyyHH:mm:ss'); -select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0845', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntax('2023-02-29 11:22:33Not/Timezone', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError BAD_ARGUMENTS } ---leap years and non-leap years -select parseDateTime64InJodaSyntax('2024-02-29 11:23:34America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -select parseDateTime64InJodaSyntax('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError CANNOT_PARSE_DATETIME } -select parseDateTime64InJodaSyntax('2024-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -select parseDateTime64InJodaSyntax('2023-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -select parseDateTime64InJodaSyntax('2024-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); -select parseDateTime64InJodaSyntax('2023-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); -select 'parseDateTime64InJodaSyntaxOrZero'; -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrZero('wrong value', 'yyyy-dd-MM HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrZero('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -select parseDateTime64InJodaSyntaxOrZero('', ''); -select parseDateTime64InJodaSyntaxOrZero('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -select 'parseDateTime64InJodaSyntaxOrNull'; -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); -select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); -select parseDateTime64InJodaSyntaxOrNull('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -select parseDateTime64InJodaSyntaxOrNull('', ''); -select parseDateTime64InJodaSyntaxOrNull('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); - -set session_timezone = 'UTC'; -select parseDateTime64InJodaSyntax('', '');