mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 07:01:59 +00:00
Merge branch 'master' into decimal
This commit is contained in:
commit
16844012e4
@ -128,6 +128,8 @@ Yu](https://github.com/yuzhichang))
|
||||
* Introduce CustomSeparated data format that supports custom escaping and
|
||||
delimiter rules. [#7118](https://github.com/ClickHouse/ClickHouse/pull/7118)
|
||||
([tavplubix](https://github.com/tavplubix))
|
||||
* Support Redis as source of external dictionary. [#4361](https://github.com/ClickHouse/ClickHouse/pull/4361) [#6962](https://github.com/ClickHouse/ClickHouse/pull/6962) ([comunodi](https://github.com/comunodi), [Anton
|
||||
Popov](https://github.com/CurtizJ))
|
||||
|
||||
### Bug Fix
|
||||
* Fix wrong query result if it has `WHERE IN (SELECT ...)` section and `optimize_read_in_order` is
|
||||
|
@ -11,7 +11,3 @@ ClickHouse is an open-source column-oriented database management system that all
|
||||
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
|
||||
* [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.
|
||||
* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person.
|
||||
|
||||
## Upcoming Events
|
||||
|
||||
* [ClickHouse Meetup in Moscow](https://yandex.ru/promo/clickhouse/moscow-december-2019) on December 11.
|
||||
|
@ -66,6 +66,7 @@ private:
|
||||
friend class COWHelper<ColumnVectorHelper, Self>;
|
||||
|
||||
public:
|
||||
using ValueType = T;
|
||||
using Container = DecimalPaddedPODArray<T>;
|
||||
|
||||
private:
|
||||
|
@ -204,7 +204,7 @@ MutableColumnPtr ColumnVector<T>::cloneResized(size_t size) const
|
||||
memcpy(new_col.data.data(), data.data(), count * sizeof(data[0]));
|
||||
|
||||
if (size > count)
|
||||
memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(value_type()), (size - count) * sizeof(value_type));
|
||||
memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(ValueType()), (size - count) * sizeof(ValueType));
|
||||
}
|
||||
|
||||
return res;
|
||||
|
@ -104,13 +104,13 @@ private:
|
||||
struct greater;
|
||||
|
||||
public:
|
||||
using value_type = T;
|
||||
using Container = PaddedPODArray<value_type>;
|
||||
using ValueType = T;
|
||||
using Container = PaddedPODArray<ValueType>;
|
||||
|
||||
private:
|
||||
ColumnVector() {}
|
||||
ColumnVector(const size_t n) : data(n) {}
|
||||
ColumnVector(const size_t n, const value_type x) : data(n, x) {}
|
||||
ColumnVector(const size_t n, const ValueType x) : data(n, x) {}
|
||||
ColumnVector(const ColumnVector & src) : data(src.data.begin(), src.data.end()) {}
|
||||
|
||||
/// Sugar constructor.
|
||||
|
@ -116,7 +116,7 @@ namespace
|
||||
return (*state.saved_hash_column)[index];
|
||||
else
|
||||
{
|
||||
using ValueType = typename ColumnType::value_type;
|
||||
using ValueType = typename ColumnType::ValueType;
|
||||
ValueType value = unalignedLoad<ValueType>(state.index_column->getDataAt(index).data);
|
||||
return DefaultHash<ValueType>()(value);
|
||||
}
|
||||
@ -367,7 +367,7 @@ private:
|
||||
{
|
||||
if constexpr (is_numeric_column)
|
||||
{
|
||||
using ValueType = typename ColumnType::value_type;
|
||||
using ValueType = typename ColumnType::ValueType;
|
||||
ValueType value = unalignedLoad<ValueType>(ref.data);
|
||||
return DefaultHash<ValueType>()(value);
|
||||
}
|
||||
|
@ -14,6 +14,15 @@ String quoteString(const StringRef & x)
|
||||
}
|
||||
|
||||
|
||||
String doubleQuoteString(const StringRef & x)
|
||||
{
|
||||
String res(x.size, '\0');
|
||||
WriteBufferFromString wb(res);
|
||||
writeDoubleQuotedString(x, wb);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
String backQuote(const StringRef & x)
|
||||
{
|
||||
String res(x.size, '\0');
|
||||
|
@ -9,6 +9,9 @@ namespace DB
|
||||
/// Quote the string.
|
||||
String quoteString(const StringRef & x);
|
||||
|
||||
/// Double quote the string.
|
||||
String doubleQuoteString(const StringRef & x);
|
||||
|
||||
/// Quote the identifier with backquotes.
|
||||
String backQuote(const StringRef & x);
|
||||
|
||||
|
@ -4,6 +4,7 @@
|
||||
#include <Core/Block.h>
|
||||
#include <Core/AccurateComparison.h>
|
||||
#include <Core/callOnTypeIndex.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
@ -22,12 +23,12 @@ namespace ErrorCodes
|
||||
///
|
||||
inline bool allowDecimalComparison(const DataTypePtr & left_type, const DataTypePtr & right_type)
|
||||
{
|
||||
if (isDecimal(left_type))
|
||||
if (isColumnedAsDecimal(left_type))
|
||||
{
|
||||
if (isDecimal(right_type) || isNotDecimalButComparableToDecimal(right_type))
|
||||
if (isColumnedAsDecimal(right_type) || isNotDecimalButComparableToDecimal(right_type))
|
||||
return true;
|
||||
}
|
||||
else if (isNotDecimalButComparableToDecimal(left_type) && isDecimal(right_type))
|
||||
else if (isNotDecimalButComparableToDecimal(left_type) && isColumnedAsDecimal(right_type))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
@ -82,7 +83,7 @@ public:
|
||||
|
||||
static bool compare(A a, B b, UInt32 scale_a, UInt32 scale_b)
|
||||
{
|
||||
static const UInt32 max_scale = maxDecimalPrecision<Decimal128>();
|
||||
static const UInt32 max_scale = DecimalUtils::maxPrecision<Decimal128>();
|
||||
if (scale_a > max_scale || scale_b > max_scale)
|
||||
throw Exception("Bad scale of decimal field", ErrorCodes::DECIMAL_OVERFLOW);
|
||||
|
||||
|
142
dbms/src/Core/DecimalFunctions.h
Normal file
142
dbms/src/Core/DecimalFunctions.h
Normal file
@ -0,0 +1,142 @@
|
||||
#pragma once
|
||||
// Moved Decimal-related functions out from Core/Types.h to reduce compilation time.
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <Common/intExp.h>
|
||||
|
||||
#include <limits>
|
||||
|
||||
class DateLUTImpl;
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace DecimalUtils
|
||||
{
|
||||
|
||||
static constexpr size_t minPrecision() { return 1; }
|
||||
template <typename T> static constexpr size_t maxPrecision() { return 0; }
|
||||
template <> constexpr size_t maxPrecision<Decimal32>() { return 9; }
|
||||
template <> constexpr size_t maxPrecision<Decimal64>() { return 18; }
|
||||
template <> constexpr size_t maxPrecision<Decimal128>() { return 38; }
|
||||
|
||||
template <typename T> T scaleMultiplier(UInt32 scale);
|
||||
template <> inline Int32 scaleMultiplier<Int32>(UInt32 scale) { return common::exp10_i32(scale); }
|
||||
template <> inline Int64 scaleMultiplier<Int64>(UInt32 scale) { return common::exp10_i64(scale); }
|
||||
template <> inline Int128 scaleMultiplier<Int128>(UInt32 scale) { return common::exp10_i128(scale); }
|
||||
|
||||
/** Components of DecimalX value:
|
||||
* whole - represents whole part of decimal, can be negatve or positive.
|
||||
* fractional - for fractional part of decimal, always positive.
|
||||
*/
|
||||
template <typename T>
|
||||
struct DecimalComponents
|
||||
{
|
||||
T whole;
|
||||
T fractional;
|
||||
};
|
||||
|
||||
/** Make a decimal value from whole and fractional components with given scale multiplier.
|
||||
* where scale_multiplier = scaleMultiplier<T>(scale)
|
||||
* this is to reduce number of calls to scaleMultiplier when scale is known.
|
||||
*
|
||||
* Sign of `whole` controls sign of result: negative whole => negative result, positive whole => positive result.
|
||||
* Sign of `fractional` is expected to be positive, otherwise result is undefined.
|
||||
* If `scale` is to big (scale > maxPrecision<DecimalType::NativeType>), result is undefined.
|
||||
*/
|
||||
template <typename DecimalType>
|
||||
DecimalType decimalFromComponentsWithMultiplier(const typename DecimalType::NativeType & whole,
|
||||
const typename DecimalType::NativeType & fractional,
|
||||
typename DecimalType::NativeType scale_multiplier)
|
||||
{
|
||||
using T = typename DecimalType::NativeType;
|
||||
const auto fractional_sign = whole < 0 ? -1 : 1;
|
||||
|
||||
const T value = whole * scale_multiplier + fractional_sign * (fractional % scale_multiplier);
|
||||
return DecimalType(value);
|
||||
}
|
||||
|
||||
/** Make a decimal value from whole and fractional components with given scale.
|
||||
*
|
||||
* @see `decimalFromComponentsWithMultiplier` for details.
|
||||
*/
|
||||
template <typename DecimalType>
|
||||
DecimalType decimalFromComponents(const typename DecimalType::NativeType & whole, const typename DecimalType::NativeType & fractional, UInt32 scale)
|
||||
{
|
||||
using T = typename DecimalType::NativeType;
|
||||
|
||||
return decimalFromComponentsWithMultiplier<DecimalType>(whole, fractional, scaleMultiplier<T>(scale));
|
||||
}
|
||||
|
||||
/** Make a decimal value from whole and fractional components with given scale.
|
||||
* @see `decimalFromComponentsWithMultiplier` for details.
|
||||
*/
|
||||
template <typename DecimalType>
|
||||
DecimalType decimalFromComponents(const DecimalComponents<typename DecimalType::NativeType> & components, UInt32 scale)
|
||||
{
|
||||
return decimalFromComponents<DecimalType>(components.whole, components.fractional, scale);
|
||||
}
|
||||
|
||||
/** Split decimal into whole and fractional parts with given scale_multiplier.
|
||||
* This is an optimization to reduce number of calls to scaleMultiplier on known scale.
|
||||
*/
|
||||
template <typename DecimalType>
|
||||
DecimalComponents<typename DecimalType::NativeType> splitWithScaleMultiplier(const DecimalType & decimal, typename DecimalType::NativeType scale_multiplier)
|
||||
{
|
||||
using T = typename DecimalType::NativeType;
|
||||
const auto whole = decimal.value / scale_multiplier;
|
||||
auto fractional = decimal.value % scale_multiplier;
|
||||
if (fractional < T(0))
|
||||
fractional *= T(-1);
|
||||
|
||||
return {whole, fractional};
|
||||
}
|
||||
|
||||
/// Split decimal into components: whole and fractional part, @see `DecimalComponents` for details.
|
||||
template <typename DecimalType>
|
||||
DecimalComponents<typename DecimalType::NativeType> split(const DecimalType & decimal, UInt32 scale)
|
||||
{
|
||||
if (scale == 0)
|
||||
{
|
||||
return {decimal.value, 0};
|
||||
}
|
||||
return splitWithScaleMultiplier(decimal, scaleMultiplier<typename DecimalType::NativeType>(scale));
|
||||
}
|
||||
|
||||
/** Get whole part from decimal.
|
||||
*
|
||||
* Sign of result follows sign of `decimal` value.
|
||||
* If scale is to big, result is undefined.
|
||||
*/
|
||||
template <typename DecimalType>
|
||||
typename DecimalType::NativeType getWholePart(const DecimalType & decimal, size_t scale)
|
||||
{
|
||||
if (scale == 0)
|
||||
return decimal.value;
|
||||
|
||||
return decimal.value / scaleMultiplier<typename DecimalType::NativeType>(scale);
|
||||
}
|
||||
|
||||
/** Get fractional part from decimal
|
||||
*
|
||||
* Result is always positive.
|
||||
* If scale is to big, result is undefined.
|
||||
*/
|
||||
template <typename DecimalType>
|
||||
typename DecimalType::NativeType getFractionalPart(const DecimalType & decimal, size_t scale)
|
||||
{
|
||||
using T = typename DecimalType::NativeType;
|
||||
|
||||
if (scale == 0)
|
||||
return 0;
|
||||
|
||||
T result = decimal.value;
|
||||
if (result < T(0))
|
||||
result *= T(-1);
|
||||
|
||||
return result % scaleMultiplier<T>(scale);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
@ -31,6 +31,8 @@ enum class TypeIndex
|
||||
Float64,
|
||||
Date,
|
||||
DateTime,
|
||||
DateTime32 = DateTime,
|
||||
DateTime64,
|
||||
String,
|
||||
FixedString,
|
||||
Enum8,
|
||||
@ -151,10 +153,15 @@ struct Decimal
|
||||
T value;
|
||||
};
|
||||
|
||||
|
||||
using Decimal32 = Decimal<Int32>;
|
||||
using Decimal64 = Decimal<Int64>;
|
||||
using Decimal128 = Decimal<Int128>;
|
||||
|
||||
// TODO (nemkov): consider making a strong typedef
|
||||
//using DateTime32 = time_t;
|
||||
using DateTime64 = Decimal64;
|
||||
|
||||
template <> struct TypeName<Decimal32> { static const char * get() { return "Decimal32"; } };
|
||||
template <> struct TypeName<Decimal64> { static const char * get() { return "Decimal64"; } };
|
||||
template <> struct TypeName<Decimal128> { static const char * get() { return "Decimal128"; } };
|
||||
@ -196,6 +203,7 @@ inline const char * getTypeName(TypeIndex idx)
|
||||
case TypeIndex::Float64: return TypeName<Float64>::get();
|
||||
case TypeIndex::Date: return "Date";
|
||||
case TypeIndex::DateTime: return "DateTime";
|
||||
case TypeIndex::DateTime64: return "DateTime64";
|
||||
case TypeIndex::String: return TypeName<String>::get();
|
||||
case TypeIndex::FixedString: return "FixedString";
|
||||
case TypeIndex::Enum8: return "Enum8";
|
||||
|
@ -71,6 +71,7 @@ bool callOnBasicType(TypeIndex number, F && f)
|
||||
{
|
||||
case TypeIndex::Date: return f(TypePair<T, UInt16>());
|
||||
case TypeIndex::DateTime: return f(TypePair<T, UInt32>());
|
||||
case TypeIndex::DateTime64: return f(TypePair<T, DateTime64>());
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@ -135,6 +136,7 @@ inline bool callOnBasicTypes(TypeIndex type_num1, TypeIndex type_num2, F && f)
|
||||
{
|
||||
case TypeIndex::Date: return callOnBasicType<UInt16, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::DateTime: return callOnBasicType<UInt32, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::DateTime64: return callOnBasicType<Decimal64, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@ -145,10 +147,11 @@ inline bool callOnBasicTypes(TypeIndex type_num1, TypeIndex type_num2, F && f)
|
||||
|
||||
|
||||
class DataTypeDate;
|
||||
class DataTypeDateTime;
|
||||
class DataTypeString;
|
||||
class DataTypeFixedString;
|
||||
class DataTypeUUID;
|
||||
class DataTypeDateTime;
|
||||
class DataTypeDateTime64;
|
||||
template <typename T> class DataTypeEnum;
|
||||
template <typename T> class DataTypeNumber;
|
||||
template <typename T> class DataTypeDecimal;
|
||||
@ -178,6 +181,7 @@ bool callOnIndexAndDataType(TypeIndex number, F && f)
|
||||
|
||||
case TypeIndex::Date: return f(TypePair<DataTypeDate, T>());
|
||||
case TypeIndex::DateTime: return f(TypePair<DataTypeDateTime, T>());
|
||||
case TypeIndex::DateTime64: return f(TypePair<DataTypeDateTime64, T>());
|
||||
|
||||
case TypeIndex::String: return f(TypePair<DataTypeString, T>());
|
||||
case TypeIndex::FixedString: return f(TypePair<DataTypeFixedString, T>());
|
||||
|
171
dbms/src/Core/tests/gtest_DecimalFunctions.cpp
Normal file
171
dbms/src/Core/tests/gtest_DecimalFunctions.cpp
Normal file
@ -0,0 +1,171 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <Core/DecimalFunctions.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
struct DecimalUtilsSplitAndCombineTestParam
|
||||
{
|
||||
const char * description;
|
||||
|
||||
Decimal64 decimal_value;
|
||||
UInt8 scale;
|
||||
|
||||
DecimalUtils::DecimalComponents<typename Decimal64::NativeType> components;
|
||||
};
|
||||
|
||||
std::ostream & operator << (std::ostream & ostr, const DecimalUtilsSplitAndCombineTestParam & param)
|
||||
{
|
||||
return ostr << param.description;
|
||||
}
|
||||
|
||||
class DecimalUtilsSplitAndCombineTest : public ::testing::TestWithParam<DecimalUtilsSplitAndCombineTestParam>
|
||||
{};
|
||||
|
||||
template <typename DecimalType>
|
||||
void testSplit(const DecimalUtilsSplitAndCombineTestParam & param)
|
||||
{
|
||||
const DecimalType decimal_value = param.decimal_value;
|
||||
const auto & actual_components = DecimalUtils::split(decimal_value, param.scale);
|
||||
|
||||
EXPECT_EQ(param.components.whole, actual_components.whole);
|
||||
EXPECT_EQ(param.components.fractional, actual_components.fractional);
|
||||
}
|
||||
|
||||
template <typename DecimalType>
|
||||
void testDecimalFromComponents(const DecimalUtilsSplitAndCombineTestParam & param)
|
||||
{
|
||||
EXPECT_EQ(param.decimal_value,
|
||||
DecimalUtils::decimalFromComponents<DecimalType>(param.components.whole, param.components.fractional, param.scale));
|
||||
}
|
||||
|
||||
template <typename DecimalType>
|
||||
void testGetWhole(const DecimalUtilsSplitAndCombineTestParam & param)
|
||||
{
|
||||
EXPECT_EQ(param.components.whole,
|
||||
DecimalUtils::getWholePart(DecimalType{param.decimal_value}, param.scale));
|
||||
}
|
||||
|
||||
template <typename DecimalType>
|
||||
void testGetFractional(const DecimalUtilsSplitAndCombineTestParam & param)
|
||||
{
|
||||
EXPECT_EQ(param.components.fractional,
|
||||
DecimalUtils::getFractionalPart(DecimalType{param.decimal_value}, param.scale));
|
||||
}
|
||||
|
||||
// unfortunatelly typed parametrized tests () are not supported in this version of gtest, so I have to emulate by hand.
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, split_Decimal32)
|
||||
{
|
||||
testSplit<Decimal32>(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, split_Decimal64)
|
||||
{
|
||||
testSplit<Decimal64>(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, split_Decimal128)
|
||||
{
|
||||
testSplit<Decimal128>(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, combine_Decimal32)
|
||||
{
|
||||
testDecimalFromComponents<Decimal32>(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, combine_Decimal64)
|
||||
{
|
||||
testDecimalFromComponents<Decimal64>(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, combine_Decimal128)
|
||||
{
|
||||
testDecimalFromComponents<Decimal64>(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, getWholePart_Decimal32)
|
||||
{
|
||||
testGetWhole<Decimal32>(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, getWholePart_Decimal64)
|
||||
{
|
||||
testGetWhole<Decimal64>(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, getWholePart_Decimal128)
|
||||
{
|
||||
testGetWhole<Decimal128>(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPart_Decimal32)
|
||||
{
|
||||
testGetFractional<Decimal32>(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPart_Decimal64)
|
||||
{
|
||||
testGetFractional<Decimal64>(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPart_Decimal128)
|
||||
{
|
||||
testGetFractional<Decimal128>(GetParam());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Intentionally small values that fit into 32-bit in order to cover Decimal32, Decimal64 and Decimal128 with single set of data.
|
||||
INSTANTIATE_TEST_CASE_P(Basic,
|
||||
DecimalUtilsSplitAndCombineTest,
|
||||
::testing::ValuesIn(std::initializer_list<DecimalUtilsSplitAndCombineTestParam>{
|
||||
{
|
||||
"Positive value with non-zero scale, whole, and fractional parts.",
|
||||
1234567'89,
|
||||
2,
|
||||
{
|
||||
1234567,
|
||||
89
|
||||
}
|
||||
},
|
||||
{
|
||||
"When scale is 0, fractional part is 0.",
|
||||
1234567'89,
|
||||
0,
|
||||
{
|
||||
123456789,
|
||||
0
|
||||
}
|
||||
},
|
||||
{
|
||||
"When scale is not 0 and fractional part is 0.",
|
||||
1234567'00,
|
||||
2,
|
||||
{
|
||||
1234567,
|
||||
0
|
||||
}
|
||||
},
|
||||
{
|
||||
"When scale is not 0 and whole part is 0.",
|
||||
123,
|
||||
3,
|
||||
{
|
||||
0,
|
||||
123
|
||||
}
|
||||
},
|
||||
{
|
||||
"For negative Decimal value whole part is negative, fractional is non-negative.",
|
||||
-1234567'89,
|
||||
2,
|
||||
{
|
||||
-1234567,
|
||||
89
|
||||
}
|
||||
}
|
||||
}
|
||||
),);
|
@ -14,6 +14,7 @@
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeEnum.h>
|
||||
#include <DataTypes/DataTypeUUID.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
|
@ -85,6 +85,8 @@ Block TTLBlockInputStream::readImpl()
|
||||
|
||||
removeValuesWithExpiredColumnTTL(block);
|
||||
|
||||
updateMovesTTL(block);
|
||||
|
||||
return block;
|
||||
}
|
||||
|
||||
@ -145,7 +147,8 @@ void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
|
||||
defaults_expression->execute(block_with_defaults);
|
||||
}
|
||||
|
||||
for (const auto & [name, ttl_entry] : storage.ttl_entries_by_name)
|
||||
std::vector<String> columns_to_remove;
|
||||
for (const auto & [name, ttl_entry] : storage.column_ttl_entries_by_name)
|
||||
{
|
||||
const auto & old_ttl_info = old_ttl_infos.columns_ttl[name];
|
||||
auto & new_ttl_info = new_ttl_infos.columns_ttl[name];
|
||||
@ -159,7 +162,10 @@ void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
|
||||
continue;
|
||||
|
||||
if (!block.has(ttl_entry.result_column))
|
||||
{
|
||||
columns_to_remove.push_back(ttl_entry.result_column);
|
||||
ttl_entry.expression->execute(block);
|
||||
}
|
||||
|
||||
ColumnPtr default_column = nullptr;
|
||||
if (block_with_defaults.has(name))
|
||||
@ -192,9 +198,34 @@ void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
|
||||
column_with_type.column = std::move(result_column);
|
||||
}
|
||||
|
||||
for (const auto & elem : storage.ttl_entries_by_name)
|
||||
if (block.has(elem.second.result_column))
|
||||
block.erase(elem.second.result_column);
|
||||
for (const String & column : columns_to_remove)
|
||||
block.erase(column);
|
||||
}
|
||||
|
||||
void TTLBlockInputStream::updateMovesTTL(Block & block)
|
||||
{
|
||||
std::vector<String> columns_to_remove;
|
||||
for (const auto & ttl_entry : storage.move_ttl_entries)
|
||||
{
|
||||
auto & new_ttl_info = new_ttl_infos.moves_ttl[ttl_entry.result_column];
|
||||
|
||||
if (!block.has(ttl_entry.result_column))
|
||||
{
|
||||
columns_to_remove.push_back(ttl_entry.result_column);
|
||||
ttl_entry.expression->execute(block);
|
||||
}
|
||||
|
||||
const IColumn * ttl_column = block.getByName(ttl_entry.result_column).column.get();
|
||||
|
||||
for (size_t i = 0; i < block.rows(); ++i)
|
||||
{
|
||||
UInt32 cur_ttl = getTimestampByIndex(ttl_column, i);
|
||||
new_ttl_info.update(cur_ttl);
|
||||
}
|
||||
}
|
||||
|
||||
for (const String & column : columns_to_remove)
|
||||
block.erase(column);
|
||||
}
|
||||
|
||||
UInt32 TTLBlockInputStream::getTimestampByIndex(const IColumn * column, size_t ind)
|
||||
|
@ -58,6 +58,9 @@ private:
|
||||
/// Removes rows with expired table ttl and computes new ttl_infos for part
|
||||
void removeRowsWithExpiredTableTTL(Block & block);
|
||||
|
||||
/// Updates TTL for moves
|
||||
void updateMovesTTL(Block & block);
|
||||
|
||||
UInt32 getTimestampByIndex(const IColumn * column, size_t ind);
|
||||
bool isTTLExpired(time_t ttl);
|
||||
};
|
||||
|
@ -9,8 +9,10 @@ namespace DB
|
||||
class DataTypeDate final : public DataTypeNumberBase<UInt16>
|
||||
{
|
||||
public:
|
||||
static constexpr auto family_name = "Date";
|
||||
|
||||
TypeIndex getTypeId() const override { return TypeIndex::Date; }
|
||||
const char * getFamilyName() const override { return "Date"; }
|
||||
const char * getFamilyName() const override { return family_name; }
|
||||
|
||||
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
|
@ -1,54 +1,25 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/parseDateTimeBestEffort.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
|
||||
#include <common/DateLUT.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <common/DateLUT.h>
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
#include <Formats/FormatSettings.h>
|
||||
#include <Formats/ProtobufReader.h>
|
||||
#include <Formats/ProtobufWriter.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/Operators.h>
|
||||
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/parseDateTimeBestEffort.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
namespace
|
||||
{
|
||||
|
||||
DataTypeDateTime::DataTypeDateTime(const std::string & time_zone_name)
|
||||
: has_explicit_time_zone(!time_zone_name.empty()),
|
||||
time_zone(DateLUT::instance(time_zone_name)),
|
||||
utc_time_zone(DateLUT::instance("UTC"))
|
||||
{
|
||||
}
|
||||
|
||||
std::string DataTypeDateTime::doGetName() const
|
||||
{
|
||||
if (!has_explicit_time_zone)
|
||||
return "DateTime";
|
||||
|
||||
WriteBufferFromOwnString out;
|
||||
out << "DateTime(" << quote << time_zone.getTimeZone() << ")";
|
||||
return out.str();
|
||||
}
|
||||
|
||||
void DataTypeDateTime::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
|
||||
{
|
||||
writeDateTimeText(assert_cast<const ColumnUInt32 &>(column).getData()[row_num], ostr, time_zone);
|
||||
}
|
||||
|
||||
void DataTypeDateTime::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
serializeText(column, row_num, ostr, settings);
|
||||
}
|
||||
|
||||
|
||||
using namespace DB;
|
||||
static inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone)
|
||||
{
|
||||
switch (settings.date_time_input_format)
|
||||
@ -61,7 +32,45 @@ static inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
TimezoneMixin::TimezoneMixin(const String & time_zone_name)
|
||||
: has_explicit_time_zone(!time_zone_name.empty()),
|
||||
time_zone(DateLUT::instance(time_zone_name)),
|
||||
utc_time_zone(DateLUT::instance("UTC"))
|
||||
{}
|
||||
|
||||
DataTypeDateTime::DataTypeDateTime(const String & time_zone_name)
|
||||
: TimezoneMixin(time_zone_name)
|
||||
{
|
||||
}
|
||||
|
||||
DataTypeDateTime::DataTypeDateTime(const TimezoneMixin & time_zone_)
|
||||
: TimezoneMixin(time_zone_)
|
||||
{}
|
||||
|
||||
String DataTypeDateTime::doGetName() const
|
||||
{
|
||||
if (!has_explicit_time_zone)
|
||||
return "DateTime";
|
||||
|
||||
WriteBufferFromOwnString out;
|
||||
out << "DateTime(" << quote << time_zone.getTimeZone() << ")";
|
||||
return out.str();
|
||||
}
|
||||
|
||||
void DataTypeDateTime::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
|
||||
{
|
||||
writeDateTimeText(assert_cast<const ColumnType &>(column).getData()[row_num], ostr, time_zone);
|
||||
}
|
||||
|
||||
void DataTypeDateTime::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
serializeText(column, row_num, ostr, settings);
|
||||
}
|
||||
|
||||
void DataTypeDateTime::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
@ -71,8 +80,8 @@ void DataTypeDateTime::deserializeWholeText(IColumn & column, ReadBuffer & istr,
|
||||
void DataTypeDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
time_t x;
|
||||
readText(x, istr, settings, time_zone, utc_time_zone);
|
||||
assert_cast<ColumnUInt32 &>(column).getData().push_back(x);
|
||||
::readText(x, istr, settings, time_zone, utc_time_zone);
|
||||
assert_cast<ColumnType &>(column).getData().push_back(x);
|
||||
}
|
||||
|
||||
void DataTypeDateTime::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
@ -87,14 +96,14 @@ void DataTypeDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr
|
||||
time_t x;
|
||||
if (checkChar('\'', istr)) /// Cases: '2017-08-31 18:36:48' or '1504193808'
|
||||
{
|
||||
readText(x, istr, settings, time_zone, utc_time_zone);
|
||||
::readText(x, istr, settings, time_zone, utc_time_zone);
|
||||
assertChar('\'', istr);
|
||||
}
|
||||
else /// Just 1504193808 or 01504193808
|
||||
{
|
||||
readIntText(x, istr);
|
||||
}
|
||||
assert_cast<ColumnUInt32 &>(column).getData().push_back(x); /// It's important to do this at the end - for exception safety.
|
||||
assert_cast<ColumnType &>(column).getData().push_back(x); /// It's important to do this at the end - for exception safety.
|
||||
}
|
||||
|
||||
void DataTypeDateTime::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
@ -109,14 +118,14 @@ void DataTypeDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr,
|
||||
time_t x;
|
||||
if (checkChar('"', istr))
|
||||
{
|
||||
readText(x, istr, settings, time_zone, utc_time_zone);
|
||||
::readText(x, istr, settings, time_zone, utc_time_zone);
|
||||
assertChar('"', istr);
|
||||
}
|
||||
else
|
||||
{
|
||||
readIntText(x, istr);
|
||||
}
|
||||
assert_cast<ColumnUInt32 &>(column).getData().push_back(x);
|
||||
assert_cast<ColumnType &>(column).getData().push_back(x);
|
||||
}
|
||||
|
||||
void DataTypeDateTime::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
@ -138,19 +147,21 @@ void DataTypeDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, c
|
||||
if (maybe_quote == '\'' || maybe_quote == '\"')
|
||||
++istr.position();
|
||||
|
||||
readText(x, istr, settings, time_zone, utc_time_zone);
|
||||
::readText(x, istr, settings, time_zone, utc_time_zone);
|
||||
|
||||
if (maybe_quote == '\'' || maybe_quote == '\"')
|
||||
assertChar(maybe_quote, istr);
|
||||
|
||||
assert_cast<ColumnUInt32 &>(column).getData().push_back(x);
|
||||
assert_cast<ColumnType &>(column).getData().push_back(x);
|
||||
}
|
||||
|
||||
void DataTypeDateTime::serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const
|
||||
{
|
||||
if (value_index)
|
||||
return;
|
||||
value_index = static_cast<bool>(protobuf.writeDateTime(assert_cast<const ColumnUInt32 &>(column).getData()[row_num]));
|
||||
|
||||
// On some platforms `time_t` is `long` but not `unsigned int` (UInt32 that we store in column), hence static_cast.
|
||||
value_index = static_cast<bool>(protobuf.writeDateTime(static_cast<time_t>(assert_cast<const ColumnType &>(column).getData()[row_num])));
|
||||
}
|
||||
|
||||
void DataTypeDateTime::deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const
|
||||
@ -160,7 +171,7 @@ void DataTypeDateTime::deserializeProtobuf(IColumn & column, ProtobufReader & pr
|
||||
if (!protobuf.readDateTime(t))
|
||||
return;
|
||||
|
||||
auto & container = assert_cast<ColumnUInt32 &>(column).getData();
|
||||
auto & container = assert_cast<ColumnType &>(column).getData();
|
||||
if (allow_add_row)
|
||||
{
|
||||
container.emplace_back(t);
|
||||
@ -177,7 +188,6 @@ bool DataTypeDateTime::equals(const IDataType & rhs) const
|
||||
return typeid(rhs) == typeid(*this);
|
||||
}
|
||||
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
@ -205,5 +215,4 @@ void registerDataTypeDateTime(DataTypeFactory & factory)
|
||||
factory.registerAlias("TIMESTAMP", "DateTime", DataTypeFactory::CaseInsensitive);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
@ -1,13 +1,31 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <DataTypes/DataTypeNumberBase.h>
|
||||
|
||||
|
||||
class DateLUTImpl;
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** Mixin-class that manages timezone info for timezone-aware DateTime implementations
|
||||
*
|
||||
* Must be used as a (second) base for class implementing IDateType-interface.
|
||||
*/
|
||||
class TimezoneMixin
|
||||
{
|
||||
public:
|
||||
explicit TimezoneMixin(const String & time_zone_name = "");
|
||||
TimezoneMixin(const TimezoneMixin &) = default;
|
||||
|
||||
const DateLUTImpl & getTimeZone() const { return time_zone; }
|
||||
|
||||
protected:
|
||||
bool has_explicit_time_zone;
|
||||
const DateLUTImpl & time_zone;
|
||||
const DateLUTImpl & utc_time_zone;
|
||||
};
|
||||
|
||||
/** DateTime stores time as unix timestamp.
|
||||
* The value itself is independent of time zone.
|
||||
*
|
||||
@ -15,7 +33,7 @@ namespace DB
|
||||
* In text format it is serialized to and parsed from YYYY-MM-DD hh:mm:ss format.
|
||||
* The text format is dependent of time zone.
|
||||
*
|
||||
* To convert from/to text format, time zone may be specified explicitly or implicit time zone may be used.
|
||||
* To cast from/to text format, time zone may be specified explicitly or implicit time zone may be used.
|
||||
*
|
||||
* Time zone may be specified explicitly as type parameter, example: DateTime('Europe/Moscow').
|
||||
* As it does not affect the internal representation of values,
|
||||
@ -28,13 +46,16 @@ namespace DB
|
||||
* Server time zone is the time zone specified in 'timezone' parameter in configuration file,
|
||||
* or system time zone at the moment of server startup.
|
||||
*/
|
||||
class DataTypeDateTime final : public DataTypeNumberBase<UInt32>
|
||||
class DataTypeDateTime final : public DataTypeNumberBase<UInt32>, public TimezoneMixin
|
||||
{
|
||||
public:
|
||||
DataTypeDateTime(const std::string & time_zone_name = "");
|
||||
explicit DataTypeDateTime(const String & time_zone_name = "");
|
||||
explicit DataTypeDateTime(const TimezoneMixin & time_zone);
|
||||
|
||||
const char * getFamilyName() const override { return "DateTime"; }
|
||||
std::string doGetName() const override;
|
||||
static constexpr auto family_name = "DateTime";
|
||||
|
||||
const char * getFamilyName() const override { return family_name; }
|
||||
String doGetName() const override;
|
||||
TypeIndex getTypeId() const override { return TypeIndex::DateTime; }
|
||||
|
||||
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
@ -54,13 +75,7 @@ public:
|
||||
bool canBeInsideNullable() const override { return true; }
|
||||
|
||||
bool equals(const IDataType & rhs) const override;
|
||||
|
||||
const DateLUTImpl & getTimeZone() const { return time_zone; }
|
||||
|
||||
private:
|
||||
bool has_explicit_time_zone;
|
||||
const DateLUTImpl & time_zone;
|
||||
const DateLUTImpl & utc_time_zone;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
252
dbms/src/DataTypes/DataTypeDateTime64.cpp
Normal file
252
dbms/src/DataTypes/DataTypeDateTime64.cpp
Normal file
@ -0,0 +1,252 @@
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <common/DateLUT.h>
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
#include <Formats/FormatSettings.h>
|
||||
#include <Formats/ProtobufReader.h>
|
||||
#include <Formats/ProtobufWriter.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/parseDateTimeBestEffort.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
DataTypeDateTime64::DataTypeDateTime64(UInt32 scale_, const std::string & time_zone_name)
|
||||
: DataTypeDecimalBase<DateTime64>(DecimalUtils::maxPrecision<DateTime64>(), scale_),
|
||||
TimezoneMixin(time_zone_name)
|
||||
{
|
||||
}
|
||||
|
||||
DataTypeDateTime64::DataTypeDateTime64(UInt32 scale_, const TimezoneMixin & time_zone_info)
|
||||
: DataTypeDecimalBase<DateTime64>(DecimalUtils::maxPrecision<DateTime64>() - scale_, scale_),
|
||||
TimezoneMixin(time_zone_info)
|
||||
{}
|
||||
|
||||
std::string DataTypeDateTime64::doGetName() const
|
||||
{
|
||||
if (!has_explicit_time_zone)
|
||||
return std::string(getFamilyName()) + "(" + std::to_string(this->scale) + ")";
|
||||
|
||||
WriteBufferFromOwnString out;
|
||||
out << "DateTime64(" << this->scale << ", " << quote << time_zone.getTimeZone() << ")";
|
||||
return out.str();
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & /*settings*/) const
|
||||
{
|
||||
writeDateTimeText(assert_cast<const ColumnType &>(column).getData()[row_num], scale, ostr, time_zone);
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
|
||||
{
|
||||
DateTime64 result = 0;
|
||||
readDateTime64Text(result, this->getScale(), istr, time_zone);
|
||||
assert_cast<ColumnType &>(column).getData().push_back(result);
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
deserializeTextEscaped(column, istr, settings);
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
serializeText(column, row_num, ostr, settings);
|
||||
}
|
||||
|
||||
static inline void readText(DateTime64 & x, UInt32 scale, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone)
|
||||
{
|
||||
switch (settings.date_time_input_format)
|
||||
{
|
||||
case FormatSettings::DateTimeInputFormat::Basic:
|
||||
readDateTime64Text(x, scale, istr, time_zone);
|
||||
return;
|
||||
case FormatSettings::DateTimeInputFormat::BestEffort:
|
||||
parseDateTime64BestEffort(x, scale, istr, time_zone, utc_time_zone);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
DateTime64 x = 0;
|
||||
readText(x, scale, istr, settings, time_zone, utc_time_zone);
|
||||
assert_cast<ColumnType &>(column).getData().push_back(x);
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
writeChar('\'', ostr);
|
||||
serializeText(column, row_num, ostr, settings);
|
||||
writeChar('\'', ostr);
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
DateTime64 x = 0;
|
||||
if (checkChar('\'', istr)) /// Cases: '2017-08-31 18:36:48' or '1504193808'
|
||||
{
|
||||
readText(x, scale, istr, settings, time_zone, utc_time_zone);
|
||||
assertChar('\'', istr);
|
||||
}
|
||||
else /// Just 1504193808 or 01504193808
|
||||
{
|
||||
readIntText(x, istr);
|
||||
}
|
||||
assert_cast<ColumnType &>(column).getData().push_back(x); /// It's important to do this at the end - for exception safety.
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
writeChar('"', ostr);
|
||||
serializeText(column, row_num, ostr, settings);
|
||||
writeChar('"', ostr);
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
DateTime64 x = 0;
|
||||
if (checkChar('"', istr))
|
||||
{
|
||||
readText(x, scale, istr, settings, time_zone, utc_time_zone);
|
||||
assertChar('"', istr);
|
||||
}
|
||||
else
|
||||
{
|
||||
readIntText(x, istr);
|
||||
}
|
||||
assert_cast<ColumnType &>(column).getData().push_back(x);
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
writeChar('"', ostr);
|
||||
serializeText(column, row_num, ostr, settings);
|
||||
writeChar('"', ostr);
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
DateTime64 x = 0;
|
||||
|
||||
if (istr.eof())
|
||||
throwReadAfterEOF();
|
||||
|
||||
char maybe_quote = *istr.position();
|
||||
|
||||
if (maybe_quote == '\'' || maybe_quote == '\"')
|
||||
++istr.position();
|
||||
|
||||
readText(x, scale, istr, settings, time_zone, utc_time_zone);
|
||||
|
||||
if (maybe_quote == '\'' || maybe_quote == '\"')
|
||||
assertChar(maybe_quote, istr);
|
||||
|
||||
assert_cast<ColumnType &>(column).getData().push_back(x);
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const
|
||||
{
|
||||
if (value_index)
|
||||
return;
|
||||
value_index = static_cast<bool>(protobuf.writeDateTime64(assert_cast<const ColumnType &>(column).getData()[row_num], scale));
|
||||
}
|
||||
|
||||
void DataTypeDateTime64::deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const
|
||||
{
|
||||
row_added = false;
|
||||
DateTime64 t = 0;
|
||||
if (!protobuf.readDateTime64(t, scale))
|
||||
return;
|
||||
|
||||
auto & container = assert_cast<ColumnType &>(column).getData();
|
||||
if (allow_add_row)
|
||||
{
|
||||
container.emplace_back(t);
|
||||
row_added = true;
|
||||
}
|
||||
else
|
||||
container.back() = t;
|
||||
}
|
||||
|
||||
bool DataTypeDateTime64::equals(const IDataType & rhs) const
|
||||
{
|
||||
if (auto * ptype = typeid_cast<const DataTypeDateTime64 *>(&rhs))
|
||||
return this->scale == ptype->getScale();
|
||||
return false;
|
||||
}
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
enum class ArgumentKind
|
||||
{
|
||||
Optional,
|
||||
Mandatory
|
||||
};
|
||||
|
||||
template <typename T, ArgumentKind Kind>
|
||||
std::conditional_t<Kind == ArgumentKind::Optional, std::optional<T>, T>
|
||||
getArgument(const ASTPtr & arguments, size_t argument_index, const char * argument_name, const std::string context_data_type_name)
|
||||
{
|
||||
using NearestResultType = NearestFieldType<T>;
|
||||
const auto fieldType = Field::TypeToEnum<NearestResultType>::value;
|
||||
const ASTLiteral * argument = nullptr;
|
||||
|
||||
auto exceptionMessage = [=](const String & message)
|
||||
{
|
||||
return std::string("Parameter #") + std::to_string(argument_index) + " '"
|
||||
+ argument_name + "' for " + context_data_type_name
|
||||
+ message
|
||||
+ ", expected: " + Field::Types::toString(fieldType) + " literal.";
|
||||
};
|
||||
|
||||
if (!arguments || arguments->children.size() <= argument_index
|
||||
|| !(argument = arguments->children[argument_index]->as<ASTLiteral>()))
|
||||
{
|
||||
if constexpr (Kind == ArgumentKind::Optional)
|
||||
return {};
|
||||
else
|
||||
throw Exception(exceptionMessage(" is missing"),
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
}
|
||||
|
||||
if (argument->value.getType() != fieldType)
|
||||
throw Exception(exceptionMessage(String(" has wrong type: ") + argument->value.getTypeName()),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return argument->value.get<NearestResultType>();
|
||||
}
|
||||
|
||||
static DataTypePtr create64(const ASTPtr & arguments)
|
||||
{
|
||||
if (!arguments || arguments->size() == 0)
|
||||
return std::make_shared<DataTypeDateTime64>(DataTypeDateTime64::default_scale);
|
||||
|
||||
const auto scale = getArgument<UInt64, ArgumentKind::Optional>(arguments, 0, "scale", "DateType64");
|
||||
const auto timezone = getArgument<String, ArgumentKind::Optional>(arguments, !!scale, "timezone", "DateType64");
|
||||
|
||||
return std::make_shared<DataTypeDateTime64>(scale.value_or(DataTypeDateTime64::default_scale), timezone.value_or(String{}));
|
||||
}
|
||||
|
||||
void registerDataTypeDateTime64(DataTypeFactory & factory)
|
||||
{
|
||||
factory.registerDataType("DateTime64", create64, DataTypeFactory::CaseInsensitive);
|
||||
}
|
||||
|
||||
}
|
84
dbms/src/DataTypes/DataTypeDateTime64.h
Normal file
84
dbms/src/DataTypes/DataTypeDateTime64.h
Normal file
@ -0,0 +1,84 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDecimalBase.h>
|
||||
|
||||
class DateLUTImpl;
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** DateTime64 is same as DateTime, but it stores values as Int64 and has configurable sub-second part.
|
||||
*
|
||||
* `scale` determines number of decimal places for sub-second part of the DateTime64.
|
||||
*/
|
||||
class DataTypeDateTime64 final : public DataTypeDecimalBase<DateTime64>, public TimezoneMixin
|
||||
{
|
||||
public:
|
||||
static constexpr UInt8 default_scale = 3;
|
||||
static constexpr auto family_name = "DateTime64";
|
||||
|
||||
explicit DataTypeDateTime64(UInt32 scale_, const std::string & time_zone_name = "");
|
||||
|
||||
// reuse timezone from other DateTime/DateTime64
|
||||
DataTypeDateTime64(UInt32 scale_, const TimezoneMixin & time_zone_info);
|
||||
|
||||
const char * getFamilyName() const override { return family_name; }
|
||||
std::string doGetName() const override;
|
||||
TypeIndex getTypeId() const override { return TypeIndex::DateTime64; }
|
||||
|
||||
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
|
||||
void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
|
||||
void serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const override;
|
||||
void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override;
|
||||
|
||||
bool equals(const IDataType & rhs) const override;
|
||||
};
|
||||
|
||||
|
||||
/** Basic wrapper for Tansform-types for DateTime64.
|
||||
*
|
||||
* Allows reusing existing Transform (that takes DateTime-values as UInt32) with DateTime64-values,
|
||||
* by discarding fractional part and producing SAME return type as original Transform.
|
||||
*
|
||||
* Such Transfotm-types are commonly used in Date/DateTime manipulation functions,
|
||||
* and implement static execute fucntion with following signature:
|
||||
* R execute(UInt32, T, const DateLUTImpl &)
|
||||
*
|
||||
* Wehere R and T could be arbitrary types.
|
||||
*/
|
||||
template <typename Transform>
|
||||
class DateTime64BasicTransformWrapper : public Transform
|
||||
{
|
||||
public:
|
||||
using Transform::execute;
|
||||
|
||||
explicit DateTime64BasicTransformWrapper(UInt32 scale_)
|
||||
: scale_multiplier(DecimalUtils::scaleMultiplier<DateTime64::NativeType>(scale_))
|
||||
{}
|
||||
|
||||
template <typename T>
|
||||
auto execute(DateTime64 t, T v, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier);
|
||||
return static_cast<const Transform *>(this)->execute(
|
||||
static_cast<UInt32>(components.whole), v, time_zone);
|
||||
}
|
||||
|
||||
private:
|
||||
UInt32 scale_multiplier = 1;
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
109
dbms/src/DataTypes/DataTypeDecimalBase.cpp
Normal file
109
dbms/src/DataTypes/DataTypeDecimalBase.cpp
Normal file
@ -0,0 +1,109 @@
|
||||
#include <DataTypes/DataTypeDecimalBase.h>
|
||||
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
#include <Formats/ProtobufReader.h>
|
||||
#include <Formats/ProtobufWriter.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/readDecimalText.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/IAST.h>
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
}
|
||||
|
||||
|
||||
bool decimalCheckComparisonOverflow(const Context & context) { return context.getSettingsRef().decimal_check_overflow; }
|
||||
bool decimalCheckArithmeticOverflow(const Context & context) { return context.getSettingsRef().decimal_check_overflow; }
|
||||
|
||||
template <typename T>
|
||||
Field DataTypeDecimalBase<T>::getDefault() const
|
||||
{
|
||||
return DecimalField(T(0), scale);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr DataTypeDecimalBase<T>::createColumn() const
|
||||
{
|
||||
return ColumnType::create(0, scale);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimalBase<T>::serializeBinary(const Field & field, WriteBuffer & ostr) const
|
||||
{
|
||||
FieldType x = get<DecimalField<T>>(field);
|
||||
writeBinary(x, ostr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimalBase<T>::serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
|
||||
{
|
||||
const FieldType & x = assert_cast<const ColumnType &>(column).getElement(row_num);
|
||||
writeBinary(x, ostr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimalBase<T>::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const
|
||||
{
|
||||
const typename ColumnType::Container & x = typeid_cast<const ColumnType &>(column).getData();
|
||||
|
||||
size_t size = x.size();
|
||||
|
||||
if (limit == 0 || offset + limit > size)
|
||||
limit = size - offset;
|
||||
|
||||
ostr.write(reinterpret_cast<const char *>(&x[offset]), sizeof(FieldType) * limit);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimalBase<T>::deserializeBinary(Field & field, ReadBuffer & istr) const
|
||||
{
|
||||
typename FieldType::NativeType x;
|
||||
readBinary(x, istr);
|
||||
field = DecimalField(T(x), this->scale);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimalBase<T>::deserializeBinary(IColumn & column, ReadBuffer & istr) const
|
||||
{
|
||||
typename FieldType::NativeType x;
|
||||
readBinary(x, istr);
|
||||
assert_cast<ColumnType &>(column).getData().push_back(FieldType(x));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimalBase<T>::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double) const
|
||||
{
|
||||
typename ColumnType::Container & x = typeid_cast<ColumnType &>(column).getData();
|
||||
size_t initial_size = x.size();
|
||||
x.resize(initial_size + limit);
|
||||
size_t size = istr.readBig(reinterpret_cast<char*>(&x[initial_size]), sizeof(FieldType) * limit);
|
||||
x.resize(initial_size + size / sizeof(FieldType));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T DataTypeDecimalBase<T>::getScaleMultiplier(UInt32 scale_)
|
||||
{
|
||||
return DecimalUtils::scaleMultiplier<typename T::NativeType>(scale_);
|
||||
}
|
||||
|
||||
|
||||
/// Explicit template instantiations.
|
||||
template class DataTypeDecimalBase<Decimal32>;
|
||||
template class DataTypeDecimalBase<Decimal64>;
|
||||
template class DataTypeDecimalBase<Decimal128>;
|
||||
|
||||
}
|
213
dbms/src/DataTypes/DataTypeDecimalBase.h
Normal file
213
dbms/src/DataTypes/DataTypeDecimalBase.h
Normal file
@ -0,0 +1,213 @@
|
||||
#pragma once
|
||||
#include <cmath>
|
||||
|
||||
#include <common/likely.h>
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <DataTypes/IDataType.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeWithSimpleSerialization.h>
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
extern const int CANNOT_CONVERT_TYPE;
|
||||
extern const int DECIMAL_OVERFLOW;
|
||||
}
|
||||
|
||||
class Context;
|
||||
bool decimalCheckComparisonOverflow(const Context & context);
|
||||
bool decimalCheckArithmeticOverflow(const Context & context);
|
||||
|
||||
inline UInt32 leastDecimalPrecisionFor(TypeIndex int_type)
|
||||
{
|
||||
switch (int_type)
|
||||
{
|
||||
case TypeIndex::Int8: [[fallthrough]];
|
||||
case TypeIndex::UInt8:
|
||||
return 3;
|
||||
case TypeIndex::Int16: [[fallthrough]];
|
||||
case TypeIndex::UInt16:
|
||||
return 5;
|
||||
case TypeIndex::Int32: [[fallthrough]];
|
||||
case TypeIndex::UInt32:
|
||||
return 10;
|
||||
case TypeIndex::Int64:
|
||||
return 19;
|
||||
case TypeIndex::UInt64:
|
||||
return 20;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/// Base class for decimals, like Decimal(P, S), where P is precision, S is scale.
|
||||
/// Maximum precisions for underlying types are:
|
||||
/// Int32 9
|
||||
/// Int64 18
|
||||
/// Int128 38
|
||||
/// Operation between two decimals leads to Decimal(P, S), where
|
||||
/// P is one of (9, 18, 38); equals to the maximum precision for the biggest underlying type of operands.
|
||||
/// S is maximum scale of operands. The allowed valuas are [0, precision]
|
||||
template <typename T>
|
||||
class DataTypeDecimalBase : public DataTypeWithSimpleSerialization
|
||||
{
|
||||
static_assert(IsDecimalNumber<T>);
|
||||
|
||||
public:
|
||||
using FieldType = T;
|
||||
using ColumnType = ColumnDecimal<T>;
|
||||
|
||||
static constexpr bool is_parametric = true;
|
||||
|
||||
static constexpr size_t maxPrecision() { return DecimalUtils::maxPrecision<T>(); }
|
||||
|
||||
DataTypeDecimalBase(UInt32 precision_, UInt32 scale_)
|
||||
: precision(precision_),
|
||||
scale(scale_)
|
||||
{
|
||||
if (unlikely(precision < 1 || precision > maxPrecision()))
|
||||
throw Exception("Precision " + std::to_string(precision) + " is out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
if (unlikely(scale < 0 || static_cast<UInt32>(scale) > maxPrecision()))
|
||||
throw Exception("Scale " + std::to_string(scale) + " is out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
}
|
||||
|
||||
TypeIndex getTypeId() const override { return TypeId<T>::value; }
|
||||
|
||||
Field getDefault() const override;
|
||||
MutableColumnPtr createColumn() const override;
|
||||
|
||||
bool isParametric() const override { return true; }
|
||||
bool haveSubtypes() const override { return false; }
|
||||
bool shouldAlignRightInPrettyFormats() const override { return true; }
|
||||
bool textCanContainOnlyValidUTF8() const override { return true; }
|
||||
bool isComparable() const override { return true; }
|
||||
bool isValueRepresentedByNumber() const override { return true; }
|
||||
bool isValueUnambiguouslyRepresentedInContiguousMemoryRegion() const override { return true; }
|
||||
bool haveMaximumSizeOfValue() const override { return true; }
|
||||
size_t getSizeOfValueInMemory() const override { return sizeof(T); }
|
||||
|
||||
bool isSummable() const override { return true; }
|
||||
bool canBeUsedInBooleanContext() const override { return true; }
|
||||
bool canBeInsideNullable() const override { return true; }
|
||||
|
||||
void serializeBinary(const Field & field, WriteBuffer & ostr) const override;
|
||||
void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
|
||||
void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override;
|
||||
|
||||
void deserializeBinary(Field & field, ReadBuffer & istr) const override;
|
||||
void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
|
||||
void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override;
|
||||
|
||||
/// Decimal specific
|
||||
|
||||
UInt32 getPrecision() const { return precision; }
|
||||
UInt32 getScale() const { return scale; }
|
||||
T getScaleMultiplier() const { return getScaleMultiplier(scale); }
|
||||
|
||||
T wholePart(T x) const
|
||||
{
|
||||
return DecimalUtils::getWholePart(x, scale);
|
||||
}
|
||||
|
||||
T fractionalPart(T x) const
|
||||
{
|
||||
return DecimalUtils::getFractionalPart(x, scale);
|
||||
}
|
||||
|
||||
T maxWholeValue() const { return getScaleMultiplier(maxPrecision() - scale) - T(1); }
|
||||
|
||||
bool canStoreWhole(T x) const
|
||||
{
|
||||
T max = maxWholeValue();
|
||||
if (x > max || x < -max)
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// @returns multiplier for U to become T with correct scale
|
||||
template <typename U>
|
||||
T scaleFactorFor(const DataTypeDecimalBase<U> & x, bool) const
|
||||
{
|
||||
if (getScale() < x.getScale())
|
||||
throw Exception("Decimal result's scale is less than argument's one", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
UInt32 scale_delta = getScale() - x.getScale(); /// scale_delta >= 0
|
||||
return getScaleMultiplier(scale_delta);
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
T scaleFactorFor(const DataTypeNumber<U> & , bool is_multiply_or_divisor) const
|
||||
{
|
||||
if (is_multiply_or_divisor)
|
||||
return 1;
|
||||
return getScaleMultiplier();
|
||||
}
|
||||
|
||||
static T getScaleMultiplier(UInt32 scale);
|
||||
|
||||
protected:
|
||||
const UInt32 precision;
|
||||
const UInt32 scale;
|
||||
};
|
||||
|
||||
|
||||
template <typename T, typename U, template <typename> typename DecimalType>
|
||||
typename std::enable_if_t<(sizeof(T) >= sizeof(U)), DecimalType<T>>
|
||||
decimalResultType(const DecimalType<T> & tx, const DecimalType<U> & ty, bool is_multiply, bool is_divide)
|
||||
{
|
||||
UInt32 scale = (tx.getScale() > ty.getScale() ? tx.getScale() : ty.getScale());
|
||||
if (is_multiply)
|
||||
scale = tx.getScale() + ty.getScale();
|
||||
else if (is_divide)
|
||||
scale = tx.getScale();
|
||||
return DecimalType<T>(DecimalUtils::maxPrecision<T>(), scale);
|
||||
}
|
||||
|
||||
template <typename T, typename U, template <typename> typename DecimalType>
|
||||
typename std::enable_if_t<(sizeof(T) < sizeof(U)), const DecimalType<U>>
|
||||
decimalResultType(const DecimalType<T> & tx, const DecimalType<U> & ty, bool is_multiply, bool is_divide)
|
||||
{
|
||||
UInt32 scale = (tx.getScale() > ty.getScale() ? tx.getScale() : ty.getScale());
|
||||
if (is_multiply)
|
||||
scale = tx.getScale() * ty.getScale();
|
||||
else if (is_divide)
|
||||
scale = tx.getScale();
|
||||
return DecimalType<U>(DecimalUtils::maxPrecision<U>(), scale);
|
||||
}
|
||||
|
||||
template <typename T, typename U, template <typename> typename DecimalType>
|
||||
const DecimalType<T> decimalResultType(const DecimalType<T> & tx, const DataTypeNumber<U> &, bool, bool)
|
||||
{
|
||||
return DecimalType<T>(DecimalUtils::maxPrecision<T>(), tx.getScale());
|
||||
}
|
||||
|
||||
template <typename T, typename U, template <typename> typename DecimalType>
|
||||
const DecimalType<U> decimalResultType(const DataTypeNumber<T> &, const DecimalType<U> & ty, bool, bool)
|
||||
{
|
||||
return DecimalType<U>(DecimalUtils::maxPrecision<U>(), ty.getScale());
|
||||
}
|
||||
|
||||
template <template <typename> typename DecimalType>
|
||||
DataTypePtr createDecimal(UInt64 precision_value, UInt64 scale_value)
|
||||
{
|
||||
if (precision_value < DecimalUtils::minPrecision() || precision_value > DecimalUtils::maxPrecision<Decimal128>())
|
||||
throw Exception("Wrong precision", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
if (static_cast<UInt64>(scale_value) > precision_value)
|
||||
throw Exception("Negative scales and scales larger than precision are not supported", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
if (precision_value <= DecimalUtils::maxPrecision<Decimal32>())
|
||||
return std::make_shared<DecimalType<Decimal32>>(precision_value, scale_value);
|
||||
else if (precision_value <= DecimalUtils::maxPrecision<Decimal64>())
|
||||
return std::make_shared<DecimalType<Decimal64>>(precision_value, scale_value);
|
||||
return std::make_shared<DecimalType<Decimal128>>(precision_value, scale_value);
|
||||
}
|
||||
|
||||
}
|
@ -127,7 +127,7 @@ void DataTypeEnum<Type>::serializeBinary(const IColumn & column, size_t row_num,
|
||||
template <typename Type>
|
||||
void DataTypeEnum<Type>::deserializeBinary(IColumn & column, ReadBuffer & istr) const
|
||||
{
|
||||
typename ColumnType::value_type x;
|
||||
typename ColumnType::ValueType x;
|
||||
readBinary(x, istr);
|
||||
assert_cast<ColumnType &>(column).getData().push_back(x);
|
||||
}
|
||||
|
@ -161,6 +161,7 @@ void registerDataTypeNumbers(DataTypeFactory & factory);
|
||||
void registerDataTypeDecimal(DataTypeFactory & factory);
|
||||
void registerDataTypeDate(DataTypeFactory & factory);
|
||||
void registerDataTypeDateTime(DataTypeFactory & factory);
|
||||
void registerDataTypeDateTime64(DataTypeFactory & factory);
|
||||
void registerDataTypeString(DataTypeFactory & factory);
|
||||
void registerDataTypeFixedString(DataTypeFactory & factory);
|
||||
void registerDataTypeEnum(DataTypeFactory & factory);
|
||||
@ -183,6 +184,7 @@ DataTypeFactory::DataTypeFactory()
|
||||
registerDataTypeDecimal(*this);
|
||||
registerDataTypeDate(*this);
|
||||
registerDataTypeDateTime(*this);
|
||||
registerDataTypeDateTime64(*this);
|
||||
registerDataTypeString(*this);
|
||||
registerDataTypeFixedString(*this);
|
||||
registerDataTypeEnum(*this);
|
||||
|
@ -153,15 +153,15 @@ Field DataTypeNumberBase<T>::getDefault() const
|
||||
template <typename T>
|
||||
void DataTypeNumberBase<T>::serializeBinary(const Field & field, WriteBuffer & ostr) const
|
||||
{
|
||||
/// ColumnVector<T>::value_type is a narrower type. For example, UInt8, when the Field type is UInt64
|
||||
typename ColumnVector<T>::value_type x = get<NearestFieldType<FieldType>>(field);
|
||||
/// ColumnVector<T>::ValueType is a narrower type. For example, UInt8, when the Field type is UInt64
|
||||
typename ColumnVector<T>::ValueType x = get<NearestFieldType<FieldType>>(field);
|
||||
writeBinary(x, ostr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeNumberBase<T>::deserializeBinary(Field & field, ReadBuffer & istr) const
|
||||
{
|
||||
typename ColumnVector<T>::value_type x;
|
||||
typename ColumnVector<T>::ValueType x;
|
||||
readBinary(x, istr);
|
||||
field = NearestFieldType<FieldType>(x);
|
||||
}
|
||||
@ -175,7 +175,7 @@ void DataTypeNumberBase<T>::serializeBinary(const IColumn & column, size_t row_n
|
||||
template <typename T>
|
||||
void DataTypeNumberBase<T>::deserializeBinary(IColumn & column, ReadBuffer & istr) const
|
||||
{
|
||||
typename ColumnVector<T>::value_type x;
|
||||
typename ColumnVector<T>::ValueType x;
|
||||
readBinary(x, istr);
|
||||
assert_cast<ColumnVector<T> &>(column).getData().push_back(x);
|
||||
}
|
||||
@ -191,7 +191,7 @@ void DataTypeNumberBase<T>::serializeBinaryBulk(const IColumn & column, WriteBuf
|
||||
limit = size - offset;
|
||||
|
||||
if (limit)
|
||||
ostr.write(reinterpret_cast<const char *>(&x[offset]), sizeof(typename ColumnVector<T>::value_type) * limit);
|
||||
ostr.write(reinterpret_cast<const char *>(&x[offset]), sizeof(typename ColumnVector<T>::ValueType) * limit);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -200,8 +200,8 @@ void DataTypeNumberBase<T>::deserializeBinaryBulk(IColumn & column, ReadBuffer &
|
||||
typename ColumnVector<T>::Container & x = typeid_cast<ColumnVector<T> &>(column).getData();
|
||||
size_t initial_size = x.size();
|
||||
x.resize(initial_size + limit);
|
||||
size_t size = istr.readBig(reinterpret_cast<char*>(&x[initial_size]), sizeof(typename ColumnVector<T>::value_type) * limit);
|
||||
x.resize(initial_size + size / sizeof(typename ColumnVector<T>::value_type));
|
||||
size_t size = istr.readBig(reinterpret_cast<char*>(&x[initial_size]), sizeof(typename ColumnVector<T>::ValueType) * limit);
|
||||
x.resize(initial_size + size / sizeof(typename ColumnVector<T>::ValueType));
|
||||
}
|
||||
|
||||
|
||||
|
@ -7,6 +7,9 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
template <typename T>
|
||||
class ColumnVector;
|
||||
|
||||
/** Implements part of the IDataType interface, common to all numbers and for Date and DateTime.
|
||||
*/
|
||||
template <typename T>
|
||||
@ -18,6 +21,8 @@ public:
|
||||
static constexpr bool is_parametric = false;
|
||||
using FieldType = T;
|
||||
|
||||
using ColumnType = ColumnVector<T>;
|
||||
|
||||
const char * getFamilyName() const override { return TypeName<T>::get(); }
|
||||
TypeIndex getTypeId() const override { return TypeId<T>::value; }
|
||||
|
||||
|
@ -1,17 +1,19 @@
|
||||
#include <type_traits>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
#include <Formats/ProtobufReader.h>
|
||||
#include <Formats/ProtobufWriter.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/readDecimalText.h>
|
||||
#include <Parsers/IAST.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/IAST.h>
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -23,34 +25,38 @@ namespace ErrorCodes
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
}
|
||||
|
||||
|
||||
bool decimalCheckComparisonOverflow(const Context & context) { return context.getSettingsRef().decimal_check_overflow; }
|
||||
bool decimalCheckArithmeticOverflow(const Context & context) { return context.getSettingsRef().decimal_check_overflow; }
|
||||
|
||||
|
||||
//
|
||||
|
||||
template <typename T>
|
||||
std::string DataTypeDecimal<T>::doGetName() const
|
||||
{
|
||||
std::stringstream ss;
|
||||
ss << "Decimal(" << precision << ", " << scale << ")";
|
||||
ss << "Decimal(" << this->precision << ", " << this->scale << ")";
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
|
||||
|
||||
template <typename T>
|
||||
bool DataTypeDecimal<T>::equals(const IDataType & rhs) const
|
||||
{
|
||||
if (auto * ptype = typeid_cast<const DataTypeDecimal<T> *>(&rhs))
|
||||
return scale == ptype->getScale();
|
||||
return this->scale == ptype->getScale();
|
||||
return false;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
DataTypePtr DataTypeDecimal<T>::promoteNumericType() const
|
||||
{
|
||||
using PromotedType = DataTypeDecimal<Decimal128>;
|
||||
return std::make_shared<PromotedType>(PromotedType::maxPrecision(), this->scale);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimal<T>::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
|
||||
{
|
||||
T value = assert_cast<const ColumnType &>(column).getData()[row_num];
|
||||
writeText(value, scale, ostr);
|
||||
writeText(value, this->scale, ostr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -58,6 +64,7 @@ bool DataTypeDecimal<T>::tryReadText(T & x, ReadBuffer & istr, UInt32 precision,
|
||||
{
|
||||
UInt32 unread_scale = scale;
|
||||
bool done = tryReadDecimalText(istr, x, precision, unread_scale);
|
||||
|
||||
x *= T::getScaleMultiplier(unread_scale);
|
||||
return done;
|
||||
}
|
||||
@ -94,74 +101,19 @@ T DataTypeDecimal<T>::parseFromString(const String & str) const
|
||||
{
|
||||
ReadBufferFromMemory buf(str.data(), str.size());
|
||||
T x;
|
||||
UInt32 unread_scale = scale;
|
||||
readDecimalText(buf, x, precision, unread_scale, true);
|
||||
UInt32 unread_scale = this->scale;
|
||||
readDecimalText(buf, x, this->precision, unread_scale, true);
|
||||
x *= T::getScaleMultiplier(unread_scale);
|
||||
|
||||
return x;
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimal<T>::serializeBinary(const Field & field, WriteBuffer & ostr) const
|
||||
{
|
||||
FieldType x = get<DecimalField<T>>(field);
|
||||
writeBinary(x, ostr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimal<T>::serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
|
||||
{
|
||||
const FieldType & x = assert_cast<const ColumnType &>(column).getElement(row_num);
|
||||
writeBinary(x, ostr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimal<T>::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const
|
||||
{
|
||||
const typename ColumnType::Container & x = typeid_cast<const ColumnType &>(column).getData();
|
||||
|
||||
size_t size = x.size();
|
||||
|
||||
if (limit == 0 || offset + limit > size)
|
||||
limit = size - offset;
|
||||
|
||||
ostr.write(reinterpret_cast<const char *>(&x[offset]), sizeof(FieldType) * limit);
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimal<T>::deserializeBinary(Field & field, ReadBuffer & istr) const
|
||||
{
|
||||
typename FieldType::NativeType x;
|
||||
readBinary(x, istr);
|
||||
field = DecimalField(T(x), scale);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimal<T>::deserializeBinary(IColumn & column, ReadBuffer & istr) const
|
||||
{
|
||||
typename FieldType::NativeType x;
|
||||
readBinary(x, istr);
|
||||
assert_cast<ColumnType &>(column).getData().push_back(FieldType(x));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimal<T>::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double) const
|
||||
{
|
||||
typename ColumnType::Container & x = typeid_cast<ColumnType &>(column).getData();
|
||||
size_t initial_size = x.size();
|
||||
x.resize(initial_size + limit);
|
||||
size_t size = istr.readBig(reinterpret_cast<char*>(&x[initial_size]), sizeof(FieldType) * limit);
|
||||
x.resize(initial_size + size / sizeof(FieldType));
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
void DataTypeDecimal<T>::serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const
|
||||
{
|
||||
if (value_index)
|
||||
return;
|
||||
value_index = static_cast<bool>(protobuf.writeDecimal(assert_cast<const ColumnType &>(column).getData()[row_num], scale));
|
||||
value_index = static_cast<bool>(protobuf.writeDecimal(assert_cast<const ColumnType &>(column).getData()[row_num], this->scale));
|
||||
}
|
||||
|
||||
|
||||
@ -170,7 +122,7 @@ void DataTypeDecimal<T>::deserializeProtobuf(IColumn & column, ProtobufReader &
|
||||
{
|
||||
row_added = false;
|
||||
T decimal;
|
||||
if (!protobuf.readDecimal(decimal, precision, scale))
|
||||
if (!protobuf.readDecimal(decimal, this->precision, this->scale))
|
||||
return;
|
||||
|
||||
auto & container = assert_cast<ColumnType &>(column).getData();
|
||||
@ -184,45 +136,6 @@ void DataTypeDecimal<T>::deserializeProtobuf(IColumn & column, ProtobufReader &
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
Field DataTypeDecimal<T>::getDefault() const
|
||||
{
|
||||
return DecimalField(T(0), scale);
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
DataTypePtr DataTypeDecimal<T>::promoteNumericType() const
|
||||
{
|
||||
using PromotedType = DataTypeDecimal<Decimal128>;
|
||||
return std::make_shared<PromotedType>(PromotedType::maxPrecision(), scale);
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr DataTypeDecimal<T>::createColumn() const
|
||||
{
|
||||
return ColumnType::create(0, scale);
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
|
||||
DataTypePtr createDecimal(UInt64 precision_value, UInt64 scale_value)
|
||||
{
|
||||
if (precision_value < minDecimalPrecision() || precision_value > maxDecimalPrecision<Decimal128>())
|
||||
throw Exception("Wrong precision", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
if (static_cast<UInt64>(scale_value) > precision_value)
|
||||
throw Exception("Negative scales and scales larger than precision are not supported", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
if (precision_value <= maxDecimalPrecision<Decimal32>())
|
||||
return std::make_shared<DataTypeDecimal<Decimal32>>(precision_value, scale_value);
|
||||
else if (precision_value <= maxDecimalPrecision<Decimal64>())
|
||||
return std::make_shared<DataTypeDecimal<Decimal64>>(precision_value, scale_value);
|
||||
return std::make_shared<DataTypeDecimal<Decimal128>>(precision_value, scale_value);
|
||||
}
|
||||
|
||||
static DataTypePtr create(const ASTPtr & arguments)
|
||||
{
|
||||
if (!arguments || arguments->children.size() != 2)
|
||||
@ -234,12 +147,12 @@ static DataTypePtr create(const ASTPtr & arguments)
|
||||
|
||||
if (!precision || precision->value.getType() != Field::Types::UInt64 ||
|
||||
!scale || !(scale->value.getType() == Field::Types::Int64 || scale->value.getType() == Field::Types::UInt64))
|
||||
throw Exception("Decimal data type family must have a two numbers as its arguments", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
throw Exception("Decimal data type family must have two numbers as its arguments", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
UInt64 precision_value = precision->value.get<UInt64>();
|
||||
UInt64 scale_value = scale->value.get<UInt64>();
|
||||
|
||||
return createDecimal(precision_value, scale_value);
|
||||
return createDecimal<DataTypeDecimal>(precision_value, scale_value);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -254,10 +167,10 @@ static DataTypePtr createExact(const ASTPtr & arguments)
|
||||
if (!scale_arg || !(scale_arg->value.getType() == Field::Types::Int64 || scale_arg->value.getType() == Field::Types::UInt64))
|
||||
throw Exception("Decimal data type family must have a two numbers as its arguments", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
UInt64 precision = maxDecimalPrecision<T>();
|
||||
UInt64 precision = DecimalUtils::maxPrecision<T>();
|
||||
UInt64 scale = scale_arg->value.get<UInt64>();
|
||||
|
||||
return createDecimal(precision, scale);
|
||||
return createDecimal<DataTypeDecimal>(precision, scale);
|
||||
}
|
||||
|
||||
void registerDataTypeDecimal(DataTypeFactory & factory)
|
||||
@ -270,7 +183,6 @@ void registerDataTypeDecimal(DataTypeFactory & factory)
|
||||
factory.registerAlias("DEC", "Decimal", DataTypeFactory::CaseInsensitive);
|
||||
}
|
||||
|
||||
|
||||
/// Explicit template instantiations.
|
||||
template class DataTypeDecimal<Decimal32>;
|
||||
template class DataTypeDecimal<Decimal64>;
|
||||
|
@ -1,12 +1,9 @@
|
||||
#pragma once
|
||||
#include <cmath>
|
||||
|
||||
#include <common/arithmeticOverflow.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
#include <DataTypes/IDataType.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeWithSimpleSerialization.h>
|
||||
#include <DataTypes/DataTypeDecimalBase.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -19,43 +16,6 @@ namespace ErrorCodes
|
||||
extern const int DECIMAL_OVERFLOW;
|
||||
}
|
||||
|
||||
class Context;
|
||||
bool decimalCheckComparisonOverflow(const Context & context);
|
||||
bool decimalCheckArithmeticOverflow(const Context & context);
|
||||
|
||||
|
||||
static constexpr size_t minDecimalPrecision() { return 1; }
|
||||
template <typename T> static constexpr size_t maxDecimalPrecision() { return 0; }
|
||||
template <> constexpr size_t maxDecimalPrecision<Decimal32>() { return 9; }
|
||||
template <> constexpr size_t maxDecimalPrecision<Decimal64>() { return 18; }
|
||||
template <> constexpr size_t maxDecimalPrecision<Decimal128>() { return 38; }
|
||||
|
||||
|
||||
DataTypePtr createDecimal(UInt64 precision, UInt64 scale);
|
||||
|
||||
inline UInt32 leastDecimalPrecisionFor(TypeIndex int_type)
|
||||
{
|
||||
switch (int_type)
|
||||
{
|
||||
case TypeIndex::Int8: [[fallthrough]];
|
||||
case TypeIndex::UInt8:
|
||||
return 3;
|
||||
case TypeIndex::Int16: [[fallthrough]];
|
||||
case TypeIndex::UInt16:
|
||||
return 5;
|
||||
case TypeIndex::Int32: [[fallthrough]];
|
||||
case TypeIndex::UInt32:
|
||||
return 10;
|
||||
case TypeIndex::Int64:
|
||||
return 19;
|
||||
case TypeIndex::UInt64:
|
||||
return 20;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/// Implements Decimal(P, S), where P is precision, S is scale.
|
||||
/// Maximum precisions for underlying types are:
|
||||
/// Int32 9
|
||||
@ -65,166 +25,40 @@ inline UInt32 leastDecimalPrecisionFor(TypeIndex int_type)
|
||||
/// P is one of (9, 18, 38); equals to the maximum precision for the biggest underlying type of operands.
|
||||
/// S is maximum scale of operands. The allowed valuas are [0, precision]
|
||||
template <typename T>
|
||||
class DataTypeDecimal final : public DataTypeWithSimpleSerialization
|
||||
class DataTypeDecimal final : public DataTypeDecimalBase<T>
|
||||
{
|
||||
using Base = DataTypeDecimalBase<T>;
|
||||
static_assert(IsDecimalNumber<T>);
|
||||
|
||||
public:
|
||||
using FieldType = T;
|
||||
using ColumnType = ColumnDecimal<T>;
|
||||
using typename Base::FieldType;
|
||||
using typename Base::ColumnType;
|
||||
using Base::Base;
|
||||
|
||||
static constexpr bool is_parametric = true;
|
||||
static constexpr auto family_name = "Decimal";
|
||||
|
||||
static constexpr size_t maxPrecision() { return maxDecimalPrecision<T>(); }
|
||||
|
||||
DataTypeDecimal(UInt32 precision_, UInt32 scale_)
|
||||
: precision(precision_),
|
||||
scale(scale_)
|
||||
{
|
||||
if (unlikely(precision < 1 || precision > maxPrecision()))
|
||||
throw Exception("Precision " + std::to_string(precision) + " is out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
if (unlikely(scale < 0 || static_cast<UInt32>(scale) > maxPrecision()))
|
||||
throw Exception("Scale " + std::to_string(scale) + " is out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
}
|
||||
|
||||
const char * getFamilyName() const override { return "Decimal"; }
|
||||
const char * getFamilyName() const override { return family_name; }
|
||||
std::string doGetName() const override;
|
||||
TypeIndex getTypeId() const override { return TypeId<T>::value; }
|
||||
bool canBePromoted() const override { return true; }
|
||||
DataTypePtr promoteNumericType() const override;
|
||||
|
||||
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
|
||||
void serializeBinary(const Field & field, WriteBuffer & ostr) const override;
|
||||
void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
|
||||
void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override;
|
||||
|
||||
void deserializeBinary(Field & field, ReadBuffer & istr) const override;
|
||||
void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
|
||||
void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override;
|
||||
|
||||
void serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const override;
|
||||
void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override;
|
||||
|
||||
Field getDefault() const override;
|
||||
bool canBePromoted() const override { return true; }
|
||||
DataTypePtr promoteNumericType() const override;
|
||||
MutableColumnPtr createColumn() const override;
|
||||
bool equals(const IDataType & rhs) const override;
|
||||
|
||||
bool isParametric() const override { return true; }
|
||||
bool haveSubtypes() const override { return false; }
|
||||
bool shouldAlignRightInPrettyFormats() const override { return true; }
|
||||
bool textCanContainOnlyValidUTF8() const override { return true; }
|
||||
bool isComparable() const override { return true; }
|
||||
bool isValueRepresentedByNumber() const override { return true; }
|
||||
bool isValueUnambiguouslyRepresentedInContiguousMemoryRegion() const override { return true; }
|
||||
bool haveMaximumSizeOfValue() const override { return true; }
|
||||
size_t getSizeOfValueInMemory() const override { return sizeof(T); }
|
||||
|
||||
bool isSummable() const override { return true; }
|
||||
bool canBeUsedInBooleanContext() const override { return true; }
|
||||
bool canBeInsideNullable() const override { return true; }
|
||||
|
||||
/// Decimal specific
|
||||
|
||||
UInt32 getPrecision() const { return precision; }
|
||||
UInt32 getScale() const { return scale; }
|
||||
T getScaleMultiplier() const { return T::getScaleMultiplier(scale); }
|
||||
|
||||
T wholePart(T x) const
|
||||
{
|
||||
if (scale == 0)
|
||||
return x;
|
||||
return x / getScaleMultiplier();
|
||||
}
|
||||
|
||||
T fractionalPart(T x) const
|
||||
{
|
||||
if (scale == 0)
|
||||
return 0;
|
||||
if (x < T(0))
|
||||
x *= T(-1);
|
||||
return x % getScaleMultiplier();
|
||||
}
|
||||
|
||||
T maxWholeValue() const { return T::getScaleMultiplier(maxPrecision() - scale) - T(1); }
|
||||
|
||||
bool canStoreWhole(T x) const
|
||||
{
|
||||
T max = maxWholeValue();
|
||||
if (x > max || x < -max)
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// @returns multiplier for U to become T with correct scale
|
||||
template <typename U>
|
||||
T scaleFactorFor(const DataTypeDecimal<U> & x, bool) const
|
||||
{
|
||||
if (getScale() < x.getScale())
|
||||
throw Exception("Decimal result's scale is less then argiment's one", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
UInt32 scale_delta = getScale() - x.getScale(); /// scale_delta >= 0
|
||||
return T::getScaleMultiplier(scale_delta);
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
T scaleFactorFor(const DataTypeNumber<U> & , bool is_multiply_or_divisor) const
|
||||
{
|
||||
if (is_multiply_or_divisor)
|
||||
return 1;
|
||||
return getScaleMultiplier();
|
||||
}
|
||||
|
||||
T parseFromString(const String & str) const;
|
||||
void readText(T & x, ReadBuffer & istr, bool csv = false) const { readText(x, istr, this->precision, this->scale, csv); }
|
||||
|
||||
void readText(T & x, ReadBuffer & istr, bool csv = false) const { readText(x, istr, precision, scale, csv); }
|
||||
static void readText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale, bool csv = false);
|
||||
static bool tryReadText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale);
|
||||
|
||||
private:
|
||||
const UInt32 precision;
|
||||
const UInt32 scale;
|
||||
static void readText(T & x, ReadBuffer & istr, UInt32 precision_, UInt32 scale_, bool csv = false);
|
||||
static bool tryReadText(T & x, ReadBuffer & istr, UInt32 precision_, UInt32 scale_);
|
||||
};
|
||||
|
||||
|
||||
template <typename T, typename U>
|
||||
typename std::enable_if_t<(sizeof(T) >= sizeof(U)), const DataTypeDecimal<T>>
|
||||
decimalResultType(const DataTypeDecimal<T> & tx, const DataTypeDecimal<U> & ty, bool is_multiply, bool is_divide)
|
||||
{
|
||||
UInt32 scale = (tx.getScale() > ty.getScale() ? tx.getScale() : ty.getScale());
|
||||
if (is_multiply)
|
||||
scale = tx.getScale() + ty.getScale();
|
||||
else if (is_divide)
|
||||
scale = tx.getScale();
|
||||
return DataTypeDecimal<T>(maxDecimalPrecision<T>(), scale);
|
||||
}
|
||||
|
||||
template <typename T, typename U>
|
||||
typename std::enable_if_t<(sizeof(T) < sizeof(U)), const DataTypeDecimal<U>>
|
||||
decimalResultType(const DataTypeDecimal<T> & tx, const DataTypeDecimal<U> & ty, bool is_multiply, bool is_divide)
|
||||
{
|
||||
UInt32 scale = (tx.getScale() > ty.getScale() ? tx.getScale() : ty.getScale());
|
||||
if (is_multiply)
|
||||
scale = tx.getScale() * ty.getScale();
|
||||
else if (is_divide)
|
||||
scale = tx.getScale();
|
||||
return DataTypeDecimal<U>(maxDecimalPrecision<U>(), scale);
|
||||
}
|
||||
|
||||
template <typename T, typename U>
|
||||
const DataTypeDecimal<T> decimalResultType(const DataTypeDecimal<T> & tx, const DataTypeNumber<U> &, bool, bool)
|
||||
{
|
||||
return DataTypeDecimal<T>(maxDecimalPrecision<T>(), tx.getScale());
|
||||
}
|
||||
|
||||
template <typename T, typename U>
|
||||
const DataTypeDecimal<U> decimalResultType(const DataTypeNumber<T> &, const DataTypeDecimal<U> & ty, bool, bool)
|
||||
{
|
||||
return DataTypeDecimal<U>(maxDecimalPrecision<U>(), ty.getScale());
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
inline const DataTypeDecimal<T> * checkDecimal(const IDataType & data_type)
|
||||
{
|
||||
@ -242,14 +76,11 @@ inline UInt32 getDecimalScale(const IDataType & data_type, UInt32 default_value
|
||||
return default_value;
|
||||
}
|
||||
|
||||
///
|
||||
|
||||
template <typename DataType> constexpr bool IsDataTypeDecimal = false;
|
||||
template <> inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal32>> = true;
|
||||
template <> inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal64>> = true;
|
||||
template <> inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal128>> = true;
|
||||
|
||||
template <typename DataType> constexpr bool IsDataTypeDecimalOrNumber = IsDataTypeDecimal<DataType> || IsDataTypeNumber<DataType>;
|
||||
template <typename T>
|
||||
inline UInt32 getDecimalScale(const DataTypeDecimal<T> & data_type)
|
||||
{
|
||||
return data_type.getScale();
|
||||
}
|
||||
|
||||
template <typename FromDataType, typename ToDataType>
|
||||
inline std::enable_if_t<IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>, typename ToDataType::FieldType>
|
||||
@ -265,7 +96,8 @@ convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_fro
|
||||
{
|
||||
converted_value = MaxFieldType::getScaleMultiplier(scale_to - scale_from);
|
||||
if (common::mulOverflow(static_cast<MaxNativeType>(value), converted_value, converted_value))
|
||||
throw Exception("Decimal convert overflow", ErrorCodes::DECIMAL_OVERFLOW);
|
||||
throw Exception(std::string(ToDataType::family_name) + " convert overflow",
|
||||
ErrorCodes::DECIMAL_OVERFLOW);
|
||||
}
|
||||
else
|
||||
converted_value = value / MaxFieldType::getScaleMultiplier(scale_from - scale_to);
|
||||
@ -274,14 +106,15 @@ convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_fro
|
||||
{
|
||||
if (converted_value < std::numeric_limits<typename ToFieldType::NativeType>::min() ||
|
||||
converted_value > std::numeric_limits<typename ToFieldType::NativeType>::max())
|
||||
throw Exception("Decimal convert overflow", ErrorCodes::DECIMAL_OVERFLOW);
|
||||
throw Exception(std::string(ToDataType::family_name) + " convert overflow",
|
||||
ErrorCodes::DECIMAL_OVERFLOW);
|
||||
}
|
||||
|
||||
return converted_value;
|
||||
}
|
||||
|
||||
template <typename FromDataType, typename ToDataType>
|
||||
inline std::enable_if_t<IsDataTypeDecimal<FromDataType> && IsDataTypeNumber<ToDataType>, typename ToDataType::FieldType>
|
||||
inline std::enable_if_t<IsDataTypeDecimal<FromDataType> && IsNumber<typename ToDataType::FieldType>, typename ToDataType::FieldType>
|
||||
convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
|
||||
{
|
||||
using FromFieldType = typename FromDataType::FieldType;
|
||||
@ -299,7 +132,8 @@ convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
|
||||
{
|
||||
if (converted_value < std::numeric_limits<ToFieldType>::min() ||
|
||||
converted_value > std::numeric_limits<ToFieldType>::max())
|
||||
throw Exception("Decimal convert overflow", ErrorCodes::DECIMAL_OVERFLOW);
|
||||
throw Exception(std::string(FromDataType::family_name) + " convert overflow",
|
||||
ErrorCodes::DECIMAL_OVERFLOW);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -307,7 +141,8 @@ convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
|
||||
|
||||
if (converted_value < 0 ||
|
||||
converted_value > static_cast<CastIntType>(std::numeric_limits<ToFieldType>::max()))
|
||||
throw Exception("Decimal convert overflow", ErrorCodes::DECIMAL_OVERFLOW);
|
||||
throw Exception(std::string(FromDataType::family_name) + " convert overflow",
|
||||
ErrorCodes::DECIMAL_OVERFLOW);
|
||||
}
|
||||
}
|
||||
return converted_value;
|
||||
@ -315,7 +150,7 @@ convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
|
||||
}
|
||||
|
||||
template <typename FromDataType, typename ToDataType>
|
||||
inline std::enable_if_t<IsDataTypeNumber<FromDataType> && IsDataTypeDecimal<ToDataType>, typename ToDataType::FieldType>
|
||||
inline std::enable_if_t<IsNumber<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>, typename ToDataType::FieldType>
|
||||
convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
|
||||
{
|
||||
using FromFieldType = typename FromDataType::FieldType;
|
||||
@ -325,7 +160,8 @@ convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
|
||||
if constexpr (std::is_floating_point_v<FromFieldType>)
|
||||
{
|
||||
if (!std::isfinite(value))
|
||||
throw Exception("Decimal convert overflow. Cannot convert infinity or NaN to decimal", ErrorCodes::DECIMAL_OVERFLOW);
|
||||
throw Exception(std::string(ToDataType::family_name) + " convert overflow. Cannot convert infinity or NaN to decimal",
|
||||
ErrorCodes::DECIMAL_OVERFLOW);
|
||||
|
||||
auto out = value * ToFieldType::getScaleMultiplier(scale);
|
||||
if constexpr (std::is_same_v<ToNativeType, Int128>)
|
||||
@ -333,12 +169,14 @@ convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
|
||||
static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64;
|
||||
static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll;
|
||||
if (out <= static_cast<ToNativeType>(min_int128) || out >= static_cast<ToNativeType>(max_int128))
|
||||
throw Exception("Decimal convert overflow. Float is out of Decimal range", ErrorCodes::DECIMAL_OVERFLOW);
|
||||
throw Exception(std::string(ToDataType::family_name) + " convert overflow. Float is out of Decimal range",
|
||||
ErrorCodes::DECIMAL_OVERFLOW);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (out <= std::numeric_limits<ToNativeType>::min() || out >= std::numeric_limits<ToNativeType>::max())
|
||||
throw Exception("Decimal convert overflow. Float is out of Decimal range", ErrorCodes::DECIMAL_OVERFLOW);
|
||||
throw Exception(std::string(ToDataType::family_name) + " convert overflow. Float is out of Decimal range",
|
||||
ErrorCodes::DECIMAL_OVERFLOW);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@ -347,6 +185,7 @@ convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
|
||||
if constexpr (std::is_same_v<FromFieldType, UInt64>)
|
||||
if (value > static_cast<UInt64>(std::numeric_limits<Int64>::max()))
|
||||
return convertDecimals<DataTypeDecimal<Decimal128>, ToDataType>(value, 0, scale);
|
||||
|
||||
return convertDecimals<DataTypeDecimal<Decimal64>, ToDataType>(value, 0, scale);
|
||||
}
|
||||
}
|
||||
|
@ -38,16 +38,4 @@ using DataTypeInt64 = DataTypeNumber<Int64>;
|
||||
using DataTypeFloat32 = DataTypeNumber<Float32>;
|
||||
using DataTypeFloat64 = DataTypeNumber<Float64>;
|
||||
|
||||
template <typename DataType> constexpr bool IsDataTypeNumber = false;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<UInt8>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<UInt16>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<UInt32>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<UInt64>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Int8>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Int16>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Int32>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Int64>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Float32>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Float64>> = true;
|
||||
|
||||
}
|
||||
|
@ -511,7 +511,8 @@ struct WhichDataType
|
||||
|
||||
bool isDate() const { return idx == TypeIndex::Date; }
|
||||
bool isDateTime() const { return idx == TypeIndex::DateTime; }
|
||||
bool isDateOrDateTime() const { return isDate() || isDateTime(); }
|
||||
bool isDateTime64() const { return idx == TypeIndex::DateTime64; }
|
||||
bool isDateOrDateTime() const { return isDate() || isDateTime() || isDateTime64(); }
|
||||
|
||||
bool isString() const { return idx == TypeIndex::String; }
|
||||
bool isFixedString() const { return idx == TypeIndex::FixedString; }
|
||||
@ -533,6 +534,7 @@ struct WhichDataType
|
||||
|
||||
inline bool isDate(const DataTypePtr & data_type) { return WhichDataType(data_type).isDate(); }
|
||||
inline bool isDateOrDateTime(const DataTypePtr & data_type) { return WhichDataType(data_type).isDateOrDateTime(); }
|
||||
inline bool isDateTime64(const DataTypePtr & data_type) { return WhichDataType(data_type).isDateTime64(); }
|
||||
inline bool isEnum(const DataTypePtr & data_type) { return WhichDataType(data_type).isEnum(); }
|
||||
inline bool isDecimal(const DataTypePtr & data_type) { return WhichDataType(data_type).isDecimal(); }
|
||||
inline bool isTuple(const DataTypePtr & data_type) { return WhichDataType(data_type).isTuple(); }
|
||||
@ -564,6 +566,14 @@ inline bool isFloat(const T & data_type)
|
||||
return which.isFloat();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline bool isNativeInteger(const T & data_type)
|
||||
{
|
||||
WhichDataType which(data_type);
|
||||
return which.isNativeInt() || which.isNativeUInt();
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
inline bool isNativeNumber(const T & data_type)
|
||||
{
|
||||
@ -585,6 +595,13 @@ inline bool isColumnedAsNumber(const T & data_type)
|
||||
return which.isInt() || which.isUInt() || which.isFloat() || which.isDateOrDateTime() || which.isUUID();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline bool isColumnedAsDecimal(const T & data_type)
|
||||
{
|
||||
WhichDataType which(data_type);
|
||||
return which.isDecimal() || which.isDateTime64();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline bool isString(const T & data_type)
|
||||
{
|
||||
@ -615,6 +632,30 @@ inline bool isCompilableType(const DataTypePtr & data_type)
|
||||
return data_type->isValueRepresentedByNumber() && !isDecimal(data_type);
|
||||
}
|
||||
|
||||
template <typename DataType> constexpr bool IsDataTypeDecimal = false;
|
||||
template <typename DataType> constexpr bool IsDataTypeNumber = false;
|
||||
template <typename DataType> constexpr bool IsDataTypeDateOrDateTime = false;
|
||||
|
||||
template <typename DataType> constexpr bool IsDataTypeDecimalOrNumber = IsDataTypeDecimal<DataType> || IsDataTypeNumber<DataType>;
|
||||
|
||||
template <typename T>
|
||||
class DataTypeDecimal;
|
||||
|
||||
template <typename T>
|
||||
class DataTypeNumber;
|
||||
|
||||
class DataTypeDate;
|
||||
class DataTypeDateTime;
|
||||
class DataTypeDateTime64;
|
||||
|
||||
template <typename T> constexpr bool IsDataTypeDecimal<DataTypeDecimal<T>> = true;
|
||||
template <> constexpr bool IsDataTypeDecimal<DataTypeDateTime64> = true;
|
||||
|
||||
template <typename T> constexpr bool IsDataTypeNumber<DataTypeNumber<T>> = true;
|
||||
|
||||
template <> inline constexpr bool IsDataTypeDateOrDateTime<DataTypeDate> = true;
|
||||
template <> inline constexpr bool IsDataTypeDateOrDateTime<DataTypeDateTime> = true;
|
||||
template <> inline constexpr bool IsDataTypeDateOrDateTime<DataTypeDateTime64> = true;
|
||||
|
||||
}
|
||||
|
||||
|
@ -12,6 +12,7 @@
|
||||
#include <DataTypes/DataTypeNothing.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
|
||||
@ -212,15 +213,47 @@ DataTypePtr getLeastSupertype(const DataTypes & types)
|
||||
{
|
||||
UInt32 have_date = type_ids.count(TypeIndex::Date);
|
||||
UInt32 have_datetime = type_ids.count(TypeIndex::DateTime);
|
||||
UInt32 have_datetime64 = type_ids.count(TypeIndex::DateTime64);
|
||||
|
||||
if (have_date || have_datetime)
|
||||
if (have_date || have_datetime || have_datetime64)
|
||||
{
|
||||
bool all_date_or_datetime = type_ids.size() == (have_date + have_datetime);
|
||||
bool all_date_or_datetime = type_ids.size() == (have_date + have_datetime + have_datetime64);
|
||||
if (!all_date_or_datetime)
|
||||
throw Exception(getExceptionMessagePrefix(types) + " because some of them are Date/DateTime and some of them are not", ErrorCodes::NO_COMMON_TYPE);
|
||||
|
||||
if (have_datetime64 == 0)
|
||||
{
|
||||
return std::make_shared<DataTypeDateTime>();
|
||||
}
|
||||
|
||||
// When DateTime64 involved, make sure that supertype has whole-part precision
|
||||
// big enough to hold max whole-value of any type from `types`.
|
||||
// That would sacrifice scale when comparing DateTime64 of different scales.
|
||||
|
||||
UInt32 max_datetime64_whole_precision = 0;
|
||||
for (const auto & t : types)
|
||||
{
|
||||
if (auto dt64 = typeid_cast<const DataTypeDateTime64 *>(t.get()))
|
||||
{
|
||||
const auto whole_precision = dt64->getPrecision() - dt64->getScale();
|
||||
max_datetime64_whole_precision = std::max(whole_precision, max_datetime64_whole_precision);
|
||||
}
|
||||
}
|
||||
|
||||
UInt32 least_decimal_precision = 0;
|
||||
if (have_datetime)
|
||||
{
|
||||
least_decimal_precision = leastDecimalPrecisionFor(TypeIndex::UInt32);
|
||||
}
|
||||
else if (have_date)
|
||||
{
|
||||
least_decimal_precision = leastDecimalPrecisionFor(TypeIndex::UInt16);
|
||||
}
|
||||
max_datetime64_whole_precision = std::max(least_decimal_precision, max_datetime64_whole_precision);
|
||||
|
||||
const UInt32 scale = DataTypeDateTime64::maxPrecision() - max_datetime64_whole_precision;
|
||||
return std::make_shared<DataTypeDateTime64>(scale);
|
||||
}
|
||||
}
|
||||
|
||||
/// Decimals
|
||||
|
0
dbms/src/DataTypes/tests/gtest_DecimalUtils.cpp
Normal file
0
dbms/src/DataTypes/tests/gtest_DecimalUtils.cpp
Normal file
@ -6,20 +6,31 @@
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
bool operator==(const IDataType & left, const IDataType & right)
|
||||
{
|
||||
return left.equals(right);
|
||||
}
|
||||
|
||||
std::ostream & operator<<(std::ostream & ostr, const IDataType & dt)
|
||||
{
|
||||
return ostr << dt.getName();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
using namespace DB;
|
||||
|
||||
|
||||
TEST(data_type, data_type_get_common_type_Test)
|
||||
auto typeFromString(const std::string & str)
|
||||
{
|
||||
auto & data_type_factory = DataTypeFactory::instance();
|
||||
auto typeFromString = [& data_type_factory](const std::string & str)
|
||||
{
|
||||
return data_type_factory.get(str);
|
||||
};
|
||||
};
|
||||
|
||||
auto typesFromString = [& typeFromString](const std::string & str)
|
||||
{
|
||||
auto typesFromString(const std::string & str)
|
||||
{
|
||||
std::istringstream data_types_stream(str);
|
||||
DataTypes data_types;
|
||||
std::string data_type;
|
||||
@ -27,96 +38,177 @@ TEST(data_type, data_type_get_common_type_Test)
|
||||
data_types.push_back(typeFromString(data_type));
|
||||
|
||||
return data_types;
|
||||
};
|
||||
};
|
||||
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString(""))->equals(*typeFromString("Nothing")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Nothing"))->equals(*typeFromString("Nothing")));
|
||||
struct TypesTestCase
|
||||
{
|
||||
const char * from_types;
|
||||
const char * expected_type = nullptr;
|
||||
};
|
||||
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("UInt8"))->equals(*typeFromString("UInt8")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("UInt8 UInt8"))->equals(*typeFromString("UInt8")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Int8 Int8"))->equals(*typeFromString("Int8")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("UInt8 Int8"))->equals(*typeFromString("Int16")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("UInt8 Int16"))->equals(*typeFromString("Int16")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("UInt8 UInt32 UInt64"))->equals(*typeFromString("UInt64")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Int8 Int32 Int64"))->equals(*typeFromString("Int64")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("UInt8 UInt32 Int64"))->equals(*typeFromString("Int64")));
|
||||
std::ostream & operator<<(std::ostream & ostr, const TypesTestCase & test_case)
|
||||
{
|
||||
ostr << "TypesTestCase{\"" << test_case.from_types << "\", ";
|
||||
if (test_case.expected_type)
|
||||
ostr << "\"" << test_case.expected_type << "\"";
|
||||
else
|
||||
ostr << "nullptr";
|
||||
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Float32 Float64"))->equals(*typeFromString("Float64")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Float32 UInt16 Int16"))->equals(*typeFromString("Float32")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Float32 UInt16 Int32"))->equals(*typeFromString("Float64")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Float32 Int16 UInt32"))->equals(*typeFromString("Float64")));
|
||||
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Date Date"))->equals(*typeFromString("Date")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Date DateTime"))->equals(*typeFromString("DateTime")));
|
||||
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("String FixedString(32) FixedString(8)"))->equals(*typeFromString("String")));
|
||||
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Array(UInt8) Array(UInt8)"))->equals(*typeFromString("Array(UInt8)")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Array(UInt8) Array(Int8)"))->equals(*typeFromString("Array(Int16)")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Array(Float32) Array(Int16) Array(UInt32)"))->equals(*typeFromString("Array(Float64)")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Array(Array(UInt8)) Array(Array(UInt8))"))->equals(*typeFromString("Array(Array(UInt8))")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Array(Array(UInt8)) Array(Array(Int8))"))->equals(*typeFromString("Array(Array(Int16))")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Array(Date) Array(DateTime)"))->equals(*typeFromString("Array(DateTime)")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Array(String) Array(FixedString(32))"))->equals(*typeFromString("Array(String)")));
|
||||
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Nullable(Nothing) Nothing"))->equals(*typeFromString("Nullable(Nothing)")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Nullable(UInt8) Int8"))->equals(*typeFromString("Nullable(Int16)")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Nullable(Nothing) UInt8 Int8"))->equals(*typeFromString("Nullable(Int16)")));
|
||||
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Tuple(Int8,UInt8) Tuple(UInt8,Int8)"))->equals(*typeFromString("Tuple(Int16,Int16)")));
|
||||
ASSERT_TRUE(getLeastSupertype(typesFromString("Tuple(Nullable(Nothing)) Tuple(Nullable(UInt8))"))->equals(*typeFromString("Tuple(Nullable(UInt8))")));
|
||||
|
||||
EXPECT_ANY_THROW(getLeastSupertype(typesFromString("Int8 String")));
|
||||
EXPECT_ANY_THROW(getLeastSupertype(typesFromString("Int64 UInt64")));
|
||||
EXPECT_ANY_THROW(getLeastSupertype(typesFromString("Float32 UInt64")));
|
||||
EXPECT_ANY_THROW(getLeastSupertype(typesFromString("Float64 Int64")));
|
||||
EXPECT_ANY_THROW(getLeastSupertype(typesFromString("Tuple(Int64) Tuple(UInt64)")));
|
||||
EXPECT_ANY_THROW(getLeastSupertype(typesFromString("Tuple(Int64, Int8) Tuple(UInt64)")));
|
||||
EXPECT_ANY_THROW(getLeastSupertype(typesFromString("Array(Int64) Array(String)")));
|
||||
|
||||
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString(""))->equals(*typeFromString("Nothing")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Nothing"))->equals(*typeFromString("Nothing")));
|
||||
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("UInt8"))->equals(*typeFromString("UInt8")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("UInt8 UInt8"))->equals(*typeFromString("UInt8")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Int8 Int8"))->equals(*typeFromString("Int8")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("UInt8 Int8"))->equals(*typeFromString("UInt8")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Int8 UInt16"))->equals(*typeFromString("Int8")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("UInt8 UInt32 UInt64"))->equals(*typeFromString("UInt8")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Int8 Int32 Int64"))->equals(*typeFromString("Int8")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("UInt8 Int64 UInt64"))->equals(*typeFromString("UInt8")));
|
||||
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Float32 Float64"))->equals(*typeFromString("Float32")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Float32 UInt16 Int16"))->equals(*typeFromString("UInt16")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Float32 UInt16 Int32"))->equals(*typeFromString("UInt16")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Float32 Int16 UInt32"))->equals(*typeFromString("Int16")));
|
||||
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("DateTime DateTime"))->equals(*typeFromString("DateTime")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Date DateTime"))->equals(*typeFromString("Date")));
|
||||
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("String FixedString(8)"))->equals(*typeFromString("FixedString(8)")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("FixedString(16) FixedString(8)"))->equals(*typeFromString("Nothing")));
|
||||
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Array(UInt8) Array(UInt8)"))->equals(*typeFromString("Array(UInt8)")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Array(UInt8) Array(Int8)"))->equals(*typeFromString("Array(UInt8)")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Array(Float32) Array(Int16) Array(UInt32)"))->equals(*typeFromString("Array(Int16)")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Array(Array(UInt8)) Array(Array(UInt8))"))->equals(*typeFromString("Array(Array(UInt8))")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Array(Array(UInt8)) Array(Array(Int8))"))->equals(*typeFromString("Array(Array(UInt8))")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Array(Date) Array(DateTime)"))->equals(*typeFromString("Array(Date)")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Array(String) Array(FixedString(32))"))->equals(*typeFromString("Array(FixedString(32))")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Array(String) Array(FixedString(32))"))->equals(*typeFromString("Array(FixedString(32))")));
|
||||
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Nullable(Nothing) Nothing"))->equals(*typeFromString("Nothing")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Nullable(UInt8) Int8"))->equals(*typeFromString("UInt8")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Nullable(Nothing) UInt8 Int8"))->equals(*typeFromString("Nothing")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Nullable(UInt8) Nullable(Int8)"))->equals(*typeFromString("Nullable(UInt8)")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Nullable(Nothing) Nullable(Int8)"))->equals(*typeFromString("Nullable(Nothing)")));
|
||||
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Tuple(Int8,UInt8) Tuple(UInt8,Int8)"))->equals(*typeFromString("Tuple(UInt8,UInt8)")));
|
||||
ASSERT_TRUE(getMostSubtype(typesFromString("Tuple(Nullable(Nothing)) Tuple(Nullable(UInt8))"))->equals(*typeFromString("Tuple(Nullable(Nothing))")));
|
||||
|
||||
EXPECT_ANY_THROW(getMostSubtype(typesFromString("Int8 String"), true));
|
||||
EXPECT_ANY_THROW(getMostSubtype(typesFromString("Nothing"), true));
|
||||
EXPECT_ANY_THROW(getMostSubtype(typesFromString("FixedString(16) FixedString(8) String"), true));
|
||||
return ostr << "}";
|
||||
}
|
||||
|
||||
class TypeTest : public ::testing::TestWithParam<TypesTestCase>
|
||||
{
|
||||
public:
|
||||
void SetUp()
|
||||
{
|
||||
const auto & p = GetParam();
|
||||
from_types = typesFromString(p.from_types);
|
||||
|
||||
if (p.expected_type)
|
||||
expected_type = typeFromString(p.expected_type);
|
||||
else
|
||||
expected_type.reset();
|
||||
}
|
||||
|
||||
public:
|
||||
DataTypes from_types;
|
||||
DataTypePtr expected_type;
|
||||
};
|
||||
|
||||
class LeastSuperTypeTest : public TypeTest {};
|
||||
|
||||
TEST_P(LeastSuperTypeTest, getLeastSupertype)
|
||||
{
|
||||
if (this->expected_type)
|
||||
{
|
||||
ASSERT_EQ(*this->expected_type, *getLeastSupertype(this->from_types));
|
||||
}
|
||||
else
|
||||
{
|
||||
EXPECT_ANY_THROW(getLeastSupertype(this->from_types));
|
||||
}
|
||||
}
|
||||
|
||||
class MostSubtypeTest : public TypeTest {};
|
||||
|
||||
TEST_P(MostSubtypeTest, getLeastSupertype)
|
||||
{
|
||||
if (this->expected_type)
|
||||
{
|
||||
ASSERT_EQ(*this->expected_type, *getMostSubtype(this->from_types));
|
||||
}
|
||||
else
|
||||
{
|
||||
EXPECT_ANY_THROW(getMostSubtype(this->from_types, true));
|
||||
}
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(data_type,
|
||||
LeastSuperTypeTest,
|
||||
::testing::ValuesIn(
|
||||
std::initializer_list<TypesTestCase>{
|
||||
{"", "Nothing"},
|
||||
{"Nothing", "Nothing"},
|
||||
|
||||
{"UInt8", "UInt8"},
|
||||
{"UInt8 UInt8", "UInt8"},
|
||||
{"Int8 Int8", "Int8"},
|
||||
{"UInt8 Int8", "Int16"},
|
||||
{"UInt8 Int16", "Int16"},
|
||||
{"UInt8 UInt32 UInt64", "UInt64"},
|
||||
{"Int8 Int32 Int64", "Int64"},
|
||||
{"UInt8 UInt32 Int64", "Int64"},
|
||||
|
||||
{"Float32 Float64", "Float64"},
|
||||
{"Float32 UInt16 Int16", "Float32"},
|
||||
{"Float32 UInt16 Int32", "Float64"},
|
||||
{"Float32 Int16 UInt32", "Float64"},
|
||||
|
||||
{"Date Date", "Date"},
|
||||
{"Date DateTime", "DateTime"},
|
||||
{"Date DateTime64(3)", "DateTime64(3)"},
|
||||
{"DateTime DateTime64(3)", "DateTime64(3)"},
|
||||
{"DateTime DateTime64(0)", "DateTime64(0)"},
|
||||
{"DateTime64(9) DateTime64(3)", "DateTime64(3)"},
|
||||
{"DateTime DateTime64(12)", "DateTime64(8)"},
|
||||
{"Date DateTime64(15)", "DateTime64(13)"},
|
||||
|
||||
{"String FixedString(32) FixedString(8)", "String"},
|
||||
|
||||
{"Array(UInt8) Array(UInt8)", "Array(UInt8)"},
|
||||
{"Array(UInt8) Array(Int8)", "Array(Int16)"},
|
||||
{"Array(Float32) Array(Int16) Array(UInt32)", "Array(Float64)"},
|
||||
{"Array(Array(UInt8)) Array(Array(UInt8))", "Array(Array(UInt8))"},
|
||||
{"Array(Array(UInt8)) Array(Array(Int8))", "Array(Array(Int16))"},
|
||||
{"Array(Date) Array(DateTime)", "Array(DateTime)"},
|
||||
{"Array(String) Array(FixedString(32))", "Array(String)"},
|
||||
|
||||
{"Nullable(Nothing) Nothing", "Nullable(Nothing)"},
|
||||
{"Nullable(UInt8) Int8", "Nullable(Int16)"},
|
||||
{"Nullable(Nothing) UInt8 Int8", "Nullable(Int16)"},
|
||||
|
||||
{"Tuple(Int8,UInt8) Tuple(UInt8,Int8)", "Tuple(Int16,Int16)"},
|
||||
{"Tuple(Nullable(Nothing)) Tuple(Nullable(UInt8))", "Tuple(Nullable(UInt8))"},
|
||||
|
||||
{"Int8 String", nullptr},
|
||||
{"Int64 UInt64", nullptr},
|
||||
{"Float32 UInt64", nullptr},
|
||||
{"Float64 Int64", nullptr},
|
||||
{"Tuple(Int64) Tuple(UInt64)", nullptr},
|
||||
{"Tuple(Int64,Int8) Tuple(UInt64)", nullptr},
|
||||
{"Array(Int64) Array(String)", nullptr},
|
||||
}
|
||||
),
|
||||
);
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(data_type,
|
||||
MostSubtypeTest,
|
||||
::testing::ValuesIn(
|
||||
std::initializer_list<TypesTestCase>{
|
||||
{"", "Nothing"},
|
||||
{"Nothing", "Nothing"},
|
||||
|
||||
{"UInt8", "UInt8"},
|
||||
{"UInt8 UInt8", "UInt8"},
|
||||
{"Int8 Int8", "Int8"},
|
||||
{"UInt8 Int8", "UInt8"},
|
||||
{"Int8 UInt16", "Int8"},
|
||||
{"UInt8 UInt32 UInt64", "UInt8"},
|
||||
{"Int8 Int32 Int64", "Int8"},
|
||||
{"UInt8 Int64 UInt64", "UInt8"},
|
||||
|
||||
{"Float32 Float64", "Float32"},
|
||||
{"Float32 UInt16 Int16", "UInt16"},
|
||||
{"Float32 UInt16 Int32", "UInt16"},
|
||||
{"Float32 Int16 UInt32", "Int16"},
|
||||
|
||||
{"DateTime DateTime", "DateTime"},
|
||||
{"Date DateTime", "Date"},
|
||||
|
||||
{"String FixedString(8)", "FixedString(8)"},
|
||||
{"FixedString(16) FixedString(8)", "Nothing"},
|
||||
|
||||
{"Array(UInt8) Array(UInt8)", "Array(UInt8)"},
|
||||
{"Array(UInt8) Array(Int8)", "Array(UInt8)"},
|
||||
{"Array(Float32) Array(Int16) Array(UInt32)", "Array(Int16)"},
|
||||
{"Array(Array(UInt8)) Array(Array(UInt8))", "Array(Array(UInt8))"},
|
||||
{"Array(Array(UInt8)) Array(Array(Int8))", "Array(Array(UInt8))"},
|
||||
{"Array(Date) Array(DateTime)", "Array(Date)"},
|
||||
{"Array(String) Array(FixedString(32))", "Array(FixedString(32))"},
|
||||
{"Array(String) Array(FixedString(32))", "Array(FixedString(32))"},
|
||||
|
||||
{"Nullable(Nothing) Nothing", "Nothing"},
|
||||
{"Nullable(UInt8) Int8", "UInt8"},
|
||||
{"Nullable(Nothing) UInt8 Int8", "Nothing"},
|
||||
{"Nullable(UInt8) Nullable(Int8)", "Nullable(UInt8)"},
|
||||
{"Nullable(Nothing) Nullable(Int8)", "Nullable(Nothing)"},
|
||||
|
||||
{"Tuple(Int8,UInt8) Tuple(UInt8,Int8)", "Tuple(UInt8,UInt8)"},
|
||||
{"Tuple(Nullable(Nothing)) Tuple(Nullable(UInt8))", "Tuple(Nullable(Nothing))"},
|
||||
|
||||
{"Int8 String", nullptr},
|
||||
{"Nothing", nullptr},
|
||||
{"FixedString(16) FixedString(8) String", nullptr},
|
||||
}),
|
||||
);
|
||||
|
@ -474,6 +474,11 @@ public:
|
||||
cannotConvertType("DateTime");
|
||||
}
|
||||
|
||||
bool readDateTime64(DateTime64 &, UInt32) override
|
||||
{
|
||||
cannotConvertType("DateTime64");
|
||||
}
|
||||
|
||||
bool readDecimal32(Decimal32 &, UInt32, UInt32) override
|
||||
{
|
||||
cannotConvertType("Decimal32");
|
||||
@ -606,6 +611,15 @@ public:
|
||||
return true;
|
||||
}
|
||||
|
||||
bool readDateTime64(DateTime64 & date_time, UInt32 scale) override
|
||||
{
|
||||
if (!readTempString())
|
||||
return false;
|
||||
ReadBufferFromString buf(temp_string);
|
||||
readDateTime64Text(date_time, scale, buf);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool readDecimal32(Decimal32 & decimal, UInt32 precision, UInt32 scale) override { return readDecimal(decimal, precision, scale); }
|
||||
bool readDecimal64(Decimal64 & decimal, UInt32 precision, UInt32 scale) override { return readDecimal(decimal, precision, scale); }
|
||||
bool readDecimal128(Decimal128 & decimal, UInt32 precision, UInt32 scale) override { return readDecimal(decimal, precision, scale); }
|
||||
@ -741,6 +755,11 @@ public:
|
||||
return true;
|
||||
}
|
||||
|
||||
bool readDateTime64(DateTime64 & date_time, UInt32 scale) override
|
||||
{
|
||||
return readDecimal(date_time, scale);
|
||||
}
|
||||
|
||||
bool readDecimal32(Decimal32 & decimal, UInt32, UInt32 scale) override { return readDecimal(decimal, scale); }
|
||||
bool readDecimal64(Decimal64 & decimal, UInt32, UInt32 scale) override { return readDecimal(decimal, scale); }
|
||||
bool readDecimal128(Decimal128 & decimal, UInt32, UInt32 scale) override { return readDecimal(decimal, scale); }
|
||||
|
@ -72,6 +72,7 @@ public:
|
||||
bool readUUID(UUID & uuid) { return current_converter->readUUID(uuid); }
|
||||
bool readDate(DayNum & date) { return current_converter->readDate(date); }
|
||||
bool readDateTime(time_t & tm) { return current_converter->readDateTime(tm); }
|
||||
bool readDateTime64(DateTime64 & tm, UInt32 scale) { return current_converter->readDateTime64(tm, scale); }
|
||||
|
||||
bool readDecimal(Decimal32 & decimal, UInt32 precision, UInt32 scale) { return current_converter->readDecimal32(decimal, precision, scale); }
|
||||
bool readDecimal(Decimal64 & decimal, UInt32 precision, UInt32 scale) { return current_converter->readDecimal64(decimal, precision, scale); }
|
||||
@ -152,6 +153,7 @@ private:
|
||||
virtual bool readUUID(UUID &) = 0;
|
||||
virtual bool readDate(DayNum &) = 0;
|
||||
virtual bool readDateTime(time_t &) = 0;
|
||||
virtual bool readDateTime64(DateTime64 &, UInt32) = 0;
|
||||
virtual bool readDecimal32(Decimal32 &, UInt32, UInt32) = 0;
|
||||
virtual bool readDecimal64(Decimal64 &, UInt32, UInt32) = 0;
|
||||
virtual bool readDecimal128(Decimal128 &, UInt32, UInt32) = 0;
|
||||
@ -227,6 +229,7 @@ public:
|
||||
bool readUUID(UUID &) { return false; }
|
||||
bool readDate(DayNum &) { return false; }
|
||||
bool readDateTime(time_t &) { return false; }
|
||||
bool readDateTime64(DateTime64 & /*tm*/, UInt32 /*scale*/) { return false; }
|
||||
bool readDecimal(Decimal32 &, UInt32, UInt32) { return false; }
|
||||
bool readDecimal(Decimal64 &, UInt32, UInt32) { return false; }
|
||||
bool readDecimal(Decimal128 &, UInt32, UInt32) { return false; }
|
||||
|
@ -329,6 +329,7 @@ public:
|
||||
virtual void writeUUID(const UUID &) override { cannotConvertType("UUID"); }
|
||||
virtual void writeDate(DayNum) override { cannotConvertType("Date"); }
|
||||
virtual void writeDateTime(time_t) override { cannotConvertType("DateTime"); }
|
||||
virtual void writeDateTime64(DateTime64, UInt32) override { cannotConvertType("DateTime64"); }
|
||||
virtual void writeDecimal32(Decimal32, UInt32) override { cannotConvertType("Decimal32"); }
|
||||
virtual void writeDecimal64(Decimal64, UInt32) override { cannotConvertType("Decimal64"); }
|
||||
virtual void writeDecimal128(const Decimal128 &, UInt32) override { cannotConvertType("Decimal128"); }
|
||||
@ -436,6 +437,13 @@ public:
|
||||
text_buffer.restart();
|
||||
}
|
||||
|
||||
void writeDateTime64(DateTime64 date_time, UInt32 scale) override
|
||||
{
|
||||
writeDateTimeText(date_time, scale, text_buffer);
|
||||
writeField(text_buffer.stringRef());
|
||||
text_buffer.restart();
|
||||
}
|
||||
|
||||
void writeDecimal32(Decimal32 decimal, UInt32 scale) override { writeDecimal(decimal, scale); }
|
||||
void writeDecimal64(Decimal64 decimal, UInt32 scale) override { writeDecimal(decimal, scale); }
|
||||
void writeDecimal128(const Decimal128 & decimal, UInt32 scale) override { writeDecimal(decimal, scale); }
|
||||
@ -533,7 +541,7 @@ public:
|
||||
|
||||
void writeDate(DayNum date) override { castNumericAndWriteField(static_cast<UInt16>(date)); }
|
||||
void writeDateTime(time_t tm) override { castNumericAndWriteField(tm); }
|
||||
|
||||
void writeDateTime64(DateTime64 date_time, UInt32 scale) override { writeDecimal(date_time, scale); }
|
||||
void writeDecimal32(Decimal32 decimal, UInt32 scale) override { writeDecimal(decimal, scale); }
|
||||
void writeDecimal64(Decimal64 decimal, UInt32 scale) override { writeDecimal(decimal, scale); }
|
||||
void writeDecimal128(const Decimal128 & decimal, UInt32 scale) override { writeDecimal(decimal, scale); }
|
||||
|
@ -5,6 +5,9 @@
|
||||
#include <common/DayNum.h>
|
||||
|
||||
#include "config_formats.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
#if USE_PROTOBUF
|
||||
|
||||
#include "ProtobufColumnMatcher.h"
|
||||
@ -70,6 +73,7 @@ public:
|
||||
bool writeUUID(const UUID & uuid) { return writeValueIfPossible(&IConverter::writeUUID, uuid); }
|
||||
bool writeDate(DayNum date) { return writeValueIfPossible(&IConverter::writeDate, date); }
|
||||
bool writeDateTime(time_t tm) { return writeValueIfPossible(&IConverter::writeDateTime, tm); }
|
||||
bool writeDateTime64(DateTime64 tm, UInt32 scale) { return writeValueIfPossible(&IConverter::writeDateTime64, tm, scale); }
|
||||
bool writeDecimal(Decimal32 decimal, UInt32 scale) { return writeValueIfPossible(&IConverter::writeDecimal32, decimal, scale); }
|
||||
bool writeDecimal(Decimal64 decimal, UInt32 scale) { return writeValueIfPossible(&IConverter::writeDecimal64, decimal, scale); }
|
||||
bool writeDecimal(const Decimal128 & decimal, UInt32 scale) { return writeValueIfPossible(&IConverter::writeDecimal128, decimal, scale); }
|
||||
@ -153,6 +157,7 @@ private:
|
||||
virtual void writeUUID(const UUID &) = 0;
|
||||
virtual void writeDate(DayNum) = 0;
|
||||
virtual void writeDateTime(time_t) = 0;
|
||||
virtual void writeDateTime64(DateTime64, UInt32 scale) = 0;
|
||||
virtual void writeDecimal32(Decimal32, UInt32) = 0;
|
||||
virtual void writeDecimal64(Decimal64, UInt32) = 0;
|
||||
virtual void writeDecimal128(const Decimal128 &, UInt32) = 0;
|
||||
@ -257,6 +262,7 @@ public:
|
||||
bool writeUUID(const UUID & /* value */) { return false; }
|
||||
bool writeDate(DayNum /* date */) { return false; }
|
||||
bool writeDateTime(time_t /* tm */) { return false; }
|
||||
bool writeDateTime64(DateTime64 /*tm*/, UInt32 /*scale*/) { return false; }
|
||||
bool writeDecimal(Decimal32 /* decimal */, UInt32 /* scale */) { return false; }
|
||||
bool writeDecimal(Decimal64 /* decimal */, UInt32 /* scale */) { return false; }
|
||||
bool writeDecimal(const Decimal128 & /* decimal */, UInt32 /* scale */) { return false; }
|
||||
|
@ -3,6 +3,7 @@
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
#include <Common/Exception.h>
|
||||
@ -90,24 +91,33 @@ struct ToStartOfWeekImpl
|
||||
template <typename FromType, typename ToType, typename Transform>
|
||||
struct Transformer
|
||||
{
|
||||
static void
|
||||
vector(const PaddedPODArray<FromType> & vec_from, PaddedPODArray<ToType> & vec_to, UInt8 week_mode, const DateLUTImpl & time_zone)
|
||||
Transformer(Transform transform_)
|
||||
: transform(std::move(transform_))
|
||||
{}
|
||||
|
||||
template <typename FromVectorType, typename ToVectorType>
|
||||
void
|
||||
vector(const FromVectorType & vec_from, ToVectorType & vec_to, UInt8 week_mode, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
size_t size = vec_from.size();
|
||||
vec_to.resize(size);
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
vec_to[i] = Transform::execute(vec_from[i], week_mode, time_zone);
|
||||
vec_to[i] = transform.execute(vec_from[i], week_mode, time_zone);
|
||||
}
|
||||
|
||||
private:
|
||||
const Transform transform;
|
||||
};
|
||||
|
||||
|
||||
template <typename FromType, typename ToType, typename Transform>
|
||||
template <typename FromDataType, typename ToDataType>
|
||||
struct CustomWeekTransformImpl
|
||||
{
|
||||
static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/)
|
||||
template <typename Transform>
|
||||
static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/, Transform transform = {})
|
||||
{
|
||||
using Op = Transformer<FromType, ToType, Transform>;
|
||||
const auto op = Transformer<typename FromDataType::FieldType, typename ToDataType::FieldType, Transform>{std::move(transform)};
|
||||
|
||||
UInt8 week_mode = DEFAULT_WEEK_MODE;
|
||||
if (arguments.size() > 1)
|
||||
@ -118,10 +128,10 @@ struct CustomWeekTransformImpl
|
||||
|
||||
const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block, arguments, 2, 0);
|
||||
const ColumnPtr source_col = block.getByPosition(arguments[0]).column;
|
||||
if (const auto * sources = checkAndGetColumn<ColumnVector<FromType>>(source_col.get()))
|
||||
if (const auto * sources = checkAndGetColumn<typename FromDataType::ColumnType>(source_col.get()))
|
||||
{
|
||||
auto col_to = ColumnVector<ToType>::create();
|
||||
Op::vector(sources->getData(), col_to->getData(), week_mode, time_zone);
|
||||
auto col_to = ToDataType::ColumnType::create();
|
||||
op.vector(sources->getData(), col_to->getData(), week_mode, time_zone);
|
||||
block.getByPosition(result).column = std::move(col_to);
|
||||
}
|
||||
else
|
||||
|
@ -1,11 +1,14 @@
|
||||
#pragma once
|
||||
#include <Core/Types.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <common/DateLUTImpl.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -629,32 +632,36 @@ struct ToYYYYMMDDhhmmssImpl
|
||||
template <typename FromType, typename ToType, typename Transform>
|
||||
struct Transformer
|
||||
{
|
||||
static void vector(const PaddedPODArray<FromType> & vec_from, PaddedPODArray<ToType> & vec_to, const DateLUTImpl & time_zone)
|
||||
template <typename FromTypeVector, typename ToTypeVector>
|
||||
static void vector(const FromTypeVector & vec_from, ToTypeVector & vec_to, const DateLUTImpl & time_zone, const Transform & transform)
|
||||
{
|
||||
size_t size = vec_from.size();
|
||||
vec_to.resize(size);
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
vec_to[i] = Transform::execute(vec_from[i], time_zone);
|
||||
vec_to[i] = transform.execute(vec_from[i], time_zone);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <typename FromType, typename ToType, typename Transform>
|
||||
template <typename FromDataType, typename ToDataType, typename Transform>
|
||||
struct DateTimeTransformImpl
|
||||
{
|
||||
static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/)
|
||||
static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/, const Transform & transform = {})
|
||||
{
|
||||
using Op = Transformer<FromType, ToType, Transform>;
|
||||
using Op = Transformer<typename FromDataType::FieldType, typename ToDataType::FieldType, Transform>;
|
||||
|
||||
const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block, arguments, 1, 0);
|
||||
|
||||
const ColumnPtr source_col = block.getByPosition(arguments[0]).column;
|
||||
if (const auto * sources = checkAndGetColumn<ColumnVector<FromType>>(source_col.get()))
|
||||
if (const auto * sources = checkAndGetColumn<typename FromDataType::ColumnType>(source_col.get()))
|
||||
{
|
||||
auto col_to = ColumnVector<ToType>::create();
|
||||
Op::vector(sources->getData(), col_to->getData(), time_zone);
|
||||
block.getByPosition(result).column = std::move(col_to);
|
||||
auto mutable_result_col = block.getByPosition(result).type->createColumn();
|
||||
auto * col_to = assert_cast<typename ToDataType::ColumnType *>(mutable_result_col.get());
|
||||
|
||||
Op::vector(sources->getData(), col_to->getData(), time_zone, transform);
|
||||
|
||||
block.getByPosition(result).column = std::move(mutable_result_col);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -9,6 +9,7 @@
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeInterval.h>
|
||||
#include <DataTypes/DataTypeAggregateFunction.h>
|
||||
#include <DataTypes/Native.h>
|
||||
|
@ -1,5 +1,6 @@
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Functions/CustomWeekTransforms.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
@ -100,11 +101,17 @@ public:
|
||||
WhichDataType which(from_type);
|
||||
|
||||
if (which.isDate())
|
||||
CustomWeekTransformImpl<DataTypeDate::FieldType, typename ToDataType::FieldType, Transform>::execute(
|
||||
block, arguments, result, input_rows_count);
|
||||
CustomWeekTransformImpl<DataTypeDate, ToDataType>::execute(
|
||||
block, arguments, result, input_rows_count, Transform{});
|
||||
else if (which.isDateTime())
|
||||
CustomWeekTransformImpl<DataTypeDateTime::FieldType, typename ToDataType::FieldType, Transform>::execute(
|
||||
block, arguments, result, input_rows_count);
|
||||
CustomWeekTransformImpl<DataTypeDateTime, ToDataType>::execute(
|
||||
block, arguments, result, input_rows_count, Transform{});
|
||||
else if (which.isDateTime64())
|
||||
{
|
||||
CustomWeekTransformImpl<DataTypeDateTime64, ToDataType>::execute(
|
||||
block, arguments, result, input_rows_count,
|
||||
DateTime64BasicTransformWrapper<Transform>{assert_cast<const DataTypeDateTime64 *>(from_type)->getScale()});
|
||||
}
|
||||
else
|
||||
throw Exception(
|
||||
"Illegal type " + block.getByPosition(arguments[0]).type->getName() + " of argument of function " + getName(),
|
||||
|
@ -2,6 +2,7 @@
|
||||
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
|
||||
#include <Columns/ColumnVector.h>
|
||||
|
||||
@ -22,9 +23,47 @@ namespace ErrorCodes
|
||||
extern const int ILLEGAL_COLUMN;
|
||||
}
|
||||
|
||||
|
||||
struct AddSecondsImpl
|
||||
/// AddOnDateTime64DefaultImpl provides default implementation of add-X functionality for DateTime64.
|
||||
///
|
||||
/// Default implementation is not to change fractional part, but only modify whole part as if it was DateTime.
|
||||
/// That means large whole values (for scale less than 9) might not fit into UInt32-range,
|
||||
/// and hence default implementation will produce incorrect results.
|
||||
template <typename T>
|
||||
struct AddOnDateTime64DefaultImpl
|
||||
{
|
||||
/*explicit*/ AddOnDateTime64DefaultImpl(UInt32 scale_ = 0)
|
||||
: scale_multiplier(DecimalUtils::scaleMultiplier<DateTime64::NativeType>(scale_))
|
||||
{}
|
||||
|
||||
// Default implementation for add/sub on DateTime64: do math on whole part (the same way as for DateTime), leave fractional as it is.
|
||||
inline DateTime64 execute(const DateTime64 & t, Int64 delta, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier);
|
||||
|
||||
const auto whole = static_cast<const T*>(this)->execute(static_cast<UInt32>(components.whole), delta, time_zone);
|
||||
return DecimalUtils::decimalFromComponentsWithMultiplier<DateTime64>(static_cast<DateTime64::NativeType>(whole), components.fractional, scale_multiplier);
|
||||
}
|
||||
|
||||
UInt32 scale_multiplier = 1;
|
||||
};
|
||||
|
||||
|
||||
/// Type of first argument of 'execute' function overload defines what INPUT DataType it is used for.
|
||||
/// Return type defines what is the OUTPUT (return) type of the CH function.
|
||||
/// Corresponding types:
|
||||
/// - UInt16 => DataTypeDate
|
||||
/// - UInt32 => DataTypeDateTime
|
||||
/// - DateTime64 => DataTypeDateTime64
|
||||
/// Please note that INPUT and OUTPUT types may differ, e.g.:
|
||||
/// - 'AddSecondsImpl::execute(UInt32, ...) -> UInt32' is available to the ClickHouse users as 'addSeconds(DateTime, ...) -> DateTime'
|
||||
/// - 'AddSecondsImpl::execute(UInt16, ...) -> UInt32' is available to the ClickHouse users as 'addSeconds(Date, ...) -> DateTime'
|
||||
|
||||
struct AddSecondsImpl : public AddOnDateTime64DefaultImpl<AddSecondsImpl>
|
||||
{
|
||||
using Base = AddOnDateTime64DefaultImpl<AddSecondsImpl>;
|
||||
using Base::Base;
|
||||
using Base::execute;
|
||||
|
||||
static constexpr auto name = "addSeconds";
|
||||
|
||||
static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &)
|
||||
@ -38,8 +77,12 @@ struct AddSecondsImpl
|
||||
}
|
||||
};
|
||||
|
||||
struct AddMinutesImpl
|
||||
struct AddMinutesImpl : public AddOnDateTime64DefaultImpl<AddMinutesImpl>
|
||||
{
|
||||
using Base = AddOnDateTime64DefaultImpl<AddMinutesImpl>;
|
||||
using Base::Base;
|
||||
using Base::execute;
|
||||
|
||||
static constexpr auto name = "addMinutes";
|
||||
|
||||
static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &)
|
||||
@ -53,8 +96,12 @@ struct AddMinutesImpl
|
||||
}
|
||||
};
|
||||
|
||||
struct AddHoursImpl
|
||||
struct AddHoursImpl : public AddOnDateTime64DefaultImpl<AddHoursImpl>
|
||||
{
|
||||
using Base = AddOnDateTime64DefaultImpl<AddHoursImpl>;
|
||||
using Base::Base;
|
||||
using Base::execute;
|
||||
|
||||
static constexpr auto name = "addHours";
|
||||
|
||||
static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &)
|
||||
@ -68,10 +115,20 @@ struct AddHoursImpl
|
||||
}
|
||||
};
|
||||
|
||||
struct AddDaysImpl
|
||||
struct AddDaysImpl : public AddOnDateTime64DefaultImpl<AddDaysImpl>
|
||||
{
|
||||
using Base = AddOnDateTime64DefaultImpl<AddDaysImpl>;
|
||||
using Base::Base;
|
||||
using Base::execute;
|
||||
|
||||
static constexpr auto name = "addDays";
|
||||
|
||||
// static inline UInt32 execute(UInt64 t, Int64 delta, const DateLUTImpl & time_zone)
|
||||
// {
|
||||
// // TODO (nemkov): LUT does not support out-of range date values for now.
|
||||
// return time_zone.addDays(t, delta);
|
||||
// }
|
||||
|
||||
static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.addDays(t, delta);
|
||||
@ -83,8 +140,12 @@ struct AddDaysImpl
|
||||
}
|
||||
};
|
||||
|
||||
struct AddWeeksImpl
|
||||
struct AddWeeksImpl : public AddOnDateTime64DefaultImpl<AddWeeksImpl>
|
||||
{
|
||||
using Base = AddOnDateTime64DefaultImpl<AddWeeksImpl>;
|
||||
using Base::Base;
|
||||
using Base::execute;
|
||||
|
||||
static constexpr auto name = "addWeeks";
|
||||
|
||||
static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone)
|
||||
@ -98,8 +159,12 @@ struct AddWeeksImpl
|
||||
}
|
||||
};
|
||||
|
||||
struct AddMonthsImpl
|
||||
struct AddMonthsImpl : public AddOnDateTime64DefaultImpl<AddMonthsImpl>
|
||||
{
|
||||
using Base = AddOnDateTime64DefaultImpl<AddMonthsImpl>;
|
||||
using Base::Base;
|
||||
using Base::execute;
|
||||
|
||||
static constexpr auto name = "addMonths";
|
||||
|
||||
static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone)
|
||||
@ -113,8 +178,12 @@ struct AddMonthsImpl
|
||||
}
|
||||
};
|
||||
|
||||
struct AddQuartersImpl
|
||||
struct AddQuartersImpl : public AddOnDateTime64DefaultImpl<AddQuartersImpl>
|
||||
{
|
||||
using Base = AddOnDateTime64DefaultImpl<AddQuartersImpl>;
|
||||
using Base::Base;
|
||||
using Base::execute;
|
||||
|
||||
static constexpr auto name = "addQuarters";
|
||||
|
||||
static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone)
|
||||
@ -128,8 +197,12 @@ struct AddQuartersImpl
|
||||
}
|
||||
};
|
||||
|
||||
struct AddYearsImpl
|
||||
struct AddYearsImpl : public AddOnDateTime64DefaultImpl<AddYearsImpl>
|
||||
{
|
||||
using Base = AddOnDateTime64DefaultImpl<AddYearsImpl>;
|
||||
using Base::Base;
|
||||
using Base::execute;
|
||||
|
||||
static constexpr auto name = "addYears";
|
||||
|
||||
static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone)
|
||||
@ -143,19 +216,16 @@ struct AddYearsImpl
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <typename Transform>
|
||||
struct SubtractIntervalImpl
|
||||
struct SubtractIntervalImpl : public Transform
|
||||
{
|
||||
static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone)
|
||||
using Transform::Transform;
|
||||
|
||||
template <typename T>
|
||||
inline auto execute(T t, Int64 delta, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
return Transform::execute(t, -delta, time_zone);
|
||||
}
|
||||
|
||||
static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return Transform::execute(d, -delta, time_zone);
|
||||
}
|
||||
};
|
||||
|
||||
struct SubtractSecondsImpl : SubtractIntervalImpl<AddSecondsImpl> { static constexpr auto name = "subtractSeconds"; };
|
||||
@ -168,68 +238,77 @@ struct SubtractQuartersImpl : SubtractIntervalImpl<AddQuartersImpl> { static con
|
||||
struct SubtractYearsImpl : SubtractIntervalImpl<AddYearsImpl> { static constexpr auto name = "subtractYears"; };
|
||||
|
||||
|
||||
template <typename FromType, typename ToType, typename Transform>
|
||||
template <typename Transform>
|
||||
struct Adder
|
||||
{
|
||||
static void vector_vector(const PaddedPODArray<FromType> & vec_from, PaddedPODArray<ToType> & vec_to, const IColumn & delta, const DateLUTImpl & time_zone)
|
||||
const Transform transform;
|
||||
|
||||
explicit Adder(Transform transform_)
|
||||
: transform(std::move(transform_))
|
||||
{}
|
||||
|
||||
template <typename FromVectorType, typename ToVectorType>
|
||||
void vector_vector(const FromVectorType & vec_from, ToVectorType & vec_to, const IColumn & delta, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
size_t size = vec_from.size();
|
||||
vec_to.resize(size);
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
vec_to[i] = Transform::execute(vec_from[i], delta.getInt(i), time_zone);
|
||||
vec_to[i] = transform.execute(vec_from[i], delta.getInt(i), time_zone);
|
||||
}
|
||||
|
||||
static void vector_constant(const PaddedPODArray<FromType> & vec_from, PaddedPODArray<ToType> & vec_to, Int64 delta, const DateLUTImpl & time_zone)
|
||||
template <typename FromVectorType, typename ToVectorType>
|
||||
void vector_constant(const FromVectorType & vec_from, ToVectorType & vec_to, Int64 delta, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
size_t size = vec_from.size();
|
||||
vec_to.resize(size);
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
vec_to[i] = Transform::execute(vec_from[i], delta, time_zone);
|
||||
vec_to[i] = transform.execute(vec_from[i], delta, time_zone);
|
||||
}
|
||||
|
||||
static void constant_vector(const FromType & from, PaddedPODArray<ToType> & vec_to, const IColumn & delta, const DateLUTImpl & time_zone)
|
||||
template <typename FromType, typename ToVectorType>
|
||||
void constant_vector(const FromType & from, ToVectorType & vec_to, const IColumn & delta, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
size_t size = delta.size();
|
||||
vec_to.resize(size);
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
vec_to[i] = Transform::execute(from, delta.getInt(i), time_zone);
|
||||
vec_to[i] = transform.execute(from, delta.getInt(i), time_zone);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <typename FromType, typename Transform>
|
||||
template <typename FromDataType, typename ToDataType, typename Transform>
|
||||
struct DateTimeAddIntervalImpl
|
||||
{
|
||||
static void execute(Block & block, const ColumnNumbers & arguments, size_t result)
|
||||
static void execute(Transform transform, Block & block, const ColumnNumbers & arguments, size_t result)
|
||||
{
|
||||
using ToType = decltype(Transform::execute(FromType(), 0, std::declval<DateLUTImpl>()));
|
||||
using Op = Adder<FromType, ToType, Transform>;
|
||||
using FromValueType = typename FromDataType::FieldType;
|
||||
using FromColumnType = typename FromDataType::ColumnType;
|
||||
using ToColumnType = typename ToDataType::ColumnType;
|
||||
|
||||
auto op = Adder<Transform>{std::move(transform)};
|
||||
|
||||
const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block, arguments, 2, 0);
|
||||
|
||||
const ColumnPtr source_col = block.getByPosition(arguments[0]).column;
|
||||
|
||||
if (const auto * sources = checkAndGetColumn<ColumnVector<FromType>>(source_col.get()))
|
||||
{
|
||||
auto col_to = ColumnVector<ToType>::create();
|
||||
auto result_col = block.getByPosition(result).type->createColumn();
|
||||
auto col_to = assert_cast<ToColumnType *>(result_col.get());
|
||||
|
||||
if (const auto * sources = checkAndGetColumn<FromColumnType>(source_col.get()))
|
||||
{
|
||||
const IColumn & delta_column = *block.getByPosition(arguments[1]).column;
|
||||
|
||||
if (const auto * delta_const_column = typeid_cast<const ColumnConst *>(&delta_column))
|
||||
Op::vector_constant(sources->getData(), col_to->getData(), delta_const_column->getField().get<Int64>(), time_zone);
|
||||
op.vector_constant(sources->getData(), col_to->getData(), delta_const_column->getField().get<Int64>(), time_zone);
|
||||
else
|
||||
Op::vector_vector(sources->getData(), col_to->getData(), delta_column, time_zone);
|
||||
|
||||
block.getByPosition(result).column = std::move(col_to);
|
||||
op.vector_vector(sources->getData(), col_to->getData(), delta_column, time_zone);
|
||||
}
|
||||
else if (const auto * sources_const = checkAndGetColumnConst<ColumnVector<FromType>>(source_col.get()))
|
||||
else if (const auto * sources_const = checkAndGetColumnConst<FromColumnType>(source_col.get()))
|
||||
{
|
||||
auto col_to = ColumnVector<ToType>::create();
|
||||
Op::constant_vector(sources_const->template getValue<FromType>(), col_to->getData(), *block.getByPosition(arguments[1]).column, time_zone);
|
||||
block.getByPosition(result).column = std::move(col_to);
|
||||
op.constant_vector(sources_const->template getValue<FromValueType>(), col_to->getData(), *block.getByPosition(arguments[1]).column, time_zone);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -237,9 +316,21 @@ struct DateTimeAddIntervalImpl
|
||||
+ " of first argument of function " + Transform::name,
|
||||
ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
|
||||
block.getByPosition(result).column = std::move(result_col);
|
||||
}
|
||||
};
|
||||
|
||||
namespace date_and_time_type_details
|
||||
{
|
||||
// Compile-time mapping of value (DataType::FieldType) types to corresponding DataType
|
||||
template <typename FieldType> struct ResultDataTypeMap {};
|
||||
template <> struct ResultDataTypeMap<UInt16> { using ResultDataType = DataTypeDate; };
|
||||
template <> struct ResultDataTypeMap<Int16> { using ResultDataType = DataTypeDate; };
|
||||
template <> struct ResultDataTypeMap<UInt32> { using ResultDataType = DataTypeDateTime; };
|
||||
template <> struct ResultDataTypeMap<Int32> { using ResultDataType = DataTypeDateTime; };
|
||||
template <> struct ResultDataTypeMap<DateTime64> { using ResultDataType = DataTypeDateTime64; };
|
||||
}
|
||||
|
||||
template <typename Transform>
|
||||
class FunctionDateOrDateTimeAddInterval : public IFunction
|
||||
@ -286,20 +377,58 @@ public:
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
|
||||
if (WhichDataType(arguments[0].type).isDate())
|
||||
switch (arguments[0].type->getTypeId())
|
||||
{
|
||||
if (std::is_same_v<decltype(Transform::execute(DataTypeDate::FieldType(), 0, std::declval<DateLUTImpl>())), UInt16>)
|
||||
case TypeIndex::Date:
|
||||
return resolveReturnType<DataTypeDate>(arguments);
|
||||
case TypeIndex::DateTime:
|
||||
return resolveReturnType<DataTypeDateTime>(arguments);
|
||||
case TypeIndex::DateTime64:
|
||||
return resolveReturnType<DataTypeDateTime64>(arguments);
|
||||
default:
|
||||
{
|
||||
throw Exception("Invalid type of 1st argument of function " + getName() + ": "
|
||||
+ arguments[0].type->getName() + ", expected: Date, DateTime or DateTime64.",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helper templates to deduce return type based on argument type, since some overloads may promote or denote types, e.g. addSeconds(Date, 1) => DateTime
|
||||
template <typename FieldType>
|
||||
using TransformExecuteReturnType = decltype(std::declval<Transform>().execute(FieldType(), 0, std::declval<DateLUTImpl>()));
|
||||
|
||||
// Deduces RETURN DataType from INTPUT DataType, based on return type of Transform{}.execute(INPUT_TYPE, UInt64, DateLUTImpl).
|
||||
// e.g. for Transform-type that has execute()-overload with 'UInt16' input and 'UInt32' return,
|
||||
// argument type is expected to be 'Date', and result type is deduced to be 'DateTime'.
|
||||
template <typename FromDataType>
|
||||
using TransformResultDataType = typename date_and_time_type_details::ResultDataTypeMap<TransformExecuteReturnType<typename FromDataType::FieldType>>::ResultDataType;
|
||||
|
||||
template <typename FromDataType>
|
||||
DataTypePtr resolveReturnType(const ColumnsWithTypeAndName & arguments) const
|
||||
{
|
||||
using ResultDataType = TransformResultDataType<FromDataType>;
|
||||
|
||||
if constexpr (std::is_same_v<ResultDataType, DataTypeDate>)
|
||||
return std::make_shared<DataTypeDate>();
|
||||
else
|
||||
else if constexpr (std::is_same_v<ResultDataType, DataTypeDateTime>)
|
||||
{
|
||||
return std::make_shared<DataTypeDateTime>(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0));
|
||||
}
|
||||
else if constexpr (std::is_same_v<ResultDataType, DataTypeDateTime64>)
|
||||
{
|
||||
// TODO (vnemkov): what if there is an overload of Transform::execute() that returns DateTime64 from DateTime or Date ?
|
||||
// Shall we use the default scale or one from optional argument ?
|
||||
const auto & datetime64_type = assert_cast<const DataTypeDateTime64 &>(*arguments[0].type);
|
||||
return std::make_shared<DataTypeDateTime64>(datetime64_type.getScale(), extractTimeZoneNameFromFunctionArguments(arguments, 2, 0));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (std::is_same_v<decltype(Transform::execute(DataTypeDateTime::FieldType(), 0, std::declval<DateLUTImpl>())), UInt16>)
|
||||
return std::make_shared<DataTypeDate>();
|
||||
else
|
||||
return std::make_shared<DataTypeDateTime>(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0));
|
||||
static_assert("Failed to resolve return type.");
|
||||
}
|
||||
|
||||
//to make PVS and GCC happy.
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
@ -311,9 +440,17 @@ public:
|
||||
WhichDataType which(from_type);
|
||||
|
||||
if (which.isDate())
|
||||
DateTimeAddIntervalImpl<DataTypeDate::FieldType, Transform>::execute(block, arguments, result);
|
||||
{
|
||||
DateTimeAddIntervalImpl<DataTypeDate, TransformResultDataType<DataTypeDate>, Transform>::execute(Transform{}, block, arguments, result);
|
||||
}
|
||||
else if (which.isDateTime())
|
||||
DateTimeAddIntervalImpl<DataTypeDateTime::FieldType, Transform>::execute(block, arguments, result);
|
||||
{
|
||||
DateTimeAddIntervalImpl<DataTypeDateTime, TransformResultDataType<DataTypeDateTime>, Transform>::execute(Transform{}, block, arguments, result);
|
||||
}
|
||||
else if (const auto * datetime64_type = assert_cast<const DataTypeDateTime64 *>(from_type))
|
||||
{
|
||||
DateTimeAddIntervalImpl<DataTypeDateTime64, TransformResultDataType<DataTypeDateTime64>, Transform>::execute(Transform{datetime64_type->getScale()}, block, arguments, result);
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal type " + block.getByPosition(arguments[0]).type->getName() + " of argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
@ -1,5 +1,6 @@
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
#include <Functions/DateTimeTransforms.h>
|
||||
@ -15,6 +16,25 @@ namespace ErrorCodes
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
}
|
||||
|
||||
template<class Transform>
|
||||
struct WithDateTime64Converter : public Transform
|
||||
{
|
||||
UInt8 scale;
|
||||
Transform transform;
|
||||
|
||||
explicit WithDateTime64Converter(UInt8 scale_, Transform transform_ = {})
|
||||
: scale(scale_),
|
||||
transform(std::move(transform_))
|
||||
{}
|
||||
|
||||
inline auto execute(DataTypeDateTime64::FieldType t, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
auto x = DateTime64(t);
|
||||
auto res = transform.execute(static_cast<UInt32>(DecimalUtils::getWholePart(x, scale)), time_zone);
|
||||
return res;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/// See DateTimeTransforms.h
|
||||
template <typename ToDataType, typename Transform>
|
||||
@ -67,8 +87,10 @@ public:
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
/// For DateTime, if time zone is specified, attach it to type.
|
||||
if (std::is_same_v<ToDataType, DataTypeDateTime>)
|
||||
return std::make_shared<DataTypeDateTime>(extractTimeZoneNameFromFunctionArguments(arguments, 1, 0));
|
||||
if constexpr (std::is_same_v<ToDataType, DataTypeDateTime>)
|
||||
return std::make_shared<ToDataType>(extractTimeZoneNameFromFunctionArguments(arguments, 1, 0));
|
||||
if constexpr (std::is_same_v<ToDataType, DataTypeDateTime64>)
|
||||
return std::make_shared<ToDataType>(extractTimeZoneNameFromFunctionArguments(arguments, 1, 0));
|
||||
else
|
||||
return std::make_shared<ToDataType>();
|
||||
}
|
||||
@ -82,9 +104,16 @@ public:
|
||||
WhichDataType which(from_type);
|
||||
|
||||
if (which.isDate())
|
||||
DateTimeTransformImpl<DataTypeDate::FieldType, typename ToDataType::FieldType, Transform>::execute(block, arguments, result, input_rows_count);
|
||||
DateTimeTransformImpl<DataTypeDate, ToDataType, Transform>::execute(block, arguments, result, input_rows_count);
|
||||
else if (which.isDateTime())
|
||||
DateTimeTransformImpl<DataTypeDateTime::FieldType, typename ToDataType::FieldType, Transform>::execute(block, arguments, result, input_rows_count);
|
||||
DateTimeTransformImpl<DataTypeDateTime, ToDataType, Transform>::execute(block, arguments, result, input_rows_count);
|
||||
else if (which.isDateTime64())
|
||||
{
|
||||
const auto scale = static_cast<const DataTypeDateTime64 *>(from_type)->getScale();
|
||||
WithDateTime64Converter<Transform> transformer(scale);
|
||||
|
||||
DateTimeTransformImpl<DataTypeDateTime64, ToDataType, WithDateTime64Converter<Transform>>::execute(block, arguments, result, input_rows_count, transformer);
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal type " + block.getByPosition(arguments[0]).type->getName() + " of argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
@ -119,6 +119,75 @@ void validateArgumentType(const IFunction & func, const DataTypes & arguments,
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
void validateArgumentsImpl(const IFunction & func,
|
||||
const ColumnsWithTypeAndName & arguments,
|
||||
size_t argument_offset,
|
||||
const FunctionArgumentTypeValidators & validators)
|
||||
{
|
||||
for (size_t i = 0; i < validators.size(); ++i)
|
||||
{
|
||||
const auto argument_index = i + argument_offset;
|
||||
if (argument_index >= arguments.size())
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
const auto & arg = arguments[i + argument_offset];
|
||||
const auto validator = validators[i];
|
||||
if (!validator.validator_func(*arg.type))
|
||||
throw Exception("Illegal type " + arg.type->getName() +
|
||||
" of " + std::to_string(i) +
|
||||
" argument of function " + func.getName() +
|
||||
" expected " + validator.expected_type_description,
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void validateFunctionArgumentTypes(const IFunction & func,
|
||||
const ColumnsWithTypeAndName & arguments,
|
||||
const FunctionArgumentTypeValidators & mandatory_args,
|
||||
const FunctionArgumentTypeValidators & optional_args)
|
||||
{
|
||||
if (arguments.size() < mandatory_args.size())
|
||||
{
|
||||
auto joinArgumentTypes = [](const auto & args, const String sep = ", ") -> String
|
||||
{
|
||||
String result;
|
||||
for (const auto & a : args)
|
||||
{
|
||||
using A = std::decay_t<decltype(a)>;
|
||||
if constexpr (std::is_same_v<A, FunctionArgumentTypeValidator>)
|
||||
result += a.expected_type_description;
|
||||
else if constexpr (std::is_same_v<A, ColumnWithTypeAndName>)
|
||||
result += a.type->getName();
|
||||
|
||||
result += sep;
|
||||
}
|
||||
|
||||
if (args.size() != 0)
|
||||
result.erase(result.end() - sep.length(), result.end());
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
throw Exception("Incorrect number of arguments of function " + func.getName()
|
||||
+ " provided " + std::to_string(arguments.size()) + " (" + joinArgumentTypes(arguments) + ")"
|
||||
+ " expected " + std::to_string(mandatory_args.size()) + (optional_args.size() ? " or " + std::to_string(mandatory_args.size() + optional_args.size()) : "")
|
||||
+ " (" + joinArgumentTypes(mandatory_args) + (optional_args.size() ? ", [" + joinArgumentTypes(mandatory_args) + "]" : "") + ")",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
}
|
||||
|
||||
validateArgumentsImpl(func, arguments, 0, mandatory_args);
|
||||
if (optional_args.size())
|
||||
{
|
||||
validateArgumentsImpl(func, arguments, mandatory_args.size(), optional_args);
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<std::vector<const IColumn *>, const ColumnArray::Offset *>
|
||||
checkAndGetNestedArrayOffset(const IColumn ** columns, size_t num_arguments)
|
||||
{
|
||||
|
@ -90,6 +90,35 @@ void validateArgumentType(const IFunction & func, const DataTypes & arguments,
|
||||
size_t argument_index, bool (* validator_func)(const IDataType &),
|
||||
const char * expected_type_description);
|
||||
|
||||
// Simple validator that is used in conjunction with validateFunctionArgumentTypes() to check if function arguments are as expected.
|
||||
struct FunctionArgumentTypeValidator
|
||||
{
|
||||
bool (* validator_func)(const IDataType &);
|
||||
const char * expected_type_description;
|
||||
};
|
||||
|
||||
using FunctionArgumentTypeValidators = std::vector<FunctionArgumentTypeValidator>;
|
||||
|
||||
/** Validate that function arguments match specification.
|
||||
*
|
||||
* Designed to simplify argument validation
|
||||
* for functions with variable arguments (e.g. depending on result type or other trait).
|
||||
* first, checks that mandatory args present and have valid type.
|
||||
* second, checks optional arguents types, skipping ones that are missing.
|
||||
*
|
||||
* Please note that if you have several optional arguments, like f([a, b, c]),
|
||||
* only these calls are considered valid:
|
||||
* f(a)
|
||||
* f(a, b)
|
||||
* f(a, b, c)
|
||||
*
|
||||
* But NOT these: f(a, c), f(b, c)
|
||||
* In other words you can't skip
|
||||
*
|
||||
* If any mandatory arg is missing, throw an exception, with explicit description of expected arguments.
|
||||
*/
|
||||
void validateFunctionArgumentTypes(const IFunction & func, const ColumnsWithTypeAndName & arguments, const FunctionArgumentTypeValidators & mandatory_args, const FunctionArgumentTypeValidators & optional_args = {});
|
||||
|
||||
/// Checks if a list of array columns have equal offsets. Return a pair of nested columns and offsets if true, otherwise throw.
|
||||
std::pair<std::vector<const IColumn *>, const ColumnArray::Offset *>
|
||||
checkAndGetNestedArrayOffset(const IColumn ** columns, size_t num_arguments);
|
||||
|
@ -13,6 +13,7 @@
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeUUID.h>
|
||||
@ -684,7 +685,7 @@ private:
|
||||
return true;
|
||||
};
|
||||
|
||||
if (!callOnBasicTypes<true, false, true, false>(left_number, right_number, call))
|
||||
if (!callOnBasicTypes<true, false, true, true>(left_number, right_number, call))
|
||||
throw Exception("Wrong call for " + getName() + " with " + col_left.type->getName() + " and " + col_right.type->getName(),
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
@ -1192,9 +1193,10 @@ public:
|
||||
{
|
||||
executeTuple(block, result, col_with_type_and_name_left, col_with_type_and_name_right, input_rows_count);
|
||||
}
|
||||
else if (isDecimal(left_type) || isDecimal(right_type))
|
||||
else if (isColumnedAsDecimal(left_type) || isColumnedAsDecimal(right_type))
|
||||
{
|
||||
if (!allowDecimalComparison(left_type, right_type))
|
||||
// compare
|
||||
if (!allowDecimalComparison(left_type, right_type) && !date_and_datetime)
|
||||
throw Exception("No operation " + getName() + " between " + left_type->getName() + " and " + right_type->getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
|
@ -46,6 +46,7 @@ void registerFunctionsConversion(FunctionFactory & factory)
|
||||
|
||||
factory.registerFunction<FunctionToDate>();
|
||||
factory.registerFunction<FunctionToDateTime>();
|
||||
factory.registerFunction<FunctionToDateTime64>();
|
||||
factory.registerFunction<FunctionToUUID>();
|
||||
factory.registerFunction<FunctionToString>();
|
||||
factory.registerFunction<FunctionToFixedString>();
|
||||
@ -65,6 +66,7 @@ void registerFunctionsConversion(FunctionFactory & factory)
|
||||
factory.registerFunction<FunctionToFloat64OrZero>();
|
||||
factory.registerFunction<FunctionToDateOrZero>();
|
||||
factory.registerFunction<FunctionToDateTimeOrZero>();
|
||||
factory.registerFunction<FunctionToDateTime64OrZero>();
|
||||
|
||||
factory.registerFunction<FunctionToDecimal32OrZero>();
|
||||
factory.registerFunction<FunctionToDecimal64OrZero>();
|
||||
@ -82,6 +84,7 @@ void registerFunctionsConversion(FunctionFactory & factory)
|
||||
factory.registerFunction<FunctionToFloat64OrNull>();
|
||||
factory.registerFunction<FunctionToDateOrNull>();
|
||||
factory.registerFunction<FunctionToDateTimeOrNull>();
|
||||
factory.registerFunction<FunctionToDateTime64OrNull>();
|
||||
|
||||
factory.registerFunction<FunctionToDecimal32OrNull>();
|
||||
factory.registerFunction<FunctionToDecimal64OrNull>();
|
||||
@ -90,6 +93,9 @@ void registerFunctionsConversion(FunctionFactory & factory)
|
||||
factory.registerFunction<FunctionParseDateTimeBestEffort>();
|
||||
factory.registerFunction<FunctionParseDateTimeBestEffortOrZero>();
|
||||
factory.registerFunction<FunctionParseDateTimeBestEffortOrNull>();
|
||||
factory.registerFunction<FunctionParseDateTime64BestEffort>();
|
||||
factory.registerFunction<FunctionParseDateTime64BestEffortOrZero>();
|
||||
factory.registerFunction<FunctionParseDateTime64BestEffortOrNull>();
|
||||
|
||||
factory.registerFunction<FunctionConvert<DataTypeInterval, NameToIntervalSecond, PositiveMonotonicity>>();
|
||||
factory.registerFunction<FunctionConvert<DataTypeInterval, NameToIntervalMinute, PositiveMonotonicity>>();
|
||||
|
@ -16,6 +16,7 @@
|
||||
#include <DataTypes/DataTypeFixedString.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeEnum.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
@ -106,15 +107,18 @@ struct ConvertImpl
|
||||
{
|
||||
const ColumnWithTypeAndName & named_from = block.getByPosition(arguments[0]);
|
||||
|
||||
using ColVecFrom = std::conditional_t<IsDecimalNumber<FromFieldType>, ColumnDecimal<FromFieldType>, ColumnVector<FromFieldType>>;
|
||||
using ColVecTo = std::conditional_t<IsDecimalNumber<ToFieldType>, ColumnDecimal<ToFieldType>, ColumnVector<ToFieldType>>;
|
||||
using ColVecFrom = typename FromDataType::ColumnType;
|
||||
using ColVecTo = typename ToDataType::ColumnType;
|
||||
|
||||
if constexpr (IsDataTypeDecimal<FromDataType> || IsDataTypeDecimal<ToDataType>)
|
||||
if constexpr ((IsDataTypeDecimal<FromDataType> || IsDataTypeDecimal<ToDataType>)
|
||||
&& !(std::is_same_v<DataTypeDateTime64, FromDataType> || std::is_same_v<DataTypeDateTime64, ToDataType>))
|
||||
{
|
||||
if constexpr (!IsDataTypeDecimalOrNumber<FromDataType> || !IsDataTypeDecimalOrNumber<ToDataType>)
|
||||
{
|
||||
throw Exception("Illegal column " + named_from.column->getName() + " of first argument of function " + Name::name,
|
||||
ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
}
|
||||
|
||||
if (const ColVecFrom * col_from = checkAndGetColumn<ColVecFrom>(named_from.column.get()))
|
||||
{
|
||||
@ -155,6 +159,11 @@ struct ConvertImpl
|
||||
}
|
||||
};
|
||||
|
||||
/** Conversion of DateTime to Date: throw off time component.
|
||||
*/
|
||||
template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDate, Name>
|
||||
: DateTimeTransformImpl<DataTypeDateTime, DataTypeDate, ToDateImpl> {};
|
||||
|
||||
|
||||
/** Conversion of Date to DateTime: adding 00:00:00 time component.
|
||||
*/
|
||||
@ -166,11 +175,16 @@ struct ToDateTimeImpl
|
||||
{
|
||||
return time_zone.fromDayNum(DayNum(d));
|
||||
}
|
||||
|
||||
// no-op conversion from DateTime to DateTime, used in DateTime64 to DateTime conversion.
|
||||
static inline UInt32 execute(UInt32 d, const DateLUTImpl & /*time_zone*/)
|
||||
{
|
||||
return d;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Name> struct ConvertImpl<DataTypeDate, DataTypeDateTime, Name>
|
||||
: DateTimeTransformImpl<UInt16, UInt32, ToDateTimeImpl> {};
|
||||
|
||||
: DateTimeTransformImpl<DataTypeDate, DataTypeDateTime, ToDateTimeImpl> {};
|
||||
|
||||
/// Implementation of toDate function.
|
||||
|
||||
@ -185,11 +199,6 @@ struct ToDateTransform32Or64
|
||||
}
|
||||
};
|
||||
|
||||
/** Conversion of DateTime to Date: throw off time component.
|
||||
*/
|
||||
template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDate, Name>
|
||||
: DateTimeTransformImpl<UInt32, UInt16, ToDateImpl> {};
|
||||
|
||||
/** Special case of converting (U)Int32 or (U)Int64 (and also, for convenience, Float32, Float64) to Date.
|
||||
* If number is less than 65536, then it is treated as DayNum, and if greater or equals, then as unix timestamp.
|
||||
* It's a bit illogical, as we actually have two functions in one.
|
||||
@ -198,17 +207,72 @@ template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDate, Name
|
||||
* (otherwise such usage would be frequent mistake).
|
||||
*/
|
||||
template <typename Name> struct ConvertImpl<DataTypeUInt32, DataTypeDate, Name>
|
||||
: DateTimeTransformImpl<UInt32, UInt16, ToDateTransform32Or64<UInt32, UInt16>> {};
|
||||
: DateTimeTransformImpl<DataTypeUInt32, DataTypeDate, ToDateTransform32Or64<UInt32, UInt16>> {};
|
||||
template <typename Name> struct ConvertImpl<DataTypeUInt64, DataTypeDate, Name>
|
||||
: DateTimeTransformImpl<UInt64, UInt16, ToDateTransform32Or64<UInt64, UInt16>> {};
|
||||
: DateTimeTransformImpl<DataTypeUInt64, DataTypeDate, ToDateTransform32Or64<UInt64, UInt16>> {};
|
||||
template <typename Name> struct ConvertImpl<DataTypeInt32, DataTypeDate, Name>
|
||||
: DateTimeTransformImpl<Int32, UInt16, ToDateTransform32Or64<Int32, UInt16>> {};
|
||||
: DateTimeTransformImpl<DataTypeInt32, DataTypeDate, ToDateTransform32Or64<Int32, UInt16>> {};
|
||||
template <typename Name> struct ConvertImpl<DataTypeInt64, DataTypeDate, Name>
|
||||
: DateTimeTransformImpl<Int64, UInt16, ToDateTransform32Or64<Int64, UInt16>> {};
|
||||
: DateTimeTransformImpl<DataTypeInt64, DataTypeDate, ToDateTransform32Or64<Int64, UInt16>> {};
|
||||
template <typename Name> struct ConvertImpl<DataTypeFloat32, DataTypeDate, Name>
|
||||
: DateTimeTransformImpl<Float32, UInt16, ToDateTransform32Or64<Float32, UInt16>> {};
|
||||
: DateTimeTransformImpl<DataTypeFloat32, DataTypeDate, ToDateTransform32Or64<Float32, UInt16>> {};
|
||||
template <typename Name> struct ConvertImpl<DataTypeFloat64, DataTypeDate, Name>
|
||||
: DateTimeTransformImpl<Float64, UInt16, ToDateTransform32Or64<Float64, UInt16>> {};
|
||||
: DateTimeTransformImpl<DataTypeFloat64, DataTypeDate, ToDateTransform32Or64<Float64, UInt16>> {};
|
||||
|
||||
|
||||
/** Conversion of Date or DateTime to DateTime64: add zero sub-second part.
|
||||
*/
|
||||
struct ToDateTime64Transform
|
||||
{
|
||||
static constexpr auto name = "toDateTime64";
|
||||
|
||||
const DateTime64::NativeType scale_multiplier = 1;
|
||||
|
||||
ToDateTime64Transform(UInt32 scale = 0)
|
||||
: scale_multiplier(DecimalUtils::scaleMultiplier<DateTime64::NativeType>(scale))
|
||||
{}
|
||||
|
||||
inline DateTime64::NativeType execute(UInt16 d, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
const auto dt = ToDateTimeImpl::execute(d, time_zone);
|
||||
return execute(dt, time_zone);
|
||||
}
|
||||
|
||||
inline DateTime64::NativeType execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) const
|
||||
{
|
||||
return DecimalUtils::decimalFromComponentsWithMultiplier<DateTime64>(dt, 0, scale_multiplier);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Name> struct ConvertImpl<DataTypeDate, DataTypeDateTime64, Name>
|
||||
: DateTimeTransformImpl<DataTypeDate, DataTypeDateTime64, ToDateTime64Transform> {};
|
||||
template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDateTime64, Name>
|
||||
: DateTimeTransformImpl<DataTypeDateTime, DataTypeDateTime64, ToDateTime64Transform> {};
|
||||
|
||||
/** Conversion of DateTime64 to Date or DateTime: discards fractional part.
|
||||
*/
|
||||
template <typename Transform>
|
||||
struct FromDateTime64Transform
|
||||
{
|
||||
static constexpr auto name = Transform::name;
|
||||
|
||||
const DateTime64::NativeType scale_multiplier = 1;
|
||||
|
||||
FromDateTime64Transform(UInt32 scale)
|
||||
: scale_multiplier(DecimalUtils::scaleMultiplier<DateTime64::NativeType>(scale))
|
||||
{}
|
||||
|
||||
inline auto execute(DateTime64::NativeType dt, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
const auto c = DecimalUtils::splitWithScaleMultiplier(DateTime64(dt), scale_multiplier);
|
||||
return Transform::execute(static_cast<UInt32>(c.whole), time_zone);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Name> struct ConvertImpl<DataTypeDateTime64, DataTypeDate, Name>
|
||||
: DateTimeTransformImpl<DataTypeDateTime64, DataTypeDate, FromDateTime64Transform<ToDateImpl>> {};
|
||||
template <typename Name> struct ConvertImpl<DataTypeDateTime64, DataTypeDateTime, Name>
|
||||
: DateTimeTransformImpl<DataTypeDateTime64, DataTypeDateTime, FromDateTime64Transform<ToDateTimeImpl>> {};
|
||||
|
||||
|
||||
/** Transformation of numbers, dates, datetimes to strings: through formatting.
|
||||
@ -240,6 +304,16 @@ struct FormatImpl<DataTypeDateTime>
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct FormatImpl<DataTypeDateTime64>
|
||||
{
|
||||
static void execute(const DataTypeDateTime64::FieldType x, WriteBuffer & wb, const DataTypeDateTime64 * type, const DateLUTImpl * time_zone)
|
||||
{
|
||||
writeDateTimeText(DateTime64(x), type->getScale(), wb, *time_zone);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <typename FieldType>
|
||||
struct FormatImpl<DataTypeEnum<FieldType>>
|
||||
{
|
||||
@ -284,7 +358,7 @@ struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType,
|
||||
const DateLUTImpl * time_zone = nullptr;
|
||||
|
||||
/// For argument of DateTime type, second argument with time zone could be specified.
|
||||
if constexpr (std::is_same_v<FromDataType, DataTypeDateTime>)
|
||||
if constexpr (std::is_same_v<FromDataType, DataTypeDateTime> || std::is_same_v<FromDataType, DataTypeDateTime64>)
|
||||
time_zone = &extractTimeZoneFromFunctionArguments(block, arguments, 1, 0);
|
||||
|
||||
if (const auto col_from = checkAndGetColumn<ColVecType>(col_with_type_and_name.column.get()))
|
||||
@ -300,6 +374,8 @@ struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType,
|
||||
data_to.resize(size * (strlen("YYYY-MM-DD") + 1));
|
||||
else if constexpr (std::is_same_v<FromDataType, DataTypeDateTime>)
|
||||
data_to.resize(size * (strlen("YYYY-MM-DD hh:mm:ss") + 1));
|
||||
else if constexpr (std::is_same_v<FromDataType, DataTypeDateTime64>)
|
||||
data_to.resize(size * (strlen("YYYY-MM-DD hh:mm:ss.") + vec_from.getScale() + 1));
|
||||
else
|
||||
data_to.resize(size * 3); /// Arbitary
|
||||
|
||||
@ -448,6 +524,8 @@ struct ConvertThroughParsing
|
||||
static_assert(std::is_same_v<FromDataType, DataTypeString> || std::is_same_v<FromDataType, DataTypeFixedString>,
|
||||
"ConvertThroughParsing is only applicable for String or FixedString data types");
|
||||
|
||||
static constexpr bool to_datetime64 = std::is_same_v<ToDataType, DataTypeDateTime64>;
|
||||
|
||||
using ToFieldType = typename ToDataType::FieldType;
|
||||
|
||||
static bool isAllRead(ReadBuffer & in)
|
||||
@ -471,15 +549,22 @@ struct ConvertThroughParsing
|
||||
static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count,
|
||||
Additions additions [[maybe_unused]] = Additions())
|
||||
{
|
||||
using ColVecTo = std::conditional_t<IsDecimalNumber<ToFieldType>, ColumnDecimal<ToFieldType>, ColumnVector<ToFieldType>>;
|
||||
using ColVecTo = typename ToDataType::ColumnType;
|
||||
|
||||
const DateLUTImpl * local_time_zone [[maybe_unused]] = nullptr;
|
||||
const DateLUTImpl * utc_time_zone [[maybe_unused]] = nullptr;
|
||||
|
||||
/// For conversion to DateTime type, second argument with time zone could be specified.
|
||||
if constexpr (std::is_same_v<ToDataType, DataTypeDateTime>)
|
||||
if constexpr (std::is_same_v<ToDataType, DataTypeDateTime> || to_datetime64)
|
||||
{
|
||||
const auto result_type = removeNullable(block.getByPosition(result).type);
|
||||
// Time zone is already figured out during result type resultion, no need to do it here.
|
||||
if (const auto dt_col = checkAndGetDataType<ToDataType>(result_type.get()))
|
||||
local_time_zone = &dt_col->getTimeZone();
|
||||
else
|
||||
{
|
||||
local_time_zone = &extractTimeZoneFromFunctionArguments(block, arguments, 1, 0);
|
||||
}
|
||||
|
||||
if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffort)
|
||||
utc_time_zone = &DateLUT::instance("UTC");
|
||||
@ -505,7 +590,14 @@ struct ConvertThroughParsing
|
||||
if constexpr (IsDataTypeDecimal<ToDataType>)
|
||||
{
|
||||
UInt32 scale = additions;
|
||||
if constexpr (to_datetime64)
|
||||
{
|
||||
ToDataType check_bounds_in_ctor(scale, local_time_zone ? local_time_zone->getTimeZone() : String{});
|
||||
}
|
||||
else
|
||||
{
|
||||
ToDataType check_bounds_in_ctor(ToDataType::maxPrecision(), scale);
|
||||
}
|
||||
col_to = ColVecTo::create(size, scale);
|
||||
}
|
||||
else
|
||||
@ -549,13 +641,28 @@ struct ConvertThroughParsing
|
||||
{
|
||||
if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffort)
|
||||
{
|
||||
time_t res;
|
||||
parseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone);
|
||||
if constexpr (to_datetime64)
|
||||
{
|
||||
DateTime64 res = 0;
|
||||
parseDateTime64BestEffort(res, vec_to.getScale(), read_buffer, *local_time_zone, *utc_time_zone);
|
||||
vec_to[i] = res;
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (IsDataTypeDecimal<ToDataType>)
|
||||
time_t res;
|
||||
parseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone);
|
||||
vec_to[i] = res;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (to_datetime64)
|
||||
{
|
||||
DateTime64 value = 0;
|
||||
readDateTime64Text(value, vec_to.getScale(), read_buffer, *local_time_zone);
|
||||
vec_to[i] = value;
|
||||
}
|
||||
else if constexpr (IsDataTypeDecimal<ToDataType>)
|
||||
ToDataType::readText(vec_to[i], read_buffer, ToDataType::maxPrecision(), vec_to.getScale());
|
||||
else
|
||||
parseImpl<ToDataType>(vec_to[i], read_buffer, local_time_zone);
|
||||
@ -570,13 +677,28 @@ struct ConvertThroughParsing
|
||||
|
||||
if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffort)
|
||||
{
|
||||
time_t res;
|
||||
parsed = tryParseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone);
|
||||
if constexpr (to_datetime64)
|
||||
{
|
||||
DateTime64 res = 0;
|
||||
parsed = tryParseDateTime64BestEffort(res, vec_to.getScale(), read_buffer, *local_time_zone, *utc_time_zone);
|
||||
vec_to[i] = res;
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (IsDataTypeDecimal<ToDataType>)
|
||||
time_t res;
|
||||
parsed = tryParseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone);
|
||||
vec_to[i] = res;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (to_datetime64)
|
||||
{
|
||||
DateTime64 value = 0;
|
||||
parsed = tryReadDateTime64Text(value, vec_to.getScale(), read_buffer, *local_time_zone);
|
||||
vec_to[i] = value;
|
||||
}
|
||||
else if constexpr (IsDataTypeDecimal<ToDataType>)
|
||||
parsed = ToDataType::tryReadText(vec_to[i], read_buffer, ToDataType::maxPrecision(), vec_to.getScale());
|
||||
else
|
||||
parsed = tryParseImpl<ToDataType>(vec_to[i], read_buffer, local_time_zone);
|
||||
@ -725,6 +847,7 @@ struct ConvertImpl<DataTypeFixedString, DataTypeString, Name>
|
||||
/// Declared early because used below.
|
||||
struct NameToDate { static constexpr auto name = "toDate"; };
|
||||
struct NameToDateTime { static constexpr auto name = "toDateTime"; };
|
||||
struct NameToDateTime64 { static constexpr auto name = "toDateTime64"; };
|
||||
struct NameToString { static constexpr auto name = "toString"; };
|
||||
struct NameToDecimal32 { static constexpr auto name = "toDecimal32"; };
|
||||
struct NameToDecimal64 { static constexpr auto name = "toDecimal64"; };
|
||||
@ -760,6 +883,8 @@ public:
|
||||
static constexpr bool to_decimal =
|
||||
std::is_same_v<Name, NameToDecimal32> || std::is_same_v<Name, NameToDecimal64> || std::is_same_v<Name, NameToDecimal128>;
|
||||
|
||||
static constexpr bool to_datetime64 = std::is_same_v<ToDataType, DataTypeDateTime64>;
|
||||
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionConvert>(); }
|
||||
|
||||
String getName() const override
|
||||
@ -773,16 +898,19 @@ public:
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
if (to_decimal && arguments.size() != 2)
|
||||
FunctionArgumentTypeValidators mandatory_args = {{[](const auto &) {return true;}, "ANY TYPE"}};
|
||||
FunctionArgumentTypeValidators optional_args;
|
||||
|
||||
if constexpr (to_decimal || to_datetime64)
|
||||
{
|
||||
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
||||
+ toString(arguments.size()) + ", should be 2.",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
mandatory_args.push_back(FunctionArgumentTypeValidator{&isNativeInteger, "Integer"}); // scale
|
||||
}
|
||||
else if (arguments.size() != 1 && arguments.size() != 2)
|
||||
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
||||
+ toString(arguments.size()) + ", should be 1 or 2. Second argument (time zone) is optional only make sense for DateTime.",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
else
|
||||
{
|
||||
optional_args.push_back(FunctionArgumentTypeValidator{&isString, "String"}); // timezone
|
||||
}
|
||||
|
||||
validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
|
||||
|
||||
if constexpr (std::is_same_v<ToDataType, DataTypeInterval>)
|
||||
{
|
||||
@ -796,42 +924,31 @@ public:
|
||||
UInt64 scale = extractToDecimalScale(arguments[1]);
|
||||
|
||||
if constexpr (std::is_same_v<Name, NameToDecimal32>)
|
||||
return createDecimal(9, scale);
|
||||
return createDecimal<DataTypeDecimal>(9, scale);
|
||||
else if constexpr (std::is_same_v<Name, NameToDecimal64>)
|
||||
return createDecimal(18, scale);
|
||||
return createDecimal<DataTypeDecimal>(18, scale);
|
||||
else if constexpr (std::is_same_v<Name, NameToDecimal128>)
|
||||
return createDecimal(38, scale);
|
||||
return createDecimal<DataTypeDecimal>(38, scale);
|
||||
|
||||
throw Exception("Someting wrong with toDecimalNN()", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
else
|
||||
{
|
||||
/** Optional second argument with time zone is supported:
|
||||
* - for functions toDateTime, toUnixTimestamp, toDate;
|
||||
* - for function toString of DateTime argument.
|
||||
*/
|
||||
// Optional second argument with time zone for DateTime.
|
||||
UInt8 timezone_arg_position = 1;
|
||||
UInt32 scale [[maybe_unused]] = DataTypeDateTime64::default_scale;
|
||||
|
||||
if (arguments.size() == 2)
|
||||
// DateTime64 requires more arguments: scale and timezone. Since timezone is optional, scale should be first.
|
||||
if constexpr (to_datetime64)
|
||||
{
|
||||
if (!checkAndGetDataType<DataTypeString>(arguments[1].type.get()))
|
||||
throw Exception("Illegal type " + arguments[1].type->getName() + " of 2nd argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
static constexpr bool to_date_or_time = std::is_same_v<Name, NameToDateTime>
|
||||
|| std::is_same_v<Name, NameToDate>
|
||||
|| std::is_same_v<Name, NameToUnixTimestamp>;
|
||||
|
||||
if (!(to_date_or_time
|
||||
|| (std::is_same_v<Name, NameToString> && WhichDataType(arguments[0].type).isDateTime())))
|
||||
{
|
||||
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
||||
+ toString(arguments.size()) + ", should be 1.",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
}
|
||||
timezone_arg_position += 1;
|
||||
scale = static_cast<UInt32>(arguments[1].column->get64(0));
|
||||
}
|
||||
|
||||
if (std::is_same_v<ToDataType, DataTypeDateTime>)
|
||||
return std::make_shared<DataTypeDateTime>(extractTimeZoneNameFromFunctionArguments(arguments, 1, 0));
|
||||
if constexpr (std::is_same_v<ToDataType, DataTypeDateTime>)
|
||||
return std::make_shared<DataTypeDateTime>(extractTimeZoneNameFromFunctionArguments(arguments, timezone_arg_position, 0));
|
||||
else if constexpr (to_datetime64)
|
||||
return std::make_shared<DataTypeDateTime64>(scale, extractTimeZoneNameFromFunctionArguments(arguments, timezone_arg_position, 0));
|
||||
else
|
||||
return std::make_shared<ToDataType>();
|
||||
}
|
||||
@ -902,17 +1019,32 @@ private:
|
||||
|
||||
if constexpr (IsDataTypeDecimal<RightDataType>)
|
||||
{
|
||||
if (arguments.size() != 2)
|
||||
if constexpr (std::is_same_v<RightDataType, DataTypeDateTime64>)
|
||||
{
|
||||
// account for optional timezone argument
|
||||
if (arguments.size() != 2 && arguments.size() != 3)
|
||||
throw Exception{"Function " + getName() + " expects 2 or 3 arguments for DataTypeDateTime64.",
|
||||
ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION};
|
||||
}
|
||||
else if (arguments.size() != 2)
|
||||
{
|
||||
throw Exception{"Function " + getName() + " expects 2 arguments for Decimal.",
|
||||
ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION};
|
||||
}
|
||||
|
||||
const ColumnWithTypeAndName & scale_column = block.getByPosition(arguments[1]);
|
||||
UInt32 scale = extractToDecimalScale(scale_column);
|
||||
|
||||
ConvertImpl<LeftDataType, RightDataType, Name>::execute(block, arguments, result, input_rows_count, scale);
|
||||
}
|
||||
else if constexpr (IsDataTypeDateOrDateTime<RightDataType> && std::is_same_v<LeftDataType, DataTypeDateTime64>)
|
||||
{
|
||||
const auto * dt64 = assert_cast<const DataTypeDateTime64 *>(block.getByPosition(arguments[0]).type.get());
|
||||
ConvertImpl<LeftDataType, RightDataType, Name>::execute(block, arguments, result, input_rows_count, dt64->getScale());
|
||||
}
|
||||
else
|
||||
ConvertImpl<LeftDataType, RightDataType, Name>::execute(block, arguments, result, input_rows_count);
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
@ -1015,15 +1147,23 @@ public:
|
||||
UInt64 scale = extractToDecimalScale(arguments[1]);
|
||||
|
||||
if constexpr (std::is_same_v<ToDataType, DataTypeDecimal<Decimal32>>)
|
||||
res = createDecimal(9, scale);
|
||||
res = createDecimal<DataTypeDecimal>(9, scale);
|
||||
else if constexpr (std::is_same_v<ToDataType, DataTypeDecimal<Decimal64>>)
|
||||
res = createDecimal(18, scale);
|
||||
res = createDecimal<DataTypeDecimal>(18, scale);
|
||||
else if constexpr (std::is_same_v<ToDataType, DataTypeDecimal<Decimal128>>)
|
||||
res = createDecimal(38, scale);
|
||||
res = createDecimal<DataTypeDecimal>(38, scale);
|
||||
|
||||
if (!res)
|
||||
throw Exception("Someting wrong with toDecimalNNOrZero() or toDecimalNNOrNull()", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
else if constexpr (std::is_same_v<ToDataType, DataTypeDateTime64>)
|
||||
{
|
||||
UInt64 scale = DataTypeDateTime64::default_scale;
|
||||
if (arguments.size() > 1)
|
||||
scale = extractToDecimalScale(arguments[1]);
|
||||
const auto timezone = extractTimeZoneNameFromFunctionArguments(arguments, 2, 0);
|
||||
res = std::make_shared<DataTypeDateTime64>(scale, timezone);
|
||||
}
|
||||
else
|
||||
res = std::make_shared<ToDataType>();
|
||||
|
||||
@ -1038,7 +1178,7 @@ public:
|
||||
const IDataType * from_type = block.getByPosition(arguments[0]).type.get();
|
||||
|
||||
bool ok = true;
|
||||
if constexpr (to_decimal)
|
||||
if constexpr (to_decimal || std::is_same_v<ToDataType, DataTypeDateTime64>)
|
||||
{
|
||||
if (arguments.size() != 2)
|
||||
throw Exception{"Function " + getName() + " expects 2 arguments for Decimal.", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION};
|
||||
@ -1380,6 +1520,7 @@ using FunctionToFloat32 = FunctionConvert<DataTypeFloat32, NameToFloat32, ToNumb
|
||||
using FunctionToFloat64 = FunctionConvert<DataTypeFloat64, NameToFloat64, ToNumberMonotonicity<Float64>>;
|
||||
using FunctionToDate = FunctionConvert<DataTypeDate, NameToDate, ToNumberMonotonicity<UInt16>>;
|
||||
using FunctionToDateTime = FunctionConvert<DataTypeDateTime, NameToDateTime, ToNumberMonotonicity<UInt32>>;
|
||||
using FunctionToDateTime64 = FunctionConvert<DataTypeDateTime64, NameToDateTime64, UnknownMonotonicity>;
|
||||
using FunctionToUUID = FunctionConvert<DataTypeUUID, NameToUUID, ToNumberMonotonicity<UInt128>>;
|
||||
using FunctionToString = FunctionConvert<DataTypeString, NameToString, ToStringMonotonicity>;
|
||||
using FunctionToUnixTimestamp = FunctionConvert<DataTypeUInt32, NameToUnixTimestamp, ToNumberMonotonicity<UInt32>>;
|
||||
@ -1402,6 +1543,7 @@ template <> struct FunctionTo<DataTypeFloat32> { using Type = FunctionToFloat32;
|
||||
template <> struct FunctionTo<DataTypeFloat64> { using Type = FunctionToFloat64; };
|
||||
template <> struct FunctionTo<DataTypeDate> { using Type = FunctionToDate; };
|
||||
template <> struct FunctionTo<DataTypeDateTime> { using Type = FunctionToDateTime; };
|
||||
template <> struct FunctionTo<DataTypeDateTime64> { using Type = FunctionToDateTime64; };
|
||||
template <> struct FunctionTo<DataTypeUUID> { using Type = FunctionToUUID; };
|
||||
template <> struct FunctionTo<DataTypeString> { using Type = FunctionToString; };
|
||||
template <> struct FunctionTo<DataTypeFixedString> { using Type = FunctionToFixedString; };
|
||||
@ -1426,6 +1568,7 @@ struct NameToFloat32OrZero { static constexpr auto name = "toFloat32OrZero"; };
|
||||
struct NameToFloat64OrZero { static constexpr auto name = "toFloat64OrZero"; };
|
||||
struct NameToDateOrZero { static constexpr auto name = "toDateOrZero"; };
|
||||
struct NameToDateTimeOrZero { static constexpr auto name = "toDateTimeOrZero"; };
|
||||
struct NameToDateTime64OrZero { static constexpr auto name = "toDateTime64OrZero"; };
|
||||
struct NameToDecimal32OrZero { static constexpr auto name = "toDecimal32OrZero"; };
|
||||
struct NameToDecimal64OrZero { static constexpr auto name = "toDecimal64OrZero"; };
|
||||
struct NameToDecimal128OrZero { static constexpr auto name = "toDecimal128OrZero"; };
|
||||
@ -1442,6 +1585,7 @@ using FunctionToFloat32OrZero = FunctionConvertFromString<DataTypeFloat32, NameT
|
||||
using FunctionToFloat64OrZero = FunctionConvertFromString<DataTypeFloat64, NameToFloat64OrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToDateOrZero = FunctionConvertFromString<DataTypeDate, NameToDateOrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToDateTimeOrZero = FunctionConvertFromString<DataTypeDateTime, NameToDateTimeOrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToDateTime64OrZero = FunctionConvertFromString<DataTypeDateTime64, NameToDateTime64OrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToDecimal32OrZero = FunctionConvertFromString<DataTypeDecimal<Decimal32>, NameToDecimal32OrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToDecimal64OrZero = FunctionConvertFromString<DataTypeDecimal<Decimal64>, NameToDecimal64OrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToDecimal128OrZero = FunctionConvertFromString<DataTypeDecimal<Decimal128>, NameToDecimal128OrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
@ -1458,6 +1602,7 @@ struct NameToFloat32OrNull { static constexpr auto name = "toFloat32OrNull"; };
|
||||
struct NameToFloat64OrNull { static constexpr auto name = "toFloat64OrNull"; };
|
||||
struct NameToDateOrNull { static constexpr auto name = "toDateOrNull"; };
|
||||
struct NameToDateTimeOrNull { static constexpr auto name = "toDateTimeOrNull"; };
|
||||
struct NameToDateTime64OrNull { static constexpr auto name = "toDateTime64OrNull"; };
|
||||
struct NameToDecimal32OrNull { static constexpr auto name = "toDecimal32OrNull"; };
|
||||
struct NameToDecimal64OrNull { static constexpr auto name = "toDecimal64OrNull"; };
|
||||
struct NameToDecimal128OrNull { static constexpr auto name = "toDecimal128OrNull"; };
|
||||
@ -1474,6 +1619,7 @@ using FunctionToFloat32OrNull = FunctionConvertFromString<DataTypeFloat32, NameT
|
||||
using FunctionToFloat64OrNull = FunctionConvertFromString<DataTypeFloat64, NameToFloat64OrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToDateOrNull = FunctionConvertFromString<DataTypeDate, NameToDateOrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToDateTimeOrNull = FunctionConvertFromString<DataTypeDateTime, NameToDateTimeOrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToDateTime64OrNull = FunctionConvertFromString<DataTypeDateTime64, NameToDateTime64OrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToDecimal32OrNull = FunctionConvertFromString<DataTypeDecimal<Decimal32>, NameToDecimal32OrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToDecimal64OrNull = FunctionConvertFromString<DataTypeDecimal<Decimal64>, NameToDecimal64OrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToDecimal128OrNull = FunctionConvertFromString<DataTypeDecimal<Decimal128>, NameToDecimal128OrNull, ConvertFromStringExceptionMode::Null>;
|
||||
@ -1481,6 +1627,10 @@ using FunctionToDecimal128OrNull = FunctionConvertFromString<DataTypeDecimal<Dec
|
||||
struct NameParseDateTimeBestEffort { static constexpr auto name = "parseDateTimeBestEffort"; };
|
||||
struct NameParseDateTimeBestEffortOrZero { static constexpr auto name = "parseDateTimeBestEffortOrZero"; };
|
||||
struct NameParseDateTimeBestEffortOrNull { static constexpr auto name = "parseDateTimeBestEffortOrNull"; };
|
||||
struct NameParseDateTime64BestEffort { static constexpr auto name = "parseDateTime64BestEffort"; };
|
||||
struct NameParseDateTime64BestEffortOrZero { static constexpr auto name = "parseDateTime64BestEffortOrZero"; };
|
||||
struct NameParseDateTime64BestEffortOrNull { static constexpr auto name = "parseDateTime64BestEffortOrNull"; };
|
||||
|
||||
|
||||
using FunctionParseDateTimeBestEffort = FunctionConvertFromString<
|
||||
DataTypeDateTime, NameParseDateTimeBestEffort, ConvertFromStringExceptionMode::Throw, ConvertFromStringParsingMode::BestEffort>;
|
||||
@ -1489,6 +1639,12 @@ using FunctionParseDateTimeBestEffortOrZero = FunctionConvertFromString<
|
||||
using FunctionParseDateTimeBestEffortOrNull = FunctionConvertFromString<
|
||||
DataTypeDateTime, NameParseDateTimeBestEffortOrNull, ConvertFromStringExceptionMode::Null, ConvertFromStringParsingMode::BestEffort>;
|
||||
|
||||
using FunctionParseDateTime64BestEffort = FunctionConvertFromString<
|
||||
DataTypeDateTime64, NameParseDateTime64BestEffort, ConvertFromStringExceptionMode::Throw, ConvertFromStringParsingMode::BestEffort>;
|
||||
using FunctionParseDateTime64BestEffortOrZero = FunctionConvertFromString<
|
||||
DataTypeDateTime64, NameParseDateTime64BestEffortOrZero, ConvertFromStringExceptionMode::Zero, ConvertFromStringParsingMode::BestEffort>;
|
||||
using FunctionParseDateTime64BestEffortOrNull = FunctionConvertFromString<
|
||||
DataTypeDateTime64, NameParseDateTime64BestEffortOrNull, ConvertFromStringExceptionMode::Null, ConvertFromStringParsingMode::BestEffort>;
|
||||
|
||||
class PreparedFunctionCast : public PreparedFunctionImpl
|
||||
{
|
||||
@ -1639,13 +1795,11 @@ private:
|
||||
};
|
||||
}
|
||||
|
||||
template <typename FieldType>
|
||||
WrapperType createDecimalWrapper(const DataTypePtr & from_type, const DataTypeDecimal<FieldType> * to_type) const
|
||||
template <typename ToDataType>
|
||||
std::enable_if_t<IsDataTypeDecimal<ToDataType>, WrapperType>
|
||||
createDecimalWrapper(const DataTypePtr & from_type, const ToDataType * to_type) const
|
||||
{
|
||||
using ToDataType = DataTypeDecimal<FieldType>;
|
||||
|
||||
TypeIndex type_index = from_type->getTypeId();
|
||||
UInt32 precision = to_type->getPrecision();
|
||||
UInt32 scale = to_type->getScale();
|
||||
|
||||
WhichDataType which(type_index);
|
||||
@ -1659,7 +1813,7 @@ private:
|
||||
throw Exception{"Conversion from " + from_type->getName() + " to " + to_type->getName() + " is not supported",
|
||||
ErrorCodes::CANNOT_CONVERT_TYPE};
|
||||
|
||||
return [type_index, precision, scale] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count)
|
||||
return [type_index, scale, to_type] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count)
|
||||
{
|
||||
auto res = callOnIndexAndDataType<ToDataType>(type_index, [&](const auto & types) -> bool
|
||||
{
|
||||
@ -1674,8 +1828,7 @@ private:
|
||||
/// Additionally check if callOnIndexAndDataType wasn't called at all.
|
||||
if (!res)
|
||||
{
|
||||
auto to = DataTypeDecimal<FieldType>(precision, scale);
|
||||
throw Exception{"Conversion from " + std::string(getTypeName(type_index)) + " to " + to.getName() +
|
||||
throw Exception{"Conversion from " + std::string(getTypeName(type_index)) + " to " + to_type->getName() +
|
||||
" is not supported", ErrorCodes::CANNOT_CONVERT_TYPE};
|
||||
}
|
||||
};
|
||||
@ -2140,7 +2293,8 @@ private:
|
||||
if constexpr (
|
||||
std::is_same_v<ToDataType, DataTypeDecimal<Decimal32>> ||
|
||||
std::is_same_v<ToDataType, DataTypeDecimal<Decimal64>> ||
|
||||
std::is_same_v<ToDataType, DataTypeDecimal<Decimal128>>)
|
||||
std::is_same_v<ToDataType, DataTypeDecimal<Decimal128>> ||
|
||||
std::is_same_v<ToDataType, DataTypeDateTime64>)
|
||||
{
|
||||
ret = createDecimalWrapper(from_type, checkAndGetDataType<ToDataType>(to_type.get()));
|
||||
return true;
|
||||
|
@ -5,10 +5,12 @@
|
||||
#include <DataTypes/DataTypeFixedString.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/memcpySmall.h>
|
||||
#include <Functions/IFunction.h>
|
||||
@ -147,7 +149,6 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <typename ToDataType, typename Name>
|
||||
class FunctionReinterpretStringAs : public IFunction
|
||||
{
|
||||
@ -156,6 +157,7 @@ public:
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionReinterpretStringAs>(); }
|
||||
|
||||
using ToFieldType = typename ToDataType::FieldType;
|
||||
using ColumnType = typename ToDataType::ColumnType;
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
@ -179,12 +181,12 @@ public:
|
||||
{
|
||||
if (const ColumnString * col_from = typeid_cast<const ColumnString *>(block.getByPosition(arguments[0]).column.get()))
|
||||
{
|
||||
auto col_res = ColumnVector<ToFieldType>::create();
|
||||
auto col_res = ColumnType::create();
|
||||
|
||||
const ColumnString::Chars & data_from = col_from->getChars();
|
||||
const ColumnString::Offsets & offsets_from = col_from->getOffsets();
|
||||
size_t size = offsets_from.size();
|
||||
typename ColumnVector<ToFieldType>::Container & vec_res = col_res->getData();
|
||||
typename ColumnType::Container & vec_res = col_res->getData();
|
||||
vec_res.resize(size);
|
||||
|
||||
size_t offset = 0;
|
||||
|
@ -6,6 +6,7 @@
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
#include "IFunction.h"
|
||||
|
@ -36,7 +36,7 @@ struct ArrayCumSumImpl
|
||||
if (which.isDecimal())
|
||||
{
|
||||
UInt32 scale = getDecimalScale(*expression_return);
|
||||
DataTypePtr nested = std::make_shared<DataTypeDecimal<Decimal128>>(maxDecimalPrecision<Decimal128>(), scale);
|
||||
DataTypePtr nested = std::make_shared<DataTypeDecimal<Decimal128>>(DecimalUtils::maxPrecision<Decimal128>(), scale);
|
||||
return std::make_shared<DataTypeArray>(nested);
|
||||
}
|
||||
|
||||
|
@ -39,7 +39,7 @@ struct ArrayCumSumNonNegativeImpl
|
||||
if (which.isDecimal())
|
||||
{
|
||||
UInt32 scale = getDecimalScale(*expression_return);
|
||||
DataTypePtr nested = std::make_shared<DataTypeDecimal<Decimal128>>(maxDecimalPrecision<Decimal128>(), scale);
|
||||
DataTypePtr nested = std::make_shared<DataTypeDecimal<Decimal128>>(DecimalUtils::maxPrecision<Decimal128>(), scale);
|
||||
return std::make_shared<DataTypeArray>(nested);
|
||||
}
|
||||
|
||||
|
@ -7,6 +7,7 @@
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/getMostSubtype.h>
|
||||
|
@ -36,7 +36,7 @@ struct ArraySumImpl
|
||||
if (which.isDecimal())
|
||||
{
|
||||
UInt32 scale = getDecimalScale(*expression_return);
|
||||
return std::make_shared<DataTypeDecimal<Decimal128>>(maxDecimalPrecision<Decimal128>(), scale);
|
||||
return std::make_shared<DataTypeDecimal<Decimal128>>(DecimalUtils::maxPrecision<Decimal128>(), scale);
|
||||
}
|
||||
|
||||
throw Exception("arraySum cannot add values of type " + expression_return->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
@ -4,6 +4,7 @@
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
|
@ -1,4 +1,5 @@
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
|
@ -2,6 +2,7 @@
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Core/Block.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <common/DateLUT.h>
|
||||
|
||||
@ -43,6 +44,8 @@ std::string extractTimeZoneNameFromFunctionArguments(const ColumnsWithTypeAndNam
|
||||
/// If time zone is attached to an argument of type DateTime.
|
||||
if (const DataTypeDateTime * type = checkAndGetDataType<DataTypeDateTime>(arguments[datetime_arg_num].type.get()))
|
||||
return type->getTimeZone().getTimeZone();
|
||||
if (const DataTypeDateTime64 * type = checkAndGetDataType<DataTypeDateTime64>(arguments[datetime_arg_num].type.get()))
|
||||
return type->getTimeZone().getTimeZone();
|
||||
|
||||
return {};
|
||||
}
|
||||
|
@ -1,4 +1,7 @@
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
|
||||
#include <Functions/IFunction.h>
|
||||
@ -11,6 +14,7 @@
|
||||
|
||||
#include <common/DateLUTImpl.h>
|
||||
#include <common/find_symbols.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
@ -27,6 +31,16 @@ namespace ErrorCodes
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
// in private namespace to avoid GCC 9 error: "explicit specialization in non-namespace scope"
|
||||
template <typename DataType> struct ActionaValueTypeMap {};
|
||||
template <> struct ActionaValueTypeMap<DataTypeDate> { using ActionValueType = UInt16; };
|
||||
template <> struct ActionaValueTypeMap<DataTypeDateTime> { using ActionValueType = UInt32; };
|
||||
// TODO(vnemkov): once there is support for Int64 in LUT, make that Int64.
|
||||
// TODO(vnemkov): to add sub-second format instruction, make that DateTime64 and do some math in Action<T>.
|
||||
template <> struct ActionaValueTypeMap<DataTypeDateTime64> { using ActionValueType = UInt32; };
|
||||
}
|
||||
|
||||
/** formatDateTime(time, 'pattern')
|
||||
* Performs formatting of time, according to provided pattern.
|
||||
@ -270,20 +284,22 @@ public:
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
|
||||
{
|
||||
if (!executeType<UInt32>(block, arguments, result)
|
||||
&& !executeType<UInt16>(block, arguments, result))
|
||||
if (!executeType<DataTypeDate>(block, arguments, result)
|
||||
&& !executeType<DataTypeDateTime>(block, arguments, result)
|
||||
&& !executeType<DataTypeDateTime64>(block, arguments, result))
|
||||
throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
|
||||
+ " of function " + getName() + ", must be Date or DateTime",
|
||||
ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <typename DataType>
|
||||
bool executeType(Block & block, const ColumnNumbers & arguments, size_t result)
|
||||
{
|
||||
if (auto * times = checkAndGetColumn<ColumnVector<T>>(block.getByPosition(arguments[0]).column.get()))
|
||||
{
|
||||
const ColumnConst * pattern_column = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
|
||||
auto * times = checkAndGetColumn<typename DataType::ColumnType>(block.getByPosition(arguments[0]).column.get());
|
||||
if (!times)
|
||||
return false;
|
||||
|
||||
const ColumnConst * pattern_column = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
|
||||
if (!pattern_column)
|
||||
throw Exception("Illegal column " + block.getByPosition(arguments[1]).column->getName()
|
||||
+ " of second ('format') argument of function " + getName()
|
||||
@ -292,6 +308,7 @@ public:
|
||||
|
||||
String pattern = pattern_column->getValue<String>();
|
||||
|
||||
using T = typename ActionaValueTypeMap<DataType>::ActionValueType;
|
||||
std::vector<Action<T>> instructions;
|
||||
String pattern_to_fill = parsePattern(pattern, instructions);
|
||||
size_t result_size = pattern_to_fill.size();
|
||||
@ -304,7 +321,13 @@ public:
|
||||
|
||||
const DateLUTImpl & time_zone = *time_zone_tmp;
|
||||
|
||||
const typename ColumnVector<T>::Container & vec = times->getData();
|
||||
const auto & vec = times->getData();
|
||||
|
||||
UInt32 scale [[maybe_unused]] = 0;
|
||||
if constexpr (std::is_same_v<DataType, DataTypeDateTime64>)
|
||||
{
|
||||
scale = vec.getScale();
|
||||
}
|
||||
|
||||
auto col_res = ColumnString::create();
|
||||
auto & dst_data = col_res->getChars();
|
||||
@ -337,9 +360,22 @@ public:
|
||||
auto pos = begin;
|
||||
|
||||
for (size_t i = 0; i < vec.size(); ++i)
|
||||
{
|
||||
if constexpr (std::is_same_v<DataType, DataTypeDateTime64>)
|
||||
{
|
||||
for (auto & instruction : instructions)
|
||||
{
|
||||
// since right now LUT does not support Int64-values and not format instructions for subsecond parts,
|
||||
// treat DatTime64 values just as DateTime values by ignoring fractional and casting to UInt32.
|
||||
const auto c = DecimalUtils::split(vec[i], scale);
|
||||
instruction.perform(pos, static_cast<UInt32>(c.whole), time_zone);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (auto & instruction : instructions)
|
||||
instruction.perform(pos, vec[i], time_zone);
|
||||
}
|
||||
|
||||
dst_offsets[i] = pos - begin;
|
||||
}
|
||||
@ -349,9 +385,6 @@ public:
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
String parsePattern(const String & pattern, std::vector<Action<T>> & instructions) const
|
||||
{
|
||||
|
@ -1,9 +1,12 @@
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Core/Field.h>
|
||||
|
||||
#include <time.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
|
98
dbms/src/Functions/now64.cpp
Normal file
98
dbms/src/Functions/now64.cpp
Normal file
@ -0,0 +1,98 @@
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
#include <time.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_COLUMN;
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
}
|
||||
|
||||
DateTime64::NativeType nowSubsecond(UInt32 scale)
|
||||
{
|
||||
const Int32 fractional_scale = 9;
|
||||
timespec spec;
|
||||
clock_gettime(CLOCK_REALTIME, &spec);
|
||||
|
||||
DecimalUtils::DecimalComponents<DateTime64::NativeType> components{spec.tv_sec, spec.tv_nsec};
|
||||
|
||||
// clock_gettime produces subsecond part in nanoseconds, but decimalFromComponents fractional is scale-dependent.
|
||||
// Andjust fractional to scale, e.g. for 123456789 nanoseconds:
|
||||
// if scale is 6 (miscoseconds) => divide by 9 - 6 = 3 to get 123456 microseconds
|
||||
// if scale is 12 (picoseconds) => multiply by abs(9 - 12) = 3 to get 123456789000 picoseconds
|
||||
const auto adjust_scale = fractional_scale - static_cast<Int32>(scale);
|
||||
if (adjust_scale < 0)
|
||||
components.fractional *= intExp10(std::abs(adjust_scale));
|
||||
else if (adjust_scale > 0)
|
||||
components.fractional /= intExp10(adjust_scale);
|
||||
|
||||
return DecimalUtils::decimalFromComponents<DateTime64>(components, scale).value;
|
||||
}
|
||||
|
||||
class FunctionNow64 : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "now64";
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionNow64>(); }
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
bool isVariadic() const override { return true; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return ColumnNumbers{0}; }
|
||||
bool isDeterministic() const override { return false; }
|
||||
|
||||
// Return type depends on argument value.
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
UInt32 scale = DataTypeDateTime64::default_scale;
|
||||
|
||||
// Type check is similar to the validateArgumentType, trying to keep error codes and messages as close to the said function as possible.
|
||||
if (arguments.size() >= 1)
|
||||
{
|
||||
const auto & argument = arguments[0];
|
||||
if (!isInteger(argument.type) || !isColumnConst(*argument.column))
|
||||
throw Exception("Illegal type " + argument.type->getName() +
|
||||
" of 0" +
|
||||
" argument of function " + getName() +
|
||||
". Expected const integer.",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
scale = argument.column->get64(0);
|
||||
}
|
||||
|
||||
return std::make_shared<DataTypeDateTime64>(scale);
|
||||
}
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & /*arguments*/, size_t result, size_t input_rows_count) override
|
||||
{
|
||||
auto & result_col = block.getByPosition(result);
|
||||
UInt32 scale = DataTypeDateTime64::default_scale;
|
||||
if (const auto * dt64 = assert_cast<const DataTypeDateTime64 *>(result_col.type.get()))
|
||||
{
|
||||
scale = dt64->getScale();
|
||||
}
|
||||
|
||||
result_col.column = DataTypeDateTime64(scale).createColumnConst(input_rows_count, nowSubsecond(scale));
|
||||
}
|
||||
};
|
||||
|
||||
void registerFunctionNow64(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionNow64>(FunctionFactory::CaseInsensitive);
|
||||
}
|
||||
|
||||
}
|
@ -37,6 +37,7 @@ void registerFunctionToRelativeMinuteNum(FunctionFactory &);
|
||||
void registerFunctionToRelativeSecondNum(FunctionFactory &);
|
||||
void registerFunctionToTime(FunctionFactory &);
|
||||
void registerFunctionNow(FunctionFactory &);
|
||||
void registerFunctionNow64(FunctionFactory &);
|
||||
void registerFunctionToday(FunctionFactory &);
|
||||
void registerFunctionYesterday(FunctionFactory &);
|
||||
void registerFunctionTimeSlot(FunctionFactory &);
|
||||
@ -101,6 +102,7 @@ void registerFunctionsDateTime(FunctionFactory & factory)
|
||||
registerFunctionToRelativeSecondNum(factory);
|
||||
registerFunctionToTime(factory);
|
||||
registerFunctionNow(factory);
|
||||
registerFunctionNow64(factory);
|
||||
registerFunctionToday(factory);
|
||||
registerFunctionYesterday(factory);
|
||||
registerFunctionTimeSlot(factory);
|
||||
|
@ -5,6 +5,7 @@
|
||||
#include <Common/assert_cast.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/NumberTraits.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
|
@ -1,5 +1,6 @@
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
|
||||
|
@ -1,6 +1,8 @@
|
||||
#include <common/DateLUTImpl.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeInterval.h>
|
||||
#include <Functions/DateTimeTransforms.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
@ -125,7 +127,6 @@ namespace
|
||||
return time_zone.toStartOfSecondInterval(t, seconds);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -233,26 +234,34 @@ private:
|
||||
ColumnPtr dispatchForColumns(
|
||||
const ColumnWithTypeAndName & time_column, const ColumnWithTypeAndName & interval_column, const DateLUTImpl & time_zone)
|
||||
{
|
||||
if (WhichDataType(time_column.type.get()).isDateTime())
|
||||
const auto & from_datatype = *time_column.type.get();
|
||||
const auto which_type = WhichDataType(from_datatype);
|
||||
if (which_type.isDateTime())
|
||||
{
|
||||
const auto * time_column_vec = checkAndGetColumn<ColumnUInt32>(time_column.column.get());
|
||||
if (time_column_vec)
|
||||
return dispatchForIntervalColumn(*time_column_vec, interval_column, time_zone);
|
||||
return dispatchForIntervalColumn(assert_cast<const DataTypeDateTime&>(from_datatype), *time_column_vec, interval_column, time_zone);
|
||||
}
|
||||
if (WhichDataType(time_column.type.get()).isDate())
|
||||
if (which_type.isDate())
|
||||
{
|
||||
const auto * time_column_vec = checkAndGetColumn<ColumnUInt16>(time_column.column.get());
|
||||
if (time_column_vec)
|
||||
return dispatchForIntervalColumn(*time_column_vec, interval_column, time_zone);
|
||||
return dispatchForIntervalColumn(assert_cast<const DataTypeDate&>(from_datatype), *time_column_vec, interval_column, time_zone);
|
||||
}
|
||||
if (which_type.isDateTime64())
|
||||
{
|
||||
const auto * time_column_vec = checkAndGetColumn<DataTypeDateTime64::ColumnType>(time_column.column.get());
|
||||
if (time_column_vec)
|
||||
return dispatchForIntervalColumn(assert_cast<const DataTypeDateTime64&>(from_datatype), *time_column_vec, interval_column, time_zone);
|
||||
}
|
||||
throw Exception(
|
||||
"Illegal column for first argument of function " + getName() + ". Must contain dates or dates with time",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
|
||||
template <typename FromType>
|
||||
template <typename ColumnType, typename FromDataType>
|
||||
ColumnPtr dispatchForIntervalColumn(
|
||||
const ColumnVector<FromType> & time_column, const ColumnWithTypeAndName & interval_column, const DateLUTImpl & time_zone)
|
||||
const FromDataType & from, const ColumnType & time_column, const ColumnWithTypeAndName & interval_column, const DateLUTImpl & time_zone)
|
||||
{
|
||||
const auto * interval_type = checkAndGetDataType<DataTypeInterval>(interval_column.type.get());
|
||||
if (!interval_type)
|
||||
@ -270,36 +279,47 @@ private:
|
||||
switch (interval_type->getKind())
|
||||
{
|
||||
case IntervalKind::Second:
|
||||
return execute<FromType, UInt32, IntervalKind::Second>(time_column, num_units, time_zone);
|
||||
return execute<FromDataType, UInt32, IntervalKind::Second>(from, time_column, num_units, time_zone);
|
||||
case IntervalKind::Minute:
|
||||
return execute<FromType, UInt32, IntervalKind::Minute>(time_column, num_units, time_zone);
|
||||
return execute<FromDataType, UInt32, IntervalKind::Minute>(from, time_column, num_units, time_zone);
|
||||
case IntervalKind::Hour:
|
||||
return execute<FromType, UInt32, IntervalKind::Hour>(time_column, num_units, time_zone);
|
||||
return execute<FromDataType, UInt32, IntervalKind::Hour>(from, time_column, num_units, time_zone);
|
||||
case IntervalKind::Day:
|
||||
return execute<FromType, UInt32, IntervalKind::Day>(time_column, num_units, time_zone);
|
||||
return execute<FromDataType, UInt32, IntervalKind::Day>(from, time_column, num_units, time_zone);
|
||||
case IntervalKind::Week:
|
||||
return execute<FromType, UInt16, IntervalKind::Week>(time_column, num_units, time_zone);
|
||||
return execute<FromDataType, UInt16, IntervalKind::Week>(from, time_column, num_units, time_zone);
|
||||
case IntervalKind::Month:
|
||||
return execute<FromType, UInt16, IntervalKind::Month>(time_column, num_units, time_zone);
|
||||
return execute<FromDataType, UInt16, IntervalKind::Month>(from, time_column, num_units, time_zone);
|
||||
case IntervalKind::Quarter:
|
||||
return execute<FromType, UInt16, IntervalKind::Quarter>(time_column, num_units, time_zone);
|
||||
return execute<FromDataType, UInt16, IntervalKind::Quarter>(from, time_column, num_units, time_zone);
|
||||
case IntervalKind::Year:
|
||||
return execute<FromType, UInt16, IntervalKind::Year>(time_column, num_units, time_zone);
|
||||
return execute<FromDataType, UInt16, IntervalKind::Year>(from, time_column, num_units, time_zone);
|
||||
}
|
||||
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
||||
template <typename FromType, typename ToType, IntervalKind::Kind unit>
|
||||
ColumnPtr execute(const ColumnVector<FromType> & time_column, UInt64 num_units, const DateLUTImpl & time_zone)
|
||||
|
||||
template <typename FromDataType, typename ToType, IntervalKind::Kind unit, typename ColumnType>
|
||||
ColumnPtr execute(const FromDataType & from_datatype, const ColumnType & time_column, UInt64 num_units, const DateLUTImpl & time_zone)
|
||||
{
|
||||
const auto & time_data = time_column.getData();
|
||||
size_t size = time_column.size();
|
||||
auto result = ColumnVector<ToType>::create();
|
||||
auto & result_data = result->getData();
|
||||
result_data.resize(size);
|
||||
|
||||
if constexpr (std::is_same_v<FromDataType, DataTypeDateTime64>)
|
||||
{
|
||||
const auto transform = DateTime64BasicTransformWrapper<Transform<unit>>{from_datatype.getScale()};
|
||||
for (size_t i = 0; i != size; ++i)
|
||||
result_data[i] = transform.execute(time_data[i], num_units, time_zone);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i != size; ++i)
|
||||
result_data[i] = Transform<unit>::execute(time_data[i], num_units, time_zone);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
@ -1,4 +1,5 @@
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
|
||||
#include <Core/Field.h>
|
||||
|
||||
@ -7,6 +8,7 @@
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -39,12 +41,17 @@ public:
|
||||
+ toString(arguments.size()) + ", should be 2",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (!WhichDataType(arguments[0].type).isDateTime())
|
||||
const auto which_type = WhichDataType(arguments[0].type);
|
||||
if (!which_type.isDateTime() && !which_type.isDateTime64())
|
||||
throw Exception{"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() +
|
||||
". Should be DateTime", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
". Should be DateTime or DateTime64", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
|
||||
String time_zone_name = extractTimeZoneNameFromFunctionArguments(arguments, 1, 0);
|
||||
if (which_type.isDateTime())
|
||||
return std::make_shared<DataTypeDateTime>(time_zone_name);
|
||||
|
||||
const auto * date_time64 = assert_cast<const DataTypeDateTime64 *>(arguments[0].type.get());
|
||||
return std::make_shared<DataTypeDateTime64>(date_time64->getScale(), time_zone_name);
|
||||
}
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
|
||||
|
@ -12,14 +12,17 @@
|
||||
#include <common/LocalDate.h>
|
||||
#include <common/LocalDateTime.h>
|
||||
#include <common/StringRef.h>
|
||||
#include <common/arithmeticOverflow.h>
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Core/UUID.h>
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/Arena.h>
|
||||
#include <Common/UInt128.h>
|
||||
#include <Common/intExp.h>
|
||||
|
||||
#include <Formats/FormatSettings.h>
|
||||
|
||||
@ -29,6 +32,8 @@
|
||||
#include <IO/VarInt.h>
|
||||
#include <IO/ZlibInflatingReadBuffer.h>
|
||||
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Wdouble-promotion"
|
||||
@ -252,7 +257,13 @@ inline void readBoolTextWord(bool & x, ReadBuffer & buf)
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename ReturnType = void>
|
||||
enum class ReadIntTextCheckOverflow
|
||||
{
|
||||
DO_NOT_CHECK_OVERFLOW,
|
||||
CHECK_OVERFLOW,
|
||||
};
|
||||
|
||||
template <typename T, typename ReturnType = void, ReadIntTextCheckOverflow check_overflow = ReadIntTextCheckOverflow::DO_NOT_CHECK_OVERFLOW>
|
||||
ReturnType readIntTextImpl(T & x, ReadBuffer & buf)
|
||||
{
|
||||
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
|
||||
@ -267,6 +278,7 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf)
|
||||
return ReturnType(false);
|
||||
}
|
||||
|
||||
const size_t initial_pos = buf.count();
|
||||
while (!buf.eof())
|
||||
{
|
||||
switch (*buf.position())
|
||||
@ -274,7 +286,7 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf)
|
||||
case '+':
|
||||
break;
|
||||
case '-':
|
||||
if (is_signed_v<T>)
|
||||
if constexpr (is_signed_v<T>)
|
||||
negative = true;
|
||||
else
|
||||
{
|
||||
@ -294,30 +306,48 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf)
|
||||
case '7': [[fallthrough]];
|
||||
case '8': [[fallthrough]];
|
||||
case '9':
|
||||
if constexpr (check_overflow == ReadIntTextCheckOverflow::CHECK_OVERFLOW)
|
||||
{
|
||||
// perform relativelly slow overflow check only when number of decimal digits so far is close to the max for given type.
|
||||
if (buf.count() - initial_pos >= std::numeric_limits<T>::max_digits10)
|
||||
{
|
||||
if (common::mulOverflow(res, static_cast<decltype(res)>(10), res)
|
||||
|| common::addOverflow(res, static_cast<decltype(res)>(*buf.position() - '0'), res))
|
||||
return ReturnType(false);
|
||||
break;
|
||||
}
|
||||
}
|
||||
res *= 10;
|
||||
res += *buf.position() - '0';
|
||||
break;
|
||||
default:
|
||||
x = negative ? -res : res;
|
||||
return ReturnType(true);
|
||||
goto end;
|
||||
}
|
||||
++buf.position();
|
||||
}
|
||||
|
||||
end:
|
||||
x = negative ? -res : res;
|
||||
|
||||
return ReturnType(true);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <ReadIntTextCheckOverflow check_overflow = ReadIntTextCheckOverflow::DO_NOT_CHECK_OVERFLOW, typename T>
|
||||
void readIntText(T & x, ReadBuffer & buf)
|
||||
{
|
||||
readIntTextImpl<T, void>(x, buf);
|
||||
readIntTextImpl<T, void, check_overflow>(x, buf);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <ReadIntTextCheckOverflow check_overflow = ReadIntTextCheckOverflow::CHECK_OVERFLOW, typename T>
|
||||
bool tryReadIntText(T & x, ReadBuffer & buf)
|
||||
{
|
||||
return readIntTextImpl<T, bool>(x, buf);
|
||||
return readIntTextImpl<T, bool, check_overflow>(x, buf);
|
||||
}
|
||||
|
||||
template <ReadIntTextCheckOverflow check_overflow = ReadIntTextCheckOverflow::DO_NOT_CHECK_OVERFLOW, typename T>
|
||||
void readIntText(Decimal<T> & x, ReadBuffer & buf)
|
||||
{
|
||||
readIntText<check_overflow>(x.value, buf);
|
||||
}
|
||||
|
||||
/** More efficient variant (about 1.5 times on real dataset).
|
||||
@ -617,22 +647,74 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons
|
||||
}
|
||||
else
|
||||
/// Why not readIntTextUnsafe? Because for needs of AdFox, parsing of unix timestamp with leading zeros is supported: 000...NNNN.
|
||||
return readIntTextImpl<time_t, ReturnType>(datetime, buf);
|
||||
return readIntTextImpl<time_t, ReturnType, ReadIntTextCheckOverflow::CHECK_OVERFLOW>(datetime, buf);
|
||||
}
|
||||
else
|
||||
return readDateTimeTextFallback<ReturnType>(datetime, buf, date_lut);
|
||||
}
|
||||
|
||||
template <typename ReturnType>
|
||||
inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut)
|
||||
{
|
||||
time_t whole;
|
||||
if (!readDateTimeTextImpl<bool>(whole, buf, date_lut))
|
||||
{
|
||||
return ReturnType(false);
|
||||
}
|
||||
|
||||
DB::DecimalUtils::DecimalComponents<DateTime64::NativeType> c{static_cast<DateTime64::NativeType>(whole), 0};
|
||||
|
||||
if (!buf.eof() && *buf.position() == '.')
|
||||
{
|
||||
buf.ignore(1); // skip separator
|
||||
const auto pos_before_fractional = buf.count();
|
||||
if (!tryReadIntText<ReadIntTextCheckOverflow::CHECK_OVERFLOW>(c.fractional, buf))
|
||||
{
|
||||
return ReturnType(false);
|
||||
}
|
||||
|
||||
// Adjust fractional part to the scale, since decimalFromComponents knows nothing
|
||||
// about convention of ommiting trailing zero on fractional part
|
||||
// and assumes that fractional part value is less than 10^scale.
|
||||
|
||||
// If scale is 3, but we read '12', promote fractional part to '120'.
|
||||
// And vice versa: if we read '1234', denote it to '123'.
|
||||
const auto fractional_length = static_cast<Int32>(buf.count() - pos_before_fractional);
|
||||
if (const auto adjust_scale = static_cast<Int32>(scale) - fractional_length; adjust_scale > 0)
|
||||
{
|
||||
c.fractional *= common::exp10_i64(adjust_scale);
|
||||
}
|
||||
else if (adjust_scale < 0)
|
||||
{
|
||||
c.fractional /= common::exp10_i64(-1 * adjust_scale);
|
||||
}
|
||||
}
|
||||
|
||||
datetime64 = DecimalUtils::decimalFromComponents<DateTime64>(c, scale);
|
||||
|
||||
return ReturnType(true);
|
||||
}
|
||||
|
||||
inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance())
|
||||
{
|
||||
readDateTimeTextImpl<void>(datetime, buf, date_lut);
|
||||
}
|
||||
|
||||
inline void readDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance())
|
||||
{
|
||||
readDateTimeTextImpl<void>(datetime64, scale, buf, date_lut);
|
||||
}
|
||||
|
||||
inline bool tryReadDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance())
|
||||
{
|
||||
return readDateTimeTextImpl<bool>(datetime, buf, date_lut);
|
||||
}
|
||||
|
||||
inline bool tryReadDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance())
|
||||
{
|
||||
return readDateTimeTextImpl<bool>(datetime64, scale, buf, date_lut);
|
||||
}
|
||||
|
||||
inline void readDateTimeText(LocalDateTime & datetime, ReadBuffer & buf)
|
||||
{
|
||||
char s[19];
|
||||
@ -858,11 +940,11 @@ void readAndThrowException(ReadBuffer & buf, const String & additional_message =
|
||||
|
||||
/** Helper function for implementation.
|
||||
*/
|
||||
template <typename T>
|
||||
template <ReadIntTextCheckOverflow check_overflow = ReadIntTextCheckOverflow::CHECK_OVERFLOW, typename T>
|
||||
static inline const char * tryReadIntText(T & x, const char * pos, const char * end)
|
||||
{
|
||||
ReadBufferFromMemory in(pos, end - pos);
|
||||
tryReadIntText(x, in);
|
||||
tryReadIntText<check_overflow>(x, in);
|
||||
return pos + in.count();
|
||||
}
|
||||
|
||||
|
@ -12,6 +12,7 @@
|
||||
#include <common/find_symbols.h>
|
||||
#include <common/StringRef.h>
|
||||
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Core/UUID.h>
|
||||
|
||||
@ -29,7 +30,6 @@
|
||||
|
||||
#include <Formats/FormatSettings.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -694,6 +694,46 @@ inline void writeDateTimeText(time_t datetime, WriteBuffer & buf, const DateLUTI
|
||||
date_lut.toHour(datetime), date_lut.toMinute(datetime), date_lut.toSecond(datetime)), buf);
|
||||
}
|
||||
|
||||
/// In the format YYYY-MM-DD HH:MM:SS.NNNNNNNNN, according to the specified time zone.
|
||||
template <char date_delimeter = '-', char time_delimeter = ':', char between_date_time_delimiter = ' ', char fractional_time_delimiter = '.'>
|
||||
inline void writeDateTimeText(DateTime64 datetime64, UInt32 scale, WriteBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance())
|
||||
{
|
||||
static constexpr UInt32 MaxScale = DecimalUtils::maxPrecision<DateTime64>();
|
||||
scale = scale > MaxScale ? MaxScale : scale;
|
||||
if (unlikely(!datetime64))
|
||||
{
|
||||
static const char s[] =
|
||||
{
|
||||
'0', '0', '0', '0', date_delimeter, '0', '0', date_delimeter, '0', '0',
|
||||
between_date_time_delimiter,
|
||||
'0', '0', time_delimeter, '0', '0', time_delimeter, '0', '0',
|
||||
fractional_time_delimiter,
|
||||
// Exactly MaxScale zeroes
|
||||
'0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0'
|
||||
};
|
||||
buf.write(s, sizeof(s) - (MaxScale - scale));
|
||||
return;
|
||||
}
|
||||
auto c = DecimalUtils::split(datetime64, scale);
|
||||
const auto & values = date_lut.getValues(c.whole);
|
||||
writeDateTimeText<date_delimeter, time_delimeter, between_date_time_delimiter>(
|
||||
LocalDateTime(values.year, values.month, values.day_of_month,
|
||||
date_lut.toHour(c.whole), date_lut.toMinute(c.whole), date_lut.toSecond(c.whole)), buf);
|
||||
|
||||
if (scale > 0)
|
||||
{
|
||||
buf.write(fractional_time_delimiter);
|
||||
|
||||
char data[20] = {'0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0'};
|
||||
static_assert(sizeof(data) >= MaxScale);
|
||||
|
||||
auto fractional = c.fractional;
|
||||
for (Int32 pos = scale - 1; pos >= 0 && fractional; --pos, fractional /= DateTime64(10))
|
||||
data[pos] += fractional % DateTime64(10);
|
||||
|
||||
writeString(&data[0], static_cast<size_t>(scale), buf);
|
||||
}
|
||||
}
|
||||
|
||||
/// In the RFC 1123 format: "Tue, 03 Dec 2019 00:11:50 GMT". You must provide GMT DateLUT.
|
||||
/// This is needed for HTTP requests.
|
||||
@ -772,9 +812,7 @@ void writeText(Decimal<T> value, UInt32 scale, WriteBuffer & ostr)
|
||||
writeChar('-', ostr); /// avoid crop leading minus when whole part is zero
|
||||
}
|
||||
|
||||
T whole_part = value;
|
||||
if (scale)
|
||||
whole_part = value / Decimal<T>::getScaleMultiplier(scale);
|
||||
const T whole_part = DecimalUtils::getWholePart(value, scale);
|
||||
|
||||
writeIntText(whole_part, ostr);
|
||||
if (scale)
|
||||
|
@ -6,6 +6,7 @@
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/parseDateTimeBestEffort.h>
|
||||
|
||||
#include <limits>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -64,9 +65,32 @@ inline void readDecimalNumber(T & res, const char * src)
|
||||
readDecimalNumberImpl<num_digits - 1, 1>(res, src);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline void readDecimalNumber(T & res, size_t num_digits, const char * src)
|
||||
{
|
||||
#define READ_DECIMAL_NUMBER(N) res *= common::exp10_i32(N); readDecimalNumber<N>(res, src); src += N; num_digits -= N; break
|
||||
|
||||
while (num_digits)
|
||||
{
|
||||
switch (num_digits)
|
||||
{
|
||||
case 3: READ_DECIMAL_NUMBER(3); break;
|
||||
case 2: READ_DECIMAL_NUMBER(2); break;
|
||||
case 1: READ_DECIMAL_NUMBER(1); break;
|
||||
default: READ_DECIMAL_NUMBER(4);
|
||||
}
|
||||
}
|
||||
#undef DECIMAL_NUMBER_CASE
|
||||
}
|
||||
|
||||
struct DateTimeSubsecondPart
|
||||
{
|
||||
Int64 value;
|
||||
UInt8 digits;
|
||||
};
|
||||
|
||||
template <typename ReturnType>
|
||||
ReturnType parseDateTimeBestEffortImpl(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone)
|
||||
ReturnType parseDateTimeBestEffortImpl(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeSubsecondPart * fractional = nullptr)
|
||||
{
|
||||
auto on_error = [](const std::string & message [[maybe_unused]], int code [[maybe_unused]])
|
||||
{
|
||||
@ -113,7 +137,7 @@ ReturnType parseDateTimeBestEffortImpl(time_t & res, ReadBuffer & in, const Date
|
||||
|
||||
while (!in.eof())
|
||||
{
|
||||
char digits[14];
|
||||
char digits[std::numeric_limits<UInt64>::digits10];
|
||||
|
||||
size_t num_digits = 0;
|
||||
|
||||
@ -358,9 +382,18 @@ ReturnType parseDateTimeBestEffortImpl(time_t & res, ReadBuffer & in, const Date
|
||||
return on_error("Cannot read DateTime: unexpected point symbol", ErrorCodes::CANNOT_PARSE_DATETIME);
|
||||
|
||||
++in.position();
|
||||
num_digits = readDigits(digits, sizeof(digits), in);
|
||||
if (fractional)
|
||||
{
|
||||
using FractionalType = typename std::decay<decltype(fractional->value)>::type;
|
||||
// Reading more decimal digits than fits into FractionalType would case an
|
||||
// overflow, so it is better to skip all digits from the right side that do not
|
||||
// fit into result type. To provide less precise value rather than bogus one.
|
||||
num_digits = std::min(static_cast<size_t>(std::numeric_limits<FractionalType>::digits10), num_digits);
|
||||
|
||||
/// Just ignore fractional part of second.
|
||||
readDigits(digits, sizeof(digits), in);
|
||||
fractional->digits = num_digits;
|
||||
readDecimalNumber(fractional->value, num_digits, digits);
|
||||
}
|
||||
}
|
||||
else if (c == '+' || c == '-')
|
||||
{
|
||||
@ -517,6 +550,28 @@ ReturnType parseDateTimeBestEffortImpl(time_t & res, ReadBuffer & in, const Date
|
||||
return ReturnType(true);
|
||||
}
|
||||
|
||||
template <typename ReturnType>
|
||||
ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone)
|
||||
{
|
||||
time_t whole;
|
||||
DateTimeSubsecondPart subsecond = {0, 0}; // needs to be explicitly initialized sine it could be missing from input string
|
||||
if (!parseDateTimeBestEffortImpl<bool>(whole, in, local_time_zone, utc_time_zone, &subsecond))
|
||||
return ReturnType(false);
|
||||
|
||||
DateTime64::NativeType fractional = subsecond.value;
|
||||
if (scale < subsecond.digits)
|
||||
{
|
||||
fractional /= common::exp10_i64(subsecond.digits - scale);
|
||||
}
|
||||
else if (scale > subsecond.digits)
|
||||
{
|
||||
fractional *= common::exp10_i64(scale - subsecond.digits);
|
||||
}
|
||||
|
||||
res = DecimalUtils::decimalFromComponents<DateTime64>(whole, fractional, scale);
|
||||
return ReturnType(true);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#if defined(__PPC__)
|
||||
@ -535,4 +590,14 @@ bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl
|
||||
return parseDateTimeBestEffortImpl<bool>(res, in, local_time_zone, utc_time_zone);
|
||||
}
|
||||
|
||||
void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone)
|
||||
{
|
||||
return parseDateTime64BestEffortImpl<void>(res, scale, in, local_time_zone, utc_time_zone);
|
||||
}
|
||||
|
||||
bool tryParseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone)
|
||||
{
|
||||
return parseDateTime64BestEffortImpl<bool>(res, scale, in, local_time_zone, utc_time_zone);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -1,6 +1,8 @@
|
||||
#include <stddef.h>
|
||||
#include <time.h>
|
||||
|
||||
#include <Core/Types.h>
|
||||
|
||||
class DateLUTImpl;
|
||||
|
||||
namespace DB
|
||||
@ -55,5 +57,7 @@ class ReadBuffer;
|
||||
|
||||
void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone);
|
||||
bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone);
|
||||
void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone);
|
||||
bool tryParseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone);
|
||||
|
||||
}
|
||||
|
185
dbms/src/IO/tests/gtest_DateTime64_parsing_and_writing.cpp
Normal file
185
dbms/src/IO/tests/gtest_DateTime64_parsing_and_writing.cpp
Normal file
@ -0,0 +1,185 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/parseDateTimeBestEffort.h>
|
||||
|
||||
#include <Common/PODArray.h>
|
||||
|
||||
/** Test formatting and parsing predefined DateTime64 values to/from string
|
||||
*/
|
||||
|
||||
using namespace DB;
|
||||
|
||||
struct DateTime64StringsTestParam
|
||||
{
|
||||
const std::string_view comment;
|
||||
const std::string_view string;
|
||||
DateTime64 dt64;
|
||||
UInt32 scale;
|
||||
const DateLUTImpl & timezone = DateLUT::instance();
|
||||
};
|
||||
|
||||
std::ostream & operator << (std::ostream & ostr, const DateTime64StringsTestParam & param)
|
||||
{
|
||||
return ostr << param.comment;
|
||||
}
|
||||
|
||||
class DateTime64StringsTest : public ::testing::TestWithParam<DateTime64StringsTestParam> {};
|
||||
class DateTime64StringParseTest : public DateTime64StringsTest{};
|
||||
class DateTime64StringParseBestEffortTest : public DateTime64StringsTest{};
|
||||
class DateTime64StringWriteTest : public DateTime64StringsTest {};
|
||||
|
||||
|
||||
TEST_P(DateTime64StringParseTest, readDateTime64Text)
|
||||
{
|
||||
const auto & param = GetParam();
|
||||
ReadBufferFromMemory read_buffer(param.string.data(), param.string.size());
|
||||
|
||||
DateTime64 actual;
|
||||
EXPECT_TRUE(tryReadDateTime64Text(actual, param.scale, read_buffer));
|
||||
|
||||
EXPECT_EQ(param.dt64, actual);
|
||||
}
|
||||
|
||||
TEST_P(DateTime64StringParseTest, parseDateTime64BestEffort)
|
||||
{
|
||||
const auto & param = GetParam();
|
||||
ReadBufferFromMemory read_buffer(param.string.data(), param.string.size());
|
||||
|
||||
DateTime64 actual;
|
||||
EXPECT_TRUE(tryParseDateTime64BestEffort(actual, param.scale, read_buffer, param.timezone, DateLUT::instance("UTC")));
|
||||
|
||||
EXPECT_EQ(param.dt64, actual);
|
||||
}
|
||||
|
||||
TEST_P(DateTime64StringWriteTest, WriteText)
|
||||
{
|
||||
const auto & param = GetParam();
|
||||
|
||||
PaddedPODArray<char> actual_string(param.string.size() * 2, '\0'); // TODO: detect overflows
|
||||
|
||||
WriteBuffer write_buffer(actual_string.data(), actual_string.size());
|
||||
EXPECT_NO_THROW(writeDateTimeText(param.dt64, param.scale, write_buffer));
|
||||
|
||||
EXPECT_STREQ(param.string.data(), actual_string.data());
|
||||
}
|
||||
|
||||
TEST_P(DateTime64StringParseBestEffortTest, parse)
|
||||
{
|
||||
const auto & param = GetParam();
|
||||
ReadBufferFromMemory read_buffer(param.string.data(), param.string.size());
|
||||
|
||||
DateTime64 actual;
|
||||
EXPECT_TRUE(tryParseDateTime64BestEffort(actual, param.scale, read_buffer, param.timezone, DateLUT::instance("UTC")));
|
||||
|
||||
EXPECT_EQ(param.dt64, actual);
|
||||
}
|
||||
|
||||
|
||||
// YYYY-MM-DD HH:MM:SS.NNNNNNNNN
|
||||
INSTANTIATE_TEST_CASE_P(Basic,
|
||||
DateTime64StringParseTest,
|
||||
::testing::ValuesIn(std::initializer_list<DateTime64StringsTestParam>{
|
||||
{
|
||||
"When subsecond part is missing from string it is set to zero.",
|
||||
"2019-09-16 19:20:17",
|
||||
1568650817'000,
|
||||
3
|
||||
},
|
||||
{
|
||||
"When subsecond part is present in string, but it is zero, it is set to zero.",
|
||||
"2019-09-16 19:20:17.0",
|
||||
1568650817'000,
|
||||
3
|
||||
},
|
||||
{
|
||||
"When scale is 0, subsecond part is not set.",
|
||||
"2019-09-16 19:20:17",
|
||||
1568650817ULL,
|
||||
0
|
||||
},
|
||||
{
|
||||
"When scale is 0, subsecond part is 0 despite beeing present in string.",
|
||||
"2019-09-16 19:20:17.123",
|
||||
1568650817ULL,
|
||||
0
|
||||
},
|
||||
{
|
||||
"When subsecond part is present in string, it is set correctly to DateTime64 value of scale 3.",
|
||||
"2019-09-16 19:20:17.123",
|
||||
1568650817'123,
|
||||
3
|
||||
},
|
||||
{
|
||||
"When subsecond part is present in string (and begins with 0), it is set correctly to DateTime64 value of scale 3.",
|
||||
"2019-09-16 19:20:17.012",
|
||||
1568650817'012,
|
||||
3
|
||||
},
|
||||
{
|
||||
"When subsecond part scale is smaller than DateTime64 scale, subsecond part is properly adjusted (as if padded from right with zeroes).",
|
||||
"2019-09-16 19:20:17.123",
|
||||
1568650817'12300ULL,
|
||||
5
|
||||
},
|
||||
{
|
||||
"When subsecond part scale is larger than DateTime64 scale, subsecond part is truncated.",
|
||||
"2019-09-16 19:20:17.123",
|
||||
1568650817'1ULL,
|
||||
1
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(BestEffort,
|
||||
DateTime64StringParseBestEffortTest,
|
||||
::testing::ValuesIn(std::initializer_list<DateTime64StringsTestParam>{
|
||||
{
|
||||
"When subsecond part is unreasonably large, it fals to parse",
|
||||
"2019-09-16 19:20:17.12345678910111213141516171819202122233435363738393031323334353637383940414243444546474849505152535455565758596061626364",
|
||||
1568650817'123456ULL,
|
||||
6
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
|
||||
// TODO: add negative test cases for invalid strings, verifying that error is reported properly
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(Basic,
|
||||
DateTime64StringWriteTest,
|
||||
::testing::ValuesIn(std::initializer_list<DateTime64StringsTestParam>{
|
||||
{
|
||||
"non-zero subsecond part on DateTime64 with scale of 3",
|
||||
"2019-09-16 19:20:17.123",
|
||||
1568650817'123,
|
||||
3
|
||||
},
|
||||
{
|
||||
"non-zero subsecond part on DateTime64 with scale of 5",
|
||||
"2019-09-16 19:20:17.12345",
|
||||
1568650817'12345ULL,
|
||||
5
|
||||
},
|
||||
{
|
||||
"Zero subsecond part is written to string",
|
||||
"2019-09-16 19:20:17.000",
|
||||
1568650817'000ULL,
|
||||
3
|
||||
},
|
||||
{
|
||||
"When scale is 0, subsecond part (and separtor) is missing from string",
|
||||
"2019-09-16 19:20:17",
|
||||
1568650817ULL,
|
||||
0
|
||||
},
|
||||
{
|
||||
"Subsecond part with leading zeroes is written to string correctly",
|
||||
"2019-09-16 19:20:17.001",
|
||||
1568650817'001ULL,
|
||||
3
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
@ -1508,7 +1508,18 @@ BackgroundProcessingPool & Context::getBackgroundMovePool()
|
||||
{
|
||||
auto lock = getLock();
|
||||
if (!shared->background_move_pool)
|
||||
shared->background_move_pool.emplace(settings.background_move_pool_size, "BackgroundMovePool", "BgMoveProcPool");
|
||||
{
|
||||
BackgroundProcessingPool::PoolSettings pool_settings;
|
||||
auto & config = getConfigRef();
|
||||
pool_settings.thread_sleep_seconds = config.getDouble("background_move_processing_pool_thread_sleep_seconds", 10);
|
||||
pool_settings.thread_sleep_seconds_random_part = config.getDouble("background_move_processing_pool_thread_sleep_seconds_random_part", 1.0);
|
||||
pool_settings.thread_sleep_seconds_if_nothing_to_do = config.getDouble("background_move_processing_pool_thread_sleep_seconds_if_nothing_to_do", 0.1);
|
||||
pool_settings.task_sleep_seconds_when_no_work_min = config.getDouble("background_move_processing_pool_task_sleep_seconds_when_no_work_min", 10);
|
||||
pool_settings.task_sleep_seconds_when_no_work_max = config.getDouble("background_move_processing_pool_task_sleep_seconds_when_no_work_max", 600);
|
||||
pool_settings.task_sleep_seconds_when_no_work_multiplier = config.getDouble("background_move_processing_pool_task_sleep_seconds_when_no_work_multiplier", 1.1);
|
||||
pool_settings.task_sleep_seconds_when_no_work_random_part = config.getDouble("background_move_processing_pool_task_sleep_seconds_when_no_work_random_part", 1.0);
|
||||
shared->background_move_pool.emplace(settings.background_move_pool_size, pool_settings, "BackgroundMovePool", "BgMoveProcPool");
|
||||
}
|
||||
return *shared->background_move_pool;
|
||||
}
|
||||
|
||||
|
@ -333,7 +333,7 @@ void Join::setSampleBlock(const Block & block)
|
||||
asof_type = AsofRowRefs::getTypeSize(asof_column, asof_size);
|
||||
if (!asof_type)
|
||||
{
|
||||
std::string msg = "ASOF join not supported for type";
|
||||
std::string msg = "ASOF join not supported for type: ";
|
||||
msg += asof_column->getFamilyName();
|
||||
throw Exception(msg, ErrorCodes::BAD_TYPE_OF_FIELD);
|
||||
}
|
||||
|
@ -1,9 +1,12 @@
|
||||
#include <Interpreters/RowRefs.h>
|
||||
|
||||
#include <Core/Block.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/ColumnsHashing.h>
|
||||
#include <Core/Block.h>
|
||||
#include <Columns/IColumn.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -18,10 +21,15 @@ void callWithType(AsofRowRefs::Type which, F && f)
|
||||
{
|
||||
switch (which)
|
||||
{
|
||||
case AsofRowRefs::Type::key32: return f(UInt32());
|
||||
case AsofRowRefs::Type::key64: return f(UInt64());
|
||||
case AsofRowRefs::Type::keyu32: return f(UInt32());
|
||||
case AsofRowRefs::Type::keyu64: return f(UInt64());
|
||||
case AsofRowRefs::Type::keyi32: return f(Int32());
|
||||
case AsofRowRefs::Type::keyi64: return f(Int64());
|
||||
case AsofRowRefs::Type::keyf32: return f(Float32());
|
||||
case AsofRowRefs::Type::keyf64: return f(Float64());
|
||||
case AsofRowRefs::Type::keyDecimal32: return f(Decimal32());
|
||||
case AsofRowRefs::Type::keyDecimal64: return f(Decimal64());
|
||||
case AsofRowRefs::Type::keyDecimal128: return f(Decimal128());
|
||||
}
|
||||
|
||||
__builtin_unreachable();
|
||||
@ -49,7 +57,9 @@ void AsofRowRefs::insert(Type type, const IColumn * asof_column, const Block * b
|
||||
using T = std::decay_t<decltype(t)>;
|
||||
using LookupPtr = typename Entry<T>::LookupPtr;
|
||||
|
||||
auto * column = typeid_cast<const ColumnVector<T> *>(asof_column);
|
||||
using ColumnType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
|
||||
auto * column = typeid_cast<const ColumnType *>(asof_column);
|
||||
|
||||
T key = column->getElement(row_num);
|
||||
auto entry = Entry<T>(key, RowRef(block, row_num));
|
||||
std::get<LookupPtr>(lookups)->insert(entry);
|
||||
@ -71,7 +81,8 @@ const RowRef * AsofRowRefs::findAsof(Type type, ASOF::Inequality inequality, con
|
||||
using EntryType = Entry<T>;
|
||||
using LookupPtr = typename EntryType::LookupPtr;
|
||||
|
||||
auto * column = typeid_cast<const ColumnVector<T> *>(asof_column);
|
||||
using ColumnType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
|
||||
auto * column = typeid_cast<const ColumnType *>(asof_column);
|
||||
T key = column->getElement(row_num);
|
||||
auto & typed_lookup = std::get<LookupPtr>(lookups);
|
||||
|
||||
@ -90,12 +101,22 @@ std::optional<AsofRowRefs::Type> AsofRowRefs::getTypeSize(const IColumn * asof_c
|
||||
if (typeid_cast<const ColumnVector<UInt32> *>(asof_column))
|
||||
{
|
||||
size = sizeof(UInt32);
|
||||
return Type::key32;
|
||||
return Type::keyu32;
|
||||
}
|
||||
else if (typeid_cast<const ColumnVector<UInt64> *>(asof_column))
|
||||
{
|
||||
size = sizeof(UInt64);
|
||||
return Type::key64;
|
||||
return Type::keyu64;
|
||||
}
|
||||
else if (typeid_cast<const ColumnVector<Int32> *>(asof_column))
|
||||
{
|
||||
size = sizeof(Int32);
|
||||
return Type::keyi32;
|
||||
}
|
||||
else if (typeid_cast<const ColumnVector<Int64> *>(asof_column))
|
||||
{
|
||||
size = sizeof(Int64);
|
||||
return Type::keyi64;
|
||||
}
|
||||
else if (typeid_cast<const ColumnVector<Float32> *>(asof_column))
|
||||
{
|
||||
@ -107,6 +128,21 @@ std::optional<AsofRowRefs::Type> AsofRowRefs::getTypeSize(const IColumn * asof_c
|
||||
size = sizeof(Float64);
|
||||
return Type::keyf64;
|
||||
}
|
||||
else if (typeid_cast<const ColumnDecimal<Decimal32> *>(asof_column))
|
||||
{
|
||||
size = sizeof(Decimal32);
|
||||
return Type::keyDecimal32;
|
||||
}
|
||||
else if (typeid_cast<const ColumnDecimal<Decimal64> *>(asof_column))
|
||||
{
|
||||
size = sizeof(Decimal64);
|
||||
return Type::keyDecimal64;
|
||||
}
|
||||
else if (typeid_cast<const ColumnDecimal<Decimal128> *>(asof_column))
|
||||
{
|
||||
size = sizeof(Decimal128);
|
||||
return Type::keyDecimal128;
|
||||
}
|
||||
|
||||
size = 0;
|
||||
return {};
|
||||
|
@ -216,15 +216,25 @@ public:
|
||||
using Lookups = std::variant<
|
||||
Entry<UInt32>::LookupPtr,
|
||||
Entry<UInt64>::LookupPtr,
|
||||
Entry<Int32>::LookupPtr,
|
||||
Entry<Int64>::LookupPtr,
|
||||
Entry<Float32>::LookupPtr,
|
||||
Entry<Float64>::LookupPtr>;
|
||||
Entry<Float64>::LookupPtr,
|
||||
Entry<Decimal32>::LookupPtr,
|
||||
Entry<Decimal64>::LookupPtr,
|
||||
Entry<Decimal128>::LookupPtr>;
|
||||
|
||||
enum class Type
|
||||
{
|
||||
key32,
|
||||
key64,
|
||||
keyu32,
|
||||
keyu64,
|
||||
keyi32,
|
||||
keyi64,
|
||||
keyf32,
|
||||
keyf64,
|
||||
keyDecimal32,
|
||||
keyDecimal64,
|
||||
keyDecimal128,
|
||||
};
|
||||
|
||||
AsofRowRefs() {}
|
||||
|
@ -11,6 +11,7 @@
|
||||
#include <DataTypes/DataTypeFixedString.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeEnum.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
|
||||
@ -146,13 +147,33 @@ UInt64 stringToDateTime(const String & s)
|
||||
return UInt64(date_time);
|
||||
}
|
||||
|
||||
DateTime64::NativeType stringToDateTime64(const String & s, UInt32 scale)
|
||||
{
|
||||
ReadBufferFromString in(s);
|
||||
DateTime64 datetime64 {0};
|
||||
|
||||
readDateTime64Text(datetime64, scale, in);
|
||||
if (!in.eof())
|
||||
throw Exception("String is too long for DateTime64: " + s, ErrorCodes::TOO_LARGE_STRING_SIZE);
|
||||
|
||||
return datetime64.value;
|
||||
}
|
||||
|
||||
Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const IDataType * from_type_hint)
|
||||
{
|
||||
WhichDataType which_type(type);
|
||||
WhichDataType which_from_type;
|
||||
if (from_type_hint)
|
||||
{
|
||||
which_from_type = WhichDataType(*from_type_hint);
|
||||
|
||||
// This was added to mitigate converting DateTime64-Field (a typedef to a Decimal64) to DataTypeDate64-compatitable type.
|
||||
if (from_type_hint && from_type_hint->equals(type))
|
||||
{
|
||||
return src;
|
||||
}
|
||||
}
|
||||
|
||||
/// Conversion between Date and DateTime and vice versa.
|
||||
if (which_type.isDate() && which_from_type.isDateTime())
|
||||
{
|
||||
@ -187,11 +208,12 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
|
||||
return dynamic_cast<const IDataTypeEnum &>(type).castToValue(src);
|
||||
}
|
||||
|
||||
if (which_type.isDateOrDateTime() && src.getType() == Field::Types::UInt64)
|
||||
if (which_type.isDateOrDateTime() && !which_type.isDateTime64() && src.getType() == Field::Types::UInt64)
|
||||
{
|
||||
/// We don't need any conversion UInt64 is under type of Date and DateTime
|
||||
return src;
|
||||
}
|
||||
// TODO (vnemkov): extra cases for DateTime64: converting from integer, converting from Decimal
|
||||
|
||||
if (src.getType() == Field::Types::String)
|
||||
{
|
||||
@ -205,6 +227,12 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
|
||||
/// Convert 'YYYY-MM-DD hh:mm:ss' Strings to DateTime
|
||||
return stringToDateTime(src.get<const String &>());
|
||||
}
|
||||
else if (which_type.isDateTime64())
|
||||
{
|
||||
const auto date_time64 = typeid_cast<const DataTypeDateTime64 *>(&type);
|
||||
/// Convert 'YYYY-MM-DD hh:mm:ss.NNNNNNNNN' Strings to DateTime
|
||||
return stringToDateTime64(src.get<const String &>(), date_time64->getScale());
|
||||
}
|
||||
else if (which_type.isUUID())
|
||||
{
|
||||
return stringToUUID(src.get<const String &>());
|
||||
|
@ -176,12 +176,14 @@ void ASTAlterCommand::formatImpl(
|
||||
settings.ostr << " TO ";
|
||||
switch (move_destination_type)
|
||||
{
|
||||
case MoveDestinationType::DISK:
|
||||
case PartDestinationType::DISK:
|
||||
settings.ostr << "DISK ";
|
||||
break;
|
||||
case MoveDestinationType::VOLUME:
|
||||
case PartDestinationType::VOLUME:
|
||||
settings.ostr << "VOLUME ";
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
settings.ostr << quoteString(move_destination_name);
|
||||
}
|
||||
|
@ -3,6 +3,7 @@
|
||||
#include <Parsers/IAST.h>
|
||||
#include <Parsers/ASTQueryWithTableAndOutput.h>
|
||||
#include <Parsers/ASTQueryWithOnCluster.h>
|
||||
#include <Parsers/ASTTTLElement.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -128,15 +129,9 @@ public:
|
||||
|
||||
bool if_exists = false; /// option for DROP_COLUMN, MODIFY_COLUMN, COMMENT_COLUMN
|
||||
|
||||
enum MoveDestinationType
|
||||
{
|
||||
DISK,
|
||||
VOLUME,
|
||||
};
|
||||
PartDestinationType move_destination_type; /// option for MOVE PART/PARTITION
|
||||
|
||||
MoveDestinationType move_destination_type;
|
||||
|
||||
String move_destination_name;
|
||||
String move_destination_name; /// option for MOVE PART/PARTITION
|
||||
|
||||
/** For FETCH PARTITION - the path in ZK to the shard, from which to download the partition.
|
||||
*/
|
||||
|
27
dbms/src/Parsers/ASTTTLElement.cpp
Normal file
27
dbms/src/Parsers/ASTTTLElement.cpp
Normal file
@ -0,0 +1,27 @@
|
||||
|
||||
#include <Columns/Collator.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <Parsers/ASTTTLElement.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
||||
{
|
||||
children.front()->formatImpl(settings, state, frame);
|
||||
if (destination_type == PartDestinationType::DISK)
|
||||
{
|
||||
settings.ostr << " TO DISK " << quoteString(destination_name);
|
||||
}
|
||||
else if (destination_type == PartDestinationType::VOLUME)
|
||||
{
|
||||
settings.ostr << " TO VOLUME " << quoteString(destination_name);
|
||||
}
|
||||
else if (destination_type == PartDestinationType::DELETE)
|
||||
{
|
||||
/// It would be better to output "DELETE" here but that will break compatibility with earlier versions.
|
||||
}
|
||||
}
|
||||
|
||||
}
|
36
dbms/src/Parsers/ASTTTLElement.h
Normal file
36
dbms/src/Parsers/ASTTTLElement.h
Normal file
@ -0,0 +1,36 @@
|
||||
#pragma once
|
||||
|
||||
#include <Parsers/IAST.h>
|
||||
#include <Storages/MergeTree/PartDestinationType.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
/** Element of TTL expression.
|
||||
*/
|
||||
class ASTTTLElement : public IAST
|
||||
{
|
||||
public:
|
||||
PartDestinationType destination_type;
|
||||
String destination_name;
|
||||
|
||||
ASTTTLElement(PartDestinationType destination_type_, const String & destination_name_)
|
||||
: destination_type(destination_type_)
|
||||
, destination_name(destination_name_)
|
||||
{
|
||||
}
|
||||
|
||||
String getID(char) const override { return "TTLElement"; }
|
||||
|
||||
ASTPtr clone() const override
|
||||
{
|
||||
auto clone = std::make_shared<ASTTTLElement>(*this);
|
||||
clone->cloneChildren();
|
||||
return clone;
|
||||
}
|
||||
|
||||
protected:
|
||||
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
|
||||
};
|
||||
|
||||
}
|
@ -16,6 +16,7 @@
|
||||
#include <Parsers/ASTAsterisk.h>
|
||||
#include <Parsers/ASTQualifiedAsterisk.h>
|
||||
#include <Parsers/ASTQueryParameter.h>
|
||||
#include <Parsers/ASTTTLElement.h>
|
||||
#include <Parsers/ASTOrderByElement.h>
|
||||
#include <Parsers/ASTSubquery.h>
|
||||
#include <Parsers/ASTFunctionWithKeyValueArguments.h>
|
||||
@ -1414,6 +1415,42 @@ bool ParserFunctionWithKeyValueArguments::parseImpl(Pos & pos, ASTPtr & node, Ex
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
{
|
||||
ParserKeyword s_to_disk("TO DISK");
|
||||
ParserKeyword s_to_volume("TO VOLUME");
|
||||
ParserKeyword s_delete("DELETE");
|
||||
ParserStringLiteral parser_string_literal;
|
||||
ParserExpression parser_exp;
|
||||
|
||||
ASTPtr expr_elem;
|
||||
if (!parser_exp.parse(pos, expr_elem, expected))
|
||||
return false;
|
||||
|
||||
PartDestinationType destination_type = PartDestinationType::DELETE;
|
||||
String destination_name;
|
||||
if (s_to_disk.ignore(pos))
|
||||
destination_type = PartDestinationType::DISK;
|
||||
else if (s_to_volume.ignore(pos))
|
||||
destination_type = PartDestinationType::VOLUME;
|
||||
else
|
||||
s_delete.ignore(pos);
|
||||
|
||||
if (destination_type == PartDestinationType::DISK || destination_type == PartDestinationType::VOLUME)
|
||||
{
|
||||
ASTPtr ast_space_name;
|
||||
if (!parser_string_literal.parse(pos, ast_space_name, expected))
|
||||
return false;
|
||||
|
||||
destination_name = ast_space_name->as<ASTLiteral &>().value.get<const String &>();
|
||||
}
|
||||
|
||||
node = std::make_shared<ASTTTLElement>(destination_type, destination_name);
|
||||
node->children.push_back(expr_elem);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ParserIdentifierWithOptionalParameters::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
{
|
||||
ParserIdentifier non_parametric;
|
||||
|
@ -320,4 +320,14 @@ protected:
|
||||
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
|
||||
};
|
||||
|
||||
/** Element of TTL expression - same as expression element, but in addition,
|
||||
* TO DISK 'xxx' | TO VOLUME 'xxx' | DELETE could be specified
|
||||
*/
|
||||
class ParserTTLElement : public IParserBase
|
||||
{
|
||||
protected:
|
||||
const char * getName() const { return "element of TTL expression"; }
|
||||
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -557,6 +557,13 @@ bool ParserOrderByExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected &
|
||||
}
|
||||
|
||||
|
||||
bool ParserTTLExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
{
|
||||
return ParserList(std::make_unique<ParserTTLElement>(), std::make_unique<ParserToken>(TokenType::Comma), false)
|
||||
.parse(pos, node, expected);
|
||||
}
|
||||
|
||||
|
||||
bool ParserNullityChecking::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
{
|
||||
ASTPtr node_comp;
|
||||
|
@ -386,6 +386,7 @@ protected:
|
||||
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
|
||||
};
|
||||
|
||||
|
||||
/// Parser for list of key-value pairs.
|
||||
class ParserKeyValuePairsList : public IParserBase
|
||||
{
|
||||
@ -394,4 +395,12 @@ protected:
|
||||
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
|
||||
};
|
||||
|
||||
|
||||
class ParserTTLExpressionList : public IParserBase
|
||||
{
|
||||
protected:
|
||||
const char * getName() const { return "ttl expression"; }
|
||||
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -87,6 +87,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
/* allow_empty = */ false);
|
||||
ParserSetQuery parser_settings(true);
|
||||
ParserNameList values_p;
|
||||
ParserTTLExpressionList parser_ttl_list;
|
||||
|
||||
if (is_live_view)
|
||||
{
|
||||
@ -236,9 +237,9 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
command->part = true;
|
||||
|
||||
if (s_to_disk.ignore(pos))
|
||||
command->move_destination_type = ASTAlterCommand::MoveDestinationType::DISK;
|
||||
command->move_destination_type = PartDestinationType::DISK;
|
||||
else if (s_to_volume.ignore(pos))
|
||||
command->move_destination_type = ASTAlterCommand::MoveDestinationType::VOLUME;
|
||||
command->move_destination_type = PartDestinationType::VOLUME;
|
||||
else
|
||||
return false;
|
||||
|
||||
@ -256,9 +257,9 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
command->type = ASTAlterCommand::MOVE_PARTITION;
|
||||
|
||||
if (s_to_disk.ignore(pos))
|
||||
command->move_destination_type = ASTAlterCommand::MoveDestinationType::DISK;
|
||||
command->move_destination_type = PartDestinationType::DISK;
|
||||
else if (s_to_volume.ignore(pos))
|
||||
command->move_destination_type = ASTAlterCommand::MoveDestinationType::VOLUME;
|
||||
command->move_destination_type = PartDestinationType::VOLUME;
|
||||
else
|
||||
return false;
|
||||
|
||||
@ -431,7 +432,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
}
|
||||
else if (s_modify_ttl.ignore(pos, expected))
|
||||
{
|
||||
if (!parser_exp_elem.parse(pos, command->ttl, expected))
|
||||
if (!parser_ttl_list.parse(pos, command->ttl, expected))
|
||||
return false;
|
||||
command->type = ASTAlterCommand::MODIFY_TTL;
|
||||
}
|
||||
|
@ -250,6 +250,7 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
ParserIdentifierWithOptionalParameters ident_with_optional_params_p;
|
||||
ParserExpression expression_p;
|
||||
ParserSetQuery settings_p(/* parse_only_internals_ = */ true);
|
||||
ParserTTLExpressionList parser_ttl_list;
|
||||
|
||||
ASTPtr engine;
|
||||
ASTPtr partition_by;
|
||||
@ -303,7 +304,7 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
|
||||
if (!ttl_table && s_ttl.ignore(pos, expected))
|
||||
{
|
||||
if (expression_p.parse(pos, ttl_table, expected))
|
||||
if (parser_ttl_list.parse(pos, ttl_table, expected))
|
||||
continue;
|
||||
else
|
||||
return false;
|
||||
|
@ -23,17 +23,6 @@ namespace CurrentMetrics
|
||||
namespace DB
|
||||
{
|
||||
|
||||
static constexpr double thread_sleep_seconds = 10;
|
||||
static constexpr double thread_sleep_seconds_random_part = 1.0;
|
||||
static constexpr double thread_sleep_seconds_if_nothing_to_do = 0.1;
|
||||
|
||||
/// For exponential backoff.
|
||||
static constexpr double task_sleep_seconds_when_no_work_min = 10;
|
||||
static constexpr double task_sleep_seconds_when_no_work_max = 600;
|
||||
static constexpr double task_sleep_seconds_when_no_work_multiplier = 1.1;
|
||||
static constexpr double task_sleep_seconds_when_no_work_random_part = 1.0;
|
||||
|
||||
|
||||
void BackgroundProcessingPoolTaskInfo::wake()
|
||||
{
|
||||
Poco::Timestamp current_time;
|
||||
@ -61,9 +50,13 @@ void BackgroundProcessingPoolTaskInfo::wake()
|
||||
}
|
||||
|
||||
|
||||
BackgroundProcessingPool::BackgroundProcessingPool(int size_, const char * log_name, const char * thread_name_)
|
||||
BackgroundProcessingPool::BackgroundProcessingPool(int size_,
|
||||
const PoolSettings & pool_settings,
|
||||
const char * log_name,
|
||||
const char * thread_name_)
|
||||
: size(size_)
|
||||
, thread_name(thread_name_)
|
||||
, settings(pool_settings)
|
||||
{
|
||||
logger = &Logger::get(log_name);
|
||||
LOG_INFO(logger, "Create " << log_name << " with " << size << " threads");
|
||||
@ -147,7 +140,7 @@ void BackgroundProcessingPool::threadFunction()
|
||||
memory_tracker->setMetric(CurrentMetrics::MemoryTrackingInBackgroundProcessingPool);
|
||||
|
||||
pcg64 rng(randomSeed());
|
||||
std::this_thread::sleep_for(std::chrono::duration<double>(std::uniform_real_distribution<double>(0, thread_sleep_seconds_random_part)(rng)));
|
||||
std::this_thread::sleep_for(std::chrono::duration<double>(std::uniform_real_distribution<double>(0, settings.thread_sleep_seconds_random_part)(rng)));
|
||||
|
||||
while (!shutdown)
|
||||
{
|
||||
@ -182,8 +175,8 @@ void BackgroundProcessingPool::threadFunction()
|
||||
{
|
||||
std::unique_lock lock(tasks_mutex);
|
||||
wake_event.wait_for(lock,
|
||||
std::chrono::duration<double>(thread_sleep_seconds
|
||||
+ std::uniform_real_distribution<double>(0, thread_sleep_seconds_random_part)(rng)));
|
||||
std::chrono::duration<double>(settings.thread_sleep_seconds
|
||||
+ std::uniform_real_distribution<double>(0, settings.thread_sleep_seconds_random_part)(rng)));
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -193,7 +186,7 @@ void BackgroundProcessingPool::threadFunction()
|
||||
{
|
||||
std::unique_lock lock(tasks_mutex);
|
||||
wake_event.wait_for(lock, std::chrono::microseconds(
|
||||
min_time - current_time + std::uniform_int_distribution<uint64_t>(0, thread_sleep_seconds_random_part * 1000000)(rng)));
|
||||
min_time - current_time + std::uniform_int_distribution<uint64_t>(0, settings.thread_sleep_seconds_random_part * 1000000)(rng)));
|
||||
}
|
||||
|
||||
std::shared_lock rlock(task->rwlock);
|
||||
@ -231,11 +224,11 @@ void BackgroundProcessingPool::threadFunction()
|
||||
Poco::Timestamp next_time_to_execute; /// current time
|
||||
if (task_result == TaskResult::ERROR)
|
||||
next_time_to_execute += 1000000 * (std::min(
|
||||
task_sleep_seconds_when_no_work_max,
|
||||
task_sleep_seconds_when_no_work_min * std::pow(task_sleep_seconds_when_no_work_multiplier, task->count_no_work_done))
|
||||
+ std::uniform_real_distribution<double>(0, task_sleep_seconds_when_no_work_random_part)(rng));
|
||||
settings.task_sleep_seconds_when_no_work_max,
|
||||
settings.task_sleep_seconds_when_no_work_min * std::pow(settings.task_sleep_seconds_when_no_work_multiplier, task->count_no_work_done))
|
||||
+ std::uniform_real_distribution<double>(0, settings.task_sleep_seconds_when_no_work_random_part)(rng));
|
||||
else if (task_result == TaskResult::NOTHING_TO_DO)
|
||||
next_time_to_execute += 1000000 * thread_sleep_seconds_if_nothing_to_do;
|
||||
next_time_to_execute += 1000000 * settings.thread_sleep_seconds_if_nothing_to_do;
|
||||
|
||||
tasks.erase(task->iterator);
|
||||
task->iterator = tasks.emplace(next_time_to_execute, task);
|
||||
|
@ -14,6 +14,7 @@
|
||||
#include <Core/Types.h>
|
||||
#include <Common/CurrentThread.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Poco/Util/AbstractConfiguration.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -46,7 +47,23 @@ public:
|
||||
using TaskHandle = std::shared_ptr<TaskInfo>;
|
||||
|
||||
|
||||
struct PoolSettings
|
||||
{
|
||||
double thread_sleep_seconds = 10;
|
||||
double thread_sleep_seconds_random_part = 1.0;
|
||||
double thread_sleep_seconds_if_nothing_to_do = 0.1;
|
||||
|
||||
/// For exponential backoff.
|
||||
double task_sleep_seconds_when_no_work_min = 10;
|
||||
double task_sleep_seconds_when_no_work_max = 600;
|
||||
double task_sleep_seconds_when_no_work_multiplier = 1.1;
|
||||
double task_sleep_seconds_when_no_work_random_part = 1.0;
|
||||
|
||||
PoolSettings() noexcept {}
|
||||
};
|
||||
|
||||
BackgroundProcessingPool(int size_,
|
||||
const PoolSettings & pool_settings = {},
|
||||
const char * log_name = "BackgroundProcessingPool",
|
||||
const char * thread_name_ = "BackgrProcPool");
|
||||
|
||||
@ -84,6 +101,9 @@ protected:
|
||||
ThreadGroupStatusPtr thread_group;
|
||||
|
||||
void threadFunction();
|
||||
|
||||
private:
|
||||
PoolSettings settings;
|
||||
};
|
||||
|
||||
|
||||
|
@ -38,6 +38,7 @@
|
||||
#include <Common/Increment.h>
|
||||
#include <Common/SimpleIncrement.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
@ -70,6 +71,12 @@ namespace CurrentMetrics
|
||||
}
|
||||
|
||||
|
||||
namespace
|
||||
{
|
||||
constexpr UInt64 RESERVATION_MIN_ESTIMATION_SIZE = 1u * 1024u * 1024u; /// 1MB
|
||||
}
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -124,7 +131,6 @@ MergeTreeData::MergeTreeData(
|
||||
, merging_params(merging_params_)
|
||||
, partition_by_ast(partition_by_ast_)
|
||||
, sample_by_ast(sample_by_ast_)
|
||||
, ttl_table_ast(ttl_table_ast_)
|
||||
, require_part_metadata(require_part_metadata_)
|
||||
, database_name(database_)
|
||||
, table_name(table_)
|
||||
@ -566,15 +572,17 @@ void checkTTLExpression(const ExpressionActionsPtr & ttl_expression, const Strin
|
||||
void MergeTreeData::setTTLExpressions(const ColumnsDescription::ColumnTTLs & new_column_ttls,
|
||||
const ASTPtr & new_ttl_table_ast, bool only_check)
|
||||
{
|
||||
auto create_ttl_entry = [this](ASTPtr ttl_ast) -> TTLEntry
|
||||
auto create_ttl_entry = [this](ASTPtr ttl_ast)
|
||||
{
|
||||
TTLEntry result;
|
||||
|
||||
auto syntax_result = SyntaxAnalyzer(global_context).analyze(ttl_ast, getColumns().getAllPhysical());
|
||||
auto expr = ExpressionAnalyzer(ttl_ast, syntax_result, global_context).getActions(false);
|
||||
result.expression = ExpressionAnalyzer(ttl_ast, syntax_result, global_context).getActions(false);
|
||||
result.destination_type = PartDestinationType::DELETE;
|
||||
result.result_column = ttl_ast->getColumnName();
|
||||
|
||||
String result_column = ttl_ast->getColumnName();
|
||||
checkTTLExpression(expr, result_column);
|
||||
|
||||
return {expr, result_column};
|
||||
checkTTLExpression(result.expression, result.result_column);
|
||||
return result;
|
||||
};
|
||||
|
||||
if (!new_column_ttls.empty())
|
||||
@ -592,24 +600,50 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription::ColumnTTLs & new
|
||||
for (const auto & [name, ast] : new_column_ttls)
|
||||
{
|
||||
if (columns_ttl_forbidden.count(name))
|
||||
throw Exception("Trying to set ttl for key column " + name, ErrorCodes::ILLEGAL_COLUMN);
|
||||
throw Exception("Trying to set TTL for key column " + name, ErrorCodes::ILLEGAL_COLUMN);
|
||||
else
|
||||
{
|
||||
auto new_ttl_entry = create_ttl_entry(ast);
|
||||
if (!only_check)
|
||||
ttl_entries_by_name.emplace(name, new_ttl_entry);
|
||||
column_ttl_entries_by_name.emplace(name, new_ttl_entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (new_ttl_table_ast)
|
||||
{
|
||||
auto new_ttl_table_entry = create_ttl_entry(new_ttl_table_ast);
|
||||
bool seen_delete_ttl = false;
|
||||
for (auto ttl_element_ptr : new_ttl_table_ast->children)
|
||||
{
|
||||
ASTTTLElement & ttl_element = static_cast<ASTTTLElement &>(*ttl_element_ptr);
|
||||
if (ttl_element.destination_type == PartDestinationType::DELETE)
|
||||
{
|
||||
if (seen_delete_ttl)
|
||||
{
|
||||
throw Exception("More than one DELETE TTL expression is not allowed", ErrorCodes::BAD_TTL_EXPRESSION);
|
||||
}
|
||||
|
||||
auto new_ttl_table_entry = create_ttl_entry(ttl_element.children[0]);
|
||||
if (!only_check)
|
||||
{
|
||||
ttl_table_ast = new_ttl_table_ast;
|
||||
ttl_table_ast = ttl_element.children[0];
|
||||
ttl_table_entry = new_ttl_table_entry;
|
||||
}
|
||||
|
||||
seen_delete_ttl = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto new_ttl_entry = create_ttl_entry(ttl_element.children[0]);
|
||||
if (!only_check)
|
||||
{
|
||||
new_ttl_entry.entry_ast = ttl_element_ptr;
|
||||
new_ttl_entry.destination_type = ttl_element.destination_type;
|
||||
new_ttl_entry.destination_name = ttl_element.destination_name;
|
||||
move_ttl_entries.emplace_back(std::move(new_ttl_entry));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -3096,20 +3130,138 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const
|
||||
return loaded_parts;
|
||||
}
|
||||
|
||||
DiskSpace::ReservationPtr MergeTreeData::reserveSpace(UInt64 expected_size)
|
||||
namespace
|
||||
{
|
||||
constexpr UInt64 RESERVATION_MIN_ESTIMATION_SIZE = 1u * 1024u * 1024u; /// 1MB
|
||||
|
||||
expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size);
|
||||
|
||||
auto reservation = storage_policy->reserve(expected_size);
|
||||
inline DiskSpace::ReservationPtr checkAndReturnReservation(UInt64 expected_size, DiskSpace::ReservationPtr reservation)
|
||||
{
|
||||
if (reservation)
|
||||
return reservation;
|
||||
|
||||
throw Exception("Cannot reserve " + formatReadableSizeWithBinarySuffix(expected_size) + ", not enough space.",
|
||||
throw Exception("Cannot reserve " + formatReadableSizeWithBinarySuffix(expected_size) + ", not enough space",
|
||||
ErrorCodes::NOT_ENOUGH_SPACE);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
DiskSpace::ReservationPtr MergeTreeData::reserveSpace(UInt64 expected_size) const
|
||||
{
|
||||
expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size);
|
||||
|
||||
auto reservation = storage_policy->reserve(expected_size);
|
||||
|
||||
return checkAndReturnReservation(expected_size, std::move(reservation));
|
||||
}
|
||||
|
||||
DiskSpace::ReservationPtr MergeTreeData::reserveSpace(UInt64 expected_size, DiskSpace::SpacePtr space) const
|
||||
{
|
||||
expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size);
|
||||
|
||||
auto reservation = tryReserveSpace(expected_size, space);
|
||||
|
||||
return checkAndReturnReservation(expected_size, std::move(reservation));
|
||||
}
|
||||
|
||||
DiskSpace::ReservationPtr MergeTreeData::tryReserveSpace(UInt64 expected_size, DiskSpace::SpacePtr space) const
|
||||
{
|
||||
expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size);
|
||||
|
||||
return space->reserve(expected_size);
|
||||
}
|
||||
|
||||
DiskSpace::ReservationPtr MergeTreeData::reserveSpacePreferringTTLRules(UInt64 expected_size,
|
||||
const MergeTreeDataPart::TTLInfos & ttl_infos,
|
||||
time_t time_of_move) const
|
||||
{
|
||||
expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size);
|
||||
|
||||
DiskSpace::ReservationPtr reservation = tryReserveSpacePreferringTTLRules(expected_size, ttl_infos, time_of_move);
|
||||
|
||||
return checkAndReturnReservation(expected_size, std::move(reservation));
|
||||
}
|
||||
|
||||
DiskSpace::ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_size,
|
||||
const MergeTreeDataPart::TTLInfos & ttl_infos,
|
||||
time_t time_of_move) const
|
||||
{
|
||||
expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size);
|
||||
|
||||
DiskSpace::ReservationPtr reservation;
|
||||
|
||||
auto ttl_entry = selectTTLEntryForTTLInfos(ttl_infos, time_of_move);
|
||||
if (ttl_entry != nullptr)
|
||||
{
|
||||
DiskSpace::SpacePtr destination_ptr = ttl_entry->getDestination(storage_policy);
|
||||
if (!destination_ptr)
|
||||
{
|
||||
if (ttl_entry->destination_type == PartDestinationType::VOLUME)
|
||||
LOG_WARNING(log, "Would like to reserve space on volume '"
|
||||
<< ttl_entry->destination_name << "' by TTL rule of table '"
|
||||
<< log_name << "' but volume was not found");
|
||||
else if (ttl_entry->destination_type == PartDestinationType::DISK)
|
||||
LOG_WARNING(log, "Would like to reserve space on disk '"
|
||||
<< ttl_entry->destination_name << "' by TTL rule of table '"
|
||||
<< log_name << "' but disk was not found");
|
||||
}
|
||||
else
|
||||
{
|
||||
reservation = destination_ptr->reserve(expected_size);
|
||||
if (reservation)
|
||||
return reservation;
|
||||
}
|
||||
}
|
||||
|
||||
reservation = storage_policy->reserve(expected_size);
|
||||
|
||||
return reservation;
|
||||
}
|
||||
|
||||
DiskSpace::SpacePtr MergeTreeData::TTLEntry::getDestination(const DiskSpace::StoragePolicyPtr & policy) const
|
||||
{
|
||||
if (destination_type == PartDestinationType::VOLUME)
|
||||
return policy->getVolumeByName(destination_name);
|
||||
else if (destination_type == PartDestinationType::DISK)
|
||||
return policy->getDiskByName(destination_name);
|
||||
else
|
||||
return {};
|
||||
}
|
||||
|
||||
bool MergeTreeData::TTLEntry::isPartInDestination(const DiskSpace::StoragePolicyPtr & policy, const MergeTreeDataPart & part) const
|
||||
{
|
||||
if (destination_type == PartDestinationType::VOLUME)
|
||||
{
|
||||
for (const auto & disk : policy->getVolumeByName(destination_name)->disks)
|
||||
if (disk->getName() == part.disk->getName())
|
||||
return true;
|
||||
}
|
||||
else if (destination_type == PartDestinationType::DISK)
|
||||
return policy->getDiskByName(destination_name)->getName() == part.disk->getName();
|
||||
return false;
|
||||
}
|
||||
|
||||
const MergeTreeData::TTLEntry * MergeTreeData::selectTTLEntryForTTLInfos(
|
||||
const MergeTreeDataPart::TTLInfos & ttl_infos,
|
||||
time_t time_of_move) const
|
||||
{
|
||||
const MergeTreeData::TTLEntry * result = nullptr;
|
||||
/// Prefer TTL rule which went into action last.
|
||||
time_t max_max_ttl = 0;
|
||||
|
||||
for (const auto & ttl_entry : move_ttl_entries)
|
||||
{
|
||||
auto ttl_info_it = ttl_infos.moves_ttl.find(ttl_entry.result_column);
|
||||
if (ttl_info_it != ttl_infos.moves_ttl.end()
|
||||
&& ttl_info_it->second.max <= time_of_move
|
||||
&& max_max_ttl <= ttl_info_it->second.max)
|
||||
{
|
||||
result = &ttl_entry;
|
||||
max_max_ttl = ttl_info_it->second.max;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const
|
||||
{
|
||||
DataParts res;
|
||||
@ -3289,12 +3441,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk(
|
||||
String dst_part_name = src_part->getNewName(dst_part_info);
|
||||
String tmp_dst_part_name = tmp_part_prefix + dst_part_name;
|
||||
|
||||
auto reservation = src_part->disk->reserve(src_part->bytes_on_disk);
|
||||
if (!reservation)
|
||||
{
|
||||
throw Exception("Cannot reserve " + formatReadableSizeWithBinarySuffix(src_part->bytes_on_disk) + ", not enough space",
|
||||
ErrorCodes::NOT_ENOUGH_SPACE);
|
||||
}
|
||||
auto reservation = reserveSpace(src_part->bytes_on_disk, src_part->disk);
|
||||
String dst_part_path = getFullPathOnDisk(reservation->getDisk());
|
||||
Poco::Path dst_part_absolute_path = Poco::Path(dst_part_path + tmp_dst_part_name).absolute();
|
||||
Poco::Path src_part_absolute_path = Poco::Path(src_part->getFullPath()).absolute();
|
||||
|
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/SimpleIncrement.h>
|
||||
#include <Common/DiskSpaceMonitor.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/ExpressionActions.h>
|
||||
#include <Storages/IStorage.h>
|
||||
@ -9,6 +10,7 @@
|
||||
#include <Storages/MergeTree/MergeTreeSettings.h>
|
||||
#include <Storages/MergeTree/MergeTreeMutationStatus.h>
|
||||
#include <Storages/MergeTree/MergeList.h>
|
||||
#include <Storages/MergeTree/PartDestinationType.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
@ -19,7 +21,6 @@
|
||||
#include <Storages/IndicesDescription.h>
|
||||
#include <Storages/MergeTree/MergeTreePartsMover.h>
|
||||
#include <Interpreters/PartLog.h>
|
||||
#include <Common/DiskSpaceMonitor.h>
|
||||
|
||||
#include <boost/multi_index_container.hpp>
|
||||
#include <boost/multi_index/ordered_index.hpp>
|
||||
@ -565,7 +566,7 @@ public:
|
||||
/// All MergeTreeData children have settings.
|
||||
void checkSettingCanBeChanged(const String & setting_name) const override;
|
||||
|
||||
/// Remove columns, that have been markedd as empty after zeroing values with expired ttl
|
||||
/// Remove columns, that have been marked as empty after zeroing values with expired ttl
|
||||
void removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part);
|
||||
|
||||
/// Freezes all parts.
|
||||
@ -587,7 +588,7 @@ public:
|
||||
bool hasPrimaryKey() const { return !primary_key_columns.empty(); }
|
||||
bool hasSkipIndices() const { return !skip_indices.empty(); }
|
||||
bool hasTableTTL() const { return ttl_table_ast != nullptr; }
|
||||
bool hasAnyColumnTTL() const { return !ttl_entries_by_name.empty(); }
|
||||
bool hasAnyColumnTTL() const { return !column_ttl_entries_by_name.empty(); }
|
||||
|
||||
/// Check that the part is not broken and calculate the checksums for it if they are not present.
|
||||
MutableDataPartPtr loadPartAndFixMetadata(const DiskSpace::DiskPtr & disk, const String & relative_path);
|
||||
@ -673,9 +674,20 @@ public:
|
||||
using PathsWithDisks = std::vector<PathWithDisk>;
|
||||
PathsWithDisks getDataPathsWithDisks() const;
|
||||
|
||||
/// Reserves space at least 1MB
|
||||
DiskSpace::ReservationPtr reserveSpace(UInt64 expected_size);
|
||||
/// Reserves space at least 1MB.
|
||||
DiskSpace::ReservationPtr reserveSpace(UInt64 expected_size) const;
|
||||
|
||||
/// Reserves space at least 1MB on specific disk or volume.
|
||||
DiskSpace::ReservationPtr reserveSpace(UInt64 expected_size, DiskSpace::SpacePtr space) const;
|
||||
DiskSpace::ReservationPtr tryReserveSpace(UInt64 expected_size, DiskSpace::SpacePtr space) const;
|
||||
|
||||
/// Reserves space at least 1MB preferring best destination according to `ttl_infos`.
|
||||
DiskSpace::ReservationPtr reserveSpacePreferringTTLRules(UInt64 expected_size,
|
||||
const MergeTreeDataPart::TTLInfos & ttl_infos,
|
||||
time_t time_of_move) const;
|
||||
DiskSpace::ReservationPtr tryReserveSpacePreferringTTLRules(UInt64 expected_size,
|
||||
const MergeTreeDataPart::TTLInfos & ttl_infos,
|
||||
time_t time_of_move) const;
|
||||
/// Choose disk with max available free space
|
||||
/// Reserves 0 bytes
|
||||
DiskSpace::ReservationPtr makeEmptyReservationOnLargestDisk() { return storage_policy->makeEmptyReservationOnLargestDisk(); }
|
||||
@ -719,12 +731,27 @@ public:
|
||||
{
|
||||
ExpressionActionsPtr expression;
|
||||
String result_column;
|
||||
|
||||
/// Name and type of a destination are only valid in table-level context.
|
||||
PartDestinationType destination_type;
|
||||
String destination_name;
|
||||
|
||||
ASTPtr entry_ast;
|
||||
|
||||
/// Returns destination disk or volume for this rule.
|
||||
DiskSpace::SpacePtr getDestination(const DiskSpace::StoragePolicyPtr & policy) const;
|
||||
|
||||
/// Checks if given part already belongs destination disk or volume for this rule.
|
||||
bool isPartInDestination(const DiskSpace::StoragePolicyPtr & policy, const MergeTreeDataPart & part) const;
|
||||
};
|
||||
|
||||
const TTLEntry * selectTTLEntryForTTLInfos(const MergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const;
|
||||
|
||||
using TTLEntriesByName = std::unordered_map<String, TTLEntry>;
|
||||
TTLEntriesByName ttl_entries_by_name;
|
||||
TTLEntriesByName column_ttl_entries_by_name;
|
||||
|
||||
TTLEntry ttl_table_entry;
|
||||
std::vector<TTLEntry> move_ttl_entries;
|
||||
|
||||
String sampling_expr_column_name;
|
||||
Names columns_required_for_sampling;
|
||||
|
@ -1,6 +1,7 @@
|
||||
#include <Storages/MergeTree/MergeTreeDataPartTTLInfo.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/quoteString.h>
|
||||
|
||||
#include <common/JSON.h>
|
||||
|
||||
@ -15,10 +16,16 @@ void MergeTreeDataPartTTLInfos::update(const MergeTreeDataPartTTLInfos & other_i
|
||||
updatePartMinMaxTTL(ttl_info.min, ttl_info.max);
|
||||
}
|
||||
|
||||
for (const auto & [expression, ttl_info] : other_infos.moves_ttl)
|
||||
{
|
||||
moves_ttl[expression].update(ttl_info);
|
||||
}
|
||||
|
||||
table_ttl.update(other_infos.table_ttl);
|
||||
updatePartMinMaxTTL(table_ttl.min, table_ttl.max);
|
||||
}
|
||||
|
||||
|
||||
void MergeTreeDataPartTTLInfos::read(ReadBuffer & in)
|
||||
{
|
||||
String json_str;
|
||||
@ -28,7 +35,7 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in)
|
||||
JSON json(json_str);
|
||||
if (json.has("columns"))
|
||||
{
|
||||
JSON columns = json["columns"];
|
||||
const JSON & columns = json["columns"];
|
||||
for (auto col : columns)
|
||||
{
|
||||
MergeTreeDataPartTTLInfo ttl_info;
|
||||
@ -42,14 +49,27 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in)
|
||||
}
|
||||
if (json.has("table"))
|
||||
{
|
||||
JSON table = json["table"];
|
||||
const JSON & table = json["table"];
|
||||
table_ttl.min = table["min"].getUInt();
|
||||
table_ttl.max = table["max"].getUInt();
|
||||
|
||||
updatePartMinMaxTTL(table_ttl.min, table_ttl.max);
|
||||
}
|
||||
if (json.has("moves"))
|
||||
{
|
||||
const JSON & moves = json["moves"];
|
||||
for (auto move : moves)
|
||||
{
|
||||
MergeTreeDataPartTTLInfo ttl_info;
|
||||
ttl_info.min = move["min"].getUInt();
|
||||
ttl_info.max = move["max"].getUInt();
|
||||
String expression = move["expression"].getString();
|
||||
moves_ttl.emplace(expression, ttl_info);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void MergeTreeDataPartTTLInfos::write(WriteBuffer & out) const
|
||||
{
|
||||
writeString("ttl format version: 1\n", out);
|
||||
@ -62,9 +82,9 @@ void MergeTreeDataPartTTLInfos::write(WriteBuffer & out) const
|
||||
if (it != columns_ttl.begin())
|
||||
writeString(",", out);
|
||||
|
||||
writeString("{\"name\":\"", out);
|
||||
writeString(it->first, out);
|
||||
writeString("\",\"min\":", out);
|
||||
writeString("{\"name\":", out);
|
||||
writeString(doubleQuoteString(it->first), out);
|
||||
writeString(",\"min\":", out);
|
||||
writeIntText(it->second.min, out);
|
||||
writeString(",\"max\":", out);
|
||||
writeIntText(it->second.max, out);
|
||||
@ -82,6 +102,26 @@ void MergeTreeDataPartTTLInfos::write(WriteBuffer & out) const
|
||||
writeIntText(table_ttl.max, out);
|
||||
writeString("}", out);
|
||||
}
|
||||
if (!moves_ttl.empty())
|
||||
{
|
||||
if (!columns_ttl.empty() || table_ttl.min)
|
||||
writeString(",", out);
|
||||
writeString("\"moves\":[", out);
|
||||
for (auto it = moves_ttl.begin(); it != moves_ttl.end(); ++it)
|
||||
{
|
||||
if (it != moves_ttl.begin())
|
||||
writeString(",", out);
|
||||
|
||||
writeString("{\"expression\":", out);
|
||||
writeString(doubleQuoteString(it->first), out);
|
||||
writeString(",\"min\":", out);
|
||||
writeIntText(it->second.min, out);
|
||||
writeString(",\"max\":", out);
|
||||
writeIntText(it->second.max, out);
|
||||
writeString("}", out);
|
||||
}
|
||||
writeString("]", out);
|
||||
}
|
||||
writeString("}", out);
|
||||
}
|
||||
|
||||
|
@ -35,9 +35,14 @@ struct MergeTreeDataPartTTLInfos
|
||||
{
|
||||
std::unordered_map<String, MergeTreeDataPartTTLInfo> columns_ttl;
|
||||
MergeTreeDataPartTTLInfo table_ttl;
|
||||
|
||||
/// `part_min_ttl` and `part_max_ttl` are TTLs which are used for selecting parts
|
||||
/// to merge in order to remove expired rows.
|
||||
time_t part_min_ttl = 0;
|
||||
time_t part_max_ttl = 0;
|
||||
|
||||
std::unordered_map<String, MergeTreeDataPartTTLInfo> moves_ttl;
|
||||
|
||||
void read(ReadBuffer & in);
|
||||
void write(WriteBuffer & out) const;
|
||||
void update(const MergeTreeDataPartTTLInfos & other_infos);
|
||||
@ -50,6 +55,11 @@ struct MergeTreeDataPartTTLInfos
|
||||
if (time_max && (!part_max_ttl || time_max > part_max_ttl))
|
||||
part_max_ttl = time_max;
|
||||
}
|
||||
|
||||
bool empty()
|
||||
{
|
||||
return !part_min_ttl && moves_ttl.empty();
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -75,12 +75,17 @@ void buildScatterSelector(
|
||||
}
|
||||
|
||||
/// Computes ttls and updates ttl infos
|
||||
void updateTTL(const MergeTreeData::TTLEntry & ttl_entry, MergeTreeDataPart::TTLInfos & ttl_infos, Block & block, const String & column_name)
|
||||
void updateTTL(const MergeTreeData::TTLEntry & ttl_entry,
|
||||
MergeTreeDataPart::TTLInfos & ttl_infos,
|
||||
DB::MergeTreeDataPartTTLInfo & ttl_info,
|
||||
Block & block, bool update_part_min_max_ttls)
|
||||
{
|
||||
bool remove_column = false;
|
||||
if (!block.has(ttl_entry.result_column))
|
||||
{
|
||||
ttl_entry.expression->execute(block);
|
||||
|
||||
auto & ttl_info = (column_name.empty() ? ttl_infos.table_ttl : ttl_infos.columns_ttl[column_name]);
|
||||
remove_column = true;
|
||||
}
|
||||
|
||||
const auto & current = block.getByName(ttl_entry.result_column);
|
||||
|
||||
@ -113,7 +118,11 @@ void updateTTL(const MergeTreeData::TTLEntry & ttl_entry, MergeTreeDataPart::TTL
|
||||
else
|
||||
throw Exception("Unexpected type of result TTL column", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (update_part_min_max_ttls)
|
||||
ttl_infos.updatePartMinMaxTTL(ttl_info.min, ttl_info.max);
|
||||
|
||||
if (remove_column)
|
||||
block.erase(ttl_entry.result_column);
|
||||
}
|
||||
|
||||
}
|
||||
@ -212,10 +221,14 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
|
||||
else
|
||||
part_name = new_part_info.getPartName();
|
||||
|
||||
/// Size of part would not be grater than block.bytes() + epsilon
|
||||
/// Size of part would not be greater than block.bytes() + epsilon
|
||||
size_t expected_size = block.bytes();
|
||||
auto reservation = data.reserveSpace(expected_size);
|
||||
|
||||
DB::MergeTreeDataPart::TTLInfos move_ttl_infos;
|
||||
for (const auto & ttl_entry : data.move_ttl_entries)
|
||||
updateTTL(ttl_entry, move_ttl_infos, move_ttl_infos.moves_ttl[ttl_entry.result_column], block, false);
|
||||
|
||||
DiskSpace::ReservationPtr reservation = data.reserveSpacePreferringTTLRules(expected_size, move_ttl_infos, time(nullptr));
|
||||
|
||||
MergeTreeData::MutableDataPartPtr new_data_part =
|
||||
std::make_shared<MergeTreeData::DataPart>(data, reservation->getDisk(), part_name, new_part_info);
|
||||
@ -251,7 +264,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterBlocks);
|
||||
|
||||
/// Sort.
|
||||
/// Sort
|
||||
IColumn::Permutation * perm_ptr = nullptr;
|
||||
IColumn::Permutation perm;
|
||||
if (!sort_description.empty())
|
||||
@ -266,10 +279,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
|
||||
}
|
||||
|
||||
if (data.hasTableTTL())
|
||||
updateTTL(data.ttl_table_entry, new_data_part->ttl_infos, block, "");
|
||||
updateTTL(data.ttl_table_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.table_ttl, block, true);
|
||||
|
||||
for (const auto & [name, ttl_entry] : data.ttl_entries_by_name)
|
||||
updateTTL(ttl_entry, new_data_part->ttl_infos, block, name);
|
||||
for (const auto & [name, ttl_entry] : data.column_ttl_entries_by_name)
|
||||
updateTTL(ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.columns_ttl[name], block, true);
|
||||
|
||||
new_data_part->ttl_infos.update(move_ttl_infos);
|
||||
|
||||
/// This effectively chooses minimal compression method:
|
||||
/// either default lz4 or compression method with zero thresholds on absolute and relative part size.
|
||||
|
@ -52,11 +52,14 @@ public:
|
||||
elems.emplace(part);
|
||||
current_size_sum += part->bytes_on_disk;
|
||||
|
||||
while (!elems.empty() && (current_size_sum - (*elems.begin())->bytes_on_disk >= required_size_sum))
|
||||
{
|
||||
current_size_sum -= (*elems.begin())->bytes_on_disk;
|
||||
elems.erase(elems.begin());
|
||||
removeRedundantElements();
|
||||
}
|
||||
|
||||
/// Weaken requirements on size
|
||||
void decreaseRequiredSizeAndRemoveRedundantParts(UInt64 size_decrease)
|
||||
{
|
||||
required_size_sum -= std::min(size_decrease, required_size_sum);
|
||||
removeRedundantElements();
|
||||
}
|
||||
|
||||
/// Returns parts ordered by size
|
||||
@ -67,6 +70,16 @@ public:
|
||||
res.push_back(elem);
|
||||
return res;
|
||||
}
|
||||
|
||||
private:
|
||||
void removeRedundantElements()
|
||||
{
|
||||
while (!elems.empty() && (current_size_sum - (*elems.begin())->bytes_on_disk >= required_size_sum))
|
||||
{
|
||||
current_size_sum -= (*elems.begin())->bytes_on_disk;
|
||||
elems.erase(elems.begin());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
@ -85,46 +98,70 @@ bool MergeTreePartsMover::selectPartsForMove(
|
||||
const auto & policy = data->getStoragePolicy();
|
||||
const auto & volumes = policy->getVolumes();
|
||||
|
||||
/// Do not check if policy has one volume
|
||||
if (volumes.size() == 1)
|
||||
return false;
|
||||
|
||||
if (volumes.size() > 0)
|
||||
{
|
||||
/// Do not check last volume
|
||||
for (size_t i = 0; i != volumes.size() - 1; ++i)
|
||||
{
|
||||
for (const auto & disk : volumes[i]->disks)
|
||||
{
|
||||
UInt64 required_available_space = disk->getTotalSpace() * policy->getMoveFactor();
|
||||
UInt64 required_maximum_available_space = disk->getTotalSpace() * policy->getMoveFactor();
|
||||
UInt64 unreserved_space = disk->getUnreservedSpace();
|
||||
|
||||
if (required_available_space > unreserved_space)
|
||||
need_to_move.emplace(disk, required_available_space - unreserved_space);
|
||||
if (unreserved_space < required_maximum_available_space)
|
||||
need_to_move.emplace(disk, required_maximum_available_space - unreserved_space);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
time_t time_of_move = time(nullptr);
|
||||
|
||||
for (const auto & part : data_parts)
|
||||
{
|
||||
String reason;
|
||||
/// Don't report message to log, because logging is excessive
|
||||
/// Don't report message to log, because logging is excessive.
|
||||
if (!can_move(part, &reason))
|
||||
continue;
|
||||
|
||||
const MergeTreeData::TTLEntry * ttl_entry_ptr = part->storage.selectTTLEntryForTTLInfos(part->ttl_infos, time_of_move);
|
||||
auto to_insert = need_to_move.find(part->disk);
|
||||
DiskSpace::ReservationPtr reservation;
|
||||
if (ttl_entry_ptr)
|
||||
{
|
||||
auto destination = ttl_entry_ptr->getDestination(policy);
|
||||
if (destination && !ttl_entry_ptr->isPartInDestination(policy, *part))
|
||||
reservation = part->storage.tryReserveSpace(part->bytes_on_disk, ttl_entry_ptr->getDestination(policy));
|
||||
}
|
||||
|
||||
if (reservation) /// Found reservation by TTL rule.
|
||||
{
|
||||
parts_to_move.emplace_back(part, std::move(reservation));
|
||||
/// If table TTL rule satisfies on this part, won't apply policy rules on it.
|
||||
/// In order to not over-move, we need to "release" required space on this disk,
|
||||
/// possibly to zero.
|
||||
if (to_insert != need_to_move.end())
|
||||
{
|
||||
to_insert->second.decreaseRequiredSizeAndRemoveRedundantParts(part->bytes_on_disk);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (to_insert != need_to_move.end())
|
||||
to_insert->second.add(part);
|
||||
}
|
||||
}
|
||||
|
||||
for (auto && move : need_to_move)
|
||||
{
|
||||
auto min_volume_priority = policy->getVolumeIndexByDisk(move.first) + 1;
|
||||
auto min_volume_index = policy->getVolumeIndexByDisk(move.first) + 1;
|
||||
for (auto && part : move.second.getAccumulatedParts())
|
||||
{
|
||||
auto reservation = policy->reserve(part->bytes_on_disk, min_volume_priority);
|
||||
auto reservation = policy->reserve(part->bytes_on_disk, min_volume_index);
|
||||
if (!reservation)
|
||||
{
|
||||
/// Next parts to move from this disk has greater size and same min volume priority
|
||||
/// There are no space for them
|
||||
/// But it can be possible to move data from other disks
|
||||
/// Next parts to move from this disk has greater size and same min volume index.
|
||||
/// There are no space for them.
|
||||
/// But it can be possible to move data from other disks.
|
||||
break;
|
||||
}
|
||||
parts_to_move.emplace_back(part, std::move(reservation));
|
||||
|
@ -186,7 +186,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart(
|
||||
checksums.files["count.txt"].file_hash = count_out_hashing.getHash();
|
||||
}
|
||||
|
||||
if (new_part->ttl_infos.part_min_ttl)
|
||||
if (!new_part->ttl_infos.empty())
|
||||
{
|
||||
/// Write a file with ttl infos in json format.
|
||||
WriteBufferFromFile out(part_path + "ttl.txt", 4096);
|
||||
|
14
dbms/src/Storages/MergeTree/PartDestinationType.h
Normal file
14
dbms/src/Storages/MergeTree/PartDestinationType.h
Normal file
@ -0,0 +1,14 @@
|
||||
#pragma once
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
enum class PartDestinationType
|
||||
{
|
||||
DISK,
|
||||
VOLUME,
|
||||
DELETE,
|
||||
};
|
||||
|
||||
}
|
@ -5,6 +5,7 @@
|
||||
#include <Parsers/ExpressionListParsers.h>
|
||||
#include <IO/Operators.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -47,6 +48,16 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr
|
||||
partition_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.partition_by_ast));
|
||||
|
||||
ttl_table = formattedAST(data.ttl_table_ast);
|
||||
|
||||
std::ostringstream ttl_move_stream;
|
||||
for (const auto & ttl_entry : data.move_ttl_entries)
|
||||
{
|
||||
if (ttl_move_stream.tellp() > 0)
|
||||
ttl_move_stream << ", ";
|
||||
ttl_move_stream << formattedAST(ttl_entry.entry_ast);
|
||||
}
|
||||
ttl_move = ttl_move_stream.str();
|
||||
|
||||
skip_indices = data.getIndices().toString();
|
||||
if (data.canUseAdaptiveGranularity())
|
||||
index_granularity_bytes = data_settings->index_granularity_bytes;
|
||||
@ -78,6 +89,9 @@ void ReplicatedMergeTreeTableMetadata::write(WriteBuffer & out) const
|
||||
if (!ttl_table.empty())
|
||||
out << "ttl: " << ttl_table << "\n";
|
||||
|
||||
if (!ttl_move.empty())
|
||||
out << "move ttl: " << ttl_move << "\n";
|
||||
|
||||
if (!skip_indices.empty())
|
||||
out << "indices: " << skip_indices << "\n";
|
||||
|
||||
@ -119,6 +133,9 @@ void ReplicatedMergeTreeTableMetadata::read(ReadBuffer & in)
|
||||
if (checkString("ttl: ", in))
|
||||
in >> ttl_table >> "\n";
|
||||
|
||||
if (checkString("move ttl: ", in))
|
||||
in >> ttl_move >> "\n";
|
||||
|
||||
if (checkString("indices: ", in))
|
||||
in >> skip_indices >> "\n";
|
||||
|
||||
@ -223,12 +240,27 @@ ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTabl
|
||||
}
|
||||
else
|
||||
throw Exception(
|
||||
"Existing table metadata in ZooKeeper differs in ttl."
|
||||
"Existing table metadata in ZooKeeper differs in TTL."
|
||||
" Stored in ZooKeeper: " + from_zk.ttl_table +
|
||||
", local: " + ttl_table,
|
||||
ErrorCodes::METADATA_MISMATCH);
|
||||
}
|
||||
|
||||
if (ttl_move != from_zk.ttl_move)
|
||||
{
|
||||
if (allow_alter)
|
||||
{
|
||||
diff.ttl_move_changed = true;
|
||||
diff.new_ttl_move = from_zk.ttl_move;
|
||||
}
|
||||
else
|
||||
throw Exception(
|
||||
"Existing table metadata in ZooKeeper differs in move TTL."
|
||||
" Stored in ZooKeeper: " + from_zk.ttl_move +
|
||||
", local: " + ttl_move,
|
||||
ErrorCodes::METADATA_MISMATCH);
|
||||
}
|
||||
|
||||
if (skip_indices != from_zk.skip_indices)
|
||||
{
|
||||
if (allow_alter)
|
||||
|
@ -28,6 +28,7 @@ struct ReplicatedMergeTreeTableMetadata
|
||||
String skip_indices;
|
||||
String constraints;
|
||||
String ttl_table;
|
||||
String ttl_move;
|
||||
UInt64 index_granularity_bytes;
|
||||
|
||||
ReplicatedMergeTreeTableMetadata() = default;
|
||||
@ -53,9 +54,12 @@ struct ReplicatedMergeTreeTableMetadata
|
||||
bool ttl_table_changed = false;
|
||||
String new_ttl_table;
|
||||
|
||||
bool ttl_move_changed = false;
|
||||
String new_ttl_move;
|
||||
|
||||
bool empty() const
|
||||
{
|
||||
return !sorting_key_changed && !skip_indices_changed && !ttl_table_changed && !constraints_changed;
|
||||
return !sorting_key_changed && !skip_indices_changed && !ttl_table_changed && !constraints_changed && !ttl_move_changed;
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -1,5 +1,6 @@
|
||||
#include <Storages/PartitionCommands.h>
|
||||
#include <Storages/IStorage.h>
|
||||
#include <Storages/MergeTree/PartDestinationType.h>
|
||||
#include <Parsers/ASTAlterQuery.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
|
||||
@ -47,12 +48,14 @@ std::optional<PartitionCommand> PartitionCommand::parse(const ASTAlterCommand *
|
||||
res.part = command_ast->part;
|
||||
switch (command_ast->move_destination_type)
|
||||
{
|
||||
case ASTAlterCommand::MoveDestinationType::DISK:
|
||||
case PartDestinationType::DISK:
|
||||
res.move_destination_type = PartitionCommand::MoveDestinationType::DISK;
|
||||
break;
|
||||
case ASTAlterCommand::MoveDestinationType::VOLUME:
|
||||
case PartDestinationType::VOLUME:
|
||||
res.move_destination_type = PartitionCommand::MoveDestinationType::VOLUME;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
res.move_destination_name = command_ast->move_destination_name;
|
||||
return res;
|
||||
|
@ -350,9 +350,15 @@ public:
|
||||
|
||||
/// if we mutate part, than we should reserve space on the same disk, because mutations possible can create hardlinks
|
||||
if (is_mutation)
|
||||
reserved_space = future_part_.parts[0]->disk->reserve(total_size);
|
||||
reserved_space = storage.tryReserveSpace(total_size, future_part_.parts[0]->disk);
|
||||
else
|
||||
reserved_space = storage.reserveSpace(total_size);
|
||||
{
|
||||
MergeTreeDataPart::TTLInfos ttl_infos;
|
||||
for (auto & part_ptr : future_part_.parts)
|
||||
ttl_infos.update(part_ptr->ttl_infos);
|
||||
|
||||
reserved_space = storage.tryReserveSpacePreferringTTLRules(total_size, ttl_infos, time(nullptr));
|
||||
}
|
||||
if (!reserved_space)
|
||||
{
|
||||
if (is_mutation)
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user