mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 07:31:57 +00:00
Add uniform snowflakeID conversion functions
This commit is contained in:
parent
ae01d9ef9b
commit
9920c3d17f
@ -5398,6 +5398,12 @@ When set to `false` than all attempts are made with identical timeouts.
|
||||
|
||||
Default value: `true`.
|
||||
|
||||
## uniform_snowflake_conversion_functions {#uniform_snowflake_conversion_functions}
|
||||
|
||||
Controls if functions `snowflakeIDToDateTime`, `snowflakeIDToDateTime64`, `dateTimeToSnowflakeID`, and `dateTime64ToSnowflakeID` are enabled (if `true`), or functions `snowflakeToDateTime`, `snowflakeToDateTime64`, `dateTimeToSnowflake`, and `dateTime64ToSnowflake` (if `false`).
|
||||
|
||||
Default value: `true`
|
||||
|
||||
## allow_experimental_variant_type {#allow_experimental_variant_type}
|
||||
|
||||
Allows creation of experimental [Variant](../../sql-reference/data-types/variant.md).
|
||||
|
@ -543,12 +543,17 @@ serverUUID()
|
||||
|
||||
Generates a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID).
|
||||
|
||||
The generated Snowflake ID contains the current Unix timestamp in milliseconds 41 (+ 1 top zero bit) bits, followed by machine id (10 bits), a counter (12 bits) to distinguish IDs within a millisecond.
|
||||
The generated Snowflake ID contains the current Unix timestamp in milliseconds (41 + 1 top zero bits), followed by a machine id (10 bits), and a counter (12 bits) to distinguish IDs within a millisecond.
|
||||
For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes.
|
||||
In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0.
|
||||
|
||||
Function `generateSnowflakeID` guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.
|
||||
|
||||
:::note
|
||||
The generated Snowflake IDs are based on the UNIX epoch 1970-01-01.
|
||||
While no standard or recommendation exists for the epoch of Snowflake IDs, implementations in other systems may use a different epoch, e.g. Twitter/X (2010-11-04) or Mastodon (2015-01-01).
|
||||
:::
|
||||
|
||||
```
|
||||
0 1 2 3
|
||||
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
|
||||
@ -605,6 +610,11 @@ SELECT generateSnowflakeID(1), generateSnowflakeID(2);
|
||||
|
||||
## snowflakeToDateTime
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Extracts the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) in [DateTime](../data-types/datetime.md) format.
|
||||
|
||||
**Syntax**
|
||||
@ -641,6 +651,11 @@ Result:
|
||||
|
||||
## snowflakeToDateTime64
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Extracts the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) in [DateTime64](../data-types/datetime64.md) format.
|
||||
|
||||
**Syntax**
|
||||
@ -677,6 +692,11 @@ Result:
|
||||
|
||||
## dateTimeToSnowflake
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
@ -711,6 +731,11 @@ Result:
|
||||
|
||||
## dateTime64ToSnowflake
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Convert a [DateTime64](../data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
@ -743,6 +768,148 @@ Result:
|
||||
└─────────────────────────────┘
|
||||
```
|
||||
|
||||
## snowflakeIDToDateTime
|
||||
|
||||
Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime](../data-types/datetime.md).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
snowflakeIDToDateTime(value[, epoch[, time_zone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Snowflake ID. [UInt64](../data-types/int-uint.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
- `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The timestamp component of `value` as a [DateTime](../data-types/datetime.md) value.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT snowflakeIDToDateTime(7204436857747984384) AS res
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌─────────────────res─┐
|
||||
│ 2024-06-06 10:59:58 │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## snowflakeIDToDateTime64
|
||||
|
||||
Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime64](../data-types/datetime64.md).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
snowflakeIDToDateTime64(value[, epoch[, time_zone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Snowflake ID. [UInt64](../data-types/int-uint.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
- `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The timestamp component of `value` as a [DateTime64](../data-types/datetime64.md) with scale = 3, i.e. millisecond precision.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT snowflakeIDToDateTime64(7204436857747984384) AS res
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌─────────────────res─┐
|
||||
│ 2024-06-06 10:59:58 │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## dateTimeToSnowflakeID
|
||||
|
||||
Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
dateTimeToSnowflakeID(value[, epoch])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Date with time. [DateTime](../data-types/datetime.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS dt, dateTimeToSnowflakeID(dt) AS res;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌──────────────────dt─┬─────────────────res─┐
|
||||
│ 2021-08-15 18:57:56 │ 6832626392367104000 │
|
||||
└─────────────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
## dateTime64ToSnowflakeID
|
||||
|
||||
Convert a [DateTime64](../data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
dateTime64ToSnowflakeID(value[, epoch])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Date with time. [DateTime64](../data-types/datetime64.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT toDateTime('2021-08-15 18:57:56.493', 3, 'Asia/Shanghai') AS dt, dateTime64ToSnowflakeID(dt) AS res;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌──────────────────────dt─┬─────────────────res─┐
|
||||
│ 2021-08-15 18:57:56.493 │ 6832626394434895872 │
|
||||
└─────────────────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [dictGetUUID](../functions/ext-dict-functions.md#ext_dict_functions-other)
|
||||
|
@ -928,6 +928,7 @@ class IColumn;
|
||||
M(Int64, prefer_warmed_unmerged_parts_seconds, 0, "Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm.", 0) \
|
||||
M(Bool, iceberg_engine_ignore_schema_evolution, false, "Ignore schema evolution in Iceberg table engine and read all data using latest schema saved on table creation. Note that it can lead to incorrect result", 0) \
|
||||
M(Bool, allow_deprecated_error_prone_window_functions, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)", 0) \
|
||||
M(Bool, uniform_snowflake_conversion_functions, true, "Enable functions snowflakeIDToDateTime[64] and dateTime[64]ToSnowflakeID.", 0) \
|
||||
|
||||
// End of COMMON_SETTINGS
|
||||
// Please add settings related to formats into the FORMAT_FACTORY_SETTINGS, move obsolete settings to OBSOLETE_SETTINGS and obsolete format settings to OBSOLETE_FORMAT_SETTINGS.
|
||||
|
@ -97,6 +97,7 @@ static const std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges
|
||||
{"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"},
|
||||
{"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"},
|
||||
{"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
|
||||
{"uniform_snowflake_conversion_functions", false, true, "Enable functions snowflakeIDToDateTime[64] and dateTime[64]ToSnowflakeID."},
|
||||
{"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
|
||||
{"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
|
||||
{"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
|
||||
|
181
src/Functions/dateTimeToSnowflakeID.cpp
Normal file
181
src/Functions/dateTimeToSnowflakeID.cpp
Normal file
@ -0,0 +1,181 @@
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnsDateTime.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int UNKNOWN_FUNCTION;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/// See generateSnowflakeID.cpp
|
||||
constexpr int time_shift = 22;
|
||||
|
||||
}
|
||||
|
||||
class FunctionDateTimeToSnowflakeID : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "dateTimeToSnowflakeID";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionDateTimeToSnowflakeID>(context); }
|
||||
explicit FunctionDateTimeToSnowflakeID(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateTime), nullptr, "DateTime"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() == 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnUInt64::create(input_rows_count);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
const auto & src_data = typeid_cast<const ColumnDateTime &>(col_src).getData();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = (static_cast<UInt64>(src_data[i]) * 1000 - epoch) << time_shift;
|
||||
return col_res;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class FunctionDateTime64ToSnowflakeID : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "dateTime64ToSnowflakeID";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionDateTime64ToSnowflakeID>(context); }
|
||||
explicit FunctionDateTime64ToSnowflakeID(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateTime64), nullptr, "DateTime64"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
const auto & src_data = typeid_cast<const ColumnDateTime64 &>(col_src).getData();
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() == 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnUInt64::create(input_rows_count);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
/// timestamps in snowflake-ids are millisecond-based, convert input to milliseconds
|
||||
UInt32 src_scale = getDecimalScale(*arguments[0].type);
|
||||
Int64 multiplier_msec = DecimalUtils::scaleMultiplier<DateTime64>(3);
|
||||
Int64 multiplier_src = DecimalUtils::scaleMultiplier<DateTime64>(src_scale);
|
||||
auto factor = multiplier_msec / static_cast<double>(multiplier_src);
|
||||
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = static_cast<UInt64>(src_data[i] * factor - epoch) << time_shift;
|
||||
|
||||
return col_res;
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_FUNCTION(DateTimeToSnowflakeID)
|
||||
{
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.)";
|
||||
FunctionDocumentation::Syntax syntax = "dateTimeToSnowflakeID(value[, epoch])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Date with time. [DateTime](../data-types/datetime.md)."},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT dateTimeToSnowflakeID(toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai'))", "6832626392367104000"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionDateTimeToSnowflakeID>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Converts a [DateTime64](../data-types/datetime64.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.)";
|
||||
FunctionDocumentation::Syntax syntax = "dateTime64ToSnowflakeID(value[, epoch])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Date with time. [DateTime64](../data-types/datetime.md)."},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT dateTime64ToSnowflakeID(toDateTime64('2021-08-15 18:57:56', 3, 'Asia/Shanghai'))", "6832626394434895872"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionDateTime64ToSnowflakeID>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
}
|
||||
|
||||
}
|
@ -207,7 +207,7 @@ public:
|
||||
|
||||
REGISTER_FUNCTION(GenerateSnowflakeID)
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Generates a Snowflake ID. The generated Snowflake ID contains the current Unix timestamp in milliseconds 41 (+ 1 top zero bit) bits, followed by machine id (10 bits), a counter (12 bits) to distinguish IDs within a millisecond. For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes. In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0. Function generateSnowflakeID guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.)";
|
||||
FunctionDocumentation::Description description = R"(Generates a Snowflake ID. The generated Snowflake ID contains the current Unix timestamp in milliseconds (41 + 1 top zero bits), followed by a machine id (10 bits), and a counter (12 bits) to distinguish IDs within a millisecond. For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes. In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0. Function generateSnowflakeID guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.)";
|
||||
FunctionDocumentation::Syntax syntax = "generateSnowflakeID([expression])";
|
||||
FunctionDocumentation::Arguments arguments = {{"expression", "The expression is used to bypass common subexpression elimination if the function is called multiple times in a query but otherwise ignored. Optional."}};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "A value of type UInt64";
|
||||
|
@ -11,11 +11,17 @@
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
|
||||
/// ------------------------------------------------------------------------------------------------------------------------------
|
||||
/// The functions in this file are deprecated and should be removed in favor of functions 'snowflakeIDToDateTime[64]' and
|
||||
/// 'dateTime[64]ToSnowflakeID' by summer 2025. Please also mark setting `uniform_snowflake_conversion_functions` as obsolete then.
|
||||
/// ------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int DEPRECATED_FUNCTION;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -34,10 +40,19 @@ constexpr int time_shift = 22;
|
||||
class FunctionDateTimeToSnowflake : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionDateTimeToSnowflake(const char * name_) : name(name_) { }
|
||||
static constexpr auto name = "dateTimeToSnowflake";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionDateTimeToSnowflake>(context);
|
||||
}
|
||||
|
||||
explicit FunctionDateTimeToSnowflake(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
@ -56,6 +71,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
const auto & src_column = *src.column;
|
||||
|
||||
@ -73,13 +91,20 @@ public:
|
||||
class FunctionSnowflakeToDateTime : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionSnowflakeToDateTime(const char * name_, ContextPtr context)
|
||||
: name(name_)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
static constexpr auto name = "snowflakeToDateTime";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionSnowflakeToDateTime>(context);
|
||||
}
|
||||
|
||||
explicit FunctionSnowflakeToDateTime(ContextPtr context)
|
||||
: allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments)
|
||||
, uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
@ -107,6 +132,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
const auto & src_column = *src.column;
|
||||
|
||||
@ -138,10 +166,19 @@ public:
|
||||
class FunctionDateTime64ToSnowflake : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionDateTime64ToSnowflake(const char * name_) : name(name_) { }
|
||||
static constexpr auto name = "dateTime64ToSnowflake";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionDateTime64ToSnowflake>(context);
|
||||
}
|
||||
|
||||
explicit FunctionDateTime64ToSnowflake(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
@ -160,6 +197,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
|
||||
const auto & src_column = *src.column;
|
||||
@ -185,13 +225,20 @@ public:
|
||||
class FunctionSnowflakeToDateTime64 : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionSnowflakeToDateTime64(const char * name_, ContextPtr context)
|
||||
: name(name_)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
static constexpr auto name = "snowflakeToDateTime64";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionSnowflakeToDateTime64>(context);
|
||||
}
|
||||
|
||||
explicit FunctionSnowflakeToDateTime64(ContextPtr context)
|
||||
: allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments)
|
||||
, uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
@ -219,6 +266,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
const auto & src_column = *src.column;
|
||||
|
||||
@ -246,27 +296,12 @@ public:
|
||||
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(DateTimeToSnowflake)
|
||||
REGISTER_FUNCTION(LegacySnowflakeConversion)
|
||||
{
|
||||
factory.registerFunction("dateTimeToSnowflake",
|
||||
[](ContextPtr){ return std::make_shared<FunctionDateTimeToSnowflake>("dateTimeToSnowflake"); });
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(DateTime64ToSnowflake)
|
||||
{
|
||||
factory.registerFunction("dateTime64ToSnowflake",
|
||||
[](ContextPtr){ return std::make_shared<FunctionDateTime64ToSnowflake>("dateTime64ToSnowflake"); });
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(SnowflakeToDateTime)
|
||||
{
|
||||
factory.registerFunction("snowflakeToDateTime",
|
||||
[](ContextPtr context){ return std::make_shared<FunctionSnowflakeToDateTime>("snowflakeToDateTime", context); });
|
||||
}
|
||||
REGISTER_FUNCTION(SnowflakeToDateTime64)
|
||||
{
|
||||
factory.registerFunction("snowflakeToDateTime64",
|
||||
[](ContextPtr context){ return std::make_shared<FunctionSnowflakeToDateTime64>("snowflakeToDateTime64", context); });
|
||||
factory.registerFunction<FunctionSnowflakeToDateTime>();
|
||||
factory.registerFunction<FunctionSnowflakeToDateTime64>();
|
||||
factory.registerFunction<FunctionDateTimeToSnowflake>();
|
||||
factory.registerFunction<FunctionDateTime64ToSnowflake>();
|
||||
}
|
||||
|
||||
}
|
||||
|
217
src/Functions/snowflakeIDToDateTime.cpp
Normal file
217
src/Functions/snowflakeIDToDateTime.cpp
Normal file
@ -0,0 +1,217 @@
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnsDateTime.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int UNKNOWN_FUNCTION;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/// See generateSnowflakeID.cpp
|
||||
constexpr int time_shift = 22;
|
||||
|
||||
}
|
||||
|
||||
class FunctionSnowflakeIDToDateTime : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "snowflakeIDToDateTime";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionSnowflakeIDToDateTime>(context); }
|
||||
explicit FunctionSnowflakeIDToDateTime(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt64), nullptr, "UInt64"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"},
|
||||
{"time_zone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
String timezone;
|
||||
if (arguments.size() == 3)
|
||||
timezone = extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, allow_nonconst_timezone_arguments);
|
||||
|
||||
return std::make_shared<DataTypeDateTime>(timezone);
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() >= 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnDateTime::create(input_rows_count);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
if (const auto * col_src_non_const = typeid_cast<const ColumnUInt64 *>(&col_src))
|
||||
{
|
||||
const auto & src_data = col_src_non_const->getData();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = static_cast<UInt32>(((src_data[i] >> time_shift) + epoch) / 1000);
|
||||
}
|
||||
else if (const auto * col_src_const = typeid_cast<const ColumnConst *>(&col_src))
|
||||
{
|
||||
UInt64 src_val = col_src_const->getValue<UInt64>();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = static_cast<UInt32>(((src_val >> time_shift) + epoch) / 1000);
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument for function {}", name);
|
||||
|
||||
return col_res;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class FunctionSnowflakeIDToDateTime64 : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "snowflakeIDToDateTime64";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionSnowflakeIDToDateTime64>(context); }
|
||||
explicit FunctionSnowflakeIDToDateTime64(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt64), nullptr, "UInt64"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"},
|
||||
{"time_zone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
String timezone;
|
||||
if (arguments.size() == 3)
|
||||
timezone = extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, allow_nonconst_timezone_arguments);
|
||||
|
||||
return std::make_shared<DataTypeDateTime64>(3, timezone);
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() >= 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnDateTime64::create(input_rows_count, 3);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
if (const auto * col_src_non_const = typeid_cast<const ColumnUInt64 *>(&col_src))
|
||||
{
|
||||
const auto & src_data = col_src_non_const->getData();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = (src_data[i] >> time_shift) + epoch;
|
||||
}
|
||||
else if (const auto * col_src_const = typeid_cast<const ColumnConst *>(&col_src))
|
||||
{
|
||||
UInt64 src_val = col_src_const->getValue<UInt64>();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = (src_val >> time_shift) + epoch;
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument for function {}", name);
|
||||
|
||||
return col_res;
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_FUNCTION(SnowflakeIDToDateTime)
|
||||
{
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime](../data-types/datetime.md).)";
|
||||
FunctionDocumentation::Syntax syntax = "snowflakeIDToDateTime(value[, epoch[, time_zone]])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Snowflake ID. [UInt64](../data-types/int-uint.md)"},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"},
|
||||
{"time_zone", "[Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "The timestamp component of `value` as a [DateTime](../data-types/datetime.md) value.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT snowflakeIDToDateTime(7204436857747984384)", "2024-06-06 10:59:58"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionSnowflakeIDToDateTime>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime64](../data-types/datetime64.md).)";
|
||||
FunctionDocumentation::Syntax syntax = "snowflakeIDToDateTime64(value[, epoch[, time_zone]])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Snowflake ID. [UInt64](../data-types/int-uint.md)"},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"},
|
||||
{"time_zone", "[Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "The timestamp component of `value` as a [DateTime64](../data-types/datetime64.md) with scale = 3, i.e. millisecond precision.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT snowflakeIDToDateTime64(7204436857747984384)", "2024-06-06 10:59:58"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionSnowflakeIDToDateTime64>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
}
|
||||
|
||||
}
|
@ -1,3 +1,5 @@
|
||||
SET uniform_snowflake_conversion_functions = 0;
|
||||
|
||||
SELECT addMonths(toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul'), 1, 'Asia/Kolkata');
|
||||
SELECT addMonths(toDateTime('2017-11-05 10:37:47', 'Asia/Kolkata'), 1);
|
||||
SELECT addMonths(toTimeZone(toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul'), 'Asia/Kolkata'), 1);
|
||||
|
@ -1,3 +1,4 @@
|
||||
SET uniform_snowflake_conversion_functions = 0; -- Force-disable uniform snowflake conversion functions (in case this is randomized in CI)
|
||||
SET session_timezone = 'Africa/Juba';
|
||||
|
||||
-- Error cases
|
||||
@ -10,6 +11,9 @@ SELECT dateTime64ToSnowflake('abc'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT dateTimeToSnowflake('abc', 123); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT dateTime64ToSnowflake('abc', 123); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
|
||||
SELECT dateTimeToSnowflake(now()) SETTINGS uniform_snowflake_conversion_functions = 1; -- { serverError DEPRECATED_FUNCTION }
|
||||
SELECT dateTime64ToSnowflake(now64()) SETTINGS uniform_snowflake_conversion_functions = 1; -- { serverError DEPRECATED_FUNCTION }
|
||||
|
||||
SELECT '-- const / non-const inputs';
|
||||
|
||||
WITH toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS dt
|
||||
|
@ -0,0 +1,29 @@
|
||||
-- Negative tests
|
||||
-- Return type
|
||||
UInt64
|
||||
UInt64
|
||||
-- Standard and twitter epoch
|
||||
Row 1:
|
||||
──────
|
||||
dt: 2021-08-15 18:57:56
|
||||
dt64: 2021-08-15 18:57:56.492
|
||||
dateTimeToSnowflakeID(dt): 6832747188322304000
|
||||
dateTime64ToSnowflakeID(dt64): 6832747190385901568
|
||||
dateTimeToSnowflakeID(dt, twitter_epoch): 1426981498778550272
|
||||
dateTime64ToSnowflakeID(dt64, twitter_epoch): 1426981500842147840
|
||||
-- Different DateTime64 scales
|
||||
Row 1:
|
||||
──────
|
||||
dateTime64ToSnowflakeID(dt64_0): 6832747188322304000
|
||||
dateTime64ToSnowflakeID(dt64_1): 6832747190000025600
|
||||
dateTime64ToSnowflakeID(dt64_2): 6832747190377512960
|
||||
dateTime64ToSnowflakeID(dt64_3): 6832747190385901568
|
||||
dateTime64ToSnowflakeID(dt64_4): 6832747190385901568
|
||||
-- Idempotency
|
||||
Row 1:
|
||||
──────
|
||||
equals(snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_0)), dt64_0): 1
|
||||
equals(snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_1)), dt64_1): 1
|
||||
equals(snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_2)), dt64_2): 1
|
||||
equals(snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_3)), dt64_3): 1
|
||||
equals(snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_4)), dt64_4): 0
|
71
tests/queries/0_stateless/01942_dateTimeToSnowflakeID.sql
Normal file
71
tests/queries/0_stateless/01942_dateTimeToSnowflakeID.sql
Normal file
@ -0,0 +1,71 @@
|
||||
SET session_timezone = 'UTC'; -- disable timezone randomization
|
||||
SET allow_experimental_analyzer = 1; -- The old path formats the result with different whitespaces
|
||||
SET uniform_snowflake_conversion_functions = 1; -- Force-enable uniform snowflake conversion functions (in case this is randomized in CI)
|
||||
|
||||
SELECT '-- Negative tests';
|
||||
SELECT dateTimeToSnowflakeID(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT dateTime64ToSnowflakeID(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT dateTimeToSnowflakeID('invalid_dt'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT dateTime64ToSnowflakeID('invalid_dt'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT dateTimeToSnowflakeID(now(), 'invalid_epoch'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT dateTime64ToSnowflakeID(now64(), 'invalid_epoch'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT dateTimeToSnowflakeID(now(), 42, 'too_many_args'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT dateTime64ToSnowflakeID(now64(), 42, 'too_many_args'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
|
||||
SELECT dateTimeToSnowflakeID(now()) SETTINGS uniform_snowflake_conversion_functions = 0; -- { serverError UNKNOWN_FUNCTION }
|
||||
SELECT dateTime64ToSnowflakeID(now64()) SETTINGS uniform_snowflake_conversion_functions = 0; -- { serverError UNKNOWN_FUNCTION }
|
||||
|
||||
SELECT '-- Return type';
|
||||
SELECT toTypeName(dateTimeToSnowflakeID(now()));
|
||||
SELECT toTypeName(dateTime64ToSnowflakeID(now64()));
|
||||
|
||||
SELECT '-- Standard and twitter epoch';
|
||||
|
||||
WITH
|
||||
toDateTime('2021-08-15 18:57:56') AS dt,
|
||||
toDateTime64('2021-08-15 18:57:56.492', 3) AS dt64,
|
||||
1288834974657 AS twitter_epoch
|
||||
SELECT
|
||||
dt,
|
||||
dt64,
|
||||
dateTimeToSnowflakeID(dt),
|
||||
dateTime64ToSnowflakeID(dt64),
|
||||
dateTimeToSnowflakeID(dt, twitter_epoch),
|
||||
dateTime64ToSnowflakeID(dt64, twitter_epoch)
|
||||
FORMAT
|
||||
Vertical;
|
||||
|
||||
SELECT '-- Different DateTime64 scales';
|
||||
|
||||
WITH
|
||||
toDateTime64('2021-08-15 18:57:56.492', 0, 'UTC') AS dt64_0,
|
||||
toDateTime64('2021-08-15 18:57:56.492', 1, 'UTC') AS dt64_1,
|
||||
toDateTime64('2021-08-15 18:57:56.492', 2, 'UTC') AS dt64_2,
|
||||
toDateTime64('2021-08-15 18:57:56.492', 3, 'UTC') AS dt64_3,
|
||||
toDateTime64('2021-08-15 18:57:56.492', 4, 'UTC') AS dt64_4
|
||||
SELECT
|
||||
dateTime64ToSnowflakeID(dt64_0),
|
||||
dateTime64ToSnowflakeID(dt64_1),
|
||||
dateTime64ToSnowflakeID(dt64_2),
|
||||
dateTime64ToSnowflakeID(dt64_3),
|
||||
dateTime64ToSnowflakeID(dt64_4)
|
||||
Format
|
||||
Vertical;
|
||||
|
||||
SELECT '-- Idempotency';
|
||||
|
||||
-- DateTime64-to-SnowflakeID-to-DateTime64 is idempotent if the scale is <=3 (millisecond precision)
|
||||
WITH
|
||||
now64(0) AS dt64_0,
|
||||
now64(1) AS dt64_1,
|
||||
now64(2) AS dt64_2,
|
||||
now64(3) AS dt64_3,
|
||||
now64(4) AS dt64_4
|
||||
SELECT
|
||||
snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_0)) == dt64_0,
|
||||
snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_1)) == dt64_1,
|
||||
snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_2)) == dt64_2,
|
||||
snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_3)) == dt64_3,
|
||||
snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_4)) == dt64_4
|
||||
FORMAT
|
||||
Vertical;
|
@ -0,0 +1,27 @@
|
||||
-- Negative tests
|
||||
-- Return type
|
||||
DateTime
|
||||
DateTime64(3)
|
||||
-- Non-const path
|
||||
Row 1:
|
||||
──────
|
||||
sf: 7204436857747984384
|
||||
dt: 2024-06-06 10:59:58
|
||||
dt64: 2024-06-06 10:59:58.851
|
||||
Row 1:
|
||||
──────
|
||||
sf: 1426981498778550272
|
||||
dt: 2021-08-15 18:57:56
|
||||
dt64: 2021-08-15 18:57:56.000
|
||||
Row 1:
|
||||
──────
|
||||
sf: 7204436857747984384
|
||||
dt: 2024-06-06 18:59:58
|
||||
dt64: 2024-06-06 18:59:58.851
|
||||
-- Const path
|
||||
Row 1:
|
||||
──────
|
||||
sf: 7204436857747984384
|
||||
dt: 2024-06-06 10:59:58
|
||||
dt64: 2024-06-06 10:59:58.851
|
||||
-- Can be combined with generateSnowflakeID
|
86
tests/queries/0_stateless/01942_snowflakeIDToDateTime.sql
Normal file
86
tests/queries/0_stateless/01942_snowflakeIDToDateTime.sql
Normal file
@ -0,0 +1,86 @@
|
||||
SET session_timezone = 'UTC'; -- disable timezone randomization
|
||||
SET allow_experimental_analyzer = 1; -- The old path formats the result with different whitespaces
|
||||
SET uniform_snowflake_conversion_functions = 1; -- Force-enable uniform snowflake conversion functions (in case this is randomized in CI)
|
||||
|
||||
SELECT '-- Negative tests';
|
||||
SELECT snowflakeIDToDateTime(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT snowflakeIDToDateTime64(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT snowflakeIDToDateTime('invalid_snowflake'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime64('invalid_snowflake'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime(123::UInt64, 'invalid_epoch'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime64(123::UInt64, 'invalid_epoch'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime(123::UInt64, materialize(42)); -- {serverError ILLEGAL_COLUMN}
|
||||
SELECT snowflakeIDToDateTime64(123::UInt64, materialize(42)); -- {serverError ILLEGAL_COLUMN}
|
||||
SELECT snowflakeIDToDateTime(123::UInt64, 42, 42); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime64(123::UInt64, 42, 42); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime(123::UInt64, 42, 'UTC', 'too_many_args'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT snowflakeIDToDateTime64(123::UInt64, 42, 'UTC', 'too_many_args'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
|
||||
SELECT snowflakeIDToDateTime(123::UInt64) SETTINGS uniform_snowflake_conversion_functions = 0; -- { serverError UNKNOWN_FUNCTION }
|
||||
SELECT snowflakeIDToDateTime64(123::UInt64) SETTINGS uniform_snowflake_conversion_functions = 0; -- { serverError UNKNOWN_FUNCTION }
|
||||
|
||||
SELECT '-- Return type';
|
||||
SELECT toTypeName(snowflakeIDToDateTime(123::UInt64));
|
||||
SELECT toTypeName(snowflakeIDToDateTime64(123::UInt64));
|
||||
|
||||
SELECT '-- Non-const path';
|
||||
-- Two const arguments are mapped to two non-const arguments ('getDefaultImplementationForConstants'), the non-const path is taken
|
||||
|
||||
WITH
|
||||
7204436857747984384 AS sf
|
||||
SELECT
|
||||
sf,
|
||||
snowflakeIDToDateTime(sf) as dt,
|
||||
snowflakeIDToDateTime64(sf) as dt64
|
||||
FORMAT
|
||||
Vertical;
|
||||
|
||||
-- With Twitter Snowflake ID and Twitter epoch
|
||||
WITH
|
||||
1426981498778550272 AS sf,
|
||||
1288834974657 AS epoch
|
||||
SELECT
|
||||
sf,
|
||||
snowflakeIDToDateTime(sf, epoch) as dt,
|
||||
snowflakeIDToDateTime64(sf, epoch) as dt64
|
||||
FORMAT
|
||||
Vertical;
|
||||
|
||||
-- non-default timezone
|
||||
WITH
|
||||
7204436857747984384 AS sf,
|
||||
0 AS epoch, -- default epoch
|
||||
'Asia/Shanghai' AS tz
|
||||
SELECT
|
||||
sf,
|
||||
snowflakeIDToDateTime(sf, epoch, tz) as dt,
|
||||
snowflakeIDToDateTime64(sf, epoch, tz) as dt64
|
||||
FORMAT
|
||||
Vertical;
|
||||
|
||||
SELECT '-- Const path';
|
||||
|
||||
-- The const path can only be tested by const snowflake + const epoch + non-const time-zone. The latter requires a special setting.
|
||||
WITH
|
||||
7204436857747984384 AS sf,
|
||||
0 AS epoch, -- default epoch
|
||||
materialize('Asia/Shanghai') AS tz
|
||||
SELECT
|
||||
sf,
|
||||
snowflakeIDToDateTime(sf, epoch, tz) as dt,
|
||||
snowflakeIDToDateTime64(sf, epoch, tz) as dt64
|
||||
FORMAT
|
||||
Vertical
|
||||
SETTINGS
|
||||
allow_nonconst_timezone_arguments = 1;
|
||||
|
||||
|
||||
SELECT '-- Can be combined with generateSnowflakeID';
|
||||
|
||||
WITH
|
||||
generateSnowflakeID() AS snowflake
|
||||
SELECT
|
||||
snowflakeIDToDateTime(snowflake),
|
||||
snowflakeIDToDateTime64(snowflake)
|
||||
FORMAT
|
||||
Null;
|
@ -1,3 +1,5 @@
|
||||
SET uniform_snowflake_conversion_functions = 0; -- Force-disable uniform snowflake conversion functions (in case this is randomized in CI)
|
||||
|
||||
-- -- Error cases
|
||||
SELECT snowflakeToDateTime(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT snowflakeToDateTime64(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
@ -8,6 +10,9 @@ SELECT snowflakeToDateTime64('abc'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeToDateTime('abc', 123); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeToDateTime64('abc', 123); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
|
||||
SELECT snowflakeToDateTime(123::Int64) SETTINGS uniform_snowflake_conversion_functions = 1; -- { serverError DEPRECATED_FUNCTION }
|
||||
SELECT snowflakeToDateTime64(123::Int64) SETTINGS uniform_snowflake_conversion_functions = 1; -- { serverError DEPRECATED_FUNCTION }
|
||||
|
||||
SELECT 'const column';
|
||||
WITH
|
||||
CAST(1426860704886947840 AS Int64) AS i64,
|
||||
|
@ -960,6 +960,7 @@ ToGeoBoundary
|
||||
ToIPv
|
||||
ToParent
|
||||
ToSnowflake
|
||||
ToSnowflakeID
|
||||
ToString
|
||||
ToUnicode
|
||||
Toolset
|
||||
@ -1453,6 +1454,7 @@ datatypes
|
||||
dateName
|
||||
dateTime
|
||||
dateTimeToSnowflake
|
||||
dateTimeToSnowflakeID
|
||||
datetime
|
||||
datetimes
|
||||
dayofyear
|
||||
@ -2468,6 +2470,7 @@ skewpop
|
||||
skewsamp
|
||||
skippingerrors
|
||||
sleepEachRow
|
||||
snowflakeIDToDateTime
|
||||
snowflakeToDateTime
|
||||
socketcache
|
||||
soundex
|
||||
|
Loading…
Reference in New Issue
Block a user