thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dotnet-integration-test' of github.com:yandex/ClickHouse into dotnet-integration-test

Browse Source
This commit is contained in:
Alexey Milovidov 2022-01-04 22:49:58 +03:00
commit 2d3f53ec5c
32 changed files with 575 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
cmake/find/blob_storage.cmake
View File

@ -1,32 +1,29 @@
option (ENABLE_AZURE_BLOB_STORAGE "Enable Azure blob storage" ${ENABLE_LIBRARIES})
if (ENABLE_AZURE_BLOB_STORAGE)
option(USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY
"Set to FALSE to use system Azure SDK instead of bundled (OFF currently not implemented)"
ON)
set(USE_AZURE_BLOB_STORAGE 1)
set(AZURE_BLOB_STORAGE_LIBRARY azure_sdk)
else()
return()
if ((NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/azure/sdk"
OR NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/azure/cmake-modules")
AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY)
message (WARNING "submodule contrib/azure is missing. to fix try run: \n git submodule update --init")
set(USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY OFF)
set(USE_AZURE_BLOB_STORAGE 0)
endif ()
if (NOT USE_INTERNAL_SSL_LIBRARY AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY)
message (FATAL_ERROR "Currently Blob Storage support can be built only with internal SSL library")
endif()
if (NOT USE_INTERNAL_CURL AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY)
message (FATAL_ERROR "Currently Blob Storage support can be built only with internal curl library")
endif()
endif()
option(USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY
"Set to FALSE to use system Azure SDK instead of bundled (OFF currently not implemented)"
ON)
if ((NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/azure/sdk"
OR NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/azure/cmake-modules")
AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY)
message (WARNING "submodule contrib/azure is missing. to fix try run: \n git submodule update --init")
set(USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY OFF)
set(USE_AZURE_BLOB_STORAGE 0)
endif ()
if (NOT USE_INTERNAL_SSL_LIBRARY AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY)
message (FATAL_ERROR "Currently Blob Storage support can be built only with internal SSL library")
endif()
if (NOT USE_INTERNAL_CURL AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY)
message (FATAL_ERROR "Currently Blob Storage support can be built only with internal curl library")
endif()
if (USE_AZURE_BLOB_STORAGE)
message (STATUS "Using Azure Blob Storage - ${USE_AZURE_BLOB_STORAGE}")
endif()
message (STATUS "Using Azure Blob Storage - ${USE_AZURE_BLOB_STORAGE}")

27
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -681,7 +681,9 @@ Queries may be limited by other settings: [max_concurrent_insert_queries](#max-c
Possible values:
- Positive integer.
- 0 — Disabled.
- 0 — No limit.
Default value: `100`.
**Example**
@ -691,7 +693,7 @@ Possible values:
## max_concurrent_insert_queries {#max-concurrent-insert-queries}
The maximum number of simultaneously processed insert queries.
The maximum number of simultaneously processed `INSERT` queries.
!!! info "Note"
These settings can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged.
@ -699,7 +701,9 @@ The maximum number of simultaneously processed insert queries.
Possible values:
- Positive integer.
- 0 — Disabled.
- 0 — No limit.
Default value: `0`.
**Example**
@ -709,7 +713,7 @@ Possible values:
## max_concurrent_select_queries {#max-concurrent-select-queries}
The maximum number of simultaneously processed select queries.
The maximum number of simultaneously processed `SELECT` queries.
!!! info "Note"
These settings can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged.
@ -717,7 +721,9 @@ The maximum number of simultaneously processed select queries.
Possible values:
- Positive integer.
- 0 — Disabled.
- 0 — No limit.
Default value: `0`.
**Example**
@ -732,7 +738,9 @@ The maximum number of simultaneously processed queries related to MergeTree tabl
Possible values:
- Positive integer.
- 0 — Disabled.
- 0 — No limit.
Default value: `0`.
**Example**
@ -748,7 +756,12 @@ Example: `max_concurrent_queries_for_all_users` can be set to 99 for all users a
Modifying the setting for one query or user does not affect other queries.
Default value: `0` that means no limit.
Possible values:
- Positive integer.
- 0 — No limit.
Default value: `0`.
**Example**

17
docs/en/operations/settings/settings.md
View File

@ -4155,3 +4155,20 @@ Default value: `''`.
Sets the character that is interpreted as a suffix after the result set for [CustomSeparated](../../interfaces/formats.md#format-customseparated) data format.
Default value: `''`.
## shutdown_wait_unfinished_queries
Enables or disables waiting unfinished queries when shutdown server.
Possible values:
- 0 — Disabled.
- 1 — Enabled. The wait time equal shutdown_wait_unfinished config.
Default value: 0.
## shutdown_wait_unfinished
The waiting time in seconds for currently handled connections when shutdown server.
Default Value: 5.

2
docs/en/sql-reference/functions/string-search-functions.md
View File

@ -351,8 +351,6 @@ Checks whether the string matches the `pattern` regular expression. A `re2` regu
Returns 0 if it does not match, or 1 if it matches.
Note that the backslash symbol (`\`) is used for escaping in the regular expression. The same symbol is used for escaping in string literals. So in order to escape the symbol in a regular expression, you must write two backslashes (\\) in a string literal.
The regular expression works with the string as if it is a set of bytes. The regular expression can’t contain null bytes.
For patterns to search for substrings in a string, it is better to use LIKE or ‘position’, since they work much faster.

57
docs/ru/operations/server-configuration-parameters/settings.md
View File

@ -673,7 +673,7 @@ ClickHouse поддерживает динамическое изменение
## max_concurrent_queries {#max-concurrent-queries}
Определяет максимальное количество одновременно обрабатываемых запросов, связанных с таблицей семейства `MergeTree`. Запросы также могут быть ограничены настройками: [max_concurrent_queries_for_user](#max-concurrent-queries-for-user), [max_concurrent_queries_for_all_users](#max-concurrent-queries-for-all-users), [min_marks_to_honor_max_concurrent_queries](#min-marks-to-honor-max-concurrent-queries).
Определяет максимальное количество одновременно обрабатываемых запросов, связанных с таблицей семейства `MergeTree`. Запросы также могут быть ограничены настройками: [max_concurrent_insert_queries](#max-concurrent-insert-queries), [max_concurrent_select_queries](#max-concurrent-select-queries), [max_concurrent_queries_for_user](#max-concurrent-queries-for-user), [max_concurrent_queries_for_all_users](#max-concurrent-queries-for-all-users), [min_marks_to_honor_max_concurrent_queries](#min-marks-to-honor-max-concurrent-queries).
!!! info "Примечание"
Параметры этих настроек могут быть изменены во время выполнения запросов и вступят в силу немедленно. Запросы, которые уже запущены, выполнятся без изменений.
@ -681,7 +681,9 @@ ClickHouse поддерживает динамическое изменение
Возможные значения:
- Положительное целое число.
- 0 — выключена.
- 0 — нет лимита.
Значение по умолчанию: `100`.
**Пример**
@ -689,6 +691,46 @@ ClickHouse поддерживает динамическое изменение
<max_concurrent_queries>100</max_concurrent_queries>
```
## max_concurrent_insert_queries {#max-concurrent-insert-queries}
Определяет максимальное количество одновременных `INSERT` запросов.
!!! info "Примечание"
Параметры этих настроек могут быть изменены во время выполнения запросов и вступят в силу немедленно. Запросы, которые уже запущены, выполнятся без изменений.
Возможные значения:
- Положительное целое число.
- 0 — нет лимита.
Значение по умолчанию: `0`.
**Example**
``` xml
<max_concurrent_insert_queries>100</max_concurrent_insert_queries>
```
## max_concurrent_select_queries {#max-concurrent-select-queries}
Определяет максимальное количество одновременных `SELECT` запросов.
!!! info "Примечание"
Параметры этих настроек могут быть изменены во время выполнения запросов и вступят в силу немедленно. Запросы, которые уже запущены, выполнятся без изменений.
Возможные значения:
- Положительное целое число.
- 0 — нет лимита.
Значение по умолчанию: `0`.
**Example**
``` xml
<max_concurrent_select_queries>100</max_concurrent_select_queries>
```
## max_concurrent_queries_for_user {#max-concurrent-queries-for-user}
Определяет максимальное количество одновременно обрабатываемых запросов, связанных с таблицей семейства `MergeTree`, для пользователя.
@ -696,7 +738,9 @@ ClickHouse поддерживает динамическое изменение
Возможные значения:
- Положительное целое число.
- 0 — выключена.
- 0 — нет лимита.
Значение по умолчанию: `0`.
**Пример**
@ -712,7 +756,12 @@ ClickHouse поддерживает динамическое изменение
Изменение настройки для одного запроса или пользователя не влияет на другие запросы.
Значение по умолчанию: `0` — отсутствие ограничений.
Возможные значения:
- Положительное целое число.
- 0 — нет лимита.
Значение по умолчанию: `0`.
**Пример**

1
docs/zh/sql-reference/statements/create/function.md
View File

@ -1 +0,0 @@
../../../../en/sql-reference/statements/create/function.md

60
docs/zh/sql-reference/statements/create/function.md Normal file
View File

@ -0,0 +1,60 @@
---
toc_priority: 38
toc_title: FUNCTION
---
# CREATE FUNCTION {#create-function}
用一个lambda表达式创建用户自定义函数。该表达式必须由函数参数、常数、运算符或其他函数调用组成。
**语法**
```sql
CREATE FUNCTION name AS (parameter0, ...) -> expression
```
一个函数可以有任意数量的参数。
存在一些限制如下:
- 函数名在用户自定义函数和系统函数中必须是唯一的。
- 递归函数是不允许的。
- 函数所使用的所有变量必须在其参数列表中指定。
如果违反了任何限制，就会产生异常。
**示例**
查询:
```sql
CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b;
SELECT number, linear_equation(number, 2, 1) FROM numbers(3);
```
结果:
``` text
┌─number─┬─plus(multiply(2, number), 1)─┐
│ 0 │ 1 │
│ 1 │ 3 │
│ 2 │ 5 │
└────────┴──────────────────────────────┘
```
在下面的查询中，[conditional function](../../../sql-reference/functions/conditional-functions.md)在用户自定义函数中被调用:
```sql
CREATE FUNCTION parity_str AS (n) -> if(n % 2, 'odd', 'even');
SELECT number, parity_str(number) FROM numbers(3);
```
结果:
``` text
┌─number─┬─if(modulo(number, 2), 'odd', 'even')─┐
│ 0 │ even │
│ 1 │ odd │
│ 2 │ even │
└────────┴──────────────────────────────────────┘
```

17
programs/client/Client.cpp
View File

@ -25,7 +25,6 @@
#include <Common/formatReadable.h>
#include <Common/TerminalSize.h>
#include <Common/Config/configReadClient.h>
#include "Common/MemoryTracker.h"
#include <Core/QueryProcessingStage.h>
#include <Client/TestHint.h>
@ -56,11 +55,6 @@
#pragma GCC optimize("-fno-var-tracking-assignments")
#endif
namespace CurrentMetrics
{
extern const Metric MemoryTracking;
}
namespace fs = std::filesystem;
@ -410,16 +404,6 @@ try
std::cout << std::fixed << std::setprecision(3);
std::cerr << std::fixed << std::setprecision(3);
/// Limit on total memory usage
size_t max_client_memory_usage = config().getInt64("max_memory_usage_in_client", 0 /*default value*/);
if (max_client_memory_usage != 0)
{
total_memory_tracker.setHardLimit(max_client_memory_usage);
total_memory_tracker.setDescription("(total)");
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
}
registerFormats();
registerFunctions();
registerAggregateFunctions();
@ -1014,7 +998,6 @@ void Client::addOptions(OptionsDescription & options_description)
("opentelemetry-tracestate", po::value<std::string>(), "OpenTelemetry tracestate header as described by W3C Trace Context recommendation")
("no-warnings", "disable warnings when client connects to server")
("max_memory_usage_in_client", po::value<int>(), "sets memory limit in client")
;
/// Commandline options related to external tables.

53
src/AggregateFunctions/AggregateFunctionIntervalLengthSum.h
View File

@ -6,6 +6,7 @@
#include <Common/ArenaAllocator.h>
#include <Common/assert_cast.h>
#include <base/arithmeticOverflow.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypesNumber.h>
@ -15,6 +16,7 @@
#include <unordered_set>
namespace DB
{
@ -23,12 +25,11 @@ namespace ErrorCodes
extern const int TOO_LARGE_ARRAY_SIZE;
}
/**
* Calculate total length of intervals without intersections. Each interval is the pair of numbers [begin, end];
* Return UInt64 for integral types (UInt/Int*, Date/DateTime) and return Float64 for Float*.
*
* Implementation simply stores intervals sorted by beginning and sums lengths at final.
*/
/** Calculate total length of intervals without intersections. Each interval is the pair of numbers [begin, end];
* Returns UInt64 for integral types (UInt/Int*, Date/DateTime) and returns Float64 for Float*.
*
* Implementation simply stores intervals sorted by beginning and sums lengths at final.
*/
template <typename T>
struct AggregateFunctionIntervalLengthSumData
{
@ -43,10 +44,14 @@ struct AggregateFunctionIntervalLengthSumData
void add(T begin, T end)
{
/// Reversed intervals are counted by absolute value of their length.
if (unlikely(end < begin))
std::swap(begin, end);
else if (unlikely(begin == end))
return;
if (sorted && !segments.empty())
{
sorted = segments.back().first <= begin;
}
segments.emplace_back(begin, end);
}
@ -130,6 +135,11 @@ template <typename T, typename Data>
class AggregateFunctionIntervalLengthSum final : public IAggregateFunctionDataHelper<Data, AggregateFunctionIntervalLengthSum<T, Data>>
{
private:
static auto NO_SANITIZE_UNDEFINED length(typename Data::Segment segment)
{
return segment.second - segment.first;
}
template <typename TResult>
TResult getIntervalLengthSum(Data & data) const
{
@ -140,21 +150,24 @@ private:
TResult res = 0;
typename Data::Segment cur_segment = data.segments[0];
typename Data::Segment curr_segment = data.segments[0];
for (size_t i = 1, sz = data.segments.size(); i < sz; ++i)
for (size_t i = 1, size = data.segments.size(); i < size; ++i)
{
/// Check if current interval intersect with next one then add length, otherwise advance interval end
if (cur_segment.second < data.segments[i].first)
{
res += cur_segment.second - cur_segment.first;
cur_segment = data.segments[i];
}
else
cur_segment.second = std::max(cur_segment.second, data.segments[i].second);
}
const typename Data::Segment & next_segment = data.segments[i];
res += cur_segment.second - cur_segment.first;
/// Check if current interval intersects with next one then add length, otherwise advance interval end.
if (curr_segment.second < next_segment.first)
{
res += length(curr_segment);
curr_segment = next_segment;
}
else if (next_segment.second > curr_segment.second)
{
curr_segment.second = next_segment.second;
}
}
res += length(curr_segment);
return res;
}

20
src/AggregateFunctions/AggregateFunctionNothing.cpp Normal file
View File

@ -0,0 +1,20 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/AggregateFunctionNothing.h>
#include <AggregateFunctions/FactoryHelpers.h>
namespace DB
{
struct Settings;
void registerAggregateFunctionNothing(AggregateFunctionFactory & factory)
{
factory.registerFunction("nothing", [](const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *)
{
assertNoParameters(name, parameters);
return std::make_shared<AggregateFunctionNothing>(argument_types, parameters);
});
}
}

12
src/AggregateFunctions/AggregateFunctionNothing.h
View File

@ -4,6 +4,8 @@
#include <DataTypes/DataTypeNothing.h>
#include <Columns/IColumn.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
namespace DB
@ -26,7 +28,7 @@ public:
DataTypePtr getReturnType() const override
{
return argument_types.front();
return argument_types.empty() ? std::make_shared<DataTypeNullable>(std::make_shared<DataTypeNothing>()) : argument_types.front();
}
bool allocatesMemoryInArena() const override { return false; }
@ -62,12 +64,16 @@ public:
{
}
void serialize(ConstAggregateDataPtr __restrict, WriteBuffer &, std::optional<size_t>) const override
void serialize(ConstAggregateDataPtr __restrict, WriteBuffer & buf, std::optional<size_t>) const override
{
writeChar('\0', buf);
}
void deserialize(AggregateDataPtr, ReadBuffer &, std::optional<size_t>, Arena *) const override
void deserialize(AggregateDataPtr, ReadBuffer & buf, std::optional<size_t>, Arena *) const override
{
[[maybe_unused]] char symbol;
readChar(symbol, buf);
assert(symbol == '\0');
}
void insertResultInto(AggregateDataPtr, IColumn & to, Arena *) const override

2
src/AggregateFunctions/registerAggregateFunctions.cpp
View File

@ -50,6 +50,7 @@ void registerAggregateFunctionWelchTTest(AggregateFunctionFactory &);
void registerAggregateFunctionStudentTTest(AggregateFunctionFactory &);
void registerAggregateFunctionSingleValueOrNull(AggregateFunctionFactory &);
void registerAggregateFunctionSequenceNextNode(AggregateFunctionFactory &);
void registerAggregateFunctionNothing(AggregateFunctionFactory &);
void registerAggregateFunctionExponentialMovingAverage(AggregateFunctionFactory &);
void registerAggregateFunctionSparkbar(AggregateFunctionFactory &);
void registerAggregateFunctionIntervalLengthSum(AggregateFunctionFactory &);
@ -114,6 +115,7 @@ void registerAggregateFunctions()
registerAggregateFunctionSequenceNextNode(factory);
registerAggregateFunctionWelchTTest(factory);
registerAggregateFunctionStudentTTest(factory);
registerAggregateFunctionNothing(factory);
registerAggregateFunctionSingleValueOrNull(factory);
registerAggregateFunctionIntervalLengthSum(factory);
registerAggregateFunctionExponentialMovingAverage(factory);

16
src/Client/ClientBase.cpp
View File

@ -10,6 +10,7 @@
#include <base/argsToConfig.h>
#include <Common/DateLUT.h>
#include <Common/LocalDate.h>
#include <Common/MemoryTracker.h>
#include <base/LineReader.h>
#include <base/scope_guard_safe.h>
#include "Common/Exception.h"
@ -65,6 +66,11 @@ namespace fs = std::filesystem;
using namespace std::literals;
namespace CurrentMetrics
{
extern const Metric MemoryTracking;
}
namespace DB
{
@ -1812,6 +1818,7 @@ void ClientBase::init(int argc, char ** argv)
("interactive", "Process queries-file or --query query and start interactive mode")
("pager", po::value<std::string>(), "Pipe all output into this command (less or similar)")
("max_memory_usage_in_client", po::value<int>(), "Set memory limit in client/local server")
;
addOptions(options_description);
@ -1917,6 +1924,15 @@ void ClientBase::init(int argc, char ** argv)
processOptions(options_description, options, external_tables_arguments);
argsToConfig(common_arguments, config(), 100);
clearPasswordFromCommandLine(argc, argv);
/// Limit on total memory usage
size_t max_client_memory_usage = config().getInt64("max_memory_usage_in_client", 0 /*default value*/);
if (max_client_memory_usage != 0)
{
total_memory_tracker.setHardLimit(max_client_memory_usage);
total_memory_tracker.setDescription("(total)");
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
}
}
}

14
src/Common/DateLUTImpl.cpp
View File

@ -174,6 +174,20 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_)
{
years_months_lut[year_months_lut_index] = first_day_of_last_month;
}
/// Fill saturated LUT.
{
ssize_t day = DATE_LUT_SIZE - 1;
for (; day >= 0; --day)
{
if (lut[day].date >= 0)
lut_saturated[day] = lut[day];
else
break;
}
for (; day >= 0; --day)
lut_saturated[day] = lut_saturated[day + 1];
}
}

195
src/Common/DateLUTImpl.h
View File

@ -61,6 +61,8 @@ private:
// has to be a separate type to support overloading
// TODO: make sure that any arithmetic on LUTIndex actually results in valid LUTIndex.
STRONG_TYPEDEF(UInt32, LUTIndex)
// Same as above but select different function overloads for zero saturation.
STRONG_TYPEDEF(UInt32, LUTIndexWithSaturation)
template <typename T>
friend inline LUTIndex operator+(const LUTIndex & index, const T v)
@ -182,6 +184,9 @@ private:
/// In comparison to std::vector, plain array is cheaper by one indirection.
Values lut[DATE_LUT_SIZE + 1];
/// Same as above but with dates < 1970-01-01 saturated to 1970-01-01.
Values lut_saturated[DATE_LUT_SIZE + 1];
/// Year number after DATE_LUT_MIN_YEAR -> LUTIndex in lut for start of year.
LUTIndex years_lut[DATE_LUT_YEARS];
@ -278,19 +283,39 @@ public:
auto getOffsetAtStartOfEpoch() const { return offset_at_start_of_epoch; }
auto getTimeOffsetAtStartOfLUT() const { return offset_at_start_of_lut; }
auto getDayNumOffsetEpoch() const { return daynum_offset_epoch; }
static auto getDayNumOffsetEpoch() { return daynum_offset_epoch; }
/// All functions below are thread-safe; arguments are not checked.
inline ExtendedDayNum toDayNum(ExtendedDayNum d) const
static ExtendedDayNum toDayNum(ExtendedDayNum d)
{
return d;
}
template <typename DateOrTime>
inline ExtendedDayNum toDayNum(DateOrTime v) const
static UInt32 saturateMinus(UInt32 x, UInt32 y)
{
return ExtendedDayNum{static_cast<ExtendedDayNum::UnderlyingType>(toLUTIndex(v).toUnderType() - daynum_offset_epoch)};
UInt32 res = x - y;
res &= -Int32(res <= x);
return res;
}
static ExtendedDayNum toDayNum(LUTIndex d)
{
return ExtendedDayNum{static_cast<ExtendedDayNum::UnderlyingType>(d.toUnderType() - daynum_offset_epoch)};
}
static DayNum toDayNum(LUTIndexWithSaturation d)
{
return DayNum{static_cast<DayNum::UnderlyingType>(saturateMinus(d.toUnderType(), daynum_offset_epoch))};
}
template <typename DateOrTime>
inline auto toDayNum(DateOrTime v) const
{
if constexpr (std::is_unsigned_v<DateOrTime> || std::is_same_v<DateOrTime, DayNum>)
return DayNum{static_cast<DayNum::UnderlyingType>(saturateMinus(toLUTIndex(v).toUnderType(), daynum_offset_epoch))};
else
return ExtendedDayNum{static_cast<ExtendedDayNum::UnderlyingType>(toLUTIndex(v).toUnderType() - daynum_offset_epoch)};
}
/// Round down to start of monday.
@ -298,14 +323,20 @@ public:
inline Time toFirstDayOfWeek(DateOrTime v) const
{
const LUTIndex i = toLUTIndex(v);
return lut[i - (lut[i].day_of_week - 1)].date;
if constexpr (std::is_unsigned_v<DateOrTime> || std::is_same_v<DateOrTime, DayNum>)
return lut_saturated[i - (lut[i].day_of_week - 1)].date;
else
return lut[i - (lut[i].day_of_week - 1)].date;
}
template <typename DateOrTime>
inline ExtendedDayNum toFirstDayNumOfWeek(DateOrTime v) const
inline auto toFirstDayNumOfWeek(DateOrTime v) const
{
const LUTIndex i = toLUTIndex(v);
return toDayNum(i - (lut[i].day_of_week - 1));
if constexpr (std::is_unsigned_v<DateOrTime> || std::is_same_v<DateOrTime, DayNum>)
return toDayNum(LUTIndexWithSaturation(i - (lut[i].day_of_week - 1)));
else
return toDayNum(LUTIndex(i - (lut[i].day_of_week - 1)));
}
/// Round down to start of month.
@ -313,21 +344,30 @@ public:
inline Time toFirstDayOfMonth(DateOrTime v) const
{
const LUTIndex i = toLUTIndex(v);
return lut[i - (lut[i].day_of_month - 1)].date;
if constexpr (std::is_unsigned_v<DateOrTime> || std::is_same_v<DateOrTime, DayNum>)
return lut_saturated[i - (lut[i].day_of_month - 1)].date;
else
return lut[i - (lut[i].day_of_month - 1)].date;
}
template <typename DateOrTime>
inline ExtendedDayNum toFirstDayNumOfMonth(DateOrTime v) const
inline auto toFirstDayNumOfMonth(DateOrTime v) const
{
const LUTIndex i = toLUTIndex(v);
return toDayNum(i - (lut[i].day_of_month - 1));
if constexpr (std::is_unsigned_v<DateOrTime> || std::is_same_v<DateOrTime, DayNum>)
return toDayNum(LUTIndexWithSaturation(i - (lut[i].day_of_month - 1)));
else
return toDayNum(LUTIndex(i - (lut[i].day_of_month - 1)));
}
/// Round down to start of quarter.
template <typename DateOrTime>
inline ExtendedDayNum toFirstDayNumOfQuarter(DateOrTime v) const
inline auto toFirstDayNumOfQuarter(DateOrTime v) const
{
return toDayNum(toFirstDayOfQuarterIndex(v));
if constexpr (std::is_unsigned_v<DateOrTime> || std::is_same_v<DateOrTime, DayNum>)
return toDayNum(LUTIndexWithSaturation(toFirstDayOfQuarterIndex(v)));
else
return toDayNum(LUTIndex(toFirstDayOfQuarterIndex(v)));
}
template <typename DateOrTime>
@ -365,9 +405,12 @@ public:
}
template <typename DateOrTime>
inline ExtendedDayNum toFirstDayNumOfYear(DateOrTime v) const
inline auto toFirstDayNumOfYear(DateOrTime v) const
{
return toDayNum(toFirstDayNumOfYearIndex(v));
if constexpr (std::is_unsigned_v<DateOrTime> || std::is_same_v<DateOrTime, DayNum>)
return toDayNum(LUTIndexWithSaturation(toFirstDayNumOfYearIndex(v)));
else
return toDayNum(LUTIndex(toFirstDayNumOfYearIndex(v)));
}
inline Time toFirstDayOfNextMonth(Time t) const
@ -514,11 +557,17 @@ public:
* because the same calendar day starts/ends at different timestamps in different time zones)
*/
inline Time fromDayNum(DayNum d) const { return lut[toLUTIndex(d)].date; }
inline Time fromDayNum(DayNum d) const { return lut_saturated[toLUTIndex(d)].date; }
inline Time fromDayNum(ExtendedDayNum d) const { return lut[toLUTIndex(d)].date; }
template <typename DateOrTime>
inline Time toDate(DateOrTime v) const { return lut[toLUTIndex(v)].date; }
inline Time toDate(DateOrTime v) const
{
if constexpr (std::is_unsigned_v<DateOrTime> || std::is_same_v<DateOrTime, DayNum>)
return lut_saturated[toLUTIndex(v)].date;
else
return lut[toLUTIndex(v)].date;
}
template <typename DateOrTime>
inline unsigned toMonth(DateOrTime v) const { return lut[toLUTIndex(v)].month; }
@ -581,9 +630,12 @@ public:
}
template <typename DateOrTime>
inline ExtendedDayNum toFirstDayNumOfISOYear(DateOrTime v) const
inline auto toFirstDayNumOfISOYear(DateOrTime v) const
{
return toDayNum(toFirstDayNumOfISOYearIndex(v));
if constexpr (std::is_unsigned_v<DateOrTime> || std::is_same_v<DateOrTime, DayNum>)
return toDayNum(LUTIndexWithSaturation(toFirstDayNumOfISOYearIndex(v)));
else
return toDayNum(LUTIndex(toFirstDayNumOfISOYearIndex(v)));
}
inline Time toFirstDayOfISOYear(Time t) const
@ -596,7 +648,7 @@ public:
template <typename DateOrTime>
inline unsigned toISOWeek(DateOrTime v) const
{
return 1 + (toFirstDayNumOfWeek(v) - toFirstDayNumOfISOYear(v)) / 7;
return 1 + (toFirstDayNumOfWeek(v) - toDayNum(toFirstDayNumOfISOYearIndex(v))) / 7;
}
/*
@ -662,7 +714,7 @@ public:
{
if (!week_year_mode && ((first_weekday_mode && weekday != 0) || (!first_weekday_mode && weekday >= 4)))
return yw;
week_year_mode = 1;
week_year_mode = true;
(yw.first)--;
first_daynr -= (days = calc_days_in_year(yw.first));
weekday = (weekday + 53 * 7 - days) % 7;
@ -724,7 +776,7 @@ public:
/// Get first day of week with week_mode, return Sunday or Monday
template <typename DateOrTime>
inline ExtendedDayNum toFirstDayNumOfWeek(DateOrTime v, UInt8 week_mode) const
inline auto toFirstDayNumOfWeek(DateOrTime v, UInt8 week_mode) const
{
bool monday_first_mode = week_mode & static_cast<UInt8>(WeekModeFlag::MONDAY_FIRST);
if (monday_first_mode)
@ -733,7 +785,10 @@ public:
}
else
{
return (toDayOfWeek(v) != 7) ? ExtendedDayNum(v - toDayOfWeek(v)) : toDayNum(v);
if constexpr (std::is_unsigned_v<DateOrTime> || std::is_same_v<DateOrTime, DayNum>)
return (toDayOfWeek(v) != 7) ? DayNum(saturateMinus(v, toDayOfWeek(v))) : toDayNum(v);
else
return (toDayOfWeek(v) != 7) ? ExtendedDayNum(v - toDayOfWeek(v)) : toDayNum(v);
}
}
@ -809,7 +864,7 @@ public:
}
template <typename DateOrTime>
inline ExtendedDayNum toStartOfYearInterval(DateOrTime v, UInt64 years) const
inline auto toStartOfYearInterval(DateOrTime v, UInt64 years) const
{
if (years == 1)
return toFirstDayNumOfYear(v);
@ -822,39 +877,59 @@ public:
if (unlikely(year < DATE_LUT_MIN_YEAR))
year = DATE_LUT_MIN_YEAR;
return toDayNum(years_lut[year - DATE_LUT_MIN_YEAR]);
if constexpr (std::is_unsigned_v<DateOrTime> || std::is_same_v<DateOrTime, DayNum>)
return toDayNum(LUTIndexWithSaturation(years_lut[year - DATE_LUT_MIN_YEAR]));
else
return toDayNum(years_lut[year - DATE_LUT_MIN_YEAR]);
}
inline ExtendedDayNum toStartOfQuarterInterval(ExtendedDayNum d, UInt64 quarters) const
template <typename Date,
typename = std::enable_if_t<std::is_same_v<Date, DayNum> || std::is_same_v<Date, ExtendedDayNum>>>
inline auto toStartOfQuarterInterval(Date d, UInt64 quarters) const
{
if (quarters == 1)
return toFirstDayNumOfQuarter(d);
return toStartOfMonthInterval(d, quarters * 3);
}
inline ExtendedDayNum toStartOfMonthInterval(ExtendedDayNum d, UInt64 months) const
template <typename Date,
typename = std::enable_if_t<std::is_same_v<Date, DayNum> || std::is_same_v<Date, ExtendedDayNum>>>
inline auto toStartOfMonthInterval(Date d, UInt64 months) const
{
if (months == 1)
return toFirstDayNumOfMonth(d);
const Values & values = lut[toLUTIndex(d)];
UInt32 month_total_index = (values.year - DATE_LUT_MIN_YEAR) * 12 + values.month - 1;
return toDayNum(years_months_lut[month_total_index / months * months]);
if constexpr (std::is_same_v<Date, DayNum>)
return toDayNum(LUTIndexWithSaturation(years_months_lut[month_total_index / months * months]));
else
return toDayNum(years_months_lut[month_total_index / months * months]);
}
inline ExtendedDayNum toStartOfWeekInterval(ExtendedDayNum d, UInt64 weeks) const
template <typename Date,
typename = std::enable_if_t<std::is_same_v<Date, DayNum> || std::is_same_v<Date, ExtendedDayNum>>>
inline auto toStartOfWeekInterval(Date d, UInt64 weeks) const
{
if (weeks == 1)
return toFirstDayNumOfWeek(d);
UInt64 days = weeks * 7;
// January 1st 1970 was Thursday so we need this 4-days offset to make weeks start on Monday.
return ExtendedDayNum(4 + (d - 4) / days * days);
if constexpr (std::is_same_v<Date, DayNum>)
return DayNum(4 + (d - 4) / days * days);
else
return ExtendedDayNum(4 + (d - 4) / days * days);
}
inline Time toStartOfDayInterval(ExtendedDayNum d, UInt64 days) const
template <typename Date,
typename = std::enable_if_t<std::is_same_v<Date, DayNum> || std::is_same_v<Date, ExtendedDayNum>>>
inline Time toStartOfDayInterval(Date d, UInt64 days) const
{
if (days == 1)
return toDate(d);
return lut[toLUTIndex(ExtendedDayNum(d / days * days))].date;
if constexpr (std::is_same_v<Date, DayNum>)
return lut_saturated[toLUTIndex(ExtendedDayNum(d / days * days))].date;
else
return lut[toLUTIndex(ExtendedDayNum(d / days * days))].date;
}
inline Time toStartOfHourInterval(Time t, UInt64 hours) const
@ -1140,7 +1215,11 @@ public:
/// If resulting month has less deys than source month, then saturation can happen.
/// Example: 31 Aug + 1 month = 30 Sep.
inline Time NO_SANITIZE_UNDEFINED addMonths(Time t, Int64 delta) const
template <
typename DateTime,
typename
= std::enable_if_t<std::is_same_v<DateTime, UInt32> || std::is_same_v<DateTime, Int64> || std::is_same_v<DateTime, time_t>>>
inline Time NO_SANITIZE_UNDEFINED addMonths(DateTime t, Int64 delta) const
{
const auto result_day = addMonthsIndex(t, delta);
@ -1154,20 +1233,28 @@ public:
if (time >= lut[result_day].time_at_offset_change())
time -= lut[result_day].amount_of_offset_change();
return lut[result_day].date + time;
auto res = lut[result_day].date + time;
if constexpr (std::is_same_v<DateTime, UInt32>)
{
/// Common compiler should generate branchless code for this saturation operation.
return res <= 0 ? 0 : res;
}
else
return res;
}
inline ExtendedDayNum NO_SANITIZE_UNDEFINED addMonths(ExtendedDayNum d, Int64 delta) const
template <typename Date,
typename = std::enable_if_t<std::is_same_v<Date, DayNum> || std::is_same_v<Date, ExtendedDayNum>>>
inline auto NO_SANITIZE_UNDEFINED addMonths(Date d, Int64 delta) const
{
return toDayNum(addMonthsIndex(d, delta));
if constexpr (std::is_same_v<Date, DayNum>)
return toDayNum(LUTIndexWithSaturation(addMonthsIndex(d, delta)));
else
return toDayNum(addMonthsIndex(d, delta));
}
inline Time NO_SANITIZE_UNDEFINED addQuarters(Time t, Int32 delta) const
{
return addMonths(t, static_cast<Int64>(delta) * 3);
}
inline ExtendedDayNum addQuarters(ExtendedDayNum d, Int32 delta) const
template <typename DateOrTime>
inline auto addQuarters(DateOrTime d, Int32 delta) const
{
return addMonths(d, static_cast<Int64>(delta) * 3);
}
@ -1189,7 +1276,11 @@ public:
}
/// Saturation can occur if 29 Feb is mapped to non-leap year.
inline Time addYears(Time t, Int64 delta) const
template <
typename DateTime,
typename
= std::enable_if_t<std::is_same_v<DateTime, UInt32> || std::is_same_v<DateTime, Int64> || std::is_same_v<DateTime, time_t>>>
inline Time addYears(DateTime t, Int64 delta) const
{
auto result_day = addYearsIndex(t, delta);
@ -1203,12 +1294,24 @@ public:
if (time >= lut[result_day].time_at_offset_change())
time -= lut[result_day].amount_of_offset_change();
return lut[result_day].date + time;
auto res = lut[result_day].date + time;
if constexpr (std::is_same_v<DateTime, UInt32>)
{
/// Common compiler should generate branchless code for this saturation operation.
return res <= 0 ? 0 : res;
}
else
return res;
}
inline ExtendedDayNum addYears(ExtendedDayNum d, Int64 delta) const
template <typename Date,
typename = std::enable_if_t<std::is_same_v<Date, DayNum> || std::is_same_v<Date, ExtendedDayNum>>>
inline auto addYears(Date d, Int64 delta) const
{
return toDayNum(addYearsIndex(d, delta));
if constexpr (std::is_same_v<Date, DayNum>)
return toDayNum(LUTIndexWithSaturation(addYearsIndex(d, delta)));
else
return toDayNum(addYearsIndex(d, delta));
}

2
src/Functions/CustomWeekTransforms.h
View File

@ -76,7 +76,7 @@ struct ToStartOfWeekImpl
}
static inline UInt16 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone)
{
return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode);
return time_zone.toFirstDayNumOfWeek(DayNum(d), week_mode);
}
using FactorTransform = ZeroTransform;

59
src/Functions/DateTimeTransforms.h
View File

@ -84,7 +84,8 @@ struct ToDate32Impl
}
static inline Int32 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return Int32(time_zone.toDayNum(t));
/// Don't saturate.
return Int32(time_zone.toDayNum<Int64>(t));
}
static inline Int32 execute(Int32 d, const DateLUTImpl &)
{
@ -117,7 +118,7 @@ struct ToStartOfDayImpl
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toDate(ExtendedDayNum(d));
return time_zone.toDate(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -143,7 +144,7 @@ struct ToMondayImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d));
return time_zone.toFirstDayNumOfWeek(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -167,7 +168,7 @@ struct ToStartOfMonthImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d));
return time_zone.toFirstDayNumOfMonth(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -191,7 +192,7 @@ struct ToStartOfQuarterImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d));
return time_zone.toFirstDayNumOfQuarter(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -215,7 +216,7 @@ struct ToStartOfYearImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toFirstDayNumOfYear(ExtendedDayNum(d));
return time_zone.toFirstDayNumOfYear(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -224,7 +225,7 @@ struct ToStartOfYearImpl
struct ToTimeImpl
{
/// When transforming to time, the date will be equated to 1970-01-01.
/// When transforming to time, the date will be equated to 1970-01-02.
static constexpr auto name = "toTime";
static UInt32 execute(const DecimalUtils::DecimalComponents<DateTime64> & t, const DateLUTImpl & time_zone)
@ -456,7 +457,7 @@ struct ToYearImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toYear(ExtendedDayNum(d));
return time_zone.toYear(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -480,7 +481,7 @@ struct ToQuarterImpl
}
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toQuarter(ExtendedDayNum(d));
return time_zone.toQuarter(DayNum(d));
}
using FactorTransform = ToStartOfYearImpl;
@ -504,7 +505,7 @@ struct ToMonthImpl
}
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toMonth(ExtendedDayNum(d));
return time_zone.toMonth(DayNum(d));
}
using FactorTransform = ToStartOfYearImpl;
@ -528,7 +529,7 @@ struct ToDayOfMonthImpl
}
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toDayOfMonth(ExtendedDayNum(d));
return time_zone.toDayOfMonth(DayNum(d));
}
using FactorTransform = ToStartOfMonthImpl;
@ -552,7 +553,7 @@ struct ToDayOfWeekImpl
}
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toDayOfWeek(ExtendedDayNum(d));
return time_zone.toDayOfWeek(DayNum(d));
}
using FactorTransform = ToMondayImpl;
@ -576,7 +577,7 @@ struct ToDayOfYearImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toDayOfYear(ExtendedDayNum(d));
return time_zone.toDayOfYear(DayNum(d));
}
using FactorTransform = ToStartOfYearImpl;
@ -699,7 +700,7 @@ struct ToISOYearImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toISOYear(ExtendedDayNum(d));
return time_zone.toISOYear(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -723,7 +724,7 @@ struct ToStartOfISOYearImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d));
return time_zone.toFirstDayNumOfISOYear(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -747,7 +748,7 @@ struct ToISOWeekImpl
}
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toISOWeek(ExtendedDayNum(d));
return time_zone.toISOWeek(DayNum(d));
}
using FactorTransform = ToISOYearImpl;
@ -771,7 +772,7 @@ struct ToRelativeYearNumImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toYear(ExtendedDayNum(d));
return time_zone.toYear(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -795,7 +796,7 @@ struct ToRelativeQuarterNumImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toRelativeQuarterNum(ExtendedDayNum(d));
return time_zone.toRelativeQuarterNum(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -819,7 +820,7 @@ struct ToRelativeMonthNumImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toRelativeMonthNum(ExtendedDayNum(d));
return time_zone.toRelativeMonthNum(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -843,7 +844,7 @@ struct ToRelativeWeekNumImpl
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toRelativeWeekNum(ExtendedDayNum(d));
return time_zone.toRelativeWeekNum(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -892,7 +893,7 @@ struct ToRelativeHourNumImpl
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toRelativeHourNum(ExtendedDayNum(d));
return time_zone.toRelativeHourNum(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -916,7 +917,7 @@ struct ToRelativeMinuteNumImpl
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toRelativeMinuteNum(ExtendedDayNum(d));
return time_zone.toRelativeMinuteNum(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -940,7 +941,7 @@ struct ToRelativeSecondNumImpl
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.fromDayNum(ExtendedDayNum(d));
return time_zone.fromDayNum(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -960,11 +961,11 @@ struct ToYYYYMMImpl
}
static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
{
return time_zone.toNumYYYYMM(static_cast<ExtendedDayNum>(d));
return time_zone.toNumYYYYMM(ExtendedDayNum(d));
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toNumYYYYMM(static_cast<DayNum>(d));
return time_zone.toNumYYYYMM(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -984,11 +985,11 @@ struct ToYYYYMMDDImpl
}
static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
{
return time_zone.toNumYYYYMMDD(static_cast<ExtendedDayNum>(d));
return time_zone.toNumYYYYMMDD(ExtendedDayNum(d));
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toNumYYYYMMDD(static_cast<DayNum>(d));
return time_zone.toNumYYYYMMDD(DayNum(d));
}
using FactorTransform = ZeroTransform;
@ -1008,11 +1009,11 @@ struct ToYYYYMMDDhhmmssImpl
}
static inline UInt64 execute(Int32 d, const DateLUTImpl & time_zone)
{
return time_zone.toNumYYYYMMDDhhmmss(time_zone.toDate(static_cast<ExtendedDayNum>(d)));
return time_zone.toNumYYYYMMDDhhmmss(time_zone.toDate(ExtendedDayNum(d)));
}
static inline UInt64 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return time_zone.toNumYYYYMMDDhhmmss(time_zone.toDate(static_cast<DayNum>(d)));
return time_zone.toNumYYYYMMDDhhmmss(time_zone.toDate(DayNum(d)));
}
using FactorTransform = ZeroTransform;

12
src/Functions/FunctionDateOrDateTimeAddInterval.h
View File

@ -58,7 +58,7 @@ struct AddSecondsImpl
}
static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
{
return time_zone.fromDayNum(ExtendedDayNum(d)) + delta;
return time_zone.fromDayNum(DayNum(d)) + delta;
}
};
@ -83,7 +83,7 @@ struct AddMinutesImpl
}
static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
{
return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 60;
return time_zone.fromDayNum(DayNum(d)) + delta * 60;
}
};
@ -107,7 +107,7 @@ struct AddHoursImpl
}
static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
{
return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 3600;
return time_zone.fromDayNum(DayNum(d)) + delta * 3600;
}
};
@ -180,7 +180,7 @@ struct AddMonthsImpl
static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
{
return time_zone.addMonths(ExtendedDayNum(d), delta);
return time_zone.addMonths(DayNum(d), delta);
}
static inline Int32 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)
@ -206,7 +206,7 @@ struct AddQuartersImpl
static inline UInt16 execute(UInt16 d, Int32 delta, const DateLUTImpl & time_zone)
{
return time_zone.addQuarters(ExtendedDayNum(d), delta);
return time_zone.addQuarters(DayNum(d), delta);
}
static inline Int32 execute(Int32 d, Int32 delta, const DateLUTImpl & time_zone)
@ -232,7 +232,7 @@ struct AddYearsImpl
static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
{
return time_zone.addYears(ExtendedDayNum(d), delta);
return time_zone.addYears(DayNum(d), delta);
}
static inline Int32 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)

4
src/Functions/FunctionsTimeWindow.h
View File

@ -48,7 +48,7 @@ struct ToStartOfTransform;
template <> \
struct ToStartOfTransform<IntervalKind::INTERVAL_KIND> \
{ \
static ExtendedDayNum execute(UInt32 t, UInt64 delta, const DateLUTImpl & time_zone) \
static auto execute(UInt32 t, UInt64 delta, const DateLUTImpl & time_zone) \
{ \
return time_zone.toStartOf##INTERVAL_KIND##Interval(time_zone.toDayNum(t), delta); \
} \
@ -89,7 +89,7 @@ struct ToStartOfTransform;
template <> \
struct AddTime<IntervalKind::INTERVAL_KIND> \
{ \
static inline ExtendedDayNum execute(UInt16 d, UInt64 delta, const DateLUTImpl & time_zone) \
static inline auto execute(UInt16 d, UInt64 delta, const DateLUTImpl & time_zone) \
{ \
return time_zone.add##INTERVAL_KIND##s(ExtendedDayNum(d), delta); \
} \

8
src/Functions/toStartOfInterval.cpp
View File

@ -37,7 +37,7 @@ namespace
static UInt16 execute(UInt16 d, UInt64 years, const DateLUTImpl & time_zone)
{
return time_zone.toStartOfYearInterval(ExtendedDayNum(d), years);
return time_zone.toStartOfYearInterval(DayNum(d), years);
}
static UInt16 execute(Int32 d, UInt64 years, const DateLUTImpl & time_zone)
@ -63,7 +63,7 @@ namespace
static UInt16 execute(UInt16 d, UInt64 quarters, const DateLUTImpl & time_zone)
{
return time_zone.toStartOfQuarterInterval(ExtendedDayNum(d), quarters);
return time_zone.toStartOfQuarterInterval(DayNum(d), quarters);
}
static UInt16 execute(Int32 d, UInt64 quarters, const DateLUTImpl & time_zone)
@ -89,7 +89,7 @@ namespace
static UInt16 execute(UInt16 d, UInt64 months, const DateLUTImpl & time_zone)
{
return time_zone.toStartOfMonthInterval(ExtendedDayNum(d), months);
return time_zone.toStartOfMonthInterval(DayNum(d), months);
}
static UInt16 execute(Int32 d, UInt64 months, const DateLUTImpl & time_zone)
@ -115,7 +115,7 @@ namespace
static UInt16 execute(UInt16 d, UInt64 weeks, const DateLUTImpl & time_zone)
{
return time_zone.toStartOfWeekInterval(ExtendedDayNum(d), weeks);
return time_zone.toStartOfWeekInterval(DayNum(d), weeks);
}
static UInt16 execute(Int32 d, UInt64 weeks, const DateLUTImpl & time_zone)

11
src/Interpreters/InterpreterInsertQuery.cpp
View File

@ -263,6 +263,10 @@ BlockIO InterpreterInsertQuery::execute()
QueryPipelineBuilder pipeline;
StoragePtr table = getTable(query);
StoragePtr inner_table;
if (const auto * mv = dynamic_cast<const StorageMaterializedView *>(table.get()))
inner_table = mv->getTargetTable();
if (query.partition_by && !table->supportsPartitionBy())
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "PARTITION BY clause is not supported by storage");
@ -450,11 +454,8 @@ BlockIO InterpreterInsertQuery::execute()
}
res.pipeline.addStorageHolder(table);
if (const auto * mv = dynamic_cast<const StorageMaterializedView *>(table.get()))
{
if (auto inner_table = mv->tryGetTargetTable())
res.pipeline.addStorageHolder(inner_table);
}
if (inner_table)
res.pipeline.addStorageHolder(inner_table);
return res;
}

8
tests/ci/pr_info.py
View File

@ -1,7 +1,6 @@
#!/usr/bin/env python3
import json
import os
import urllib
import requests
from unidiff import PatchSet
@ -140,16 +139,15 @@ class PRInfo:
if not self.diff_url:
raise Exception("Diff URL cannot be find for event")
response = requests.get(self.diff_url)
response.raise_for_status()
if 'commits' in self.event and self.number == 0:
response = requests.get(self.diff_url)
response.raise_for_status()
diff = response.json()
if 'files' in diff:
self.changed_files = [f['filename'] for f in diff['files']]
else:
diff = urllib.request.urlopen(self.diff_url)
diff_object = PatchSet(diff, diff.headers.get_charsets()[0])
diff_object = PatchSet(response.text)
self.changed_files = {f.path for f in diff_object}
def get_dict(self):

24
tests/integration/test_storage_kafka/test.py
View File

@ -445,15 +445,21 @@ def test_kafka_formats(kafka_cluster):
# /src/Processors/Formats/IRowInputFormat.cpp:0: DB::IRowInputFormat::generate() @ 0x1de72710 in /usr/bin/clickhouse
],
},
# 'Template' : {
# 'data_sample' : [
# '(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)',
# # '(id = 1, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 2, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 3, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 4, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 5, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 6, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 7, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 8, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 9, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 10, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 11, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 12, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 13, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 14, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 15, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)',
# # '(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)',
# # '' # tolerates
# ],
# 'extra_settings': ", format_template_row='template_row.format'"
# },
'CustomSeparated' : {
'data_sample' : [
'0\t0\tAM\t0.5\t1\n',
'1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n',
'0\t0\tAM\t0.5\t1\n',
],
},
'Template' : {
'data_sample' : [
'(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)',
'(id = 1, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 2, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 3, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 4, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 5, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 6, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 7, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 8, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 9, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 10, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 11, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 12, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 13, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 14, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 15, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)',
'(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)',
],
'extra_settings': ", format_template_row='template_row.format'"
},
'Regexp': {
'data_sample': [
'(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)',

6
tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper_long.reference
View File

@ -8,10 +8,10 @@ Hello 1
Word 1
date1 date2 value1 value2
2019-10-02 2018-10-02 1 1
CREATE TABLE default.table_rename_with_ttl\n(\n `date1` Date,\n `date2` Date,\n `value1` String,\n `value2` String TTL date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/default/test_01213/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192
CREATE TABLE default.table_rename_with_ttl\n(\n `date1` Date,\n `date2` Date,\n `value1` String,\n `value2` String TTL date1 + toIntervalMonth(500)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/default/test_01213/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL date2 + toIntervalMonth(500)\nSETTINGS index_granularity = 8192
renamed_date1 date2 value1 value2
2019-10-02 2018-10-02 1 1
CREATE TABLE default.table_rename_with_ttl\n(\n `renamed_date1` Date,\n `date2` Date,\n `value1` String,\n `value2` String TTL renamed_date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/default/test_01213/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192
CREATE TABLE default.table_rename_with_ttl\n(\n `renamed_date1` Date,\n `date2` Date,\n `value1` String,\n `value2` String TTL renamed_date1 + toIntervalMonth(500)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/default/test_01213/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL date2 + toIntervalMonth(500)\nSETTINGS index_granularity = 8192
renamed_date1 renamed_date2 value1 value2
2019-10-02 2018-10-02 1 1
CREATE TABLE default.table_rename_with_ttl\n(\n `renamed_date1` Date,\n `renamed_date2` Date,\n `value1` String,\n `value2` String TTL renamed_date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/default/test_01213/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL renamed_date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192
CREATE TABLE default.table_rename_with_ttl\n(\n `renamed_date1` Date,\n `renamed_date2` Date,\n `value1` String,\n `value2` String TTL renamed_date1 + toIntervalMonth(500)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/default/test_01213/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL renamed_date2 + toIntervalMonth(500)\nSETTINGS index_granularity = 8192

4
tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper_long.sql
View File

@ -38,11 +38,11 @@ CREATE TABLE table_rename_with_ttl
date1 Date,
date2 Date,
value1 String,
value2 String TTL date1 + INTERVAL 10000 MONTH
value2 String TTL date1 + INTERVAL 500 MONTH
)
ENGINE = ReplicatedMergeTree('/clickhouse/{database}/test_01213/table_rename_with_ttl', '1')
ORDER BY tuple()
TTL date2 + INTERVAL 10000 MONTH;
TTL date2 + INTERVAL 500 MONTH;
INSERT INTO table_rename_with_ttl SELECT toDateTime(toDate('2019-10-01') + number % 3, 'Europe/Moscow'), toDateTime(toDate('2018-10-01') + number % 3, 'Europe/Moscow'), toString(number), toString(number) from numbers(9);

1
tests/queries/0_stateless/01922_sum_null_for_remote.reference Normal file
View File

@ -0,0 +1 @@
\N

1
tests/queries/0_stateless/01922_sum_null_for_remote.sql Normal file
View File

@ -0,0 +1 @@
select sum(null) from remote('127.0.0.{1,2}', 'system', 'one')

23
tests/queries/0_stateless/02003_memory_limit_in_client.expect
View File

@ -16,6 +16,10 @@ expect_after {
}
set basedir [file dirname $argv0]
#
# Check that the query will fail in clickhouse-client
#
spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1"
expect ":) "
@ -28,7 +32,24 @@ expect ":) "
send -- "\4"
expect eof
set basedir [file dirname $argv0]
#
# Check that the query will fail in clickhouse-client
#
spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1"
expect ":) "
send -- "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000)\r"
expect "Code: 241"
expect ":) "
# Exit.
send -- "\4"
expect eof
#
# Check that the query will not fail (due to max_untracked_memory)
#
spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1"
expect ":) "

30
tests/queries/0_stateless/02096_date_time_1970_saturation.reference Normal file
View File

@ -0,0 +1,30 @@
1970-01-01
1970-01-01 03:00:00
1970-01-01
1970-01-01
1970-01-01
1970-01-01
1970-01-01
1970-01-01
1970-01-01
1970-01-01
1970-01-01
1970-01-01
1970-01-02 03:00:00
1970-01-01 03:00:00
1970-01-01 03:00:00
1970-01-01 03:00:00
1970-01-01 03:00:00
1970-01-01 03:00:00
1969-12-31 16:00:00
1970-01-01
1970-01-01
1970-01-01
1970-01-01
1970-01-01
1970-01-02 16:00:00
1969-12-31 16:00:00
1969-12-31 16:00:00
1969-12-31 16:00:00
1969-12-31 16:00:00
1969-12-31 16:00:00

31
tests/queries/0_stateless/02096_date_time_1970_saturation.sql Normal file
View File

@ -0,0 +1,31 @@
select toDate(0);
select toDateTime(0, 'Europe/Moscow');
select toMonday(toDate(0));
select toMonday(toDateTime(0, 'Europe/Moscow'));
select toStartOfWeek(toDate(0));
select toStartOfWeek(toDateTime(0, 'Europe/Moscow'));
select toStartOfMonth(toDate(0));
select toStartOfMonth(toDateTime(0, 'Europe/Moscow'));
select toStartOfQuarter(toDate(0));
select toStartOfQuarter(toDateTime(0, 'Europe/Moscow'));
select toStartOfYear(toDate(0));
select toStartOfYear(toDateTime(0, 'Europe/Moscow'));
select toTime(toDateTime(0, 'Europe/Moscow'));
select toStartOfMinute(toDateTime(0, 'Europe/Moscow'));
select toStartOfFiveMinute(toDateTime(0, 'Europe/Moscow'));
select toStartOfTenMinutes(toDateTime(0, 'Europe/Moscow'));
select toStartOfFifteenMinutes(toDateTime(0, 'Europe/Moscow'));
select toStartOfHour(toDateTime(0, 'Europe/Moscow'));
select toDateTime(0, 'America/Los_Angeles');
select toMonday(toDateTime(0, 'America/Los_Angeles'));
select toStartOfWeek(toDateTime(0, 'America/Los_Angeles'));
select toStartOfMonth(toDateTime(0, 'America/Los_Angeles'));
select toStartOfQuarter(toDateTime(0, 'America/Los_Angeles'));
select toStartOfYear(toDateTime(0, 'America/Los_Angeles'));
select toTime(toDateTime(0, 'America/Los_Angeles'), 'America/Los_Angeles');
select toStartOfMinute(toDateTime(0, 'America/Los_Angeles'));
select toStartOfFiveMinute(toDateTime(0, 'America/Los_Angeles'));
select toStartOfTenMinutes(toDateTime(0, 'America/Los_Angeles'));
select toStartOfFifteenMinutes(toDateTime(0, 'America/Los_Angeles'));
select toStartOfHour(toDateTime(0, 'America/Los_Angeles'));

1
tests/queries/0_stateless/02158_interval_length_sum.reference Normal file
View File

@ -0,0 +1 @@
11

1
tests/queries/0_stateless/02158_interval_length_sum.sql Normal file
View File

@ -0,0 +1 @@
SELECT intervalLengthSum(x, y) FROM values('x Int64, y Int64', (0, 10), (5, 5), (5, 6), (1, -1));

2
website/templates/index/success.html
View File

@ -62,7 +62,7 @@
</div>
<div class="col-lg-auto pb-5 pb-lg-0 px-2">
<p class="mb-3 text-dark">{{ _('Uber moved it’s logging platform to ClickHouse increasing developer productivity and overall reliability of the platform while seeing 3x data compression, 10x performance increase, and ½ the reduction in hardware cost.') }}</p>
<p class="mb-3 text-dark">{{ _('Uber moved its logging platform to ClickHouse increasing developer productivity and overall reliability of the platform while seeing 3x data compression, 10x performance increase, and ½ the reduction in hardware cost.') }}</p>
<a class="trailing-link" href="https://eng.uber.com/logging/" rel="external nofollow noreferrer">{{ _('Read the Case Study') }}</a>