From f0163c2acfe41c78124e49582301e896ee3f8240 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Fri, 12 Feb 2021 17:02:04 +0300 Subject: [PATCH 001/233] Don't create empty parts on INSERT --- src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp | 6 ++++++ src/Storages/MergeTree/MergeTreeDataWriter.cpp | 5 +++++ .../MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp | 5 +++++ .../queries/0_stateless/01560_optimize_on_insert.reference | 1 + tests/queries/0_stateless/01560_optimize_on_insert.sql | 7 +++++++ 5 files changed, 24 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp index 904081cc1df..bb5644567ae 100644 --- a/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp @@ -29,6 +29,12 @@ void MergeTreeBlockOutputStream::write(const Block & block) Stopwatch watch; MergeTreeData::MutableDataPartPtr part = storage.writer.writeTempPart(current_block, metadata_snapshot, optimize_on_insert); + + /// If optimize_on_insert setting is true, current_block could become empty after merge + /// and we didn't create part. + if (!part) + continue; + storage.renameTempPartAndAdd(part, &storage.increment); PartLog::addNewPart(storage.global_context, part, watch.elapsed()); diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 5a9bdd90bc8..5929293d714 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -327,6 +327,11 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// Size of part would not be greater than block.bytes() + epsilon size_t expected_size = block.bytes(); + /// If optimize_on_insert is true, block may become empty after merge. + /// There is no need to create empty part. + if (expected_size == 0) + return nullptr; + DB::IMergeTreeDataPart::TTLInfos move_ttl_infos; const auto & move_ttl_entries = metadata_snapshot->getMoveTTLs(); for (const auto & ttl_entry : move_ttl_entries) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 7046a510f75..6f90d9f00a9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -144,6 +144,11 @@ void ReplicatedMergeTreeBlockOutputStream::write(const Block & block) MergeTreeData::MutableDataPartPtr part = storage.writer.writeTempPart(current_block, metadata_snapshot, optimize_on_insert); + /// If optimize_on_insert setting is true, current_block could become empty after merge + /// and we didn't create part. + if (!part) + continue; + String block_id; if (deduplicate) diff --git a/tests/queries/0_stateless/01560_optimize_on_insert.reference b/tests/queries/0_stateless/01560_optimize_on_insert.reference index 7ace2043be0..477f48be7a9 100644 --- a/tests/queries/0_stateless/01560_optimize_on_insert.reference +++ b/tests/queries/0_stateless/01560_optimize_on_insert.reference @@ -11,3 +11,4 @@ Summing Merge Tree Aggregating Merge Tree 1 5 2020-01-01 00:00:00 2 5 2020-01-02 00:00:00 +Check creating empty parts diff --git a/tests/queries/0_stateless/01560_optimize_on_insert.sql b/tests/queries/0_stateless/01560_optimize_on_insert.sql index 9f6dac686bb..f64f4c75cfe 100644 --- a/tests/queries/0_stateless/01560_optimize_on_insert.sql +++ b/tests/queries/0_stateless/01560_optimize_on_insert.sql @@ -33,3 +33,10 @@ INSERT INTO aggregating_merge_tree VALUES (1, 1, '2020-01-01'), (2, 1, '2020-01- SELECT * FROM aggregating_merge_tree ORDER BY key; DROP TABLE aggregating_merge_tree; +SELECT 'Check creating empty parts'; +DROP TABLE IF EXISTS empty; +CREATE TABLE empty (key UInt32, val UInt32, date Datetime) ENGINE=SummingMergeTree(val) PARTITION BY date ORDER BY key; +INSERT INTO empty VALUES (1, 1, '2020-01-01'), (1, 1, '2020-01-01'), (1, -2, '2020-01-01'); +SELECT * FROM empty ORDER BY key; +SELECT table, partition, active FROM system.parts where table = 'empty' and active = 1; +DROP TABLE empty; From 6eeef74d4389d97fcd614d3ae0b49025c6ac1a91 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Feb 2021 00:32:39 +0300 Subject: [PATCH 002/233] first draft --- docs/en/sql-reference/statements/detach.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 62a7c0cc1e0..f3f8b053724 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -5,7 +5,9 @@ toc_title: DETACH # DETACH Statement {#detach} -Deletes information about the ‘name’ table from the server. The server stops knowing about the table’s existence. +Deletes information about the `name` table from the server. The server stops knowing about the table’s existence. + +Syntax: ``` sql DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] @@ -13,4 +15,20 @@ DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] This does not delete the table’s data or metadata. On the next server launch, the server will read the metadata and find out about the table again. -Similarly, a “detached” table can be re-attached using the `ATTACH` query (with the exception of system tables, which do not have metadata stored for them). +Similarly, a “detached” table can be re-attached using the [ATTACH](../../sql-reference/statements/attach) query (with the exception of system tables, which do not have metadata stored for them). + +## DETACH PERMAMENTLY {#detach-permamently} + +Deletes information about `name` table or view from the server. Permamently detached tables won't automatically reappear after the server restart. + +Syntax: + +``` sql +DETACH TABLE/VIEW [IF EXISTS] [db.]name PERMAMENTLY [ON CLUSTER cluster] +``` + +This statement does not delete the table’s data or metadata. + +Permamently detached table or view can be reattached with [ATTACH](../../sql-reference/statements/attach) query and can be shown with [SHOW CREATE TABLE](../../sql-reference/statements/show.md#show-create-table) query. + +[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From 341e7bc8482e99478a0e40ea1afa446ca15f9312 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Feb 2021 13:23:52 +0300 Subject: [PATCH 003/233] Fixed links --- docs/en/sql-reference/statements/detach.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index f3f8b053724..b2720acaaa5 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -15,7 +15,7 @@ DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] This does not delete the table’s data or metadata. On the next server launch, the server will read the metadata and find out about the table again. -Similarly, a “detached” table can be re-attached using the [ATTACH](../../sql-reference/statements/attach) query (with the exception of system tables, which do not have metadata stored for them). +Similarly, a “detached” table can be re-attached using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). ## DETACH PERMAMENTLY {#detach-permamently} @@ -29,6 +29,6 @@ DETACH TABLE/VIEW [IF EXISTS] [db.]name PERMAMENTLY [ON CLUSTER cluster] This statement does not delete the table’s data or metadata. -Permamently detached table or view can be reattached with [ATTACH](../../sql-reference/statements/attach) query and can be shown with [SHOW CREATE TABLE](../../sql-reference/statements/show.md#show-create-table) query. +Permamently detached table or view can be reattached with [ATTACH](../../sql-reference/statements/attach.md) query and can be shown with [SHOW CREATE TABLE](../../sql-reference/statements/show.md#show-create-table) query. [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From 17d7a49106342536a0348c020ca92e1cafc52434 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Feb 2021 13:33:20 +0300 Subject: [PATCH 004/233] Fixed typos --- docs/en/sql-reference/statements/detach.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index b2720acaaa5..adb2df570d7 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -17,9 +17,9 @@ This does not delete the table’s data or metadata. On the next server launch, Similarly, a “detached” table can be re-attached using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). -## DETACH PERMAMENTLY {#detach-permamently} +## DETACH PERMANENTLY {#detach-permanently} -Deletes information about `name` table or view from the server. Permamently detached tables won't automatically reappear after the server restart. +Deletes information about `name` table or view from the server. Permanently detached tables won't automatically reappear after the server restart. Syntax: @@ -29,6 +29,6 @@ DETACH TABLE/VIEW [IF EXISTS] [db.]name PERMAMENTLY [ON CLUSTER cluster] This statement does not delete the table’s data or metadata. -Permamently detached table or view can be reattached with [ATTACH](../../sql-reference/statements/attach.md) query and can be shown with [SHOW CREATE TABLE](../../sql-reference/statements/show.md#show-create-table) query. +Permanently detached table or view can be reattached with [ATTACH](../../sql-reference/statements/attach.md) query and can be shown with [SHOW CREATE TABLE](../../sql-reference/statements/show.md#show-create-table) query. [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From 2d03d330bcc400a0b61c8028b01587de072aa60e Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 17 Apr 2020 16:26:44 +0300 Subject: [PATCH 005/233] Extended range of DateTime64 to years 1925 - 2238 The Year 1925 is a starting point because most of the timezones switched to saner (mostly 15-minutes based) offsets somewhere during 1924 or before. And that significantly simplifies implementation. 2238 is to simplify arithmetics for sanitizing LUT index access; there are less than 0x1ffff days from 1925. * Extended DateLUTImpl internal LUT to 0x1ffff items, some of which represent negative (pre-1970) time values. As a collateral benefit, Date now correctly supports dates up to 2149 (instead of 2106). * Added a new strong typedef ExtendedDayNum, which represents dates pre-1970 and post 2149. * Functions that used to return DayNum now return ExtendedDayNum. * Refactored DateLUTImpl to untie DayNum from the dual role of being a value and an index (due to negative time). Index is now a different type LUTIndex with explicit conversion functions from DatNum, time_t, and ExtendedDayNum. * Updated DateLUTImpl to properly support values close to epoch start (1970-01-01 00:00), including negative ones. * Reduced resolution of DateLUTImpl::Values::time_at_offset_change to multiple of 15-minutes to allow storing 64-bits of time_t in DateLUTImpl::Value while keeping same size. * Minor performance updates to DateLUTImpl when building month LUT by skipping non-start-of-month days. * Fixed extractTimeZoneFromFunctionArguments to work correctly with DateTime64. * New unit-tests and stateless integration tests for both DateTime and DateTime64. --- base/common/DateLUT.h | 1 - base/common/DateLUTImpl.cpp | 42 +- base/common/DateLUTImpl.h | 683 +++++++++++------- base/common/DayNum.h | 5 + base/common/LocalDate.h | 3 +- base/common/strong_typedef.h | 1 + base/common/tests/CMakeLists.txt | 4 +- base/common/tests/gtest_DateLutImpl.cpp | 515 +++++++++++++ programs/client/Client.cpp | 2 +- src/Core/DecimalFunctions.h | 18 +- src/Core/MySQL/MySQLReplication.cpp | 8 +- src/Core/tests/gtest_DecimalFunctions.cpp | 2 +- src/DataStreams/MongoDBBlockInputStream.cpp | 4 +- src/DataTypes/DataTypeDateTime64.h | 61 -- src/Functions/CustomWeekTransforms.h | 24 +- src/Functions/DateTimeTransforms.h | 225 +++++- src/Functions/FunctionCustomWeekToSomething.h | 1 + .../FunctionDateOrDateTimeAddInterval.h | 143 ++-- .../FunctionDateOrDateTimeToSomething.h | 3 +- src/Functions/FunctionsConversion.h | 106 +-- src/Functions/TransformDateTime64.h | 92 +++ src/Functions/dateDiff.cpp | 4 +- .../extractTimeZoneFromFunctionArguments.cpp | 5 +- src/Functions/formatDateTime.cpp | 6 +- src/Functions/now64.cpp | 2 +- src/Functions/toStartOfInterval.cpp | 50 +- src/Functions/today.cpp | 2 +- src/IO/ReadHelpers.h | 10 +- src/IO/WriteHelpers.h | 16 +- src/IO/parseDateTimeBestEffort.cpp | 15 +- src/IO/parseDateTimeBestEffort.h | 1 + src/Interpreters/CrashLog.cpp | 2 +- src/Interpreters/MetricLog.cpp | 2 +- src/Interpreters/OpenTelemetrySpanLog.cpp | 2 +- src/Interpreters/PartLog.cpp | 2 +- src/Interpreters/QueryLog.cpp | 2 +- src/Interpreters/QueryThreadLog.cpp | 2 +- src/Interpreters/TextLog.cpp | 2 +- src/Interpreters/TraceLog.cpp | 2 +- src/Interpreters/convertFieldToType.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/tests/part_name.cpp | 2 +- .../00921_datetime64_compatibility.python | 5 +- .../00921_datetime64_compatibility.reference | 152 ---- .../00921_datetime64_compatibility.sh | 2 +- .../01252_weird_time_zone.reference | 14 +- .../0_stateless/01252_weird_time_zone.sql | 28 +- .../01440_to_date_monotonicity.reference | 2 +- .../01561_Date_and_DateTime64_comparision.sql | 6 +- ...1_date_overflow_as_partition_key.reference | 4 +- .../01631_date_overflow_as_partition_key.sql | 2 +- .../01691_DateTime64_clamp.reference | 23 + .../0_stateless/01691_DateTime64_clamp.sql | 7 + .../convert-month-partitioned-parts/main.cpp | 5 +- 54 files changed, 1585 insertions(+), 741 deletions(-) create mode 100644 base/common/tests/gtest_DateLutImpl.cpp create mode 100644 src/Functions/TransformDateTime64.h diff --git a/base/common/DateLUT.h b/base/common/DateLUT.h index 93c6cb403e2..378b4360f3b 100644 --- a/base/common/DateLUT.h +++ b/base/common/DateLUT.h @@ -32,7 +32,6 @@ public: return date_lut.getImplementation(time_zone); } - static void setDefaultTimezone(const std::string & time_zone) { auto & date_lut = getInstance(); diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 50620e21b8f..906f88fa90f 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -46,19 +46,26 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) if (&inside_main) assert(inside_main); - size_t i = 0; - time_t start_of_day = 0; cctz::time_zone cctz_time_zone; if (!cctz::load_time_zone(time_zone, &cctz_time_zone)) throw Poco::Exception("Cannot load time zone " + time_zone_); - cctz::time_zone::absolute_lookup start_of_epoch_lookup = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(start_of_day)); - offset_at_start_of_epoch = start_of_epoch_lookup.offset; + const cctz::civil_day epoch{1970, 1, 1}; + const cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1}; + time_t start_of_day = std::chrono::system_clock::to_time_t(cctz_time_zone.lookup(lut_start).pre); + time_offset_epoch = cctz::convert(cctz::civil_second(lut_start), cctz_time_zone).time_since_epoch().count(); + + // Note validated this against all timezones in the system. + assert((epoch - lut_start) == daynum_offset_epoch); + + offset_at_start_of_epoch = cctz_time_zone.lookup(cctz_time_zone.lookup(epoch).pre).offset; + offset_at_start_of_lut = cctz_time_zone.lookup(cctz_time_zone.lookup(lut_start).pre).offset; offset_is_whole_number_of_hours_everytime = true; - cctz::civil_day date{1970, 1, 1}; + cctz::civil_day date = lut_start; + UInt32 i = 0; do { cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date); @@ -72,7 +79,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.day_of_week = getDayOfWeek(date); values.date = start_of_day; - assert(values.year >= DATE_LUT_MIN_YEAR && values.year <= DATE_LUT_MAX_YEAR); + assert(values.year >= DATE_LUT_MIN_YEAR && values.year <= DATE_LUT_MAX_YEAR + 1); assert(values.month >= 1 && values.month <= 12); assert(values.day_of_month >= 1 && values.day_of_month <= 31); assert(values.day_of_week >= 1 && values.day_of_week <= 7); @@ -85,10 +92,13 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) else values.days_in_month = i != 0 ? lut[i - 1].days_in_month : 31; - values.time_at_offset_change = 0; - values.amount_of_offset_change = 0; + values.time_at_offset_change_value = 0; + values.amount_of_offset_change_value = 0; - if (start_of_day % 3600) + // TODO: this partially ignores fractional pre-epoch offsets, which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe\Minsk + // when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. + // https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 + if (start_of_day > 0 && start_of_day % 3600) offset_is_whole_number_of_hours_everytime = false; /// If UTC offset was changed in previous day. @@ -97,7 +107,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) auto amount_of_offset_change_at_prev_day = 86400 - (lut[i].date - lut[i - 1].date); if (amount_of_offset_change_at_prev_day) { - lut[i - 1].amount_of_offset_change = amount_of_offset_change_at_prev_day; + lut[i - 1].amount_of_offset_change_value = amount_of_offset_change_at_prev_day / Values::OffsetChangeFactor; const auto utc_offset_at_beginning_of_day = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(lut[i - 1].date)).offset; @@ -116,11 +126,11 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) time_at_offset_change += 900; } - lut[i - 1].time_at_offset_change = time_at_offset_change; + lut[i - 1].time_at_offset_change_value = time_at_offset_change / Values::OffsetChangeFactor; - /// We doesn't support cases when time change results in switching to previous day. - if (static_cast(lut[i - 1].time_at_offset_change) + static_cast(lut[i - 1].amount_of_offset_change) < 0) - lut[i - 1].time_at_offset_change = -lut[i - 1].amount_of_offset_change; + /// We don't support cases when time change results in switching to previous day. + if (static_cast(lut[i - 1].time_at_offset_change()) + static_cast(lut[i - 1].amount_of_offset_change()) < 0) + lut[i - 1].time_at_offset_change_value = -lut[i - 1].amount_of_offset_change_value; } } @@ -128,7 +138,9 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) ++date; ++i; } - while (start_of_day <= DATE_LUT_MAX && i <= DATE_LUT_MAX_DAY_NUM); + while (i < DATE_LUT_SIZE && lut[i - 1].year <= DATE_LUT_MAX_YEAR); + +// date_lut_max = start_of_day; /// Fill excessive part of lookup table. This is needed only to simplify handling of overflow cases. while (i < DATE_LUT_SIZE) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 064787fb64e..adfffb04681 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -11,9 +11,9 @@ #define DATE_LUT_MAX (0xFFFFFFFFU - 86400) #define DATE_LUT_MAX_DAY_NUM (0xFFFFFFFFU / 86400) /// Table size is bigger than DATE_LUT_MAX_DAY_NUM to fill all indices within UInt16 range: this allows to remove extra check. -#define DATE_LUT_SIZE 0x10000 -#define DATE_LUT_MIN_YEAR 1970 -#define DATE_LUT_MAX_YEAR 2106 /// Last supported year (incomplete) +#define DATE_LUT_SIZE 0x20000 +#define DATE_LUT_MIN_YEAR 1925 /// 1925 since wast majority of timezones changed to 15-minute aligned offsets somewhere in 1924 or earlier. +#define DATE_LUT_MAX_YEAR 2283 /// Last supported year (complete) #define DATE_LUT_YEARS (1 + DATE_LUT_MAX_YEAR - DATE_LUT_MIN_YEAR) /// Number of years in lookup table #if defined(__PPC__) @@ -45,14 +45,68 @@ public: DateLUTImpl(const DateLUTImpl &&) = delete; DateLUTImpl & operator=(const DateLUTImpl &&) = delete; + // Normalized and bound-checked index of element in lut, + // has to be a separate type to support overloading + // TODO: make sure that any arithmetic on LUTIndex actually results in valid LUTIndex. + STRONG_TYPEDEF(UInt32, LUTIndex) + template + friend inline LUTIndex operator+(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() + v) & date_lut_mask}; + } + template + friend inline LUTIndex operator+(const T v, const LUTIndex & index) + { + return LUTIndex{(v + index.toUnderType()) & date_lut_mask}; + } + friend inline LUTIndex operator+(const LUTIndex & index, const LUTIndex & v) + { + return LUTIndex{(index.toUnderType() + v.toUnderType()) & date_lut_mask}; + } + + template + friend inline LUTIndex operator-(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() - v) & date_lut_mask}; + } + template + friend inline LUTIndex operator-(const T v, const LUTIndex & index) + { + return LUTIndex{(v - index.toUnderType()) & date_lut_mask}; + } + friend inline LUTIndex operator-(const LUTIndex & index, const LUTIndex & v) + { + return LUTIndex{(index.toUnderType() - v.toUnderType()) & date_lut_mask}; + } + + template + friend inline LUTIndex operator*(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() * v) & date_lut_mask}; + } + template + friend inline LUTIndex operator*(const T v, const LUTIndex & index) + { + return LUTIndex{(v * index.toUnderType()) & date_lut_mask}; + } + + template + friend inline LUTIndex operator/(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() / v) & date_lut_mask}; + } + template + friend inline LUTIndex operator/(const T v, const LUTIndex & index) + { + return LUTIndex{(v / index.toUnderType()) & date_lut_mask}; + } + public: /// The order of fields matters for alignment and sizeof. struct Values { - /// Least significat 32 bits from time_t at beginning of the day. - /// If the unix timestamp of beginning of the day is negative (example: 1970-01-01 MSK, where time_t == -10800), then value will overflow. - /// Change to time_t; change constants above; and recompile the sources if you need to support time after 2105 year. - UInt32 date; + /// Least significat 64 bits from time_t at beginning of the day. + Int64 date; /// Properties of the day. UInt16 year; @@ -65,107 +119,175 @@ public: UInt8 days_in_month; /// For days, when offset from UTC was changed due to daylight saving time or permanent change, following values could be non zero. - Int16 amount_of_offset_change; /// Usually -3600 or 3600, but look at Lord Howe Island. - UInt32 time_at_offset_change; /// In seconds from beginning of the day. + Int8 amount_of_offset_change_value; /// Usually -3600 or 3600, but look at Lord Howe Island. multiply by OffsetChangeFactor + UInt8 time_at_offset_change_value; /// In seconds from beginning of the day. multiply by OffsetChangeFactor + + inline Int32 amount_of_offset_change() const + { + return static_cast(amount_of_offset_change_value) * OffsetChangeFactor; + } + + inline UInt32 time_at_offset_change() const + { + return static_cast(time_at_offset_change_value) * OffsetChangeFactor; + } + + /// Since most of the modern timezones have a DST change aligned to 15 minutes, to save as much space as possible inside Value, + /// we are dividing any offset change related value by this factor before setting it to Value, + /// hence it has to be explicitly multiplied back by this factor before being used. + static const UInt16 OffsetChangeFactor = 900; }; static_assert(sizeof(Values) == 16); private: - /// Lookup table is indexed by DayNum. + + // Mask is all-ones to allow efficient protection against overflow. + static const UInt32 date_lut_mask = 0x1ffff; + static_assert(date_lut_mask == DATE_LUT_SIZE - 1); + + const UInt32 daynum_offset_epoch = 16436; // offset to epoch in days (ExtendedDayNum) of the first day in LUT. + + /// Lookup table is indexed by LUTIndex. /// Day nums are the same in all time zones. 1970-01-01 is 0 and so on. /// Table is relatively large, so better not to place the object on stack. /// In comparison to std::vector, plain array is cheaper by one indirection. - Values lut[DATE_LUT_SIZE]; + Values lut[DATE_LUT_SIZE + 1]; - /// Year number after DATE_LUT_MIN_YEAR -> day num for start of year. - DayNum years_lut[DATE_LUT_YEARS]; + /// Year number after DATE_LUT_MIN_YEAR -> LUTIndex in lut for start of year. + LUTIndex years_lut[DATE_LUT_YEARS]; /// Year number after DATE_LUT_MIN_YEAR * month number starting at zero -> day num for first day of month - DayNum years_months_lut[DATE_LUT_YEARS * 12]; + LUTIndex years_months_lut[DATE_LUT_YEARS * 12]; /// UTC offset at beginning of the Unix epoch. The same as unix timestamp of 1970-01-01 00:00:00 local time. time_t offset_at_start_of_epoch; + time_t offset_at_start_of_lut; bool offset_is_whole_number_of_hours_everytime; + time_t time_offset_epoch; /// Time zone name. std::string time_zone; - - /// We can correctly process only timestamps that less DATE_LUT_MAX (i.e. up to 2105 year inclusively) - /// We don't care about overflow. - inline DayNum findIndex(time_t t) const + inline LUTIndex findIndex(time_t t) const { /// First guess. - DayNum guess(t / 86400); + const UInt32 guess = ((t / 86400) + daynum_offset_epoch) & date_lut_mask; /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. - - if ((guess == 0 || t >= lut[guess].date) && t < lut[DayNum(guess + 1)].date) - return guess; + if ((guess == daynum_offset_epoch || t >= lut[guess].date) && t < lut[UInt32(guess + 1)].date) + return LUTIndex{guess}; /// Time zones that have offset 0 from UTC do daylight saving time change (if any) towards increasing UTC offset (example: British Standard Time). - if (t >= lut[DayNum(guess + 1)].date) - return DayNum(guess + 1); + if (t >= lut[UInt32(guess + 1)].date) + return LUTIndex(guess + 1); - return DayNum(guess - 1); + if (lut[guess - 1].date <= t) + return LUTIndex(guess - 1); + return LUTIndex(guess - 2); } - inline const Values & find(time_t t) const + inline LUTIndex toLUTIndex(DayNum d) const { - return lut[findIndex(t)]; + return LUTIndex{(d + daynum_offset_epoch) & date_lut_mask}; + } + + inline LUTIndex toLUTIndex(ExtendedDayNum d) const + { + return LUTIndex{static_cast(d + daynum_offset_epoch) & date_lut_mask}; + } + + inline LUTIndex toLUTIndex(time_t t) const + { + return findIndex(t); + } + + inline LUTIndex toLUTIndex(LUTIndex i) const + { + return i; + } + +// template +// inline LUTIndex toLUTIndex(T t) const +// { +// return LUTIndex{static_cast(t) & date_lut_mask}; +// } + + template + inline const Values & find(V v) const + { + return lut[toLUTIndex(v)]; } public: const std::string & getTimeZone() const { return time_zone; } + // Methods only for unit-testing, it makes very little sense to use it from user code. + auto getOffsetAtStartOfEpoch() const { return offset_at_start_of_epoch; } + auto getOffsetIsWholNumberOfHoursEveryWhere() const { return offset_is_whole_number_of_hours_everytime; } + auto getTimeOffsetEpoch() const { return time_offset_epoch; } + auto getTimeOffsetAtStartOfLUT() const { return offset_at_start_of_lut; } + /// All functions below are thread-safe; arguments are not checked. - inline time_t toDate(time_t t) const { return find(t).date; } - inline unsigned toMonth(time_t t) const { return find(t).month; } - inline unsigned toQuarter(time_t t) const { return (find(t).month - 1) / 3 + 1; } - inline unsigned toYear(time_t t) const { return find(t).year; } - inline unsigned toDayOfWeek(time_t t) const { return find(t).day_of_week; } - inline unsigned toDayOfMonth(time_t t) const { return find(t).day_of_month; } + inline ExtendedDayNum toDayNum(ExtendedDayNum d) const + { + return d; + } + + template + inline ExtendedDayNum toDayNum(V v) const + { + return ExtendedDayNum{static_cast(toLUTIndex(v).toUnderType()) - daynum_offset_epoch}; + } /// Round down to start of monday. - inline time_t toFirstDayOfWeek(time_t t) const + template + inline time_t toFirstDayOfWeek(V v) const { - DayNum index = findIndex(t); - return lut[DayNum(index - (lut[index].day_of_week - 1))].date; + const auto i = toLUTIndex(v); + return lut[i - (lut[i].day_of_week - 1)].date; } - inline DayNum toFirstDayNumOfWeek(DayNum d) const + template + inline ExtendedDayNum toFirstDayNumOfWeek(V v) const { - return DayNum(d - (lut[d].day_of_week - 1)); - } - - inline DayNum toFirstDayNumOfWeek(time_t t) const - { - return toFirstDayNumOfWeek(toDayNum(t)); + const auto i = toLUTIndex(v); + return toDayNum(i - (lut[i].day_of_week - 1)); } /// Round down to start of month. - inline time_t toFirstDayOfMonth(time_t t) const + template + inline time_t toFirstDayOfMonth(V v) const { - DayNum index = findIndex(t); - return lut[index - (lut[index].day_of_month - 1)].date; + const auto i = toLUTIndex(v); + return lut[i - (lut[i].day_of_month - 1)].date; } - inline DayNum toFirstDayNumOfMonth(DayNum d) const + template + inline ExtendedDayNum toFirstDayNumOfMonth(V v) const { - return DayNum(d - (lut[d].day_of_month - 1)); + const auto i = toLUTIndex(v); + return toDayNum(i - (lut[i].day_of_month - 1)); } - inline DayNum toFirstDayNumOfMonth(time_t t) const - { - return toFirstDayNumOfMonth(toDayNum(t)); - } +// inline DayNum toFirstDayNumOfMonth(time_t t) const +// { +// return toFirstDayNumOfMonth(toDayNum(t)); +// } /// Round down to start of quarter. - inline DayNum toFirstDayNumOfQuarter(DayNum d) const + template + inline ExtendedDayNum toFirstDayNumOfQuarter(V v) const { - DayNum index = d; + return toDayNum(toFirstDayOfQuarterIndex(v)); + } + + template + inline LUTIndex toFirstDayOfQuarterIndex(V v) const + { + //return fromDayNum(toFirstDayNumOfQuarter(v)); + auto index = toLUTIndex(v); size_t month_inside_quarter = (lut[index].month - 1) % 3; index -= lut[index].day_of_month; @@ -175,17 +297,13 @@ public: --month_inside_quarter; } - return DayNum(index + 1); + return index + 1; } - inline DayNum toFirstDayNumOfQuarter(time_t t) const + template + inline time_t toFirstDayOfQuarter(V v) const { - return toFirstDayNumOfQuarter(toDayNum(t)); - } - - inline time_t toFirstDayOfQuarter(time_t t) const - { - return fromDayNum(toFirstDayNumOfQuarter(t)); + return toDate(toFirstDayOfQuarterIndex(v)); } /// Round down to start of year. @@ -194,48 +312,47 @@ public: return lut[years_lut[lut[findIndex(t)].year - DATE_LUT_MIN_YEAR]].date; } - inline DayNum toFirstDayNumOfYear(DayNum d) const + template + inline LUTIndex toFirstDayNumOfYearIndex(V v) const { - return years_lut[lut[d].year - DATE_LUT_MIN_YEAR]; + return years_lut[lut[toLUTIndex(v)].year - DATE_LUT_MIN_YEAR]; } - inline DayNum toFirstDayNumOfYear(time_t t) const + template + inline ExtendedDayNum toFirstDayNumOfYear(V v) const { - return toFirstDayNumOfYear(toDayNum(t)); + return toDayNum(toFirstDayNumOfYearIndex(v)); } inline time_t toFirstDayOfNextMonth(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); index += 32 - lut[index].day_of_month; return lut[index - (lut[index].day_of_month - 1)].date; } inline time_t toFirstDayOfPrevMonth(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); index -= lut[index].day_of_month; return lut[index - (lut[index].day_of_month - 1)].date; } - inline UInt8 daysInMonth(DayNum d) const + template + inline UInt8 daysInMonth(V v) const { - return lut[d].days_in_month; + const auto i = toLUTIndex(v); + return lut[i].days_in_month; } - inline UInt8 daysInMonth(time_t t) const - { - return find(t).days_in_month; - } - - inline UInt8 daysInMonth(UInt16 year, UInt8 month) const + inline UInt8 daysInMonth(Int16 year, UInt8 month) const { UInt16 idx = year - DATE_LUT_MIN_YEAR; if (unlikely(idx >= DATE_LUT_YEARS)) return 31; /// Implementation specific behaviour on overflow. /// 32 makes arithmetic more simple. - DayNum any_day_of_month = DayNum(years_lut[idx] + 32 * (month - 1)); + const auto any_day_of_month = years_lut[year - DATE_LUT_MIN_YEAR] + 32 * (month - 1); return lut[any_day_of_month].days_in_month; } @@ -243,37 +360,38 @@ public: */ inline time_t toDateAndShift(time_t t, Int32 days) const { - return lut[DayNum(findIndex(t) + days)].date; + return lut[findIndex(t) + days].date; } inline time_t toTime(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); - if (unlikely(index == 0 || index > DATE_LUT_MAX_DAY_NUM)) + if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) return t + offset_at_start_of_epoch; time_t res = t - lut[index].date; - if (res >= lut[index].time_at_offset_change) - res += lut[index].amount_of_offset_change; + if (res >= lut[index].time_at_offset_change()) + res += lut[index].amount_of_offset_change(); return res - offset_at_start_of_epoch; /// Starting at 1970-01-01 00:00:00 local time. } inline unsigned toHour(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); /// If it is overflow case, - /// then limit number of hours to avoid insane results like 1970-01-01 89:28:15 - if (unlikely(index == 0 || index > DATE_LUT_MAX_DAY_NUM)) + /// than limit number of hours to avoid insane results like 1970-01-01 89:28:15 + if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) return static_cast((t + offset_at_start_of_epoch) / 3600) % 24; time_t time = t - lut[index].date; - if (time >= lut[index].time_at_offset_change) - time += lut[index].amount_of_offset_change; + /// Data is cleaned to avoid possibility of underflow. + if (time >= lut[index].time_at_offset_change()) + time += lut[index].amount_of_offset_change(); unsigned res = time / 3600; return res <= 23 ? res : 0; @@ -286,24 +404,32 @@ public: */ inline time_t timezoneOffset(time_t t) const { - DayNum index = findIndex(t); + const auto index = findIndex(t); /// Calculate daylight saving offset first. /// Because the "amount_of_offset_change" in LUT entry only exists in the change day, it's costly to scan it from the very begin. /// but we can figure out all the accumulated offsets from 1970-01-01 to that day just by get the whole difference between lut[].date, /// and then, we can directly subtract multiple 86400s to get the real DST offsets for the leap seconds is not considered now. - time_t res = (lut[index].date - lut[0].date) % 86400; + time_t res = (lut[index].date - lut[daynum_offset_epoch].date) % 86400; /// As so far to know, the maximal DST offset couldn't be more than 2 hours, so after the modulo operation the remainder /// will sits between [-offset --> 0 --> offset] which respectively corresponds to moving clock forward or backward. res = res > 43200 ? (86400 - res) : (0 - res); /// Check if has a offset change during this day. Add the change when cross the line - if (lut[index].amount_of_offset_change != 0 && t >= lut[index].date + lut[index].time_at_offset_change) - res += lut[index].amount_of_offset_change; + if (lut[index].amount_of_offset_change() != 0 && t >= lut[index].date + lut[index].time_at_offset_change()) + res += lut[index].amount_of_offset_change(); return res + offset_at_start_of_epoch; } + static inline time_t toSecondsSinceTheDayStart(time_t t) + { + t %= 86400; + t = (t < 0 ? t + 86400 : t); + + return t; + } + /** Only for time zones with/when offset from UTC is multiple of five minutes. * This is true for all time zones: right now, all time zones have an offset that is multiple of 15 minutes. * @@ -314,13 +440,15 @@ public: * Also please note, that unix timestamp doesn't count "leap seconds": * each minute, with added or subtracted leap second, spans exactly 60 unix timestamps. */ - - inline unsigned toSecond(time_t t) const { return UInt32(t) % 60; } + inline unsigned toSecond(time_t t) const + { + return toSecondsSinceTheDayStart(t) % 60; + } inline unsigned toMinute(time_t t) const { if (offset_is_whole_number_of_hours_everytime) - return (UInt32(t) / 60) % 60; + return (toSecondsSinceTheDayStart(t) / 60) % 60; UInt32 date = find(t).date; return (UInt32(t) - date) / 60 % 60; @@ -348,80 +476,85 @@ public: * because the same calendar day starts/ends at different timestamps in different time zones) */ - inline DayNum toDayNum(time_t t) const { return findIndex(t); } - inline time_t fromDayNum(DayNum d) const { return lut[d].date; } +// inline DayNum toDayNum(time_t t) const { return DayNum{findIndex(t) - daynum_offset_epoch}; } +// inline ExtendedDayNum toExtendedDayNum(time_t t) const { return ExtendedDayNum{findIndex(t) - daynum_offset_epoch}; } + inline time_t fromDayNum(DayNum d) const { return lut[toLUTIndex(d)].date; } + inline time_t fromDayNum(ExtendedDayNum d) const { return lut[toLUTIndex(d)].date; } - inline time_t toDate(DayNum d) const { return lut[d].date; } - inline unsigned toMonth(DayNum d) const { return lut[d].month; } - inline unsigned toQuarter(DayNum d) const { return (lut[d].month - 1) / 3 + 1; } - inline unsigned toYear(DayNum d) const { return lut[d].year; } - inline unsigned toDayOfWeek(DayNum d) const { return lut[d].day_of_week; } - inline unsigned toDayOfMonth(DayNum d) const { return lut[d].day_of_month; } - inline unsigned toDayOfYear(DayNum d) const { return d + 1 - toFirstDayNumOfYear(d); } - - inline unsigned toDayOfYear(time_t t) const { return toDayOfYear(toDayNum(t)); } + template + inline time_t toDate(V v) const { return lut[toLUTIndex(v)].date; } + template + inline unsigned toMonth(V v) const { return lut[toLUTIndex(v)].month; } + template + inline unsigned toQuarter(V v) const { return (lut[toLUTIndex(v)].month - 1) / 3 + 1; } + template + inline Int16 toYear(V v) const { return lut[toLUTIndex(v)].year; } + template + inline unsigned toDayOfWeek(V v) const { return lut[toLUTIndex(v)].day_of_week; } + template + inline unsigned toDayOfMonth(V v) const { return lut[toLUTIndex(v)].day_of_month; } + template + inline unsigned toDayOfYear(V v) const + { + // TODO: different overload for ExtendedDayNum + const auto i = toLUTIndex(v); + return i + 1 - toFirstDayNumOfYearIndex(i); + } /// Number of week from some fixed moment in the past. Week begins at monday. /// (round down to monday and divide DayNum by 7; we made an assumption, /// that in domain of the function there was no weeks with any other number of days than 7) - inline unsigned toRelativeWeekNum(DayNum d) const + template + inline unsigned toRelativeWeekNum(V v) const { + const auto i = toLUTIndex(v); /// We add 8 to avoid underflow at beginning of unix epoch. - return (d + 8 - toDayOfWeek(d)) / 7; - } - - inline unsigned toRelativeWeekNum(time_t t) const - { - return toRelativeWeekNum(toDayNum(t)); + return toDayNum(i + 8 - toDayOfWeek(i)) / 7; } /// Get year that contains most of the current week. Week begins at monday. - inline unsigned toISOYear(DayNum d) const + template + inline unsigned toISOYear(V v) const { + const auto i = toLUTIndex(v); /// That's effectively the year of thursday of current week. - return toYear(DayNum(d + 4 - toDayOfWeek(d))); - } - - inline unsigned toISOYear(time_t t) const - { - return toISOYear(toDayNum(t)); + return toYear(toLUTIndex(i + 4 - toDayOfWeek(i))); } /// ISO year begins with a monday of the week that is contained more than by half in the corresponding calendar year. /// Example: ISO year 2019 begins at 2018-12-31. And ISO year 2017 begins at 2017-01-02. /// https://en.wikipedia.org/wiki/ISO_week_date - inline DayNum toFirstDayNumOfISOYear(DayNum d) const + template + inline LUTIndex toFirstDayNumOfISOYearIndex(V v) const { - auto iso_year = toISOYear(d); + const auto i = toLUTIndex(v); + auto iso_year = toISOYear(i); - DayNum first_day_of_year = years_lut[iso_year - DATE_LUT_MIN_YEAR]; + const auto first_day_of_year = years_lut[iso_year - DATE_LUT_MIN_YEAR]; auto first_day_of_week_of_year = lut[first_day_of_year].day_of_week; - return DayNum(first_day_of_week_of_year <= 4 + return LUTIndex{first_day_of_week_of_year <= 4 ? first_day_of_year + 1 - first_day_of_week_of_year - : first_day_of_year + 8 - first_day_of_week_of_year); + : first_day_of_year + 8 - first_day_of_week_of_year}; } - inline DayNum toFirstDayNumOfISOYear(time_t t) const + template + inline ExtendedDayNum toFirstDayNumOfISOYear(V v) const { - return toFirstDayNumOfISOYear(toDayNum(t)); + return toDayNum(toFirstDayNumOfISOYearIndex(v)); } inline time_t toFirstDayOfISOYear(time_t t) const { - return fromDayNum(toFirstDayNumOfISOYear(t)); + return lut[toFirstDayNumOfISOYearIndex(t)].date; } /// ISO 8601 week number. Week begins at monday. /// The week number 1 is the first week in year that contains 4 or more days (that's more than half). - inline unsigned toISOWeek(DayNum d) const + template + inline unsigned toISOWeek(V v) const { - return 1 + DayNum(toFirstDayNumOfWeek(d) - toFirstDayNumOfISOYear(d)) / 7; - } - - inline unsigned toISOWeek(time_t t) const - { - return toISOWeek(toDayNum(t)); + return 1 + (toFirstDayNumOfWeek(v) - toFirstDayNumOfISOYear(v)) / 7; } /* @@ -457,30 +590,33 @@ public: Otherwise it is the last week of the previous year, and the next week is week 1. */ - inline YearWeek toYearWeek(DayNum d, UInt8 week_mode) const + template + inline YearWeek toYearWeek(V v, UInt8 week_mode) const { - bool newyear_day_mode = week_mode & static_cast(WeekModeFlag::NEWYEAR_DAY); + const bool newyear_day_mode = week_mode & static_cast(WeekModeFlag::NEWYEAR_DAY); week_mode = check_week_mode(week_mode); - bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); + const bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); bool week_year_mode = week_mode & static_cast(WeekModeFlag::YEAR); - bool first_weekday_mode = week_mode & static_cast(WeekModeFlag::FIRST_WEEKDAY); + const bool first_weekday_mode = week_mode & static_cast(WeekModeFlag::FIRST_WEEKDAY); + + const auto i = toLUTIndex(v); // Calculate week number of WeekModeFlag::NEWYEAR_DAY mode if (newyear_day_mode) { - return toYearWeekOfNewyearMode(d, monday_first_mode); + return toYearWeekOfNewyearMode(i, monday_first_mode); } - YearWeek yw(toYear(d), 0); + YearWeek yw(toYear(i), 0); UInt16 days = 0; - UInt16 daynr = makeDayNum(yw.first, toMonth(d), toDayOfMonth(d)); - UInt16 first_daynr = makeDayNum(yw.first, 1, 1); + const auto daynr = makeDayNum(yw.first, toMonth(i), toDayOfMonth(i)); + auto first_daynr = makeDayNum(yw.first, 1, 1); // 0 for monday, 1 for tuesday ... // get weekday from first day in year. - UInt16 weekday = calc_weekday(DayNum(first_daynr), !monday_first_mode); + UInt16 weekday = calc_weekday(first_daynr, !monday_first_mode); - if (toMonth(d) == 1 && toDayOfMonth(d) <= static_cast(7 - weekday)) + if (toMonth(i) == 1 && toDayOfMonth(i) <= static_cast(7 - weekday)) { if (!week_year_mode && ((first_weekday_mode && weekday != 0) || (!first_weekday_mode && weekday >= 4))) return yw; @@ -511,30 +647,34 @@ public: /// Calculate week number of WeekModeFlag::NEWYEAR_DAY mode /// The week number 1 is the first week in year that contains January 1, - inline YearWeek toYearWeekOfNewyearMode(DayNum d, bool monday_first_mode) const + template + inline YearWeek toYearWeekOfNewyearMode(V v, bool monday_first_mode) const { YearWeek yw(0, 0); UInt16 offset_day = monday_first_mode ? 0U : 1U; + const auto i = LUTIndex(v); + // Checking the week across the year - yw.first = toYear(DayNum(d + 7 - toDayOfWeek(DayNum(d + offset_day)))); + yw.first = toYear(i + 7 - toDayOfWeek(i + offset_day)); - DayNum first_day = makeDayNum(yw.first, 1, 1); - DayNum this_day = d; + auto first_day = makeLUTIndex(yw.first, 1, 1); + auto this_day = i; + //TODO: do not perform calculations in terms of DayNum, since that would under/overflow for extended range. if (monday_first_mode) { // Rounds down a date to the nearest Monday. first_day = toFirstDayNumOfWeek(first_day); - this_day = toFirstDayNumOfWeek(d); + this_day = toFirstDayNumOfWeek(i); } else { // Rounds down a date to the nearest Sunday. if (toDayOfWeek(first_day) != 7) - first_day = DayNum(first_day - toDayOfWeek(first_day)); - if (toDayOfWeek(d) != 7) - this_day = DayNum(d - toDayOfWeek(d)); + first_day = ExtendedDayNum(first_day - toDayOfWeek(first_day)); + if (toDayOfWeek(i) != 7) + this_day = ExtendedDayNum(i - toDayOfWeek(i)); } yw.second = (this_day - first_day) / 7 + 1; return yw; @@ -543,16 +683,17 @@ public: /** * get first day of week with week_mode, return Sunday or Monday */ - inline DayNum toFirstDayNumOfWeek(DayNum d, UInt8 week_mode) const + template + inline ExtendedDayNum toFirstDayNumOfWeek(V v, UInt8 week_mode) const { bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); if (monday_first_mode) { - return toFirstDayNumOfWeek(d); + return toFirstDayNumOfWeek(v); } else { - return (toDayOfWeek(d) != 7) ? DayNum(d - toDayOfWeek(d)) : d; + return (toDayOfWeek(v) != 7) ? ExtendedDayNum(v - toDayOfWeek(v)) : toDayNum(v); } } @@ -568,39 +709,35 @@ public: /** Calculate weekday from d. * Returns 0 for monday, 1 for tuesday... */ - inline unsigned calc_weekday(DayNum d, bool sunday_first_day_of_week) const + template + inline unsigned calc_weekday(V v, bool sunday_first_day_of_week) const { + const auto i = toLUTIndex(v); if (!sunday_first_day_of_week) - return toDayOfWeek(d) - 1; + return toDayOfWeek(i) - 1; else - return toDayOfWeek(DayNum(d + 1)) - 1; + return toDayOfWeek(i + 1) - 1; } /// Calculate days in one year. - inline unsigned calc_days_in_year(UInt16 year) const + inline unsigned calc_days_in_year(Int32 year) const { return ((year & 3) == 0 && (year % 100 || (year % 400 == 0 && year)) ? 366 : 365); } /// Number of month from some fixed moment in the past (year * 12 + month) - inline unsigned toRelativeMonthNum(DayNum d) const + template + inline unsigned toRelativeMonthNum(V v) const { - return lut[d].year * 12 + lut[d].month; + const auto i = toLUTIndex(v); + return lut[i].year * 12 + lut[i].month; } - inline unsigned toRelativeMonthNum(time_t t) const + template + inline unsigned toRelativeQuarterNum(V v) const { - return toRelativeMonthNum(toDayNum(t)); - } - - inline unsigned toRelativeQuarterNum(DayNum d) const - { - return lut[d].year * 4 + (lut[d].month - 1) / 3; - } - - inline unsigned toRelativeQuarterNum(time_t t) const - { - return toRelativeQuarterNum(toDayNum(t)); + const auto i = toLUTIndex(v); + return lut[i].year * 4 + (lut[i].month - 1) / 3; } /// We count all hour-length intervals, unrelated to offset changes. @@ -614,9 +751,10 @@ public: return (t + 86400 - offset_at_start_of_epoch) / 3600; } - inline time_t toRelativeHourNum(DayNum d) const + template + inline time_t toRelativeHourNum(V v) const { - return toRelativeHourNum(lut[d].date); + return toRelativeHourNum(lut[toLUTIndex(v)].date); } inline time_t toRelativeMinuteNum(time_t t) const @@ -624,48 +762,52 @@ public: return t / 60; } - inline time_t toRelativeMinuteNum(DayNum d) const + template + inline time_t toRelativeMinuteNum(V v) const { - return toRelativeMinuteNum(lut[d].date); + return toRelativeMinuteNum(lut[toLUTIndex(v)].date); } - inline DayNum toStartOfYearInterval(DayNum d, UInt64 years) const + template + inline ExtendedDayNum toStartOfYearInterval(V v, UInt64 years) const { if (years == 1) - return toFirstDayNumOfYear(d); - return years_lut[(lut[d].year - DATE_LUT_MIN_YEAR) / years * years]; + return toFirstDayNumOfYear(v); + + const auto i = toLUTIndex(v); + return toDayNum(years_lut[lut[i].year / years * years - DATE_LUT_MIN_YEAR]); } - inline DayNum toStartOfQuarterInterval(DayNum d, UInt64 quarters) const + inline ExtendedDayNum toStartOfQuarterInterval(ExtendedDayNum d, UInt64 quarters) const { if (quarters == 1) return toFirstDayNumOfQuarter(d); return toStartOfMonthInterval(d, quarters * 3); } - inline DayNum toStartOfMonthInterval(DayNum d, UInt64 months) const + inline ExtendedDayNum toStartOfMonthInterval(ExtendedDayNum d, UInt64 months) const { if (months == 1) return toFirstDayNumOfMonth(d); - const auto & date = lut[d]; + const auto & date = lut[toLUTIndex(d)]; UInt32 month_total_index = (date.year - DATE_LUT_MIN_YEAR) * 12 + date.month - 1; - return years_months_lut[month_total_index / months * months]; + return toDayNum(years_months_lut[month_total_index / months * months]); } - inline DayNum toStartOfWeekInterval(DayNum d, UInt64 weeks) const + inline ExtendedDayNum toStartOfWeekInterval(ExtendedDayNum d, UInt64 weeks) const { if (weeks == 1) return toFirstDayNumOfWeek(d); UInt64 days = weeks * 7; // January 1st 1970 was Thursday so we need this 4-days offset to make weeks start on Monday. - return DayNum(4 + (d - 4) / days * days); + return ExtendedDayNum(4 + (d - 4) / days * days); } - inline time_t toStartOfDayInterval(DayNum d, UInt64 days) const + inline time_t toStartOfDayInterval(ExtendedDayNum d, UInt64 days) const { if (days == 1) return toDate(d); - return lut[d / days * days].date; + return lut[toLUTIndex(ExtendedDayNum(d / days * days))].date; } inline time_t toStartOfHourInterval(time_t t, UInt64 hours) const @@ -694,33 +836,41 @@ public: return t / seconds * seconds; } - /// Create DayNum from year, month, day of month. - inline DayNum makeDayNum(UInt16 year, UInt8 month, UInt8 day_of_month) const + inline LUTIndex makeLUTIndex(Int16 year, UInt8 month, UInt8 day_of_month) const { if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31)) - return DayNum(0); // TODO (nemkov, DateTime64 phase 2): implement creating real date for year outside of LUT range. + return LUTIndex(0); - // The day after 2106-02-07 will not stored fully as struct Values, so just overflow it as 0 - if (unlikely(year == DATE_LUT_MAX_YEAR && (month > 2 || (month == 2 && day_of_month > 7)))) - return DayNum(0); - - return DayNum(years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1); + return LUTIndex{years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1}; } - inline time_t makeDate(UInt16 year, UInt8 month, UInt8 day_of_month) const + /// Create DayNum from year, month, day of month. + inline ExtendedDayNum makeDayNum(Int16 year, UInt8 month, UInt8 day_of_month) const { - return lut[makeDayNum(year, month, day_of_month)].date; + if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31)) + return ExtendedDayNum(0); + + // The day after 2283 are not stored fully as struct Values, so just overflow it as 0 + if (unlikely(year > DATE_LUT_MAX_YEAR)) + return ExtendedDayNum(0); + + return toDayNum(makeLUTIndex(year, month, day_of_month)); + } + + inline time_t makeDate(Int16 year, UInt8 month, UInt8 day_of_month) const + { + return lut[makeLUTIndex(year, month, day_of_month)].date; } /** Does not accept daylight saving time as argument: in case of ambiguity, it choose greater timestamp. */ - inline time_t makeDateTime(UInt16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const + inline time_t makeDateTime(Int16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const { - size_t index = makeDayNum(year, month, day_of_month); + size_t index = makeLUTIndex(year, month, day_of_month); UInt32 time_offset = hour * 3600 + minute * 60 + second; - if (time_offset >= lut[index].time_at_offset_change) - time_offset -= lut[index].amount_of_offset_change; + if (time_offset >= lut[index].time_at_offset_change()) + time_offset -= lut[index].amount_of_offset_change(); UInt32 res = lut[index].date + time_offset; @@ -730,30 +880,20 @@ public: return res; } - inline const Values & getValues(DayNum d) const { return lut[d]; } - inline const Values & getValues(time_t t) const { return lut[findIndex(t)]; } + template + inline const Values & getValues(V v) const { return lut[toLUTIndex(v)]; } - inline UInt32 toNumYYYYMM(time_t t) const + template + inline UInt32 toNumYYYYMM(V v) const { - const Values & values = find(t); + const Values & values = getValues(v); return values.year * 100 + values.month; } - inline UInt32 toNumYYYYMM(DayNum d) const + template + inline UInt32 toNumYYYYMMDD(V v) const { - const Values & values = lut[d]; - return values.year * 100 + values.month; - } - - inline UInt32 toNumYYYYMMDD(time_t t) const - { - const Values & values = find(t); - return values.year * 10000 + values.month * 100 + values.day_of_month; - } - - inline UInt32 toNumYYYYMMDD(DayNum d) const - { - const Values & values = lut[d]; + const Values & values = getValues(v); return values.year * 10000 + values.month * 100 + values.day_of_month; } @@ -762,7 +902,7 @@ public: return makeDate(num / 10000, num / 100 % 100, num % 100); } - inline DayNum YYYYMMDDToDayNum(UInt32 num) const + inline ExtendedDayNum YYYYMMDDToDayNum(UInt32 num) const { return makeDayNum(num / 10000, num / 100 % 100, num % 100); } @@ -796,13 +936,14 @@ public: inline NO_SANITIZE_UNDEFINED time_t addDays(time_t t, Int64 delta) const { - DayNum index = findIndex(t); + auto index = findIndex(t); time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); index += delta; + index &= date_lut_mask; - if (time_offset >= lut[index].time_at_offset_change) - time_offset -= lut[index].amount_of_offset_change; + if (time_offset >= lut[index].time_at_offset_change()) + time_offset -= lut[index].amount_of_offset_change(); return lut[index].date + time_offset; } @@ -812,7 +953,7 @@ public: return addDays(t, delta * 7); } - inline UInt8 saturateDayOfMonth(UInt16 year, UInt8 month, UInt8 day_of_month) const + inline UInt8 saturateDayOfMonth(Int16 year, UInt8 month, UInt8 day_of_month) const { if (likely(day_of_month <= 28)) return day_of_month; @@ -825,23 +966,10 @@ public: return day_of_month; } - /// If resulting month has less deys than source month, then saturation can happen. - /// Example: 31 Aug + 1 month = 30 Sep. - inline time_t addMonths(time_t t, Int64 delta) const + template + inline LUTIndex addMonthsIndex(V v, Int64 delta) const { - DayNum result_day = addMonths(toDayNum(t), delta); - - time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); - - if (time_offset >= lut[result_day].time_at_offset_change) - time_offset -= lut[result_day].amount_of_offset_change; - - return lut[result_day].date + time_offset; - } - - inline NO_SANITIZE_UNDEFINED DayNum addMonths(DayNum d, Int64 delta) const - { - const Values & values = lut[d]; + const Values & values = lut[toLUTIndex(v)]; Int64 month = static_cast(values.month) + delta; @@ -851,7 +979,7 @@ public: month = ((month - 1) % 12) + 1; auto day_of_month = saturateDayOfMonth(year, month, values.day_of_month); - return makeDayNum(year, month, day_of_month); + return makeLUTIndex(year, month, day_of_month); } else { @@ -859,36 +987,43 @@ public: month = 12 - (-month % 12); auto day_of_month = saturateDayOfMonth(year, month, values.day_of_month); - return makeDayNum(year, month, day_of_month); + return makeLUTIndex(year, month, day_of_month); } } - inline NO_SANITIZE_UNDEFINED time_t addQuarters(time_t t, Int64 delta) const + /// If resulting month has less deys than source month, then saturation can happen. + /// Example: 31 Aug + 1 month = 30 Sep. + inline time_t NO_SANITIZE_UNDEFINED addMonths(time_t t, Int64 delta) const { - return addMonths(t, delta * 3); - } - - inline NO_SANITIZE_UNDEFINED DayNum addQuarters(DayNum d, Int64 delta) const - { - return addMonths(d, delta * 3); - } - - /// Saturation can occur if 29 Feb is mapped to non-leap year. - inline NO_SANITIZE_UNDEFINED time_t addYears(time_t t, Int64 delta) const - { - DayNum result_day = addYears(toDayNum(t), delta); + const auto result_day = addMonthsIndex(t, delta); time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); - if (time_offset >= lut[result_day].time_at_offset_change) - time_offset -= lut[result_day].amount_of_offset_change; + if (time_offset >= lut[result_day].time_at_offset_change()) + time_offset -= lut[result_day].amount_of_offset_change(); return lut[result_day].date + time_offset; } - inline NO_SANITIZE_UNDEFINED DayNum addYears(DayNum d, Int64 delta) const + inline ExtendedDayNum NO_SANITIZE_UNDEFINED addMonths(ExtendedDayNum d, Int64 delta) const { - const Values & values = lut[d]; + return toDayNum(addMonthsIndex(d, delta)); + } + + inline time_t NO_SANITIZE_UNDEFINED addQuarters(time_t t, Int64 delta) const + { + return addMonths(t, delta * 3); + } + + inline ExtendedDayNum addQuarters(ExtendedDayNum d, Int64 delta) const + { + return addMonths(d, delta * 3); + } + + template + inline LUTIndex NO_SANITIZE_UNDEFINED addYearsIndex(V v, Int64 delta) const + { + const Values & values = lut[toLUTIndex(v)]; auto year = values.year + delta; auto month = values.month; @@ -898,13 +1033,31 @@ public: if (unlikely(day_of_month == 29 && month == 2)) day_of_month = saturateDayOfMonth(year, month, day_of_month); - return makeDayNum(year, month, day_of_month); + return makeLUTIndex(year, month, day_of_month); + } + + /// Saturation can occur if 29 Feb is mapped to non-leap year. + inline time_t addYears(time_t t, Int64 delta) const + { + auto result_day = addYearsIndex(t, delta); + + time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); + + if (time_offset >= lut[result_day].time_at_offset_change()) + time_offset -= lut[result_day].amount_of_offset_change(); + + return lut[result_day].date + time_offset; + } + + inline ExtendedDayNum addYears(ExtendedDayNum d, Int64 delta) const + { + return toDayNum(addYearsIndex(d, delta)); } inline std::string timeToString(time_t t) const { - const Values & values = find(t); + const Values & values = getValues(t); std::string s {"0000-00-00 00:00:00"}; @@ -933,7 +1086,7 @@ public: inline std::string dateToString(time_t t) const { - const Values & values = find(t); + const Values & values = getValues(t); std::string s {"0000-00-00"}; @@ -949,9 +1102,9 @@ public: return s; } - inline std::string dateToString(DayNum d) const + inline std::string dateToString(ExtendedDayNum d) const { - const Values & values = lut[d]; + const Values & values = getValues(d); std::string s {"0000-00-00"}; diff --git a/base/common/DayNum.h b/base/common/DayNum.h index a4ef0c43b69..5cf4d4635c8 100644 --- a/base/common/DayNum.h +++ b/base/common/DayNum.h @@ -7,3 +7,8 @@ * See DateLUTImpl for usage examples. */ STRONG_TYPEDEF(UInt16, DayNum) + +/** Represent number of days since 1970-01-01 but in extended range, + * for dates before 1970-01-01 and after 2105 + */ +STRONG_TYPEDEF(Int32, ExtendedDayNum) diff --git a/base/common/LocalDate.h b/base/common/LocalDate.h index e5ebe877bc5..7e1260c1385 100644 --- a/base/common/LocalDate.h +++ b/base/common/LocalDate.h @@ -105,7 +105,8 @@ public: DayNum getDayNum() const { - return DateLUT::instance().makeDayNum(m_year, m_month, m_day); + const auto & lut = DateLUT::instance(); + return DayNum(lut.makeDayNum(m_year, m_month, m_day).toUnderType()); } operator DayNum() const diff --git a/base/common/strong_typedef.h b/base/common/strong_typedef.h index d9850a25c37..77b83bfa6e5 100644 --- a/base/common/strong_typedef.h +++ b/base/common/strong_typedef.h @@ -12,6 +12,7 @@ private: T t; public: + using UnderlyingType = T; template ::type> explicit StrongTypedef(const T & t_) : t(t_) {} template ::type> diff --git a/base/common/tests/CMakeLists.txt b/base/common/tests/CMakeLists.txt index b7082ee9900..b335b302cb0 100644 --- a/base/common/tests/CMakeLists.txt +++ b/base/common/tests/CMakeLists.txt @@ -16,7 +16,9 @@ target_link_libraries (realloc-perf PRIVATE common) add_check(local_date_time_comparison) if(USE_GTEST) - add_executable(unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp gtest_find_symbols.cpp) + add_executable(unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp gtest_find_symbols.cpp gtest_DateLutImpl.cpp + ${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp + ) target_link_libraries(unit_tests_libcommon PRIVATE common ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) add_check(unit_tests_libcommon) endif() diff --git a/base/common/tests/gtest_DateLutImpl.cpp b/base/common/tests/gtest_DateLutImpl.cpp new file mode 100644 index 00000000000..395e2eddb00 --- /dev/null +++ b/base/common/tests/gtest_DateLutImpl.cpp @@ -0,0 +1,515 @@ +#include +#include + +#include + +#include +#include + +/// For the expansion of gtest macros. +#if defined(__clang__) + #pragma clang diagnostic ignored "-Wused-but-marked-unused" +#endif + +// All timezones present at build time and embedded into CH binary. +extern const char * auto_time_zones[]; + +namespace +{ + +cctz::civil_day YYYYMMDDToDay(unsigned value) +{ + return cctz::civil_day( + value / 10000, // year + (value % 10000) / 100, // month + value % 100); // day +} + +cctz::civil_second YYYYMMDDHMMSSToSecond(std::uint64_t value) +{ + return cctz::civil_second( + value / 10000000000, + value / 100000000 % 100, + value / 1000000 % 100, + value / 10000 % 100, + value / 100 % 100, + value % 100); +} + + +std::vector allTimezones() +{ + std::vector result; + + auto timezone_name = auto_time_zones; + while (*timezone_name) + { + result.push_back(*timezone_name); + ++timezone_name; + } + + return result; +} + +struct FailuresCount +{ + size_t non_fatal = 0; + size_t fatal = 0; + size_t total = 0; +}; + +FailuresCount countFailures(const ::testing::TestResult & test_result) +{ + FailuresCount failures{0, 0, 0}; + const size_t count = test_result.total_part_count(); + for (size_t i = 0; i < count; ++i) + { + const auto & part = test_result.GetTestPartResult(i); + if (part.nonfatally_failed()) + { + ++failures.non_fatal; + ++failures.total; + } + if (part.fatally_failed()) + { + ++failures.fatal; + ++failures.total; + } + } + + return failures; +} + +} + +TEST(YYYYMMDDToDay, Test) +{ + std::cerr << YYYYMMDDHMMSSToSecond(19700101'00'00'00) << std::endl; +} + +TEST(DateLUTTest, TimeValuesInMiddleOfRange) +{ + const DateLUTImpl lut("Europe/Minsk"); + const time_t time = 1568650811; // 2019-09-16 19:20:11 (Monday) + + EXPECT_EQ(lut.getTimeZone(), "Europe/Minsk"); + EXPECT_EQ(lut.getOffsetAtStartOfEpoch(), 3600*3); // UTC-3 + + EXPECT_EQ(lut.toDate(time), 1568581200); + EXPECT_EQ(lut.toMonth(time), 9); + EXPECT_EQ(lut.toQuarter(time), 3); + EXPECT_EQ(lut.toYear(time), 2019); + EXPECT_EQ(lut.toDayOfMonth(time), 16); + + EXPECT_EQ(lut.toFirstDayOfWeek(time), 1568581200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfWeek(time), DayNum(18155) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfMonth(time), 1567285200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfMonth(time), DayNum(18140) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayNumOfQuarter(time), DayNum(18078) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfQuarter(time), 1561928400 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfYear(time), 1546290000 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfYear(time), DayNum(17897) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfNextMonth(time), 1569877200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfPrevMonth(time), 1564606800 /*time_t*/); + EXPECT_EQ(lut.daysInMonth(time), 30 /*UInt8*/); + EXPECT_EQ(lut.toDateAndShift(time, 10), 1569445200 /*time_t*/); + EXPECT_EQ(lut.toTime(time), 58811 /*time_t*/); + EXPECT_EQ(lut.toHour(time), 19 /*unsigned*/); + EXPECT_EQ(lut.toSecond(time), 11 /*unsigned*/); + EXPECT_EQ(lut.toMinute(time), 20 /*unsigned*/); + EXPECT_EQ(lut.toStartOfMinute(time), 1568650800 /*time_t*/); + EXPECT_EQ(lut.toStartOfFiveMinute(time), 1568650800 /*time_t*/); + EXPECT_EQ(lut.toStartOfFifteenMinutes(time), 1568650500 /*time_t*/); + EXPECT_EQ(lut.toStartOfTenMinutes(time), 1568650800 /*time_t*/); + EXPECT_EQ(lut.toStartOfHour(time), 1568649600 /*time_t*/); + EXPECT_EQ(lut.toDayNum(time), DayNum(18155) /*DayNum*/); + EXPECT_EQ(lut.toDayOfYear(time), 259 /*unsigned*/); + EXPECT_EQ(lut.toRelativeWeekNum(time), 2594 /*unsigned*/); + EXPECT_EQ(lut.toISOYear(time), 2019 /*unsigned*/); + EXPECT_EQ(lut.toFirstDayNumOfISOYear(time), DayNum(17896) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfISOYear(time), 1546203600 /*time_t*/); + EXPECT_EQ(lut.toISOWeek(time), 38 /*unsigned*/); + EXPECT_EQ(lut.toRelativeMonthNum(time), 24237 /*unsigned*/); + EXPECT_EQ(lut.toRelativeQuarterNum(time), 8078 /*unsigned*/); + EXPECT_EQ(lut.toRelativeHourNum(time), 435736 /*time_t*/); + EXPECT_EQ(lut.toRelativeMinuteNum(time), 26144180 /*time_t*/); + EXPECT_EQ(lut.toStartOfHourInterval(time, 5), 1568646000 /*time_t*/); + EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 1568650680 /*time_t*/); + EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 1568650811 /*time_t*/); + EXPECT_EQ(lut.toNumYYYYMM(time), 201909 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDD(time), 20190916 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDDhhmmss(time), 20190916192011 /*UInt64*/); + EXPECT_EQ(lut.addDays(time, 100), 1577290811 /*time_t*/); + EXPECT_EQ(lut.addWeeks(time, 100), 1629130811 /*time_t*/); + EXPECT_EQ(lut.addMonths(time, 100), 1831652411 /*time_t*/); + EXPECT_EQ(lut.addQuarters(time, 100), 2357655611 /*time_t*/); + EXPECT_EQ(lut.addYears(time, 10), 1884270011 /*time_t*/); + EXPECT_EQ(lut.timeToString(time), "2019-09-16 19:20:11" /*std::string*/); + EXPECT_EQ(lut.dateToString(time), "2019-09-16" /*std::string*/); +} + + +TEST(DateLUTTest, TimeValuesAtLeftBoderOfRange) +{ + const DateLUTImpl lut("UTC"); + const time_t time = 0; // 1970-01-01 00:00:00 (Thursday) + + EXPECT_EQ(lut.getTimeZone(), "UTC"); + + EXPECT_EQ(lut.toDate(time), 0); + EXPECT_EQ(lut.toMonth(time), 1); + EXPECT_EQ(lut.toQuarter(time), 1); + EXPECT_EQ(lut.toYear(time), 1970); + EXPECT_EQ(lut.toDayOfMonth(time), 1); + + EXPECT_EQ(lut.toFirstDayOfWeek(time), -259200 /*time_t*/); // 1969-12-29 00:00:00 + EXPECT_EQ(lut.toFirstDayNumOfWeek(time), ExtendedDayNum(-3) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfMonth(time), 0 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfMonth(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayNumOfQuarter(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfQuarter(time), 0 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfYear(time), 0 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfYear(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfNextMonth(time), 2678400 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfPrevMonth(time), -2678400 /*time_t*/); // 1969-12-01 00:00:00 + EXPECT_EQ(lut.daysInMonth(time), 31 /*UInt8*/); + EXPECT_EQ(lut.toDateAndShift(time, 10), 864000 /*time_t*/); + EXPECT_EQ(lut.toTime(time), 0 /*time_t*/); + EXPECT_EQ(lut.toHour(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toSecond(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toMinute(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toStartOfMinute(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfFiveMinute(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfFifteenMinutes(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfTenMinutes(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfHour(time), 0 /*time_t*/); + EXPECT_EQ(lut.toDayNum(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toDayOfYear(time), 1 /*unsigned*/); + EXPECT_EQ(lut.toRelativeWeekNum(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toISOYear(time), 1970 /*unsigned*/); + EXPECT_EQ(lut.toFirstDayNumOfISOYear(time), ExtendedDayNum(-3) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfISOYear(time), -259200 /*time_t*/); // 1969-12-29 00:00:00 + EXPECT_EQ(lut.toISOWeek(time), 1 /*unsigned*/); + EXPECT_EQ(lut.toRelativeMonthNum(time), 23641 /*unsigned*/); // ? + EXPECT_EQ(lut.toRelativeQuarterNum(time), 7880 /*unsigned*/); // ? + EXPECT_EQ(lut.toRelativeHourNum(time), 0 /*time_t*/); + EXPECT_EQ(lut.toRelativeMinuteNum(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfHourInterval(time, 5), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 0 /*time_t*/); + EXPECT_EQ(lut.toNumYYYYMM(time), 197001 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDD(time), 19700101 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDDhhmmss(time), 19700101000000 /*UInt64*/); + EXPECT_EQ(lut.addDays(time, 100), 8640000 /*time_t*/); + EXPECT_EQ(lut.addWeeks(time, 100), 60480000 /*time_t*/); + EXPECT_EQ(lut.addMonths(time, 100), 262828800 /*time_t*/); + EXPECT_EQ(lut.addQuarters(time, 100), 788918400 /*time_t*/); + EXPECT_EQ(lut.addYears(time, 10), 315532800 /*time_t*/); + EXPECT_EQ(lut.timeToString(time), "1970-01-01 00:00:00" /*std::string*/); + EXPECT_EQ(lut.dateToString(time), "1970-01-01" /*std::string*/); +} + +TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) +{ + // Value is at the right border of the OLD (small) LUT, and provides meaningful values where OLD LUT would provide garbage. + const DateLUTImpl lut("UTC"); + + const time_t time = 4294343873; // 2106-01-31T01:17:53 (Sunday) + + EXPECT_EQ(lut.getTimeZone(), "UTC"); + + EXPECT_EQ(lut.toDate(time), 4294339200); + EXPECT_EQ(lut.toMonth(time), 1); + EXPECT_EQ(lut.toQuarter(time), 1); + EXPECT_EQ(lut.toYear(time), 2106); + EXPECT_EQ(lut.toDayOfMonth(time), 31); + + EXPECT_EQ(lut.toFirstDayOfWeek(time), 4293820800 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfWeek(time), DayNum(49697)); + EXPECT_EQ(lut.toFirstDayOfMonth(time), 4291747200 /*time_t*/); // 2016-01-01 + EXPECT_EQ(lut.toFirstDayNumOfMonth(time), DayNum(49673)); + EXPECT_EQ(lut.toFirstDayNumOfQuarter(time), DayNum(49673) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfQuarter(time), 4291747200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfYear(time), 4291747200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfYear(time), DayNum(49673) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfNextMonth(time), 4294425600 /*time_t*/); // 2106-02-01 + EXPECT_EQ(lut.toFirstDayOfPrevMonth(time), 4289068800 /*time_t*/); // 2105-12-01 + EXPECT_EQ(lut.daysInMonth(time), 31 /*UInt8*/); + EXPECT_EQ(lut.toDateAndShift(time, 10), 4295203200 /*time_t*/); // 2106-02-10 + EXPECT_EQ(lut.toTime(time), 4673 /*time_t*/); + EXPECT_EQ(lut.toHour(time), 1 /*unsigned*/); + EXPECT_EQ(lut.toMinute(time), 17 /*unsigned*/); + EXPECT_EQ(lut.toSecond(time), 53 /*unsigned*/); + EXPECT_EQ(lut.toStartOfMinute(time), 4294343820 /*time_t*/); + EXPECT_EQ(lut.toStartOfFiveMinute(time), 4294343700 /*time_t*/); + EXPECT_EQ(lut.toStartOfFifteenMinutes(time), 4294343700 /*time_t*/); + EXPECT_EQ(lut.toStartOfTenMinutes(time), 4294343400 /*time_t*/); + EXPECT_EQ(lut.toStartOfHour(time), 4294342800 /*time_t*/); + EXPECT_EQ(lut.toDayNum(time), DayNum(49703) /*DayNum*/); + EXPECT_EQ(lut.toDayOfYear(time), 31 /*unsigned*/); + EXPECT_EQ(lut.toRelativeWeekNum(time), 7100 /*unsigned*/); + EXPECT_EQ(lut.toISOYear(time), 2106 /*unsigned*/); + EXPECT_EQ(lut.toFirstDayNumOfISOYear(time), DayNum(49676) /*DayNum*/); // 2106-01-04 + EXPECT_EQ(lut.toFirstDayOfISOYear(time), 4292006400 /*time_t*/); + EXPECT_EQ(lut.toISOWeek(time), 4 /*unsigned*/); + EXPECT_EQ(lut.toRelativeMonthNum(time), 25273 /*unsigned*/); + EXPECT_EQ(lut.toRelativeQuarterNum(time), 8424 /*unsigned*/); + EXPECT_EQ(lut.toRelativeHourNum(time), 1192873 /*time_t*/); + EXPECT_EQ(lut.toRelativeMinuteNum(time), 71572397 /*time_t*/); + EXPECT_EQ(lut.toStartOfHourInterval(time, 5), 4294332000 /*time_t*/); + EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 4294343520 /*time_t*/); + EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 4294343872 /*time_t*/); + EXPECT_EQ(lut.toNumYYYYMM(time), 210601 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDD(time), 21060131 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDDhhmmss(time), 21060131011753 /*UInt64*/); + EXPECT_EQ(lut.addDays(time, 100), 4302983873 /*time_t*/); + EXPECT_EQ(lut.addWeeks(time, 10), 4300391873 /*time_t*/); + EXPECT_EQ(lut.addMonths(time, 10), 4320523073 /*time_t*/); // 2106-11-30 01:17:53 + EXPECT_EQ(lut.addQuarters(time, 10), 4373140673 /*time_t*/); // 2108-07-31 01:17:53 + EXPECT_EQ(lut.addYears(time, 10), 4609876673 /*time_t*/); // 2116-01-31 01:17:53 + + EXPECT_EQ(lut.timeToString(time), "2106-01-31 01:17:53" /*std::string*/); + EXPECT_EQ(lut.dateToString(time), "2106-01-31" /*std::string*/); +} + + +class DateLUT_TimeZone : public ::testing::TestWithParam +{}; + +TEST_P(DateLUT_TimeZone, DISABLED_LoadAllTimeZones) +{ + // There are some assumptions and assertions about TZ data made in DateLUTImpl which are verified upon loading, + // to make sure that those assertions are true for all timezones we are going to load all of them one by one. + DateLUTImpl{GetParam()}; +} + +// Another long running test, shouldn't be run to often +TEST_P(DateLUT_TimeZone, VaidateTimeComponentsAroundEpoch) +{ + // Converting time around 1970-01-01 to hour-minute-seconds time components + // could be problematic. + const size_t max_failures_per_tz = 3; + const auto timezone_name = GetParam(); + + const auto * test_info = ::testing::UnitTest::GetInstance()->current_test_info(); + const auto lut = DateLUTImpl(timezone_name); + + for (time_t i = -856147870; i < 86400 * 10000; i += 11 * 13 * 17 * 19) + { + SCOPED_TRACE(::testing::Message() + << "\n\tTimezone: " << timezone_name + << "\n\ttimestamp: " << i + << "\n\t offset at start of epoch : " << lut.getOffsetAtStartOfEpoch() + << "\n\t offset_is_whole_number_of_hours_everytime : " << lut.getOffsetIsWholNumberOfHoursEveryWhere() + << "\n\t time_offset_epoch : " << lut.getTimeOffsetEpoch() + << "\n\t offset_at_start_of_lut : " << lut.getTimeOffsetAtStartOfLUT()); + + EXPECT_GE(24, lut.toHour(i)); + EXPECT_GT(60, lut.toMinute(i)); + EXPECT_GT(60, lut.toSecond(i)); + + const auto current_failures = countFailures(*test_info->result()); + if (current_failures.total > 0) + { + if (i < 0) + i = -1; + } + + if (current_failures.total >= max_failures_per_tz) + break; + } +} + +TEST_P(DateLUT_TimeZone, getTimeZone) +{ + const auto & lut = DateLUT::instance(GetParam()); + + EXPECT_EQ(GetParam(), lut.getTimeZone()); +} + +TEST_P(DateLUT_TimeZone, ZeroTime) +{ + const auto & lut = DateLUT::instance(GetParam()); + + EXPECT_EQ(0, lut.toDayNum(time_t{0})); + EXPECT_EQ(0, lut.toDayNum(DayNum{0})); + EXPECT_EQ(0, lut.toDayNum(ExtendedDayNum{0})); +} + +// Group of tests for timezones that have or had some time ago an offset which is not multiple of 15 minutes. +INSTANTIATE_TEST_SUITE_P(ExoticTimezones, + DateLUT_TimeZone, + ::testing::ValuesIn(std::initializer_list{ + "Africa/El_Aaiun", + "Pacific/Apia", + "Pacific/Enderbury", + "Pacific/Fakaofo", + "Pacific/Kiritimati", + }) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimeZones, + DateLUT_TimeZone, + ::testing::ValuesIn(allTimezones()) +); + +std::ostream & operator<<(std::ostream & ostr, const DateLUTImpl::Values & v) +{ + return ostr << "DateLUTImpl::Values{" + << "\n\t date : " << v.date + << "\n\t year : " << static_cast(v.year) + << "\n\t month : " << static_cast(v.month) + << "\n\t day : " << static_cast(v.day_of_month) + << "\n\t weekday : " << static_cast(v.day_of_week) + << "\n\t days in month : " << static_cast(v.days_in_month) + << "\n\t offset change : " << v.amount_of_offset_change() + << "\n\t offfset change at : " << v.time_at_offset_change() + << "\n}"; +} + +struct TimeRangeParam +{ + const cctz::civil_second begin; + const cctz::civil_second end; + const int step_in_seconds; +}; + +std::ostream & operator<<(std::ostream & ostr, const TimeRangeParam & param) +{ + const auto approximate_step = [](const int step) -> std::string + { + // Convert seconds to a string of seconds or fractional count of minutes/hours/days. + static const size_t multipliers[] = {1 /*seconds to seconds*/, 60 /*seconds to minutes*/, 60 /*minutes to hours*/, 24 /*hours to days*/, 0 /*terminator*/}; + static const char* names[] = {"s", "m", "h", "d", nullptr}; + double result = step; + size_t i = 0; + for (; i < sizeof(multipliers)/sizeof(multipliers[0]) && result > multipliers[i]; ++i) + result /= multipliers[i]; + + char buffer[256] = {'\0'}; + std::snprintf(buffer, sizeof(buffer), "%.1f%s", result, names[i - 1]); + return std::string{buffer}; + }; + + return ostr << param.begin << " : " << param.end << " step: " << param.step_in_seconds << "s (" << approximate_step(param.step_in_seconds) << ")"; +} + +class DateLUT_Timezone_TimeRange : public ::testing::TestWithParam> +{}; + +// refactored test from tests/date_lut3.cpp +TEST_P(DateLUT_Timezone_TimeRange, InRange) +{ + // for a time_t values in range [begin, end) to match with reference obtained from cctz: + // compare date and time components: year, month, day, hours, minutes, seconds, formatted time string. + const auto & [timezone_name, range_data] = GetParam(); + const auto & [begin, end, step] = range_data; + + const auto * test_info = ::testing::UnitTest::GetInstance()->current_test_info(); + static const size_t max_failures_per_case = 3; + cctz::time_zone tz; + ASSERT_TRUE(cctz::load_time_zone(timezone_name, &tz)); + + const auto & lut = DateLUT::instance(timezone_name); + const auto start = cctz::convert(begin, tz).time_since_epoch().count(); + const auto stop = cctz::convert(end, tz).time_since_epoch().count(); + + for (time_t expected_time_t = start; expected_time_t < stop; expected_time_t += step) + { + SCOPED_TRACE(expected_time_t); + + const auto tz_time = cctz::convert(std::chrono::system_clock::from_time_t(expected_time_t), tz); + + EXPECT_EQ(tz_time.year(), lut.toYear(expected_time_t)); + EXPECT_EQ(tz_time.month(), lut.toMonth(expected_time_t)); + EXPECT_EQ(tz_time.day(), lut.toDayOfMonth(expected_time_t)); + EXPECT_EQ(static_cast(cctz::get_weekday(tz_time)) + 1, lut.toDayOfWeek(expected_time_t)); // tm.tm_wday Sunday is 0, while for DateLUTImpl it is 7 + EXPECT_EQ(cctz::get_yearday(tz_time), lut.toDayOfYear(expected_time_t)); + EXPECT_EQ(tz_time.hour(), lut.toHour(expected_time_t)); + EXPECT_EQ(tz_time.minute(), lut.toMinute(expected_time_t)); + EXPECT_EQ(tz_time.second(), lut.toSecond(expected_time_t)); + + const auto time_string = cctz::format("%E4Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(expected_time_t), tz); + EXPECT_EQ(time_string, lut.timeToString(expected_time_t)); + + // it makes sense to let test execute all checks above to simplify debugging, + // but once we've found a bad apple, no need to dig deeper. + if (countFailures(*test_info->result()).total >= max_failures_per_case) + break; + } +} + +/** Next tests are disabled due to following reasons: + * 1. They are huge and take enormous amount of time to run + * 2. Current implementation of DateLUTImpl is inprecise and some cases fail and it seems impractical to try to fix those. + * 3. Many failures (~300) were fixed while refactoring, about ~40 remain the same and 3 new introduced: + * "Asia/Gaza" + * "Pacific/Enderbury" + * "Pacific/Kiritimati" + * So it would be tricky to skip knonw failures to allow all unit tests to pass. + */ +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(20101031), YYYYMMDDToDay(20101101), 15 * 60}, + {YYYYMMDDToDay(20100328), YYYYMMDDToDay(20100330), 15 * 60} + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19701231), 3191 /*53m 11s*/}, + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(20100101), YYYYMMDDToDay(20101231), 3191 /*53m 11s*/}, + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(20200101), YYYYMMDDToDay(20201231), 3191 /*53m 11s*/}, + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + {YYYYMMDDToDay(19500101), YYYYMMDDToDay(19600101), 15 * 60}, + {YYYYMMDDToDay(19300101), YYYYMMDDToDay(19350101), 11 * 15 * 60} + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19700201), 15 * 60}, + {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19701231), 11 * 13 * 17} +// // 11 was chosen as a number which can't divide product of 2-combinarions of (7, 24, 60), +// // to reduce likelehood of hitting same hour/minute/second values for different days. +// // + 12 is just to make sure that last day is covered fully. +// {0, 0 + 11 * 3600 * 24 + 12, 11}, + })) +); + diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 3c27908741c..63fa5d8a5c9 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -390,7 +390,7 @@ private: for (auto d : chineseNewYearIndicators) { /// Let's celebrate until Lantern Festival - if (d <= days && d + 25u >= days) + if (d <= days && d + 25 >= days) return true; else if (d > days) return false; diff --git a/src/Core/DecimalFunctions.h b/src/Core/DecimalFunctions.h index 355cf1d378a..2cd50ab8d08 100644 --- a/src/Core/DecimalFunctions.h +++ b/src/Core/DecimalFunctions.h @@ -50,9 +50,10 @@ inline auto scaleMultiplier(UInt32 scale) * whole - represents whole part of decimal, can be negative or positive. * fractional - for fractional part of decimal, always positive. */ -template +template struct DecimalComponents { + using T = typename DecimalType::NativeType; T whole; T fractional; }; @@ -106,6 +107,15 @@ inline DecimalType decimalFromComponentsWithMultiplier( return DecimalType(value); } +template +inline DecimalType decimalFromComponentsWithMultiplier( + const DecimalComponents & components, + typename DecimalType::NativeType scale_multiplier) +{ + return decimalFromComponentsWithMultiplier(components.whole, components.fractional, scale_multiplier); +} + + /** Make a decimal value from whole and fractional components with given scale. * * @see `decimalFromComponentsWithMultiplier` for details. @@ -126,7 +136,7 @@ inline DecimalType decimalFromComponents( */ template inline DecimalType decimalFromComponents( - const DecimalComponents & components, + const DecimalComponents & components, UInt32 scale) { return decimalFromComponents(components.whole, components.fractional, scale); @@ -136,7 +146,7 @@ inline DecimalType decimalFromComponents( * This is an optimization to reduce number of calls to scaleMultiplier on known scale. */ template -inline DecimalComponents splitWithScaleMultiplier( +inline DecimalComponents splitWithScaleMultiplier( const DecimalType & decimal, typename DecimalType::NativeType scale_multiplier) { @@ -151,7 +161,7 @@ inline DecimalComponents splitWithScaleMultipl /// Split decimal into components: whole and fractional part, @see `DecimalComponents` for details. template -inline DecimalComponents split(const DecimalType & decimal, UInt32 scale) +inline DecimalComponents split(const DecimalType & decimal, UInt32 scale) { if (scale == 0) { diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 1b202c4edb4..3e9c5230955 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -420,8 +420,8 @@ namespace MySQLReplication UInt32 i24 = 0; payload.readStrict(reinterpret_cast(&i24), 3); - DayNum date_day_number = DateLUT::instance().makeDayNum( - static_cast((i24 >> 9) & 0x7fff), static_cast((i24 >> 5) & 0xf), static_cast(i24 & 0x1f)); + const DayNum date_day_number{DateLUT::instance().makeDayNum( + static_cast((i24 >> 9) & 0x7fff), static_cast((i24 >> 5) & 0xf), static_cast(i24 & 0x1f)).toUnderType()}; row.push_back(Field(date_day_number.toUnderType())); break; @@ -443,7 +443,7 @@ namespace MySQLReplication row.push_back(Field{UInt32(date_time)}); else { - DB::DecimalUtils::DecimalComponents components{ + DB::DecimalUtils::DecimalComponents components{ static_cast(date_time), 0}; components.fractional = fsp; @@ -462,7 +462,7 @@ namespace MySQLReplication row.push_back(Field{sec}); else { - DB::DecimalUtils::DecimalComponents components{ + DB::DecimalUtils::DecimalComponents components{ static_cast(sec), 0}; components.fractional = fsp; diff --git a/src/Core/tests/gtest_DecimalFunctions.cpp b/src/Core/tests/gtest_DecimalFunctions.cpp index be64661176b..1069a810d64 100644 --- a/src/Core/tests/gtest_DecimalFunctions.cpp +++ b/src/Core/tests/gtest_DecimalFunctions.cpp @@ -14,7 +14,7 @@ struct DecimalUtilsSplitAndCombineTestParam Decimal64 decimal_value; uint8_t scale; - DecimalUtils::DecimalComponents components; + DecimalUtils::DecimalComponents components; }; std::ostream & operator << (std::ostream & ostr, const DecimalUtilsSplitAndCombineTestParam & param) diff --git a/src/DataStreams/MongoDBBlockInputStream.cpp b/src/DataStreams/MongoDBBlockInputStream.cpp index 5463d95151b..e4ddcd09ede 100644 --- a/src/DataStreams/MongoDBBlockInputStream.cpp +++ b/src/DataStreams/MongoDBBlockInputStream.cpp @@ -270,8 +270,8 @@ namespace throw Exception{"Type mismatch, expected Timestamp, got type id = " + toString(value.type()) + " for column " + name, ErrorCodes::TYPE_MISMATCH}; - assert_cast(column).getData().push_back(UInt16{DateLUT::instance().toDayNum( - static_cast &>(value).value().epochTime())}); + assert_cast(column).getData().push_back(static_cast(DateLUT::instance().toDayNum( + static_cast &>(value).value().epochTime()))); break; } diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/DataTypeDateTime64.h index 198c3739f58..ec3f2fde889 100644 --- a/src/DataTypes/DataTypeDateTime64.h +++ b/src/DataTypes/DataTypeDateTime64.h @@ -48,66 +48,5 @@ public: bool canBePromoted() const override { return false; } }; -/** Tansform-type wrapper for DateTime64, applies given Transform to DateTime64 value or only to a whole part of it. - * - * Depending on what overloads of Transform::execute() are available, when called with DateTime64 value, - * invokes Transform::execute() with: - * * whole part of DateTime64 value, discarding fractional part. - * * DateTime64 value and scale factor. - * - * Suitable Transfotm-types are commonly used in Date/DateTime manipulation functions, - * and should implement static (or const) function with following signatures: - * R execute(UInt32 whole_value, ... , const TimeZoneImpl &) - * OR - * R execute(DateTime64 value, Int64 scale_factor, ... , const TimeZoneImpl &) - * - * Where R and T could be arbitrary types. -*/ -template -class TransformDateTime64 : public Transform -{ -private: - // Detect if Transform::execute is const or static method - // with signature defined by template args (ignoring result type). - template - struct TransformHasExecuteOverload : std::false_type {}; - - template - struct TransformHasExecuteOverload().execute(std::declval()...))>, Args...> - : std::true_type {}; - - template - static constexpr bool TransformHasExecuteOverload_v = TransformHasExecuteOverload::value; - -public: - static constexpr auto name = Transform::name; - - using Transform::execute; - - // non-explicit constructor to allow creating from scale value (or with no scale at all), indispensable in some contexts. - TransformDateTime64(UInt32 scale_ = 0) - : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) - {} - - template - inline auto execute(const DateTime64 & t, Args && ... args) const - { - const auto transform = static_cast(this); - - if constexpr (TransformHasExecuteOverload_v) - { - return transform->execute(t, scale_multiplier, std::forward(args)...); - } - else - { - const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); - return transform->execute(static_cast(components.whole), std::forward(args)...); - } - } - -private: - DateTime64::NativeType scale_multiplier = 1; -}; - } diff --git a/src/Functions/CustomWeekTransforms.h b/src/Functions/CustomWeekTransforms.h index afcbadc835c..28da546eb93 100644 --- a/src/Functions/CustomWeekTransforms.h +++ b/src/Functions/CustomWeekTransforms.h @@ -33,14 +33,21 @@ static inline UInt32 dateIsNotSupported(const char * name) /// This factor transformation will say that the function is monotone everywhere. struct ZeroTransform { - static inline UInt16 execute(UInt32, UInt8, const DateLUTImpl &) { return 0; } static inline UInt16 execute(UInt16, UInt8, const DateLUTImpl &) { return 0; } + static inline UInt16 execute(UInt32, UInt8, const DateLUTImpl &) { return 0; } + static inline UInt16 execute(Int64, UInt8, const DateLUTImpl &) { return 0; } }; struct ToWeekImpl { static constexpr auto name = "toWeek"; + static inline UInt8 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + // TODO: ditch conversion to DayNum, since it doesn't support extended range. + YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode); + return yw.second; + } static inline UInt8 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) { YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode); @@ -59,6 +66,13 @@ struct ToYearWeekImpl { static constexpr auto name = "toYearWeek"; + static inline UInt32 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + // TODO: ditch toDayNum() + YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode | static_cast(WeekModeFlag::YEAR)); + return yw.first * 100 + yw.second; + } + static inline UInt32 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) { YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode | static_cast(WeekModeFlag::YEAR)); @@ -77,13 +91,19 @@ struct ToStartOfWeekImpl { static constexpr auto name = "toStartOfWeek"; + static inline UInt16 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode); +// return time_zone.toFirstDayNumOfWeek(t, week_mode); + } static inline UInt16 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode); +// return time_zone.toFirstDayNumOfWeek(t, week_mode); } static inline UInt16 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfWeek(DayNum(d), week_mode); + return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode); } using FactorTransform = ZeroTransform; diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 333b397312d..c299b9c4169 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -3,6 +3,7 @@ #include #include #include +//#include #include #include #include @@ -33,14 +34,15 @@ namespace ErrorCodes * factor-transformation F is "round to the nearest month" (2015-02-03 -> 2015-02-01). */ -static inline UInt32 dateIsNotSupported(const char * name) -{ - throw Exception("Illegal type Date of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); -} + static inline UInt32 dateIsNotSupported(const char * name) + { + throw Exception("Illegal type Date of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } /// This factor transformation will say that the function is monotone everywhere. struct ZeroTransform { + static inline UInt16 execute(Int64, const DateLUTImpl &) { return 0; } static inline UInt16 execute(UInt32, const DateLUTImpl &) { return 0; } static inline UInt16 execute(UInt16, const DateLUTImpl &) { return 0; } }; @@ -49,6 +51,10 @@ struct ToDateImpl { static constexpr auto name = "toDate"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return UInt16(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return UInt16(time_zone.toDayNum(t)); @@ -65,13 +71,18 @@ struct ToStartOfDayImpl { static constexpr auto name = "toStartOfDay"; + //TODO: right now it is hardcoded to produce DateTime only, needs fixing later. See date_and_time_type_details::ResultDataTypeMap for deduction of result type example. + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toDate(static_cast(t.whole)); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDate(t); } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDate(DayNum(d)); + return time_zone.toDate(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -81,13 +92,19 @@ struct ToMondayImpl { static constexpr auto name = "toMonday"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + //return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); + return time_zone.toFirstDayNumOfWeek(t); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); + //return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); + return time_zone.toFirstDayNumOfWeek(t); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfWeek(DayNum(d)); + return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -97,13 +114,17 @@ struct ToStartOfMonthImpl { static constexpr auto name = "toStartOfMonth"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfMonth(DayNum(d)); + return time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -113,13 +134,17 @@ struct ToStartOfQuarterImpl { static constexpr auto name = "toStartOfQuarter"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfQuarter(DayNum(d)); + return time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -129,13 +154,17 @@ struct ToStartOfYearImpl { static constexpr auto name = "toStartOfYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfYear(DayNum(d)); + return time_zone.toFirstDayNumOfYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -144,9 +173,13 @@ struct ToStartOfYearImpl struct ToTimeImpl { + /// When transforming to time, the date will be equated to 1970-01-01. static constexpr auto name = "toTime"; - /// When transforming to time, the date will be equated to 1970-01-02. + static UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toTime(t.whole) + 86400; + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toTime(t) + 86400; @@ -164,6 +197,10 @@ struct ToStartOfMinuteImpl { static constexpr auto name = "toStartOfMinute"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfMinute(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfMinute(t); @@ -215,6 +252,10 @@ struct ToStartOfFiveMinuteImpl { static constexpr auto name = "toStartOfFiveMinute"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfFiveMinute(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfFiveMinute(t); @@ -231,6 +272,10 @@ struct ToStartOfTenMinutesImpl { static constexpr auto name = "toStartOfTenMinutes"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfTenMinutes(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfTenMinutes(t); @@ -247,6 +292,10 @@ struct ToStartOfFifteenMinutesImpl { static constexpr auto name = "toStartOfFifteenMinutes"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfFifteenMinutes(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfFifteenMinutes(t); @@ -264,6 +313,12 @@ struct TimeSlotImpl { static constexpr auto name = "timeSlot"; + //static inline DecimalUtils::DecimalComponents execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl &) + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl &) + { + return t.whole / 1800 * 1800; + } + static inline UInt32 execute(UInt32 t, const DateLUTImpl &) { return t / 1800 * 1800; @@ -281,6 +336,11 @@ struct ToStartOfHourImpl { static constexpr auto name = "toStartOfHour"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfHour(t.whole); + } + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfHour(t); @@ -298,13 +358,17 @@ struct ToYearImpl { static constexpr auto name = "toYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toYear(t); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toYear(t); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toYear(DayNum(d)); + return time_zone.toYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -314,13 +378,17 @@ struct ToQuarterImpl { static constexpr auto name = "toQuarter"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toQuarter(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toQuarter(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toQuarter(DayNum(d)); + return time_zone.toQuarter(ExtendedDayNum(d)); } using FactorTransform = ToStartOfYearImpl; @@ -330,13 +398,17 @@ struct ToMonthImpl { static constexpr auto name = "toMonth"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toMonth(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toMonth(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toMonth(DayNum(d)); + return time_zone.toMonth(ExtendedDayNum(d)); } using FactorTransform = ToStartOfYearImpl; @@ -346,13 +418,17 @@ struct ToDayOfMonthImpl { static constexpr auto name = "toDayOfMonth"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayOfMonth(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayOfMonth(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDayOfMonth(DayNum(d)); + return time_zone.toDayOfMonth(ExtendedDayNum(d)); } using FactorTransform = ToStartOfMonthImpl; @@ -362,13 +438,17 @@ struct ToDayOfWeekImpl { static constexpr auto name = "toDayOfWeek"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayOfWeek(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayOfWeek(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDayOfWeek(DayNum(d)); + return time_zone.toDayOfWeek(ExtendedDayNum(d)); } using FactorTransform = ToMondayImpl; @@ -378,13 +458,17 @@ struct ToDayOfYearImpl { static constexpr auto name = "toDayOfYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayOfYear(t); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayOfYear(t); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDayOfYear(DayNum(d)); + return time_zone.toDayOfYear(ExtendedDayNum(d)); } using FactorTransform = ToStartOfYearImpl; @@ -394,6 +478,10 @@ struct ToHourImpl { static constexpr auto name = "toHour"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toHour(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toHour(t); @@ -411,6 +499,11 @@ struct TimezoneOffsetImpl { static constexpr auto name = "timezoneOffset"; + static inline time_t execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.timezoneOffset(t); + } + static inline time_t execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.timezoneOffset(t); @@ -428,6 +521,10 @@ struct ToMinuteImpl { static constexpr auto name = "toMinute"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toMinute(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toMinute(t); @@ -444,6 +541,10 @@ struct ToSecondImpl { static constexpr auto name = "toSecond"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toSecond(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toSecond(t); @@ -460,13 +561,17 @@ struct ToISOYearImpl { static constexpr auto name = "toISOYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toISOYear(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toISOYear(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toISOYear(DayNum(d)); + return time_zone.toISOYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -476,13 +581,17 @@ struct ToStartOfISOYearImpl { static constexpr auto name = "toStartOfISOYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfISOYear(DayNum(d)); + return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -492,13 +601,17 @@ struct ToISOWeekImpl { static constexpr auto name = "toISOWeek"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toISOWeek(time_zone.toDayNum(t)); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toISOWeek(time_zone.toDayNum(t)); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toISOWeek(DayNum(d)); + return time_zone.toISOWeek(ExtendedDayNum(d)); } using FactorTransform = ToISOYearImpl; @@ -508,13 +621,17 @@ struct ToRelativeYearNumImpl { static constexpr auto name = "toRelativeYearNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toYear(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toYear(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toYear(DayNum(d)); + return time_zone.toYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -524,13 +641,17 @@ struct ToRelativeQuarterNumImpl { static constexpr auto name = "toRelativeQuarterNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeQuarterNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeQuarterNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeQuarterNum(DayNum(d)); + return time_zone.toRelativeQuarterNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -540,13 +661,17 @@ struct ToRelativeMonthNumImpl { static constexpr auto name = "toRelativeMonthNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeMonthNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeMonthNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeMonthNum(DayNum(d)); + return time_zone.toRelativeMonthNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -556,13 +681,17 @@ struct ToRelativeWeekNumImpl { static constexpr auto name = "toRelativeWeekNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeWeekNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeWeekNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeWeekNum(DayNum(d)); + return time_zone.toRelativeWeekNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -572,10 +701,14 @@ struct ToRelativeDayNumImpl { static constexpr auto name = "toRelativeDayNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toDayNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl &) { return static_cast(d); @@ -589,13 +722,17 @@ struct ToRelativeHourNumImpl { static constexpr auto name = "toRelativeHourNum"; - static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeHourNum(t); } + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeHourNum(static_cast(t)); + } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeHourNum(DayNum(d)); + return time_zone.toRelativeHourNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -605,13 +742,17 @@ struct ToRelativeMinuteNumImpl { static constexpr auto name = "toRelativeMinuteNum"; - static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeMinuteNum(t); } + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeMinuteNum(static_cast(t)); + } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeMinuteNum(DayNum(d)); + return time_zone.toRelativeMinuteNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -621,13 +762,17 @@ struct ToRelativeSecondNumImpl { static constexpr auto name = "toRelativeSecondNum"; + static inline Int64 execute(Int64 t, const DateLUTImpl &) + { + return t; + } static inline UInt32 execute(UInt32 t, const DateLUTImpl &) { return t; } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)); + return time_zone.fromDayNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -637,6 +782,10 @@ struct ToYYYYMMImpl { static constexpr auto name = "toYYYYMM"; + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toNumYYYYMM(t); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMM(t); @@ -653,6 +802,10 @@ struct ToYYYYMMDDImpl { static constexpr auto name = "toYYYYMMDD"; + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toNumYYYYMMDD(t); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMMDD(t); @@ -669,6 +822,10 @@ struct ToYYYYMMDDhhmmssImpl { static constexpr auto name = "toYYYYMMDDhhmmss"; + static inline UInt64 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toNumYYYYMMDDhhmmss(t); + } static inline UInt64 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMMDDhhmmss(t); diff --git a/src/Functions/FunctionCustomWeekToSomething.h b/src/Functions/FunctionCustomWeekToSomething.h index 8a343cffb95..5634ea11584 100644 --- a/src/Functions/FunctionCustomWeekToSomething.h +++ b/src/Functions/FunctionCustomWeekToSomething.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index 5f964b899b4..2b0082f4334 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -25,31 +26,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -/// AddOnDateTime64DefaultImpl provides default implementation of add-X functionality for DateTime64. -/// -/// Default implementation is not to change fractional part, but only modify whole part as if it was DateTime. -/// That means large whole values (for scale less than 9) might not fit into UInt32-range, -/// and hence default implementation will produce incorrect results. -template -struct AddOnDateTime64DefaultImpl -{ - AddOnDateTime64DefaultImpl(UInt32 scale_ = 0) - : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) - {} - - // Default implementation for add/sub on DateTime64: do math on whole part (the same way as for DateTime), leave fractional as it is. - inline DateTime64 execute(const DateTime64 & t, Int64 delta, const DateLUTImpl & time_zone) const - { - const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); - - const auto whole = static_cast(this)->execute(static_cast(components.whole), delta, time_zone); - return DecimalUtils::decimalFromComponentsWithMultiplier(static_cast(whole), components.fractional, scale_multiplier); - } - - UInt32 scale_multiplier = 1; -}; - - /// Type of first argument of 'execute' function overload defines what INPUT DataType it is used for. /// Return type defines what is the OUTPUT (return) type of the CH function. /// Corresponding types: @@ -60,14 +36,15 @@ struct AddOnDateTime64DefaultImpl /// - 'AddSecondsImpl::execute(UInt32, ...) -> UInt32' is available to the ClickHouse users as 'addSeconds(DateTime, ...) -> DateTime' /// - 'AddSecondsImpl::execute(UInt16, ...) -> UInt32' is available to the ClickHouse users as 'addSeconds(Date, ...) -> DateTime' -struct AddSecondsImpl : public AddOnDateTime64DefaultImpl +struct AddSecondsImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addSeconds"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + { + return {t.whole + delta, t.fractional}; + } + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta; @@ -75,18 +52,19 @@ struct AddSecondsImpl : public AddOnDateTime64DefaultImpl static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)) + delta; + return time_zone.fromDayNum(ExtendedDayNum(d)) + delta; } }; -struct AddMinutesImpl : public AddOnDateTime64DefaultImpl +struct AddMinutesImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addMinutes"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + { + return {t.whole + delta * 60, t.fractional}; + } + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta * 60; @@ -94,18 +72,18 @@ struct AddMinutesImpl : public AddOnDateTime64DefaultImpl static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)) + delta * 60; + return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 60; } }; -struct AddHoursImpl : public AddOnDateTime64DefaultImpl +struct AddHoursImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addHours"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + { + return {t.whole + delta * 3600, t.fractional}; + } static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta * 3600; @@ -113,19 +91,20 @@ struct AddHoursImpl : public AddOnDateTime64DefaultImpl static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)) + delta * 3600; + return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 3600; } }; -struct AddDaysImpl : public AddOnDateTime64DefaultImpl +struct AddDaysImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addDays"; - static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addDays(t.whole, delta), t.fractional}; + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addDays(t, delta); } @@ -136,14 +115,15 @@ struct AddDaysImpl : public AddOnDateTime64DefaultImpl } }; -struct AddWeeksImpl : public AddOnDateTime64DefaultImpl +struct AddWeeksImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addWeeks"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addWeeks(t.whole, delta), t.fractional}; + } + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addWeeks(t, delta); @@ -155,14 +135,15 @@ struct AddWeeksImpl : public AddOnDateTime64DefaultImpl } }; -struct AddMonthsImpl : public AddOnDateTime64DefaultImpl +struct AddMonthsImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addMonths"; + static inline DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addMonths(t.whole, delta), t.fractional}; + } + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addMonths(t, delta); @@ -170,18 +151,19 @@ struct AddMonthsImpl : public AddOnDateTime64DefaultImpl static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.addMonths(DayNum(d), delta); + return time_zone.addMonths(ExtendedDayNum(d), delta); } }; -struct AddQuartersImpl : public AddOnDateTime64DefaultImpl +struct AddQuartersImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addQuarters"; + static inline DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addQuarters(t.whole, delta), t.fractional}; + } + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addQuarters(t, delta); @@ -189,18 +171,19 @@ struct AddQuartersImpl : public AddOnDateTime64DefaultImpl static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.addQuarters(DayNum(d), delta); + return time_zone.addQuarters(ExtendedDayNum(d), delta); } }; -struct AddYearsImpl : public AddOnDateTime64DefaultImpl +struct AddYearsImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addYears"; + static inline DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addYears(t.whole, delta), t.fractional}; + } + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addYears(t, delta); @@ -208,7 +191,7 @@ struct AddYearsImpl : public AddOnDateTime64DefaultImpl static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.addYears(DayNum(d), delta); + return time_zone.addYears(ExtendedDayNum(d), delta); } }; @@ -351,6 +334,7 @@ template <> struct ResultDataTypeMap { using ResultDataType = DataTy template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime; }; template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime; }; template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime64; }; +template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime64; }; } template @@ -417,10 +401,18 @@ public: } } + // TransformDateTime64 helps choosing correct overload of exec and does some transformations + // on input and output parameters to simplify support of DateTime64 in concrete Transform. + template + using TransformType = std::conditional_t< + std::is_same_v, + TransformDateTime64, + Transform>; + /// Helper templates to deduce return type based on argument type, since some overloads may promote or denote types, /// e.g. addSeconds(Date, 1) => DateTime template - using TransformExecuteReturnType = decltype(std::declval().execute(FieldType(), 0, std::declval())); + using TransformExecuteReturnType = decltype(std::declval>().execute(FieldType(), 0, std::declval())); // Deduces RETURN DataType from INPUT DataType, based on return type of Transform{}.execute(INPUT_TYPE, UInt64, DateLUTImpl). // e.g. for Transform-type that has execute()-overload with 'UInt16' input and 'UInt32' return, @@ -475,8 +467,9 @@ public: } else if (const auto * datetime64_type = assert_cast(from_type)) { - return DateTimeAddIntervalImpl, Transform>::execute( - Transform{datetime64_type->getScale()}, arguments, result_type); + using WrappedTransformType = TransformType; + return DateTimeAddIntervalImpl, WrappedTransformType>::execute( + WrappedTransformType{datetime64_type->getScale()}, arguments, result_type); } else throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName(), diff --git a/src/Functions/FunctionDateOrDateTimeToSomething.h b/src/Functions/FunctionDateOrDateTimeToSomething.h index e0676f3dc0f..abe859e2f29 100644 --- a/src/Functions/FunctionDateOrDateTimeToSomething.h +++ b/src/Functions/FunctionDateOrDateTimeToSomething.h @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -107,6 +108,7 @@ public: else if (which.isDateTime64()) { const auto scale = static_cast(from_type)->getScale(); + const TransformDateTime64 transformer(scale); return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count, transformer); } @@ -133,7 +135,6 @@ public: /// This method is called only if the function has one argument. Therefore, we do not care about the non-local time zone. const DateLUTImpl & date_lut = DateLUT::instance(); - if (left.isNull() || right.isNull()) return is_not_monotonic; diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 2e2a4ce9cfa..62577b8b402 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -310,10 +311,15 @@ struct ToDateTimeImpl return time_zone.fromDayNum(DayNum(d)); } - // no-op conversion from DateTime to DateTime, used in DateTime64 to DateTime conversion. - static inline UInt32 execute(UInt32 d, const DateLUTImpl & /*time_zone*/) + static inline UInt32 execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) { - return d; + return dt; + } + + // TODO: return UInt32 ??? + static inline Int64 execute(Int64 dt64, const DateLUTImpl & /*time_zone*/) + { + return dt64; } }; @@ -329,6 +335,7 @@ struct ToDateTransform32Or64 static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone) { + // since converting to Date, no need in values outside of default LUT range. return (from < 0xFFFF) ? from : time_zone.toDayNum(std::min(time_t(from), time_t(0xFFFFFFFF))); @@ -342,6 +349,7 @@ struct ToDateTransform32Or64Signed static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone) { + // TODO: decide narrow or extended range based on FromType /// The function should be monotonic (better for query optimizations), so we saturate instead of overflow. if (from < 0) return 0; @@ -447,35 +455,8 @@ template struct ConvertImpl struct ConvertImpl : DateTimeTransformImpl> {}; - -/** Conversion of Date or DateTime to DateTime64: add zero sub-second part. - */ -struct ToDateTime64Transform -{ - static constexpr auto name = "toDateTime64"; - - const DateTime64::NativeType scale_multiplier = 1; - - ToDateTime64Transform(UInt32 scale = 0) - : scale_multiplier(DecimalUtils::scaleMultiplier(scale)) - {} - - inline DateTime64::NativeType execute(UInt16 d, const DateLUTImpl & time_zone) const - { - const auto dt = ToDateTimeImpl::execute(d, time_zone); - return execute(dt, time_zone); - } - - inline DateTime64::NativeType execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) const - { - return DecimalUtils::decimalFromComponentsWithMultiplier(dt, 0, scale_multiplier); - } -}; - -template struct ConvertImpl - : DateTimeTransformImpl {}; -template struct ConvertImpl - : DateTimeTransformImpl {}; +const time_t LUT_MIN_TIME = -1420070400l; // 1925-01-01 UTC +const time_t LUT_MAX_TIME = 9877248000l; // 2282-12-31 UTC /** Conversion of numeric to DateTime64 */ @@ -493,7 +474,7 @@ struct ToDateTime64TransformUnsigned inline NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const { - from = std::min(time_t(from), time_t(0xFFFFFFFF)); + from = std::min(from, LUT_MAX_TIME); return DecimalUtils::decimalFromComponentsWithMultiplier(from, 0, scale_multiplier); } }; @@ -510,9 +491,8 @@ struct ToDateTime64TransformSigned inline NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const { - if (from < 0) - return 0; - from = std::min(time_t(from), time_t(0xFFFFFFFF)); + from = std::max(from, LUT_MIN_TIME); + from = std::min(from, LUT_MAX_TIME); return DecimalUtils::decimalFromComponentsWithMultiplier(from, 0, scale_multiplier); } }; @@ -551,6 +531,7 @@ template struct ConvertImpl struct ConvertImpl : DateTimeTransformImpl> {}; + /** Conversion of DateTime64 to Date or DateTime: discards fractional part. */ template @@ -571,10 +552,41 @@ struct FromDateTime64Transform } }; +/** Conversion of DateTime64 to Date or DateTime: discards fractional part. + */ template struct ConvertImpl - : DateTimeTransformImpl> {}; + : DateTimeTransformImpl> {}; template struct ConvertImpl - : DateTimeTransformImpl> {}; + : DateTimeTransformImpl> {}; + +struct ToDateTime64Transform +{ + static constexpr auto name = "toDateTime64"; + + const DateTime64::NativeType scale_multiplier = 1; + + ToDateTime64Transform(UInt32 scale = 0) + : scale_multiplier(DecimalUtils::scaleMultiplier(scale)) + {} + + inline DateTime64::NativeType execute(UInt16 d, const DateLUTImpl & time_zone) const + { + const auto dt = ToDateTimeImpl::execute(d, time_zone); + return execute(dt, time_zone); + } + + inline DateTime64::NativeType execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) const + { + return DecimalUtils::decimalFromComponentsWithMultiplier(dt, 0, scale_multiplier); + } +}; + +/** Conversion of Date or DateTime to DateTime64: add zero sub-second part. + */ +template struct ConvertImpl + : DateTimeTransformImpl {}; +template struct ConvertImpl + : DateTimeTransformImpl {}; /** Transformation of numbers, dates, datetimes to strings: through formatting. @@ -658,7 +670,6 @@ struct ConvertImpl(*col_with_type_and_name.type); const DateLUTImpl * time_zone = nullptr; - /// For argument of DateTime type, second argument with time zone could be specified. if constexpr (std::is_same_v || std::is_same_v) time_zone = &extractTimeZoneFromFunctionArguments(arguments, 1, 0); @@ -754,6 +765,7 @@ inline void parseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb x = tmp; } +// NOTE: no need of extra overload of DateTime64, since readDateTimeText64 has different signature and that case is explicitly handled in the calling code. template <> inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone) { @@ -762,6 +774,7 @@ inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuf x = tmp; } + template <> inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *) { @@ -989,9 +1002,18 @@ struct ConvertThroughParsing } else if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffortUS) { - time_t res; - parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); - vec_to[i] = res; + if constexpr (to_datetime64) + { + DateTime64 res = 0; + parseDateTime64BestEffortUS(res, vec_to.getScale(), read_buffer, *local_time_zone, *utc_time_zone); + vec_to[i] = res; + } + else + { + time_t res; + parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); + vec_to[i] = res; + } } else { diff --git a/src/Functions/TransformDateTime64.h b/src/Functions/TransformDateTime64.h new file mode 100644 index 00000000000..0a5e36cd2bd --- /dev/null +++ b/src/Functions/TransformDateTime64.h @@ -0,0 +1,92 @@ +#pragma once + +#include +#include + +namespace DB +{ +/** Tansform-type wrapper for DateTime64, simplifies DateTime64 support for given Transform. + * + * Depending on what overloads of Transform::execute() are available, when called with DateTime64 value, + * invokes Transform::execute() with either: + * * whole part of DateTime64 value, discarding fractional part (1) + * * DateTime64 value and scale factor (2) + * * DateTime64 broken down to components, result of execute is then re-assembled back into DateTime64 value (3) + * + * Suitable Transfotm-types are commonly used in Date/DateTime manipulation functions, + * and should implement static (or const) function with following signatures: + * 1: + * R execute(Int64 whole_value, ... ) + * 2: + * R execute(DateTime64 value, Int64 scale_multiplier, ... ) + * 3: + * R execute(DecimalUtils::DecimalComponents components, ... ) + * + * Where R could be of arbitrary type, in case of (3) if R is DecimalUtils::DecimalComponents, result is re-assembed back into DateTime64. +*/ +template +class TransformDateTime64 +{ +private: + // Detect if Transform::execute is const or static method + // with signature defined by template args (ignoring result type). + template + struct TransformHasExecuteOverload : std::false_type {}; + + template + struct TransformHasExecuteOverload().execute(std::declval()...))>, Args...> + : std::true_type {}; + + template + static constexpr bool TransformHasExecuteOverload_v = TransformHasExecuteOverload::value; + +public: + static constexpr auto name = Transform::name; + + // non-explicit constructor to allow creating from scale value (or with no scale at all), indispensable in some contexts. + TransformDateTime64(UInt32 scale_ = 0) + : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) + {} + + template + inline auto execute(const DateTime64 & t, Args && ... args) const + { + if constexpr (TransformHasExecuteOverload_v) + { + return wrapped_transform.execute(t, scale_multiplier, std::forward(args)...); + } + else if constexpr (TransformHasExecuteOverload_v, Args...>) + { + auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + + const auto result = wrapped_transform.execute(components, std::forward(args)...); + using ResultType = std::decay_t; + + if constexpr (std::is_same_v, ResultType>) + { + return DecimalUtils::decimalFromComponentsWithMultiplier(result, scale_multiplier); + } + else + { + return result; + } + } + else + { + const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + return wrapped_transform.execute(static_cast(components.whole), std::forward(args)...); + } + } + + template >>> + inline auto execute(const T & t, Args && ... args) const + { + return wrapped_transform.execute(t, std::forward(args)...); + } + +private: + DateTime64::NativeType scale_multiplier = 1; + Transform wrapped_transform = {}; +}; + +} diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp index 54833eb359f..f660b92efc5 100644 --- a/src/Functions/dateDiff.cpp +++ b/src/Functions/dateDiff.cpp @@ -97,8 +97,8 @@ public: size_t rows = input_rows_count; auto res = ColumnInt64::create(rows); - const DateLUTImpl & timezone_x = extractTimeZoneFromFunctionArguments(arguments, 3, 1); - const DateLUTImpl & timezone_y = extractTimeZoneFromFunctionArguments(arguments, 3, 2); + const auto & timezone_x = extractTimeZoneFromFunctionArguments(arguments, 3, 1); + const auto & timezone_y = extractTimeZoneFromFunctionArguments(arguments, 3, 2); if (unit == "year" || unit == "yy" || unit == "yyyy") dispatchForColumns(x, y, timezone_x, timezone_y, res->getData()); diff --git a/src/Functions/extractTimeZoneFromFunctionArguments.cpp b/src/Functions/extractTimeZoneFromFunctionArguments.cpp index 9d6e54a599e..0ba08b3c612 100644 --- a/src/Functions/extractTimeZoneFromFunctionArguments.cpp +++ b/src/Functions/extractTimeZoneFromFunctionArguments.cpp @@ -66,10 +66,11 @@ const DateLUTImpl & extractTimeZoneFromFunctionArguments(const ColumnsWithTypeAn if (arguments.empty()) return DateLUT::instance(); + const auto & dt_arg = arguments[datetime_arg_num].type.get(); /// If time zone is attached to an argument of type DateTime. - if (const auto * type = checkAndGetDataType(arguments[datetime_arg_num].type.get())) + if (const auto * type = checkAndGetDataType(dt_arg)) return type->getTimeZone(); - if (const auto * type = checkAndGetDataType(arguments[datetime_arg_num].type.get())) + if (const auto * type = checkAndGetDataType(dt_arg)) return type->getTimeZone(); return DateLUT::instance(); diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index fd909ed6fce..5128f077c5a 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -46,9 +46,8 @@ template <> struct ActionValueTypeMap { using ActionValueTyp template <> struct ActionValueTypeMap { using ActionValueType = UInt32; }; template <> struct ActionValueTypeMap { using ActionValueType = UInt16; }; template <> struct ActionValueTypeMap { using ActionValueType = UInt32; }; -// TODO(vnemkov): once there is support for Int64 in LUT, make that Int64. // TODO(vnemkov): to add sub-second format instruction, make that DateTime64 and do some math in Action. -template <> struct ActionValueTypeMap { using ActionValueType = UInt32; }; +template <> struct ActionValueTypeMap { using ActionValueType = Int64; }; /** formatDateTime(time, 'pattern') @@ -434,7 +433,6 @@ public: time_zone_tmp = &DateLUT::instance(); const DateLUTImpl & time_zone = *time_zone_tmp; - const auto & vec = times->getData(); UInt32 scale [[maybe_unused]] = 0; @@ -519,6 +517,8 @@ public: { if constexpr (std::is_same_v) instructions.emplace_back(func, shift); + else if constexpr (std::is_same_v) + instructions.emplace_back(func, shift); else add_shift(shift); }; diff --git a/src/Functions/now64.cpp b/src/Functions/now64.cpp index feb821fde82..32c7a95de17 100644 --- a/src/Functions/now64.cpp +++ b/src/Functions/now64.cpp @@ -30,7 +30,7 @@ Field nowSubsecond(UInt32 scale) if (clock_gettime(CLOCK_REALTIME, &spec)) throwFromErrno("Cannot clock_gettime.", ErrorCodes::CANNOT_CLOCK_GETTIME); - DecimalUtils::DecimalComponents components{spec.tv_sec, spec.tv_nsec}; + DecimalUtils::DecimalComponents components{spec.tv_sec, spec.tv_nsec}; // clock_gettime produces subsecond part in nanoseconds, but decimalFromComponents fractional is scale-dependent. // Andjust fractional to scale, e.g. for 123456789 nanoseconds: diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 6f5a52ca182..f194da166aa 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -35,13 +36,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 years, const DateLUTImpl & time_zone) { - return time_zone.toStartOfYearInterval(DayNum(d), years); + return time_zone.toStartOfYearInterval(ExtendedDayNum(d), years); } static UInt16 execute(UInt32 t, UInt64 years, const DateLUTImpl & time_zone) { return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); } + + static UInt16 execute(Int64 t, UInt64 years, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); + } }; template <> @@ -51,13 +57,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 quarters, const DateLUTImpl & time_zone) { - return time_zone.toStartOfQuarterInterval(DayNum(d), quarters); + return time_zone.toStartOfQuarterInterval(ExtendedDayNum(d), quarters); } static UInt16 execute(UInt32 t, UInt64 quarters, const DateLUTImpl & time_zone) { return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); } + + static UInt16 execute(Int64 t, UInt64 quarters, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); + } }; template <> @@ -67,13 +78,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 months, const DateLUTImpl & time_zone) { - return time_zone.toStartOfMonthInterval(DayNum(d), months); + return time_zone.toStartOfMonthInterval(ExtendedDayNum(d), months); } static UInt16 execute(UInt32 t, UInt64 months, const DateLUTImpl & time_zone) { return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); } + + static UInt16 execute(Int64 t, UInt64 months, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); + } }; template <> @@ -83,13 +99,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 weeks, const DateLUTImpl & time_zone) { - return time_zone.toStartOfWeekInterval(DayNum(d), weeks); + return time_zone.toStartOfWeekInterval(ExtendedDayNum(d), weeks); } static UInt16 execute(UInt32 t, UInt64 weeks, const DateLUTImpl & time_zone) { return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); } + + static UInt16 execute(Int64 t, UInt64 weeks, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); + } }; template <> @@ -99,13 +120,18 @@ namespace static UInt32 execute(UInt16 d, UInt64 days, const DateLUTImpl & time_zone) { - return time_zone.toStartOfDayInterval(DayNum(d), days); + return time_zone.toStartOfDayInterval(ExtendedDayNum(d), days); } static UInt32 execute(UInt32 t, UInt64 days, const DateLUTImpl & time_zone) { return time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days); } + + static UInt32 execute(Int64 t, UInt64 days, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days); + } }; template <> @@ -114,8 +140,8 @@ namespace static constexpr auto name = function_name; static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } - static UInt32 execute(UInt32 t, UInt64 hours, const DateLUTImpl & time_zone) { return time_zone.toStartOfHourInterval(t, hours); } + static UInt32 execute(Int64 t, UInt64 hours, const DateLUTImpl & time_zone) { return time_zone.toStartOfHourInterval(t, hours); } }; template <> @@ -129,6 +155,11 @@ namespace { return time_zone.toStartOfMinuteInterval(t, minutes); } + + static UInt32 execute(Int64 t, UInt64 minutes, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfMinuteInterval(t, minutes); + } }; template <> @@ -142,6 +173,11 @@ namespace { return time_zone.toStartOfSecondInterval(t, seconds); } + + static Int64 execute(Int64 t, UInt64 seconds, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfSecondInterval(t, seconds); + } }; @@ -230,7 +266,7 @@ public: { const auto & time_column = arguments[0]; const auto & interval_column = arguments[1]; - const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(arguments, 2, 0); + const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, 2, 0); auto result_column = dispatchForColumns(time_column, interval_column, time_zone); return result_column; } diff --git a/src/Functions/today.cpp b/src/Functions/today.cpp index 65373058540..511af881d73 100644 --- a/src/Functions/today.cpp +++ b/src/Functions/today.cpp @@ -77,7 +77,7 @@ public: FunctionBaseImplPtr build(const ColumnsWithTypeAndName &, const DataTypePtr &) const override { - return std::make_unique(DateLUT::instance().toDayNum(time(nullptr))); + return std::make_unique(DayNum(DateLUT::instance().toDayNum(time(nullptr)).toUnderType())); } }; diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index e33de04f322..369237f329d 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -747,7 +747,7 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re return ReturnType(false); } - DB::DecimalUtils::DecimalComponents components{static_cast(whole), 0}; + DB::DecimalUtils::DecimalComponents components{static_cast(whole), 0}; if (!buf.eof() && *buf.position() == '.') { @@ -791,9 +791,9 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re return ReturnType(true); } -inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - readDateTimeTextImpl(datetime, buf, date_lut); + readDateTimeTextImpl(datetime, buf, time_zone); } inline void readDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) @@ -801,9 +801,9 @@ inline void readDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer readDateTimeTextImpl(datetime64, scale, buf, date_lut); } -inline bool tryReadDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline bool tryReadDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - return readDateTimeTextImpl(datetime, buf, date_lut); + return readDateTimeTextImpl(datetime, buf, time_zone); } inline bool tryReadDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index a382ae13cdd..72c6b69114f 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -819,12 +819,12 @@ inline void writeDateTimeText(const LocalDateTime & datetime, WriteBuffer & buf) /// In the format YYYY-MM-DD HH:MM:SS, according to the specified time zone. template -inline void writeDateTimeText(time_t datetime, WriteBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void writeDateTimeText(time_t datetime, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - const auto & values = date_lut.getValues(datetime); + const auto & values = time_zone.getValues(datetime); writeDateTimeText( LocalDateTime(values.year, values.month, values.day_of_month, - date_lut.toHour(datetime), date_lut.toMinute(datetime), date_lut.toSecond(datetime)), buf); + time_zone.toHour(datetime), time_zone.toMinute(datetime), time_zone.toSecond(datetime)), buf); } /// In the format YYYY-MM-DD HH:MM:SS.NNNNNNNNN, according to the specified time zone. @@ -849,9 +849,9 @@ inline void writeDateTimeText(DateTime64 datetime64, UInt32 scale, WriteBuffer & /// In the RFC 1123 format: "Tue, 03 Dec 2019 00:11:50 GMT". You must provide GMT DateLUT. /// This is needed for HTTP requests. -inline void writeDateTimeTextRFC1123(time_t datetime, WriteBuffer & buf, const DateLUTImpl & date_lut) +inline void writeDateTimeTextRFC1123(time_t datetime, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - const auto & values = date_lut.getValues(datetime); + const auto & values = time_zone.getValues(datetime); static const char week_days[3 * 8 + 1] = "XXX" "Mon" "Tue" "Wed" "Thu" "Fri" "Sat" "Sun"; static const char months[3 * 13 + 1] = "XXX" "Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug" "Sep" "Oct" "Nov" "Dec"; @@ -865,11 +865,11 @@ inline void writeDateTimeTextRFC1123(time_t datetime, WriteBuffer & buf, const D buf.write(&digits100[values.year / 100 * 2], 2); buf.write(&digits100[values.year % 100 * 2], 2); buf.write(' '); - buf.write(&digits100[date_lut.toHour(datetime) * 2], 2); + buf.write(&digits100[time_zone.toHour(datetime) * 2], 2); buf.write(':'); - buf.write(&digits100[date_lut.toMinute(datetime) * 2], 2); + buf.write(&digits100[time_zone.toMinute(datetime) * 2], 2); buf.write(':'); - buf.write(&digits100[date_lut.toSecond(datetime) * 2], 2); + buf.write(&digits100[time_zone.toSecond(datetime) * 2], 2); buf.write(" GMT", 4); } diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index 47a298ede29..26745a8f138 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -600,7 +600,7 @@ ReturnType parseDateTimeBestEffortImpl( return ReturnType(true); } -template +template ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { time_t whole; @@ -608,12 +608,12 @@ ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuf if constexpr (std::is_same_v) { - if (!parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond)) + if (!parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond)) return false; } else { - parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond); + parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond); } @@ -661,12 +661,17 @@ bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTIm void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { - return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); + return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); +} + +void parseDateTime64BestEffortUS(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) +{ + return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); } bool tryParseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { - return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); + return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); } } diff --git a/src/IO/parseDateTimeBestEffort.h b/src/IO/parseDateTimeBestEffort.h index 65e92cbee42..fe3da24a797 100644 --- a/src/IO/parseDateTimeBestEffort.h +++ b/src/IO/parseDateTimeBestEffort.h @@ -61,6 +61,7 @@ bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); +void parseDateTime64BestEffortUS(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); bool tryParseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); } diff --git a/src/Interpreters/CrashLog.cpp b/src/Interpreters/CrashLog.cpp index bf81a2e8aba..5067acd4a5c 100644 --- a/src/Interpreters/CrashLog.cpp +++ b/src/Interpreters/CrashLog.cpp @@ -40,7 +40,7 @@ void CrashLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(timestamp_ns); columns[i++]->insert(signal); diff --git a/src/Interpreters/MetricLog.cpp b/src/Interpreters/MetricLog.cpp index ce5d5793b87..fd1c120f18c 100644 --- a/src/Interpreters/MetricLog.cpp +++ b/src/Interpreters/MetricLog.cpp @@ -41,7 +41,7 @@ void MetricLogElement::appendToBlock(MutableColumns & columns) const { size_t column_idx = 0; - columns[column_idx++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[column_idx++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[column_idx++]->insert(event_time); columns[column_idx++]->insert(event_time_microseconds); columns[column_idx++]->insert(milliseconds); diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index e1df145cf51..f9ae6518af0 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -49,7 +49,7 @@ void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(operation_name); columns[i++]->insert(start_time_us); columns[i++]->insert(finish_time_us); - columns[i++]->insert(DateLUT::instance().toDayNum(finish_time_us / 1000000)); + columns[i++]->insert(DateLUT::instance().toDayNum(finish_time_us / 1000000).toUnderType()); columns[i++]->insert(attribute_names); // The user might add some ints values, and we will have Int Field, and the // insert will fail because the column requires Strings. Convert the fields diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index 860666a0035..c180a4dd254 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -71,7 +71,7 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(query_id); columns[i++]->insert(event_type); - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(duration_ms); diff --git a/src/Interpreters/QueryLog.cpp b/src/Interpreters/QueryLog.cpp index 82b957f895b..b6902468242 100644 --- a/src/Interpreters/QueryLog.cpp +++ b/src/Interpreters/QueryLog.cpp @@ -119,7 +119,7 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const size_t i = 0; columns[i++]->insert(type); - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(query_start_time); diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp index f1cce1a3da9..31f1fddc87f 100644 --- a/src/Interpreters/QueryThreadLog.cpp +++ b/src/Interpreters/QueryThreadLog.cpp @@ -76,7 +76,7 @@ void QueryThreadLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(query_start_time); diff --git a/src/Interpreters/TextLog.cpp b/src/Interpreters/TextLog.cpp index f60b6acae6f..489bb302ad0 100644 --- a/src/Interpreters/TextLog.cpp +++ b/src/Interpreters/TextLog.cpp @@ -55,7 +55,7 @@ void TextLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(microseconds); diff --git a/src/Interpreters/TraceLog.cpp b/src/Interpreters/TraceLog.cpp index 40bcc0db445..fe7512f2f00 100644 --- a/src/Interpreters/TraceLog.cpp +++ b/src/Interpreters/TraceLog.cpp @@ -42,7 +42,7 @@ void TraceLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(timestamp_ns); diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 1d93ef56dea..d47f64cb1dc 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -141,7 +141,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID /// Conversion between Date and DateTime and vice versa. if (which_type.isDate() && which_from_type.isDateTime()) { - return static_cast(*from_type_hint).getTimeZone().toDayNum(src.get()); + return static_cast(static_cast(*from_type_hint).getTimeZone().toDayNum(src.get()).toUnderType()); } else if (which_type.isDateTime() && which_from_type.isDate()) { diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f2c88cdedd9..8bf785afa0f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4369,7 +4369,7 @@ static String getPartNamePossiblyFake(MergeTreeDataFormatVersion format_version, /// The date range is all month long. const auto & lut = DateLUT::instance(); time_t start_time = lut.YYYYMMDDToDate(parse(part_info.partition_id + "01")); - DayNum left_date = lut.toDayNum(start_time); + DayNum left_date = DayNum{lut.toDayNum(start_time).toUnderType()}; DayNum right_date = DayNum(static_cast(left_date) + lut.daysInMonth(start_time) - 1); return part_info.getPartNameV0(left_date, right_date); } diff --git a/src/Storages/tests/part_name.cpp b/src/Storages/tests/part_name.cpp index 79c5578a8ca..aeadfd208cc 100644 --- a/src/Storages/tests/part_name.cpp +++ b/src/Storages/tests/part_name.cpp @@ -5,7 +5,7 @@ int main(int, char **) { - DayNum today = DateLUT::instance().toDayNum(time(nullptr)); + const DayNum today{DateLUT::instance().toDayNum(time(nullptr)).toUnderType()}; for (DayNum date = today; DayNum(date + 10) > today; --date) { diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.python b/tests/queries/0_stateless/00921_datetime64_compatibility.python index bf0ae8a72ac..c8b9620629d 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.python +++ b/tests/queries/0_stateless/00921_datetime64_compatibility.python @@ -86,8 +86,7 @@ CAST(N as DateTime64(9, 'Europe/Minsk')) formatDateTime(N, '%C %d %D %e %F %H %I %j %m %M %p %R %S %T %u %V %w %y %Y %%') """.splitlines() -# Expanded later to cartesian product of all arguments. -# NOTE: {N} to be turned into N after str.format() for keys (format string), but not for list of values! +# Expanded later to cartesian product of all arguments, using format string. extra_ops = [ # With same type: ( @@ -179,7 +178,7 @@ def escape_string(s): def execute_functions_for_types(functions, types): - # TODO: use string.Template here to allow lines that do not contain type, like: SELECT CAST(toDateTime64(1234567890), 'DateTime64') + # NOTE: use string.Template here to allow lines with missing keys, like type, e.g. SELECT CAST(toDateTime64(1234567890), 'DateTime64') for func in functions: print(("""SELECT 'SELECT {func}';""".format(func=escape_string(func)))) for dt in types: diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.reference b/tests/queries/0_stateless/00921_datetime64_compatibility.reference index 004f4f5e824..67413512e06 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.reference +++ b/tests/queries/0_stateless/00921_datetime64_compatibility.reference @@ -1,5 +1,4 @@ SELECT toTimeZone(N, \'UTC\') - Code: 43 "DateTime('UTC')","2019-09-16 16:20:11" "DateTime64(3, 'UTC')","2019-09-16 16:20:11.234" @@ -35,25 +34,21 @@ SELECT toDayOfWeek(N) "UInt8",1 ------------------------------------------ SELECT toHour(N) - Code: 43 "UInt8",19 "UInt8",19 ------------------------------------------ SELECT toMinute(N) - Code: 43 "UInt8",20 "UInt8",20 ------------------------------------------ SELECT toSecond(N) - Code: 43 "UInt8",11 "UInt8",11 ------------------------------------------ SELECT toUnixTimestamp(N) - Code: 44 "UInt32",1568650811 "UInt32",1568650811 @@ -94,31 +89,26 @@ SELECT toStartOfDay(N) "DateTime('Europe/Minsk')","2019-09-16 00:00:00" ------------------------------------------ SELECT toStartOfHour(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:00:00" "DateTime('Europe/Minsk')","2019-09-16 19:00:00" ------------------------------------------ SELECT toStartOfMinute(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toStartOfFiveMinute(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toStartOfTenMinutes(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toStartOfFifteenMinutes(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:15:00" "DateTime('Europe/Minsk')","2019-09-16 19:15:00" @@ -139,7 +129,6 @@ SELECT toStartOfInterval(N, INTERVAL 1 day) "DateTime('Europe/Minsk')","2019-09-16 00:00:00" ------------------------------------------ SELECT toStartOfInterval(N, INTERVAL 15 minute) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:15:00" "DateTime('Europe/Minsk')","2019-09-16 19:15:00" @@ -160,13 +149,11 @@ SELECT date_trunc(\'day\', N) "DateTime('Europe/Minsk')","2019-09-16 00:00:00" ------------------------------------------ SELECT date_trunc(\'minute\', N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toTime(N) - Code: 43 "DateTime('Europe/Minsk')","1970-01-02 19:20:11" "DateTime('Europe/Minsk')","1970-01-02 19:20:11" @@ -232,7 +219,6 @@ SELECT toYearWeek(N) "UInt32",201937 ------------------------------------------ SELECT timeSlot(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:00:00" "DateTime('Europe/Minsk')","2019-09-16 19:00:00" @@ -375,15 +361,11 @@ SELECT formatDateTime(N, \'%C %d %D %e %F %H %I %j %m %M %p %R %S %T %u %V %w %y SELECT N - N "Int32",0 "Int32",0 - Code: 43 ------------------------------------------ SELECT N + N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N != N @@ -417,47 +399,33 @@ SELECT N >= N "UInt8",1 ------------------------------------------ SELECT N - DT - Code: 43 "Int32",0 - Code: 43 ------------------------------------------ SELECT DT - N - Code: 43 "Int32",0 - Code: 43 ------------------------------------------ SELECT N - D "Int32",0 - Code: 43 - Code: 43 ------------------------------------------ SELECT D - N "Int32",0 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - DT64 - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT DT64 - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N != DT @@ -726,11 +694,8 @@ SELECT N - toUInt8(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt8(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt8(-1) @@ -739,11 +704,8 @@ SELECT N - toInt8(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt8(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toUInt16(1) @@ -752,11 +714,8 @@ SELECT N - toUInt16(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt16(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt16(-1) @@ -765,11 +724,8 @@ SELECT N - toInt16(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt16(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toUInt32(1) @@ -778,11 +734,8 @@ SELECT N - toUInt32(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt32(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt32(-1) @@ -791,11 +744,8 @@ SELECT N - toInt32(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt32(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toUInt64(1) @@ -804,11 +754,8 @@ SELECT N - toUInt64(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt64(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt64(-1) @@ -817,585 +764,486 @@ SELECT N - toInt64(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt64(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N == toUInt8(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt8(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt8(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt8(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toUInt16(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt16(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt16(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt16(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toUInt32(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt32(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt32(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt32(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toUInt64(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt64(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt64(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt64(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N != toUInt8(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt8(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt8(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt8(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toUInt16(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt16(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt16(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt16(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toUInt32(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt32(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt32(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt32(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toUInt64(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt64(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt64(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt64(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt8(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt8(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt8(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt8(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt16(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt16(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt16(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt16(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt32(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt32(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt32(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt32(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt64(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt64(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt64(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt64(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt8(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt8(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt8(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt8(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt16(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt16(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt16(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt16(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt32(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt32(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt32(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt32(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt64(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt64(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt64(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt64(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N > toUInt8(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt8(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt8(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt8(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toUInt16(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt16(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt16(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt16(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toUInt32(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt32(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt32(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt32(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toUInt64(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt64(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt64(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt64(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt8(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt8(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt8(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt8(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt16(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt16(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt16(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt16(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt32(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt32(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt32(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt32(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt64(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt64(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt64(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt64(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.sh b/tests/queries/0_stateless/00921_datetime64_compatibility.sh index 1617e5b1f77..5f5034819e4 100755 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.sh +++ b/tests/queries/0_stateless/00921_datetime64_compatibility.sh @@ -13,4 +13,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) python3 "${CURDIR}"/00921_datetime64_compatibility.python \ | ${CLICKHOUSE_CLIENT} --ignore-error -T -nm --calculate_text_stack_trace 0 --log-level 'error' 2>&1 \ - | sed 's/Received exception .*//g; s/^\(Code: [0-9]\+\).*$/\1/g' + | grep -v 'Received exception .*$' | sed 's/^\(Code: [0-9]\+\).*$/\1/g' diff --git a/tests/queries/0_stateless/01252_weird_time_zone.reference b/tests/queries/0_stateless/01252_weird_time_zone.reference index f2968d4efa6..90f5bf0e30d 100644 --- a/tests/queries/0_stateless/01252_weird_time_zone.reference +++ b/tests/queries/0_stateless/01252_weird_time_zone.reference @@ -1,7 +1,7 @@ -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Kiritimati 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Africa/El_Aaiun 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Asia/Pyongyang 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Kwajalein 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Apia 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Enderbury 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Fakaofo 2020-01-02 03:04:05 2020-01-02 00:00:00 3 diff --git a/tests/queries/0_stateless/01252_weird_time_zone.sql b/tests/queries/0_stateless/01252_weird_time_zone.sql index 68ea903a797..c4919ca4fe0 100644 --- a/tests/queries/0_stateless/01252_weird_time_zone.sql +++ b/tests/queries/0_stateless/01252_weird_time_zone.sql @@ -1,15 +1,15 @@ -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Kiritimati') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Africa/El_Aaiun') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Asia/Pyongyang') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Kwajalein') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Apia') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Enderbury') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Fakaofo') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Kiritimati', toDateTime('2020-01-02 03:04:05', 'Pacific/Kiritimati') AS x, toStartOfDay(x), toHour(x); +SELECT 'Africa/El_Aaiun', toDateTime('2020-01-02 03:04:05', 'Africa/El_Aaiun') AS x, toStartOfDay(x), toHour(x); +SELECT 'Asia/Pyongyang', toDateTime('2020-01-02 03:04:05', 'Asia/Pyongyang') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Kwajalein', toDateTime('2020-01-02 03:04:05', 'Pacific/Kwajalein') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Apia', toDateTime('2020-01-02 03:04:05', 'Pacific/Apia') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Enderbury', toDateTime('2020-01-02 03:04:05', 'Pacific/Enderbury') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Fakaofo', toDateTime('2020-01-02 03:04:05', 'Pacific/Fakaofo') AS x, toStartOfDay(x), toHour(x); -SELECT toHour(toDateTime(rand(), 'Pacific/Kiritimati') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Africa/El_Aaiun') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Asia/Pyongyang') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Kwajalein') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Apia') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Enderbury') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Fakaofo') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Kiritimati', rand() as r, toHour(toDateTime(r, 'Pacific/Kiritimati') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Africa/El_Aaiun', rand() as r, toHour(toDateTime(r, 'Africa/El_Aaiun') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Asia/Pyongyang', rand() as r, toHour(toDateTime(r, 'Asia/Pyongyang') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Kwajalein', rand() as r, toHour(toDateTime(r, 'Pacific/Kwajalein') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Apia', rand() as r, toHour(toDateTime(r, 'Pacific/Apia') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Enderbury', rand() as r, toHour(toDateTime(r, 'Pacific/Enderbury') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Fakaofo', rand() as r, toHour(toDateTime(r, 'Pacific/Fakaofo') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; diff --git a/tests/queries/0_stateless/01440_to_date_monotonicity.reference b/tests/queries/0_stateless/01440_to_date_monotonicity.reference index 96732e5996c..74716fe6223 100644 --- a/tests/queries/0_stateless/01440_to_date_monotonicity.reference +++ b/tests/queries/0_stateless/01440_to_date_monotonicity.reference @@ -1,4 +1,4 @@ 0 -1970-01-01 2106-02-07 1970-04-11 1970-01-01 2106-02-07 +1970-01-01 2106-02-07 1970-04-11 1970-01-01 2149-06-06 1970-01-01 03:00:00 2106-02-07 09:28:15 1970-01-01 03:16:40 2000-01-01 13:12:12 diff --git a/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql b/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql index 7e75d871e07..a61bcff4db7 100644 --- a/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql +++ b/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql @@ -6,7 +6,7 @@ SELECT dt64 < d, toDate(dt64) < d, dt64 < toDateTime64(d, 1, 'UTC'), - + '<=', dt64 <= d, toDate(dt64) <= d, @@ -16,7 +16,7 @@ SELECT dt64 = d, toDate(dt64) = d, dt64 = toDateTime64(d, 1, 'UTC'), - + '>=', dt64 >= d, toDate(dt64) >= d, @@ -31,7 +31,7 @@ SELECT dt64 != d, toDate(dt64) != d, dt64 != toDateTime64(d, 1, 'UTC') -FROM +FROM ( WITH toDateTime('2019-09-16 19:20:11') as val SELECT diff --git a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference index dbcd92da11c..62f620f3ba9 100644 --- a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference +++ b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference @@ -1,2 +1,2 @@ -1970-01-01 1 -1970-01-01 1 +2106-11-11 1 +2106-11-12 1 diff --git a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql index f252e10806a..9a8d37084fb 100644 --- a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql +++ b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql @@ -6,6 +6,6 @@ insert into dt_overflow values('2106-11-11', 1); insert into dt_overflow values('2106-11-12', 1); -select * from dt_overflow; +select * from dt_overflow ORDER BY d; drop table if exists dt_overflow; diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.reference b/tests/queries/0_stateless/01691_DateTime64_clamp.reference index 3adc9a17e5c..da80de59e50 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.reference +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.reference @@ -1,4 +1,5 @@ -- { echo } +<<<<<<< HEAD SELECT toTimeZone(toDateTime(-2, 2), 'Europe/Moscow'); 1970-01-01 03:00:00.00 SELECT toDateTime64(-2, 2, 'Europe/Moscow'); @@ -15,3 +16,25 @@ SELECT toDateTime64(-2., 2, 'Europe/Moscow'); SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow'); 2106-02-07 09:00:00.00 SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; +======= +-- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) +SELECT toDateTime(-2, 2); +1970-01-01 02:59:58.00 +SELECT toDateTime64(-2, 2); +1970-01-01 02:59:58.00 +SELECT CAST(-1 AS DateTime64); +1970-01-01 02:59:59.000 +SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64); +2020-01-01 00:00:00.300 +SELECT toDateTime64(bitShiftLeft(toUInt64(1),33), 2); +2242-03-16 15:56:32.00 +-- These are outsize of extended range and hence clamped +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); +1925-01-01 02:00:00.00 +SELECT CAST(-1 * bitShiftLeft(toUInt64(1),35) AS DateTime64); +1925-01-01 02:00:00.000 +SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); +2282-12-31 03:00:00.000 +SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); +2282-12-31 03:00:00.00 +>>>>>>> af31042451... Extended range of DateTime64 to years 1925 - 2238 diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.sql b/tests/queries/0_stateless/01691_DateTime64_clamp.sql index 92d5a33328f..958de4edada 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.sql +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.sql @@ -1,4 +1,5 @@ -- { echo } +-- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) SELECT toTimeZone(toDateTime(-2, 2), 'Europe/Moscow'); SELECT toDateTime64(-2, 2, 'Europe/Moscow'); SELECT CAST(-1 AS DateTime64(0, 'Europe/Moscow')); @@ -8,3 +9,9 @@ SELECT toTimeZone(toDateTime(-2., 2), 'Europe/Moscow'); SELECT toDateTime64(-2., 2, 'Europe/Moscow'); SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow'); SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; + +-- These are outsize of extended range and hence clamped +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); +SELECT CAST(-1 * bitShiftLeft(toUInt64(1),35) AS DateTime64); +SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); +SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); diff --git a/utils/convert-month-partitioned-parts/main.cpp b/utils/convert-month-partitioned-parts/main.cpp index 0a697937eb6..a6829d79726 100644 --- a/utils/convert-month-partitioned-parts/main.cpp +++ b/utils/convert-month-partitioned-parts/main.cpp @@ -47,8 +47,9 @@ void run(String part_path, String date_column, String dest_path) DayNum max_date; MergeTreePartInfo::parseMinMaxDatesFromPartName(old_part_name, min_date, max_date); - UInt32 yyyymm = DateLUT::instance().toNumYYYYMM(min_date); - if (yyyymm != DateLUT::instance().toNumYYYYMM(max_date)) + const auto & time_zone = DateLUT::instance(); + UInt32 yyyymm = time_zone.toNumYYYYMM(min_date); + if (yyyymm != time_zone.toNumYYYYMM(max_date)) throw Exception("Part " + old_part_name + " spans different months", ErrorCodes::BAD_DATA_PART_NAME); From 4fcc23ec9ae35eff445089858804dc92d465b499 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sun, 14 Feb 2021 13:00:40 +0200 Subject: [PATCH 006/233] Fixed build for GCC-10 --- base/common/DateLUTImpl.h | 2 +- src/Core/MySQL/MySQLReplication.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/tests/part_name.cpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index adfffb04681..f10c62e9865 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -238,7 +238,7 @@ public: template inline ExtendedDayNum toDayNum(V v) const { - return ExtendedDayNum{static_cast(toLUTIndex(v).toUnderType()) - daynum_offset_epoch}; + return ExtendedDayNum{static_cast(toLUTIndex(v).toUnderType() - daynum_offset_epoch)}; } /// Round down to start of monday. diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 3e9c5230955..4cb885d4c34 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -420,8 +420,8 @@ namespace MySQLReplication UInt32 i24 = 0; payload.readStrict(reinterpret_cast(&i24), 3); - const DayNum date_day_number{DateLUT::instance().makeDayNum( - static_cast((i24 >> 9) & 0x7fff), static_cast((i24 >> 5) & 0xf), static_cast(i24 & 0x1f)).toUnderType()}; + const DayNum date_day_number(DateLUT::instance().makeDayNum( + static_cast((i24 >> 9) & 0x7fff), static_cast((i24 >> 5) & 0xf), static_cast(i24 & 0x1f)).toUnderType()); row.push_back(Field(date_day_number.toUnderType())); break; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8bf785afa0f..150f2aa577f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4369,7 +4369,7 @@ static String getPartNamePossiblyFake(MergeTreeDataFormatVersion format_version, /// The date range is all month long. const auto & lut = DateLUT::instance(); time_t start_time = lut.YYYYMMDDToDate(parse(part_info.partition_id + "01")); - DayNum left_date = DayNum{lut.toDayNum(start_time).toUnderType()}; + DayNum left_date = DayNum(lut.toDayNum(start_time).toUnderType()); DayNum right_date = DayNum(static_cast(left_date) + lut.daysInMonth(start_time) - 1); return part_info.getPartNameV0(left_date, right_date); } diff --git a/src/Storages/tests/part_name.cpp b/src/Storages/tests/part_name.cpp index aeadfd208cc..227e19cf17c 100644 --- a/src/Storages/tests/part_name.cpp +++ b/src/Storages/tests/part_name.cpp @@ -5,7 +5,7 @@ int main(int, char **) { - const DayNum today{DateLUT::instance().toDayNum(time(nullptr)).toUnderType()}; + const DayNum today(DateLUT::instance().toDayNum(time(nullptr)).toUnderType()); for (DayNum date = today; DayNum(date + 10) > today; --date) { From 7a53daaefdbfd760cee1099e8f706e5684aac948 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 16 Feb 2021 12:41:08 +0200 Subject: [PATCH 007/233] Fixed issues reported by PVS-Studio and Clang11/GCC10 --- base/common/DateLUTImpl.h | 4 --- base/common/tests/gtest_DateLutImpl.cpp | 48 +++++++------------------ 2 files changed, 12 insertions(+), 40 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index f10c62e9865..898fb7bf843 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -850,10 +850,6 @@ public: if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31)) return ExtendedDayNum(0); - // The day after 2283 are not stored fully as struct Values, so just overflow it as 0 - if (unlikely(year > DATE_LUT_MAX_YEAR)) - return ExtendedDayNum(0); - return toDayNum(makeLUTIndex(year, month, day_of_month)); } diff --git a/base/common/tests/gtest_DateLutImpl.cpp b/base/common/tests/gtest_DateLutImpl.cpp index 395e2eddb00..2355c117328 100644 --- a/base/common/tests/gtest_DateLutImpl.cpp +++ b/base/common/tests/gtest_DateLutImpl.cpp @@ -25,18 +25,6 @@ cctz::civil_day YYYYMMDDToDay(unsigned value) value % 100); // day } -cctz::civil_second YYYYMMDDHMMSSToSecond(std::uint64_t value) -{ - return cctz::civil_second( - value / 10000000000, - value / 100000000 % 100, - value / 1000000 % 100, - value / 10000 % 100, - value / 100 % 100, - value % 100); -} - - std::vector allTimezones() { std::vector result; @@ -82,14 +70,17 @@ FailuresCount countFailures(const ::testing::TestResult & test_result) } -TEST(YYYYMMDDToDay, Test) +TEST(DateLUTTest, Test_makeDayNum) { - std::cerr << YYYYMMDDHMMSSToSecond(19700101'00'00'00) << std::endl; + const DateLUTImpl & lut = DateLUT::instance("UTC"); + EXPECT_EQ(0, lut.makeDayNum(2500, 12, 25)); + EXPECT_EQ(0, lut.makeDayNum(1924, 12, 31)); } + TEST(DateLUTTest, TimeValuesInMiddleOfRange) { - const DateLUTImpl lut("Europe/Minsk"); + const DateLUTImpl & lut = DateLUT::instance("Europe/Minsk"); const time_t time = 1568650811; // 2019-09-16 19:20:11 (Monday) EXPECT_EQ(lut.getTimeZone(), "Europe/Minsk"); @@ -151,7 +142,7 @@ TEST(DateLUTTest, TimeValuesInMiddleOfRange) TEST(DateLUTTest, TimeValuesAtLeftBoderOfRange) { - const DateLUTImpl lut("UTC"); + const DateLUTImpl & lut = DateLUT::instance("UTC"); const time_t time = 0; // 1970-01-01 00:00:00 (Thursday) EXPECT_EQ(lut.getTimeZone(), "UTC"); @@ -212,7 +203,7 @@ TEST(DateLUTTest, TimeValuesAtLeftBoderOfRange) TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) { // Value is at the right border of the OLD (small) LUT, and provides meaningful values where OLD LUT would provide garbage. - const DateLUTImpl lut("UTC"); + const DateLUTImpl & lut = DateLUT::instance("UTC"); const time_t time = 4294343873; // 2106-01-31T01:17:53 (Sunday) @@ -276,11 +267,11 @@ TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) class DateLUT_TimeZone : public ::testing::TestWithParam {}; -TEST_P(DateLUT_TimeZone, DISABLED_LoadAllTimeZones) +TEST_P(DateLUT_TimeZone, DISABLED_LoadLut) { // There are some assumptions and assertions about TZ data made in DateLUTImpl which are verified upon loading, // to make sure that those assertions are true for all timezones we are going to load all of them one by one. - DateLUTImpl{GetParam()}; + DateLUT::instance(GetParam()); } // Another long running test, shouldn't be run to often @@ -292,7 +283,7 @@ TEST_P(DateLUT_TimeZone, VaidateTimeComponentsAroundEpoch) const auto timezone_name = GetParam(); const auto * test_info = ::testing::UnitTest::GetInstance()->current_test_info(); - const auto lut = DateLUTImpl(timezone_name); + const DateLUTImpl & lut = DateLUT::instance(timezone_name); for (time_t i = -856147870; i < 86400 * 10000; i += 11 * 13 * 17 * 19) { @@ -376,22 +367,7 @@ struct TimeRangeParam std::ostream & operator<<(std::ostream & ostr, const TimeRangeParam & param) { - const auto approximate_step = [](const int step) -> std::string - { - // Convert seconds to a string of seconds or fractional count of minutes/hours/days. - static const size_t multipliers[] = {1 /*seconds to seconds*/, 60 /*seconds to minutes*/, 60 /*minutes to hours*/, 24 /*hours to days*/, 0 /*terminator*/}; - static const char* names[] = {"s", "m", "h", "d", nullptr}; - double result = step; - size_t i = 0; - for (; i < sizeof(multipliers)/sizeof(multipliers[0]) && result > multipliers[i]; ++i) - result /= multipliers[i]; - - char buffer[256] = {'\0'}; - std::snprintf(buffer, sizeof(buffer), "%.1f%s", result, names[i - 1]); - return std::string{buffer}; - }; - - return ostr << param.begin << " : " << param.end << " step: " << param.step_in_seconds << "s (" << approximate_step(param.step_in_seconds) << ")"; + return ostr << param.begin << " : " << param.end << " step: " << param.step_in_seconds << "s"; } class DateLUT_Timezone_TimeRange : public ::testing::TestWithParam> From d321c13cbf688f3fd526b1ec36ae296e8939ab2e Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 18 Feb 2021 00:33:34 +0200 Subject: [PATCH 008/233] Fixed special build --- base/common/DateLUTImpl.cpp | 2 +- base/common/tests/gtest_DateLutImpl.cpp | 36 ++++++++++++------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 906f88fa90f..6f4fb3dd5fc 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -53,7 +53,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) const cctz::civil_day epoch{1970, 1, 1}; const cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1}; - time_t start_of_day = std::chrono::system_clock::to_time_t(cctz_time_zone.lookup(lut_start).pre); + time_t start_of_day; time_offset_epoch = cctz::convert(cctz::civil_second(lut_start), cctz_time_zone).time_since_epoch().count(); // Note validated this against all timezones in the system. diff --git a/base/common/tests/gtest_DateLutImpl.cpp b/base/common/tests/gtest_DateLutImpl.cpp index 2355c117328..9169d9e768f 100644 --- a/base/common/tests/gtest_DateLutImpl.cpp +++ b/base/common/tests/gtest_DateLutImpl.cpp @@ -29,7 +29,7 @@ std::vector allTimezones() { std::vector result; - auto timezone_name = auto_time_zones; + const auto * timezone_name = auto_time_zones; while (*timezone_name) { result.push_back(*timezone_name); @@ -70,7 +70,7 @@ FailuresCount countFailures(const ::testing::TestResult & test_result) } -TEST(DateLUTTest, Test_makeDayNum) +TEST(DateLUTTest, makeDayNumTest) { const DateLUTImpl & lut = DateLUT::instance("UTC"); EXPECT_EQ(0, lut.makeDayNum(2500, 12, 25)); @@ -264,10 +264,10 @@ TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) } -class DateLUT_TimeZone : public ::testing::TestWithParam +class DateLUTWithTimeZone : public ::testing::TestWithParam {}; -TEST_P(DateLUT_TimeZone, DISABLED_LoadLut) +TEST_P(DateLUTWithTimeZone, DISABLED_LoadLut) { // There are some assumptions and assertions about TZ data made in DateLUTImpl which are verified upon loading, // to make sure that those assertions are true for all timezones we are going to load all of them one by one. @@ -275,12 +275,12 @@ TEST_P(DateLUT_TimeZone, DISABLED_LoadLut) } // Another long running test, shouldn't be run to often -TEST_P(DateLUT_TimeZone, VaidateTimeComponentsAroundEpoch) +TEST_P(DateLUTWithTimeZone, VaidateTimeComponentsAroundEpoch) { // Converting time around 1970-01-01 to hour-minute-seconds time components // could be problematic. const size_t max_failures_per_tz = 3; - const auto timezone_name = GetParam(); + const auto * timezone_name = GetParam(); const auto * test_info = ::testing::UnitTest::GetInstance()->current_test_info(); const DateLUTImpl & lut = DateLUT::instance(timezone_name); @@ -311,14 +311,14 @@ TEST_P(DateLUT_TimeZone, VaidateTimeComponentsAroundEpoch) } } -TEST_P(DateLUT_TimeZone, getTimeZone) +TEST_P(DateLUTWithTimeZone, getTimeZone) { const auto & lut = DateLUT::instance(GetParam()); EXPECT_EQ(GetParam(), lut.getTimeZone()); } -TEST_P(DateLUT_TimeZone, ZeroTime) +TEST_P(DateLUTWithTimeZone, ZeroTime) { const auto & lut = DateLUT::instance(GetParam()); @@ -329,7 +329,7 @@ TEST_P(DateLUT_TimeZone, ZeroTime) // Group of tests for timezones that have or had some time ago an offset which is not multiple of 15 minutes. INSTANTIATE_TEST_SUITE_P(ExoticTimezones, - DateLUT_TimeZone, + DateLUTWithTimeZone, ::testing::ValuesIn(std::initializer_list{ "Africa/El_Aaiun", "Pacific/Apia", @@ -340,7 +340,7 @@ INSTANTIATE_TEST_SUITE_P(ExoticTimezones, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimeZones, - DateLUT_TimeZone, + DateLUTWithTimeZone, ::testing::ValuesIn(allTimezones()) ); @@ -370,11 +370,11 @@ std::ostream & operator<<(std::ostream & ostr, const TimeRangeParam & param) return ostr << param.begin << " : " << param.end << " step: " << param.step_in_seconds << "s"; } -class DateLUT_Timezone_TimeRange : public ::testing::TestWithParam> +class DateLUTWithTimeZoneAndTimeRange : public ::testing::TestWithParam> {}; // refactored test from tests/date_lut3.cpp -TEST_P(DateLUT_Timezone_TimeRange, InRange) +TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) { // for a time_t values in range [begin, end) to match with reference obtained from cctz: // compare date and time components: year, month, day, hours, minutes, seconds, formatted time string. @@ -425,7 +425,7 @@ TEST_P(DateLUT_Timezone_TimeRange, InRange) * So it would be tricky to skip knonw failures to allow all unit tests to pass. */ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ @@ -436,7 +436,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ @@ -446,7 +446,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ @@ -456,7 +456,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ @@ -466,7 +466,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ @@ -476,7 +476,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ From d568ba5ec7f966842075a66fc9ff554db8430701 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sun, 21 Feb 2021 12:04:29 +0200 Subject: [PATCH 009/233] Marked 00921_datetime64_compatibility as long test --- ...bility.python => 00921_datetime64_compatibility_long.python} | 0 ....reference => 00921_datetime64_compatibility_long.reference} | 0 ..._compatibility.sh => 00921_datetime64_compatibility_long.sh} | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename tests/queries/0_stateless/{00921_datetime64_compatibility.python => 00921_datetime64_compatibility_long.python} (100%) rename tests/queries/0_stateless/{00921_datetime64_compatibility.reference => 00921_datetime64_compatibility_long.reference} (100%) rename tests/queries/0_stateless/{00921_datetime64_compatibility.sh => 00921_datetime64_compatibility_long.sh} (91%) diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.python b/tests/queries/0_stateless/00921_datetime64_compatibility_long.python similarity index 100% rename from tests/queries/0_stateless/00921_datetime64_compatibility.python rename to tests/queries/0_stateless/00921_datetime64_compatibility_long.python diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.reference b/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference similarity index 100% rename from tests/queries/0_stateless/00921_datetime64_compatibility.reference rename to tests/queries/0_stateless/00921_datetime64_compatibility_long.reference diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.sh b/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh similarity index 91% rename from tests/queries/0_stateless/00921_datetime64_compatibility.sh rename to tests/queries/0_stateless/00921_datetime64_compatibility_long.sh index 5f5034819e4..52a29c19be1 100755 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.sh +++ b/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh @@ -11,6 +11,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # ${CURDIR}/00921_datetime64_compatibility.python -python3 "${CURDIR}"/00921_datetime64_compatibility.python \ +python3 "${CURDIR}"/00921_datetime64_compatibility_long.python \ | ${CLICKHOUSE_CLIENT} --ignore-error -T -nm --calculate_text_stack_trace 0 --log-level 'error' 2>&1 \ | grep -v 'Received exception .*$' | sed 's/^\(Code: [0-9]\+\).*$/\1/g' From b8b916008c75e041c7d44c69d65fa0c1135d124d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 22 Feb 2021 20:50:17 +0300 Subject: [PATCH 010/233] Update DateLUTImpl.h --- base/common/DateLUTImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 898fb7bf843..1c897080e3a 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -105,7 +105,7 @@ public: /// The order of fields matters for alignment and sizeof. struct Values { - /// Least significat 64 bits from time_t at beginning of the day. + /// time_t at beginning of the day. Int64 date; /// Properties of the day. From d5757c67e612b1846e71536a965fde7efda2ac2a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 22 Feb 2021 22:45:55 +0300 Subject: [PATCH 011/233] Update DateLUTImpl.h --- base/common/DateLUTImpl.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 1c897080e3a..5a12ad5dc13 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -119,8 +119,9 @@ public: UInt8 days_in_month; /// For days, when offset from UTC was changed due to daylight saving time or permanent change, following values could be non zero. - Int8 amount_of_offset_change_value; /// Usually -3600 or 3600, but look at Lord Howe Island. multiply by OffsetChangeFactor - UInt8 time_at_offset_change_value; /// In seconds from beginning of the day. multiply by OffsetChangeFactor + /// All in OffsetChangeFactor (15 minute) intervals. + Int8 amount_of_offset_change_value; /// Usually -4 or 4, but look at Lord Howe Island. Multiply by OffsetChangeFactor + UInt8 time_at_offset_change_value; /// In seconds from beginning of the day. Multiply by OffsetChangeFactor inline Int32 amount_of_offset_change() const { From f4b2cbc30f66371ab83eb0d59a7ecb55825c99f1 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sat, 27 Feb 2021 13:21:31 +0200 Subject: [PATCH 012/233] Fixed tests --- .../01691_DateTime64_clamp.reference | 23 ++++--------------- ..._toDateTime_from_string_clamping.reference | 8 +++---- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.reference b/tests/queries/0_stateless/01691_DateTime64_clamp.reference index da80de59e50..f29a9e2d1d5 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.reference +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.reference @@ -1,11 +1,11 @@ -- { echo } -<<<<<<< HEAD +-- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) SELECT toTimeZone(toDateTime(-2, 2), 'Europe/Moscow'); -1970-01-01 03:00:00.00 +1970-01-01 02:59:58.00 SELECT toDateTime64(-2, 2, 'Europe/Moscow'); -1970-01-01 03:00:00.00 +1970-01-01 02:59:58.00 SELECT CAST(-1 AS DateTime64(0, 'Europe/Moscow')); -1970-01-01 03:00:00 +1970-01-01 02:59:59 SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64(0, 'Europe/Moscow')); 2020-01-01 00:00:00 SELECT toDateTime64(bitShiftLeft(toUInt64(1), 33), 2, 'Europe/Moscow') FORMAT Null; @@ -14,20 +14,8 @@ SELECT toTimeZone(toDateTime(-2., 2), 'Europe/Moscow'); SELECT toDateTime64(-2., 2, 'Europe/Moscow'); 1970-01-01 03:00:00.00 SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow'); -2106-02-07 09:00:00.00 +2106-02-07 09:28:16.00 SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; -======= --- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) -SELECT toDateTime(-2, 2); -1970-01-01 02:59:58.00 -SELECT toDateTime64(-2, 2); -1970-01-01 02:59:58.00 -SELECT CAST(-1 AS DateTime64); -1970-01-01 02:59:59.000 -SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64); -2020-01-01 00:00:00.300 -SELECT toDateTime64(bitShiftLeft(toUInt64(1),33), 2); -2242-03-16 15:56:32.00 -- These are outsize of extended range and hence clamped SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); 1925-01-01 02:00:00.00 @@ -37,4 +25,3 @@ SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); 2282-12-31 03:00:00.000 SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); 2282-12-31 03:00:00.00 ->>>>>>> af31042451... Extended range of DateTime64 to years 1925 - 2238 diff --git a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference index 228086615da..92639948fbc 100644 --- a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference +++ b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference @@ -1,9 +1,9 @@ -- { echo } SELECT toString(toDateTime('-922337203.6854775808', 1)); -2106-02-07 15:41:33.6 +1940-10-09 22:13:17.6 SELECT toString(toDateTime('9922337203.6854775808', 1)); -2104-12-30 00:50:11.6 +1925-07-26 00:46:43.6 SELECT toDateTime64(CAST('10000000000.1' AS Decimal64(1)), 1); -2106-02-07 20:50:08.1 +1928-01-11 00:46:40.1 SELECT toDateTime64(CAST('-10000000000.1' AS Decimal64(1)), 1); -2011-12-23 00:38:08.1 +2011-12-22 00:13:20.1 From 6e6ae1db5706e497a27e700772c835b032b9d3ad Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 2 Mar 2021 15:55:26 +0200 Subject: [PATCH 013/233] Nudge CI From 04672a4365409c3eedbce5d79b3f20692fa7f226 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 22:29:26 +0300 Subject: [PATCH 014/233] Update DateLUTImpl.cpp --- base/common/DateLUTImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 6f4fb3dd5fc..d9dde3a8d0d 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -56,7 +56,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) time_t start_of_day; time_offset_epoch = cctz::convert(cctz::civil_second(lut_start), cctz_time_zone).time_since_epoch().count(); - // Note validated this against all timezones in the system. + // Note: it's validated against all timezones in the system. assert((epoch - lut_start) == daynum_offset_epoch); offset_at_start_of_epoch = cctz_time_zone.lookup(cctz_time_zone.lookup(epoch).pre).offset; From e412bcb490c3519373fab7a69369be82bbf248bf Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 22:30:12 +0300 Subject: [PATCH 015/233] Update DateLUTImpl.cpp --- base/common/DateLUTImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index d9dde3a8d0d..563b744e073 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -95,7 +95,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = 0; values.amount_of_offset_change_value = 0; - // TODO: this partially ignores fractional pre-epoch offsets, which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe\Minsk + // TODO: this partially ignores fractional pre-epoch offsets, which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe/Minsk // when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. // https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 if (start_of_day > 0 && start_of_day % 3600) From 80c17d55376e29c3a73aa231aac78d341af03271 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 22:41:10 +0300 Subject: [PATCH 016/233] Update DateLUTImpl.cpp --- base/common/DateLUTImpl.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 563b744e073..68a3aa97e51 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -140,8 +140,6 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) } while (i < DATE_LUT_SIZE && lut[i - 1].year <= DATE_LUT_MAX_YEAR); -// date_lut_max = start_of_day; - /// Fill excessive part of lookup table. This is needed only to simplify handling of overflow cases. while (i < DATE_LUT_SIZE) { From 2f23f1b123517251b5661a971e7499f0e9fc99b7 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 22:49:20 +0300 Subject: [PATCH 017/233] Update DateLUTImpl.h --- base/common/DateLUTImpl.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 5a12ad5dc13..72786e31cbc 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -136,18 +136,20 @@ public: /// Since most of the modern timezones have a DST change aligned to 15 minutes, to save as much space as possible inside Value, /// we are dividing any offset change related value by this factor before setting it to Value, /// hence it has to be explicitly multiplied back by this factor before being used. - static const UInt16 OffsetChangeFactor = 900; + static constexpr UInt16 OffsetChangeFactor = 900; }; static_assert(sizeof(Values) == 16); private: - // Mask is all-ones to allow efficient protection against overflow. - static const UInt32 date_lut_mask = 0x1ffff; + /// Mask is all-ones to allow efficient protection against overflow. + static constexpr UInt32 date_lut_mask = 0x1ffff; static_assert(date_lut_mask == DATE_LUT_SIZE - 1); - const UInt32 daynum_offset_epoch = 16436; // offset to epoch in days (ExtendedDayNum) of the first day in LUT. + /// Offset to epoch in days (ExtendedDayNum) of the first day in LUT. + static constexpr UInt32 daynum_offset_epoch = 16436; + static_assert(daynum_offset_epoch == (DATE_LUT_MIN_YEAR - 1970) * 365 + (1970 - DATE_LUT_MIN_YEAR / 4 * 4) / 4); /// Lookup table is indexed by LUTIndex. /// Day nums are the same in all time zones. 1970-01-01 is 0 and so on. From 0889e0da87ea511cc30669b6c2aec888052d00ee Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 22:52:07 +0300 Subject: [PATCH 018/233] Update DateLUTImpl.h --- base/common/DateLUTImpl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 72786e31cbc..9b36c60eb5e 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -148,6 +148,7 @@ private: static_assert(date_lut_mask == DATE_LUT_SIZE - 1); /// Offset to epoch in days (ExtendedDayNum) of the first day in LUT. + /// "epoch" is the Unix Epoch (starts at unix timestamp zero) static constexpr UInt32 daynum_offset_epoch = 16436; static_assert(daynum_offset_epoch == (DATE_LUT_MIN_YEAR - 1970) * 365 + (1970 - DATE_LUT_MIN_YEAR / 4 * 4) / 4); From 3e19f4a00fc03d466b3648305e3b236653485e8b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 23:09:59 +0300 Subject: [PATCH 019/233] Update DateLUTImpl.h --- base/common/DateLUTImpl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 9b36c60eb5e..79bf1a9f127 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -166,6 +166,7 @@ private: /// UTC offset at beginning of the Unix epoch. The same as unix timestamp of 1970-01-01 00:00:00 local time. time_t offset_at_start_of_epoch; + /// UTC offset at the beginning of the first supported year. time_t offset_at_start_of_lut; bool offset_is_whole_number_of_hours_everytime; time_t time_offset_epoch; From cf0912ba02c8e89fd1d913318a9c117ab336a5df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:14:33 +0300 Subject: [PATCH 020/233] Remove unused field --- base/common/DateLUTImpl.cpp | 1 - base/common/DateLUTImpl.h | 2 -- base/common/tests/gtest_DateLutImpl.cpp | 1 - 3 files changed, 4 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 68a3aa97e51..bf180acb835 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -54,7 +54,6 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) const cctz::civil_day epoch{1970, 1, 1}; const cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1}; time_t start_of_day; - time_offset_epoch = cctz::convert(cctz::civil_second(lut_start), cctz_time_zone).time_since_epoch().count(); // Note: it's validated against all timezones in the system. assert((epoch - lut_start) == daynum_offset_epoch); diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 79bf1a9f127..429db332b49 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -169,7 +169,6 @@ private: /// UTC offset at the beginning of the first supported year. time_t offset_at_start_of_lut; bool offset_is_whole_number_of_hours_everytime; - time_t time_offset_epoch; /// Time zone name. std::string time_zone; @@ -230,7 +229,6 @@ public: // Methods only for unit-testing, it makes very little sense to use it from user code. auto getOffsetAtStartOfEpoch() const { return offset_at_start_of_epoch; } auto getOffsetIsWholNumberOfHoursEveryWhere() const { return offset_is_whole_number_of_hours_everytime; } - auto getTimeOffsetEpoch() const { return time_offset_epoch; } auto getTimeOffsetAtStartOfLUT() const { return offset_at_start_of_lut; } /// All functions below are thread-safe; arguments are not checked. diff --git a/base/common/tests/gtest_DateLutImpl.cpp b/base/common/tests/gtest_DateLutImpl.cpp index 9169d9e768f..8cc4fbbbfb0 100644 --- a/base/common/tests/gtest_DateLutImpl.cpp +++ b/base/common/tests/gtest_DateLutImpl.cpp @@ -292,7 +292,6 @@ TEST_P(DateLUTWithTimeZone, VaidateTimeComponentsAroundEpoch) << "\n\ttimestamp: " << i << "\n\t offset at start of epoch : " << lut.getOffsetAtStartOfEpoch() << "\n\t offset_is_whole_number_of_hours_everytime : " << lut.getOffsetIsWholNumberOfHoursEveryWhere() - << "\n\t time_offset_epoch : " << lut.getTimeOffsetEpoch() << "\n\t offset_at_start_of_lut : " << lut.getTimeOffsetAtStartOfLUT()); EXPECT_GE(24, lut.toHour(i)); From aaef0c5ebd1246fa633371ee2d3635b2539bdcee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:19:26 +0300 Subject: [PATCH 021/233] Fix build --- base/common/DateLUTImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 429db332b49..5275cc83abb 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -150,7 +150,7 @@ private: /// Offset to epoch in days (ExtendedDayNum) of the first day in LUT. /// "epoch" is the Unix Epoch (starts at unix timestamp zero) static constexpr UInt32 daynum_offset_epoch = 16436; - static_assert(daynum_offset_epoch == (DATE_LUT_MIN_YEAR - 1970) * 365 + (1970 - DATE_LUT_MIN_YEAR / 4 * 4) / 4); + static_assert(daynum_offset_epoch == (1970 - DATE_LUT_MIN_YEAR) * 365 + (1970 - DATE_LUT_MIN_YEAR / 4 * 4) / 4); /// Lookup table is indexed by LUTIndex. /// Day nums are the same in all time zones. 1970-01-01 is 0 and so on. From ed0099a11308719bc8319c9aac5707e092379f29 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:24:56 +0300 Subject: [PATCH 022/233] Maybe unused condition --- base/common/DateLUTImpl.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 5275cc83abb..66df4744b72 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -179,10 +179,11 @@ private: const UInt32 guess = ((t / 86400) + daynum_offset_epoch) & date_lut_mask; /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. - if ((guess == daynum_offset_epoch || t >= lut[guess].date) && t < lut[UInt32(guess + 1)].date) + if (t >= lut[guess].date && t < lut[UInt32(guess + 1)].date) return LUTIndex{guess}; - /// Time zones that have offset 0 from UTC do daylight saving time change (if any) towards increasing UTC offset (example: British Standard Time). + /// Time zones that have offset 0 from UTC do daylight saving time change (if any) + /// towards increasing UTC offset (example: British Standard Time). if (t >= lut[UInt32(guess + 1)].date) return LUTIndex(guess + 1); From b1b3db09d765c008bdb053bdc2693d02f546055e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:34:15 +0300 Subject: [PATCH 023/233] Remove commented out code --- base/common/DateLUTImpl.h | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 66df4744b72..890ed3ae173 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -189,6 +189,7 @@ private: if (lut[guess - 1].date <= t) return LUTIndex(guess - 1); + return LUTIndex(guess - 2); } @@ -212,12 +213,6 @@ private: return i; } -// template -// inline LUTIndex toLUTIndex(T t) const -// { -// return LUTIndex{static_cast(t) & date_lut_mask}; -// } - template inline const Values & find(V v) const { @@ -275,11 +270,6 @@ public: return toDayNum(i - (lut[i].day_of_month - 1)); } -// inline DayNum toFirstDayNumOfMonth(time_t t) const -// { -// return toFirstDayNumOfMonth(toDayNum(t)); -// } - /// Round down to start of quarter. template inline ExtendedDayNum toFirstDayNumOfQuarter(V v) const From 6f5877abdfcf686913526c013ad09921c995d6df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:53:52 +0300 Subject: [PATCH 024/233] Fix build after merge with master --- base/common/DateLUTImpl.h | 221 +++++++++++++++++++------------------- 1 file changed, 111 insertions(+), 110 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 2fdf293cef4..740411b7113 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -213,8 +213,8 @@ private: return i; } - template - inline const Values & find(V v) const + template + inline const Values & find(DateOrTime v) const { return lut[toLUTIndex(v)]; } @@ -234,54 +234,53 @@ public: return d; } - template - inline ExtendedDayNum toDayNum(V v) const + template + inline ExtendedDayNum toDayNum(DateOrTime v) const { return ExtendedDayNum{static_cast(toLUTIndex(v).toUnderType() - daynum_offset_epoch)}; } /// Round down to start of monday. - template - inline time_t toFirstDayOfWeek(V v) const + template + inline time_t toFirstDayOfWeek(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return lut[i - (lut[i].day_of_week - 1)].date; } - template - inline ExtendedDayNum toFirstDayNumOfWeek(V v) const + template + inline ExtendedDayNum toFirstDayNumOfWeek(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return toDayNum(i - (lut[i].day_of_week - 1)); } /// Round down to start of month. - template - inline time_t toFirstDayOfMonth(V v) const + template + inline time_t toFirstDayOfMonth(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return lut[i - (lut[i].day_of_month - 1)].date; } - template - inline ExtendedDayNum toFirstDayNumOfMonth(V v) const + template + inline ExtendedDayNum toFirstDayNumOfMonth(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return toDayNum(i - (lut[i].day_of_month - 1)); } /// Round down to start of quarter. - template - inline ExtendedDayNum toFirstDayNumOfQuarter(V v) const + template + inline ExtendedDayNum toFirstDayNumOfQuarter(DateOrTime v) const { return toDayNum(toFirstDayOfQuarterIndex(v)); } - template - inline LUTIndex toFirstDayOfQuarterIndex(V v) const + template + inline LUTIndex toFirstDayOfQuarterIndex(DateOrTime v) const { - //return fromDayNum(toFirstDayNumOfQuarter(v)); - auto index = toLUTIndex(v); + LUTIndex index = toLUTIndex(v); size_t month_inside_quarter = (lut[index].month - 1) % 3; index -= lut[index].day_of_month; @@ -294,8 +293,8 @@ public: return index + 1; } - template - inline time_t toFirstDayOfQuarter(V v) const + template + inline time_t toFirstDayOfQuarter(DateOrTime v) const { return toDate(toFirstDayOfQuarterIndex(v)); } @@ -306,36 +305,36 @@ public: return lut[years_lut[lut[findIndex(t)].year - DATE_LUT_MIN_YEAR]].date; } - template - inline LUTIndex toFirstDayNumOfYearIndex(V v) const + template + inline LUTIndex toFirstDayNumOfYearIndex(DateOrTime v) const { return years_lut[lut[toLUTIndex(v)].year - DATE_LUT_MIN_YEAR]; } - template - inline ExtendedDayNum toFirstDayNumOfYear(V v) const + template + inline ExtendedDayNum toFirstDayNumOfYear(DateOrTime v) const { return toDayNum(toFirstDayNumOfYearIndex(v)); } inline time_t toFirstDayOfNextMonth(time_t t) const { - auto index = findIndex(t); + LUTIndex index = findIndex(t); index += 32 - lut[index].day_of_month; return lut[index - (lut[index].day_of_month - 1)].date; } inline time_t toFirstDayOfPrevMonth(time_t t) const { - auto index = findIndex(t); + LUTIndex index = findIndex(t); index -= lut[index].day_of_month; return lut[index - (lut[index].day_of_month - 1)].date; } - template - inline UInt8 daysInMonth(V v) const + template + inline UInt8 daysInMonth(DateOrTime value) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(value); return lut[i].days_in_month; } @@ -359,7 +358,7 @@ public: inline time_t toTime(time_t t) const { - auto index = findIndex(t); + const LUTIndex index = findIndex(t); if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) return t + offset_at_start_of_epoch; @@ -374,7 +373,7 @@ public: inline unsigned toHour(time_t t) const { - auto index = findIndex(t); + const LUTIndex index = findIndex(t); /// If it is overflow case, /// than limit number of hours to avoid insane results like 1970-01-01 89:28:15 @@ -398,7 +397,7 @@ public: */ inline time_t timezoneOffset(time_t t) const { - const auto index = findIndex(t); + const LUTIndex index = findIndex(t); /// Calculate daylight saving offset first. /// Because the "amount_of_offset_change" in LUT entry only exists in the change day, it's costly to scan it from the very begin. @@ -446,10 +445,10 @@ public: /// To consider the DST changing situation within this day. /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account - DayNum index = findIndex(t); + LUTIndex index = findIndex(t); UInt32 res = t - lut[index].date; - if (lut[index].amount_of_offset_change != 0 && t >= lut[index].date + lut[index].time_at_offset_change) - res += lut[index].amount_of_offset_change; + if (lut[index].amount_of_offset_change() != 0 && t >= lut[index].date + lut[index].time_at_offset_change()) + res += lut[index].amount_of_offset_change(); return res / 60 % 60; } @@ -476,47 +475,51 @@ public: * because the same calendar day starts/ends at different timestamps in different time zones) */ -// inline DayNum toDayNum(time_t t) const { return DayNum{findIndex(t) - daynum_offset_epoch}; } -// inline ExtendedDayNum toExtendedDayNum(time_t t) const { return ExtendedDayNum{findIndex(t) - daynum_offset_epoch}; } inline time_t fromDayNum(DayNum d) const { return lut[toLUTIndex(d)].date; } inline time_t fromDayNum(ExtendedDayNum d) const { return lut[toLUTIndex(d)].date; } - template - inline time_t toDate(V v) const { return lut[toLUTIndex(v)].date; } - template - inline unsigned toMonth(V v) const { return lut[toLUTIndex(v)].month; } - template - inline unsigned toQuarter(V v) const { return (lut[toLUTIndex(v)].month - 1) / 3 + 1; } - template - inline Int16 toYear(V v) const { return lut[toLUTIndex(v)].year; } - template - inline unsigned toDayOfWeek(V v) const { return lut[toLUTIndex(v)].day_of_week; } - template - inline unsigned toDayOfMonth(V v) const { return lut[toLUTIndex(v)].day_of_month; } - template - inline unsigned toDayOfYear(V v) const + template + inline time_t toDate(DateOrTime v) const { return lut[toLUTIndex(v)].date; } + + template + inline unsigned toMonth(DateOrTime v) const { return lut[toLUTIndex(v)].month; } + + template + inline unsigned toQuarter(DateOrTime v) const { return (lut[toLUTIndex(v)].month - 1) / 3 + 1; } + + template + inline Int16 toYear(DateOrTime v) const { return lut[toLUTIndex(v)].year; } + + template + inline unsigned toDayOfWeek(DateOrTime v) const { return lut[toLUTIndex(v)].day_of_week; } + + template + inline unsigned toDayOfMonth(DateOrTime v) const { return lut[toLUTIndex(v)].day_of_month; } + + template + inline unsigned toDayOfYear(DateOrTime v) const { // TODO: different overload for ExtendedDayNum - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return i + 1 - toFirstDayNumOfYearIndex(i); } /// Number of week from some fixed moment in the past. Week begins at monday. /// (round down to monday and divide DayNum by 7; we made an assumption, /// that in domain of the function there was no weeks with any other number of days than 7) - template - inline unsigned toRelativeWeekNum(V v) const + template + inline unsigned toRelativeWeekNum(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); /// We add 8 to avoid underflow at beginning of unix epoch. return toDayNum(i + 8 - toDayOfWeek(i)) / 7; } /// Get year that contains most of the current week. Week begins at monday. - template - inline unsigned toISOYear(V v) const + template + inline unsigned toISOYear(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); /// That's effectively the year of thursday of current week. return toYear(toLUTIndex(i + 4 - toDayOfWeek(i))); } @@ -524,10 +527,10 @@ public: /// ISO year begins with a monday of the week that is contained more than by half in the corresponding calendar year. /// Example: ISO year 2019 begins at 2018-12-31. And ISO year 2017 begins at 2017-01-02. /// https://en.wikipedia.org/wiki/ISO_week_date - template - inline LUTIndex toFirstDayNumOfISOYearIndex(V v) const + template + inline LUTIndex toFirstDayNumOfISOYearIndex(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); auto iso_year = toISOYear(i); const auto first_day_of_year = years_lut[iso_year - DATE_LUT_MIN_YEAR]; @@ -538,8 +541,8 @@ public: : first_day_of_year + 8 - first_day_of_week_of_year}; } - template - inline ExtendedDayNum toFirstDayNumOfISOYear(V v) const + template + inline ExtendedDayNum toFirstDayNumOfISOYear(DateOrTime v) const { return toDayNum(toFirstDayNumOfISOYearIndex(v)); } @@ -551,8 +554,8 @@ public: /// ISO 8601 week number. Week begins at monday. /// The week number 1 is the first week in year that contains 4 or more days (that's more than half). - template - inline unsigned toISOWeek(V v) const + template + inline unsigned toISOWeek(DateOrTime v) const { return 1 + (toFirstDayNumOfWeek(v) - toFirstDayNumOfISOYear(v)) / 7; } @@ -590,8 +593,8 @@ public: Otherwise it is the last week of the previous year, and the next week is week 1. */ - template - inline YearWeek toYearWeek(V v, UInt8 week_mode) const + template + inline YearWeek toYearWeek(DateOrTime v, UInt8 week_mode) const { const bool newyear_day_mode = week_mode & static_cast(WeekModeFlag::NEWYEAR_DAY); week_mode = check_week_mode(week_mode); @@ -599,7 +602,7 @@ public: bool week_year_mode = week_mode & static_cast(WeekModeFlag::YEAR); const bool first_weekday_mode = week_mode & static_cast(WeekModeFlag::FIRST_WEEKDAY); - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); // Calculate week number of WeekModeFlag::NEWYEAR_DAY mode if (newyear_day_mode) @@ -647,13 +650,13 @@ public: /// Calculate week number of WeekModeFlag::NEWYEAR_DAY mode /// The week number 1 is the first week in year that contains January 1, - template - inline YearWeek toYearWeekOfNewyearMode(V v, bool monday_first_mode) const + template + inline YearWeek toYearWeekOfNewyearMode(DateOrTime v, bool monday_first_mode) const { YearWeek yw(0, 0); UInt16 offset_day = monday_first_mode ? 0U : 1U; - const auto i = LUTIndex(v); + const LUTIndex i = LUTIndex(v); // Checking the week across the year yw.first = toYear(i + 7 - toDayOfWeek(i + offset_day)); @@ -661,7 +664,7 @@ public: auto first_day = makeLUTIndex(yw.first, 1, 1); auto this_day = i; - //TODO: do not perform calculations in terms of DayNum, since that would under/overflow for extended range. + // TODO: do not perform calculations in terms of DayNum, since that would under/overflow for extended range. if (monday_first_mode) { // Rounds down a date to the nearest Monday. @@ -680,11 +683,9 @@ public: return yw; } - /** - * get first day of week with week_mode, return Sunday or Monday - */ - template - inline ExtendedDayNum toFirstDayNumOfWeek(V v, UInt8 week_mode) const + /// Get first day of week with week_mode, return Sunday or Monday + template + inline ExtendedDayNum toFirstDayNumOfWeek(DateOrTime v, UInt8 week_mode) const { bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); if (monday_first_mode) @@ -709,10 +710,10 @@ public: /** Calculate weekday from d. * Returns 0 for monday, 1 for tuesday... */ - template - inline unsigned calc_weekday(V v, bool sunday_first_day_of_week) const + template + inline unsigned calc_weekday(DateOrTime v, bool sunday_first_day_of_week) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); if (!sunday_first_day_of_week) return toDayOfWeek(i) - 1; else @@ -726,17 +727,17 @@ public: } /// Number of month from some fixed moment in the past (year * 12 + month) - template - inline unsigned toRelativeMonthNum(V v) const + template + inline unsigned toRelativeMonthNum(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return lut[i].year * 12 + lut[i].month; } - template - inline unsigned toRelativeQuarterNum(V v) const + template + inline unsigned toRelativeQuarterNum(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return lut[i].year * 4 + (lut[i].month - 1) / 3; } @@ -751,8 +752,8 @@ public: return (t + 86400 - offset_at_start_of_epoch) / 3600; } - template - inline time_t toRelativeHourNum(V v) const + template + inline time_t toRelativeHourNum(DateOrTime v) const { return toRelativeHourNum(lut[toLUTIndex(v)].date); } @@ -762,19 +763,19 @@ public: return t / 60; } - template - inline time_t toRelativeMinuteNum(V v) const + template + inline time_t toRelativeMinuteNum(DateOrTime v) const { return toRelativeMinuteNum(lut[toLUTIndex(v)].date); } - template - inline ExtendedDayNum toStartOfYearInterval(V v, UInt64 years) const + template + inline ExtendedDayNum toStartOfYearInterval(DateOrTime v, UInt64 years) const { if (years == 1) return toFirstDayNumOfYear(v); - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return toDayNum(years_lut[lut[i].year / years * years - DATE_LUT_MIN_YEAR]); } @@ -789,8 +790,8 @@ public: { if (months == 1) return toFirstDayNumOfMonth(d); - const auto & date = lut[toLUTIndex(d)]; - UInt32 month_total_index = (date.year - DATE_LUT_MIN_YEAR) * 12 + date.month - 1; + const Values & values = lut[toLUTIndex(d)]; + UInt32 month_total_index = (values.year - DATE_LUT_MIN_YEAR) * 12 + values.month - 1; return toDayNum(years_months_lut[month_total_index / months * months]); } @@ -876,18 +877,18 @@ public: return res; } - template - inline const Values & getValues(V v) const { return lut[toLUTIndex(v)]; } + template + inline const Values & getValues(DateOrTime v) const { return lut[toLUTIndex(v)]; } - template - inline UInt32 toNumYYYYMM(V v) const + template + inline UInt32 toNumYYYYMM(DateOrTime v) const { const Values & values = getValues(v); return values.year * 100 + values.month; } - template - inline UInt32 toNumYYYYMMDD(V v) const + template + inline UInt32 toNumYYYYMMDD(DateOrTime v) const { const Values & values = getValues(v); return values.year * 10000 + values.month * 100 + values.day_of_month; @@ -932,7 +933,7 @@ public: inline NO_SANITIZE_UNDEFINED time_t addDays(time_t t, Int64 delta) const { - auto index = findIndex(t); + LUTIndex index = findIndex(t); time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); index += delta; @@ -962,8 +963,8 @@ public: return day_of_month; } - template - inline LUTIndex addMonthsIndex(V v, Int64 delta) const + template + inline LUTIndex addMonthsIndex(DateOrTime v, Int64 delta) const { const Values & values = lut[toLUTIndex(v)]; @@ -1016,8 +1017,8 @@ public: return addMonths(d, delta * 3); } - template - inline LUTIndex NO_SANITIZE_UNDEFINED addYearsIndex(V v, Int64 delta) const + template + inline LUTIndex NO_SANITIZE_UNDEFINED addYearsIndex(DateOrTime v, Int64 delta) const { const Values & values = lut[toLUTIndex(v)]; From fdc00beb772a0c83032f6c63e61da0d5b6ac9ff9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:56:59 +0300 Subject: [PATCH 025/233] Whitespaces --- base/common/DateLUTImpl.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 740411b7113..159219bab83 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -49,16 +49,19 @@ public: // has to be a separate type to support overloading // TODO: make sure that any arithmetic on LUTIndex actually results in valid LUTIndex. STRONG_TYPEDEF(UInt32, LUTIndex) + template friend inline LUTIndex operator+(const LUTIndex & index, const T v) { return LUTIndex{(index.toUnderType() + v) & date_lut_mask}; } + template friend inline LUTIndex operator+(const T v, const LUTIndex & index) { return LUTIndex{(v + index.toUnderType()) & date_lut_mask}; } + friend inline LUTIndex operator+(const LUTIndex & index, const LUTIndex & v) { return LUTIndex{(index.toUnderType() + v.toUnderType()) & date_lut_mask}; @@ -69,11 +72,13 @@ public: { return LUTIndex{(index.toUnderType() - v) & date_lut_mask}; } + template friend inline LUTIndex operator-(const T v, const LUTIndex & index) { return LUTIndex{(v - index.toUnderType()) & date_lut_mask}; } + friend inline LUTIndex operator-(const LUTIndex & index, const LUTIndex & v) { return LUTIndex{(index.toUnderType() - v.toUnderType()) & date_lut_mask}; @@ -84,6 +89,7 @@ public: { return LUTIndex{(index.toUnderType() * v) & date_lut_mask}; } + template friend inline LUTIndex operator*(const T v, const LUTIndex & index) { @@ -95,6 +101,7 @@ public: { return LUTIndex{(index.toUnderType() / v) & date_lut_mask}; } + template friend inline LUTIndex operator/(const T v, const LUTIndex & index) { From 51d51e474858cddaadc5600995efec37ce6eaa67 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 00:44:58 +0300 Subject: [PATCH 026/233] Return private --- base/common/DateLUT.cpp | 2 +- base/common/DateLUTImpl.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/base/common/DateLUT.cpp b/base/common/DateLUT.cpp index 6ff0884701c..d14b63cd70a 100644 --- a/base/common/DateLUT.cpp +++ b/base/common/DateLUT.cpp @@ -152,7 +152,7 @@ const DateLUTImpl & DateLUT::getImplementation(const std::string & time_zone) co auto it = impls.emplace(time_zone, nullptr).first; if (!it->second) - it->second = std::make_unique(time_zone); + it->second = std::unique_ptr(new DateLUTImpl(time_zone)); return *it->second; } diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 159219bab83..8c2bbb3262d 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -37,7 +37,8 @@ using YearWeek = std::pair; */ class DateLUTImpl { -public: +private: + friend class DateLUT; explicit DateLUTImpl(const std::string & time_zone); DateLUTImpl(const DateLUTImpl &) = delete; From 2632b568ae2ce56a635c40d2a4e119c731d2b91c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 01:34:30 +0300 Subject: [PATCH 027/233] Move tests to appropriate place --- base/common/tests/CMakeLists.txt | 14 ----- base/common/tests/date_lut2.cpp | 53 ---------------- base/common/tests/date_lut3.cpp | 62 ------------------- .../tests/date_lut_default_timezone.cpp | 31 ---------- src/CMakeLists.txt | 11 +++- .../Common}/tests/gtest_DateLutImpl.cpp | 2 +- .../Common}/tests/gtest_find_symbols.cpp | 0 .../gtest_global_register_functions.h.bak | 17 +++++ .../Common}/tests/gtest_json_test.cpp | 10 +-- .../Common}/tests/gtest_strong_typedef.cpp | 0 10 files changed, 33 insertions(+), 167 deletions(-) delete mode 100644 base/common/tests/date_lut2.cpp delete mode 100644 base/common/tests/date_lut3.cpp delete mode 100644 base/common/tests/date_lut_default_timezone.cpp rename {base/common => src/Common}/tests/gtest_DateLutImpl.cpp (99%) rename {base/common => src/Common}/tests/gtest_find_symbols.cpp (100%) create mode 100644 src/Common/tests/gtest_global_register_functions.h.bak rename {base/common => src/Common}/tests/gtest_json_test.cpp (99%) rename {base/common => src/Common}/tests/gtest_strong_typedef.cpp (100%) diff --git a/base/common/tests/CMakeLists.txt b/base/common/tests/CMakeLists.txt index b335b302cb0..92be2f67c94 100644 --- a/base/common/tests/CMakeLists.txt +++ b/base/common/tests/CMakeLists.txt @@ -1,27 +1,13 @@ include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) -add_executable (date_lut2 date_lut2.cpp) -add_executable (date_lut3 date_lut3.cpp) -add_executable (date_lut_default_timezone date_lut_default_timezone.cpp) add_executable (local_date_time_comparison local_date_time_comparison.cpp) add_executable (realloc-perf allocator.cpp) set(PLATFORM_LIBS ${CMAKE_DL_LIBS}) -target_link_libraries (date_lut2 PRIVATE common ${PLATFORM_LIBS}) -target_link_libraries (date_lut3 PRIVATE common ${PLATFORM_LIBS}) -target_link_libraries (date_lut_default_timezone PRIVATE common ${PLATFORM_LIBS}) target_link_libraries (local_date_time_comparison PRIVATE common) target_link_libraries (realloc-perf PRIVATE common) add_check(local_date_time_comparison) -if(USE_GTEST) - add_executable(unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp gtest_find_symbols.cpp gtest_DateLutImpl.cpp - ${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp - ) - target_link_libraries(unit_tests_libcommon PRIVATE common ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) - add_check(unit_tests_libcommon) -endif() - add_executable (dump_variable dump_variable.cpp) target_link_libraries (dump_variable PRIVATE clickhouse_common_io) diff --git a/base/common/tests/date_lut2.cpp b/base/common/tests/date_lut2.cpp deleted file mode 100644 index 6dcf5e8adf2..00000000000 --- a/base/common/tests/date_lut2.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include - -#include - - -static std::string toString(time_t Value) -{ - struct tm tm; - char buf[96]; - - localtime_r(&Value, &tm); - snprintf(buf, sizeof(buf), "%04d-%02d-%02d %02d:%02d:%02d", - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - - return buf; -} - -static time_t orderedIdentifierToDate(unsigned value) -{ - struct tm tm; - - memset(&tm, 0, sizeof(tm)); - - tm.tm_year = value / 10000 - 1900; - tm.tm_mon = (value % 10000) / 100 - 1; - tm.tm_mday = value % 100; - tm.tm_isdst = -1; - - return mktime(&tm); -} - - -void loop(time_t begin, time_t end, int step) -{ - const auto & date_lut = DateLUT::instance(); - - for (time_t t = begin; t < end; t += step) - std::cout << toString(t) - << ", " << toString(date_lut.toTime(t)) - << ", " << date_lut.toHour(t) - << std::endl; -} - - -int main(int, char **) -{ - loop(orderedIdentifierToDate(20101031), orderedIdentifierToDate(20101101), 15 * 60); - loop(orderedIdentifierToDate(20100328), orderedIdentifierToDate(20100330), 15 * 60); - loop(orderedIdentifierToDate(20141020), orderedIdentifierToDate(20141106), 15 * 60); - - return 0; -} diff --git a/base/common/tests/date_lut3.cpp b/base/common/tests/date_lut3.cpp deleted file mode 100644 index 411765d2b2a..00000000000 --- a/base/common/tests/date_lut3.cpp +++ /dev/null @@ -1,62 +0,0 @@ -#include -#include - -#include - -#include - - -static std::string toString(time_t Value) -{ - struct tm tm; - char buf[96]; - - localtime_r(&Value, &tm); - snprintf(buf, sizeof(buf), "%04d-%02d-%02d %02d:%02d:%02d", - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - - return buf; -} - -static time_t orderedIdentifierToDate(unsigned value) -{ - struct tm tm; - - memset(&tm, 0, sizeof(tm)); - - tm.tm_year = value / 10000 - 1900; - tm.tm_mon = (value % 10000) / 100 - 1; - tm.tm_mday = value % 100; - tm.tm_isdst = -1; - - return mktime(&tm); -} - - -void loop(time_t begin, time_t end, int step) -{ - const auto & date_lut = DateLUT::instance(); - - for (time_t t = begin; t < end; t += step) - { - time_t t2 = date_lut.makeDateTime(date_lut.toYear(t), date_lut.toMonth(t), date_lut.toDayOfMonth(t), - date_lut.toHour(t), date_lut.toMinute(t), date_lut.toSecond(t)); - - std::string s1 = toString(t); - std::string s2 = toString(t2); - - std::cerr << s1 << ", " << s2 << std::endl; - - if (s1 != s2) - throw Poco::Exception("Test failed."); - } -} - - -int main(int, char **) -{ - loop(orderedIdentifierToDate(20101031), orderedIdentifierToDate(20101101), 15 * 60); - loop(orderedIdentifierToDate(20100328), orderedIdentifierToDate(20100330), 15 * 60); - - return 0; -} diff --git a/base/common/tests/date_lut_default_timezone.cpp b/base/common/tests/date_lut_default_timezone.cpp deleted file mode 100644 index b8e5aa08931..00000000000 --- a/base/common/tests/date_lut_default_timezone.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include -#include -#include - -int main(int, char **) -{ - try - { - const auto & date_lut = DateLUT::instance(); - std::cout << "Detected default timezone: `" << date_lut.getTimeZone() << "'" << std::endl; - time_t now = time(nullptr); - std::cout << "Current time: " << date_lut.timeToString(now) - << ", UTC: " << DateLUT::instance("UTC").timeToString(now) << std::endl; - } - catch (const Poco::Exception & e) - { - std::cerr << e.displayText() << std::endl; - return 1; - } - catch (std::exception & e) - { - std::cerr << "std::exception: " << e.what() << std::endl; - return 2; - } - catch (...) - { - std::cerr << "Some exception" << std::endl; - return 3; - } - return 0; -} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b80bcfdf4d4..a6a7d280479 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -478,6 +478,15 @@ if (ENABLE_TESTS AND USE_GTEST) -Wno-gnu-zero-variadic-macro-arguments ) - target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_aggregate_functions clickhouse_parsers dbms clickhouse_common_zookeeper string_utils) + target_link_libraries(unit_tests_dbms PRIVATE + ${GTEST_BOTH_LIBRARIES} + clickhouse_functions + clickhouse_aggregate_functions + clickhouse_parsers + clickhouse_storages_system + dbms + clickhouse_common_zookeeper + string_utils) + add_check(unit_tests_dbms) endif () diff --git a/base/common/tests/gtest_DateLutImpl.cpp b/src/Common/tests/gtest_DateLutImpl.cpp similarity index 99% rename from base/common/tests/gtest_DateLutImpl.cpp rename to src/Common/tests/gtest_DateLutImpl.cpp index 8cc4fbbbfb0..3a0da1ee1ee 100644 --- a/base/common/tests/gtest_DateLutImpl.cpp +++ b/src/Common/tests/gtest_DateLutImpl.cpp @@ -11,7 +11,7 @@ #pragma clang diagnostic ignored "-Wused-but-marked-unused" #endif -// All timezones present at build time and embedded into CH binary. +// All timezones present at build time and embedded into ClickHouse binary. extern const char * auto_time_zones[]; namespace diff --git a/base/common/tests/gtest_find_symbols.cpp b/src/Common/tests/gtest_find_symbols.cpp similarity index 100% rename from base/common/tests/gtest_find_symbols.cpp rename to src/Common/tests/gtest_find_symbols.cpp diff --git a/src/Common/tests/gtest_global_register_functions.h.bak b/src/Common/tests/gtest_global_register_functions.h.bak new file mode 100644 index 00000000000..197ce5838b9 --- /dev/null +++ b/src/Common/tests/gtest_global_register_functions.h.bak @@ -0,0 +1,17 @@ +#include +#include + +struct RegisteredFunctionsState +{ + RegisteredFunctionsState() + { + DB::registerFunctions(); + } + + RegisteredFunctionsState(RegisteredFunctionsState &&) = default; +}; + +inline void tryRegisterFunctions() +{ + static RegisteredFunctionsState registered_functions_state; +} diff --git a/base/common/tests/gtest_json_test.cpp b/src/Common/tests/gtest_json_test.cpp similarity index 99% rename from base/common/tests/gtest_json_test.cpp rename to src/Common/tests/gtest_json_test.cpp index 189a1a03d99..7b810504952 100644 --- a/base/common/tests/gtest_json_test.cpp +++ b/src/Common/tests/gtest_json_test.cpp @@ -500,14 +500,14 @@ TEST(JSONSuite, SimpleTest) { R"("detail")", ResultType::Return, "detail" }, { R"("actionField")", ResultType::Return, "actionField" }, { R"("list")", ResultType::Return, "list" }, - { "\0\"", ResultType::Throw, "JSON: expected \", got \0" }, + { "\0\"", ResultType::Throw, "JSON: begin >= end." }, { "\"/igrushki/konstruktory\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/Творчество/Рисование/Инструменты и кра\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0t", ResultType::Throw, "JSON: expected \", got \0" }, + { "\0t", ResultType::Throw, "JSON: begin >= end." }, { "\"/Хозтовары/Хранение вещей и организа\xD1\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/Хозтовары/Товары для стир\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"li\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, @@ -572,10 +572,10 @@ TEST(JSONSuite, SimpleTest) { "\"/Игр\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/Игрушки/Игрушки для девочек/Игровые модули дл\xD1\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Крупная бытовая техника/Стиральные машины/С фронт\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0 ", ResultType::Throw, "JSON: expected \", got \0" }, + { "\0 ", ResultType::Throw, "JSON: begin >= end." }, { "\"Светодиодная лента SMD3528, 5 м. IP33, 60LED, зеленый, 4,8W/мет\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Сантехника/Мебель для ванных комнат/Стол\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0o", ResultType::Throw, "JSON: expected \", got \0" }, + { "\0o", ResultType::Throw, "JSON: begin >= end." }, { "\"/igrushki/konstruktory\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/posuda/kuhonnye-prinadlezhnosti-i-instrumenty/kuhonnye-pr\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, @@ -583,7 +583,7 @@ TEST(JSONSuite, SimpleTest) { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0 ", ResultType::Throw, "JSON: expected \", got \0" }, + { "\0 ", ResultType::Throw, "JSON: begin >= end." }, { "\"/Хозтовары/Хранение вещей и организа\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/Хозтовары/Товары для стир\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"li\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, diff --git a/base/common/tests/gtest_strong_typedef.cpp b/src/Common/tests/gtest_strong_typedef.cpp similarity index 100% rename from base/common/tests/gtest_strong_typedef.cpp rename to src/Common/tests/gtest_strong_typedef.cpp From 329074bd029a397adbda1fa67751e2a35cb2d48b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 01:55:56 +0300 Subject: [PATCH 028/233] Fix test --- src/Common/tests/gtest_json_test.cpp | 1230 +++++++++++++------------- 1 file changed, 615 insertions(+), 615 deletions(-) diff --git a/src/Common/tests/gtest_json_test.cpp b/src/Common/tests/gtest_json_test.cpp index 7b810504952..726fb836030 100644 --- a/src/Common/tests/gtest_json_test.cpp +++ b/src/Common/tests/gtest_json_test.cpp @@ -1,14 +1,13 @@ #include #include #include +#include #include #include - -using namespace std::literals::string_literals; - #include + enum class ResultType { Return, @@ -17,620 +16,622 @@ enum class ResultType struct GetStringTestRecord { - const char * input; + std::string_view input; ResultType result_type; - const char * result; + std::string_view result; }; TEST(JSONSuite, SimpleTest) { + using namespace std::literals; + std::vector test_data = { - { R"("name")", ResultType::Return, "name" }, - { R"("Вафельница Vitek WX-1102 FL")", ResultType::Return, "Вафельница Vitek WX-1102 FL" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("184509")", ResultType::Return, "184509" }, - { R"("category")", ResultType::Return, "category" }, - { R"("Все для детей/Детская техника/Vitek")", ResultType::Return, "Все для детей/Детская техника/Vitek" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("В наличии")", ResultType::Return, "В наличии" }, - { R"("price")", ResultType::Return, "price" }, - { R"("2390.00")", ResultType::Return, "2390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("Карточка")", ResultType::Return, "Карточка" }, - { R"("position")", ResultType::Return, "position" }, - { R"("detail")", ResultType::Return, "detail" }, - { R"("actionField")", ResultType::Return, "actionField" }, - { R"("list")", ResultType::Return, "list" }, - { R"("http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc")", ResultType::Return, "http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc" }, - { R"("action")", ResultType::Return, "action" }, - { R"("detail")", ResultType::Return, "detail" }, - { R"("products")", ResultType::Return, "products" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Вафельница Vitek WX-1102 FL")", ResultType::Return, "Вафельница Vitek WX-1102 FL" }, - { R"("id")", ResultType::Return, "id" }, - { R"("184509")", ResultType::Return, "184509" }, - { R"("price")", ResultType::Return, "price" }, - { R"("2390.00")", ResultType::Return, "2390.00" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("Vitek")", ResultType::Return, "Vitek" }, - { R"("category")", ResultType::Return, "category" }, - { R"("Все для детей/Детская техника/Vitek")", ResultType::Return, "Все для детей/Детская техника/Vitek" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("В наличии")", ResultType::Return, "В наличии" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("isAuthorized")", ResultType::Return, "isAuthorized" }, - { R"("isSubscriber")", ResultType::Return, "isSubscriber" }, - { R"("postType")", ResultType::Return, "postType" }, - { R"("Новости")", ResultType::Return, "Новости" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("Электроплита GEFEST Брест ЭПНД 5140-01 0001")", ResultType::Return, "Электроплита GEFEST Брест ЭПНД 5140-01 0001" }, - { R"("price")", ResultType::Return, "price" }, - { R"("currencyCode")", ResultType::Return, "currencyCode" }, - { R"("RUB")", ResultType::Return, "RUB" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("trash_login")", ResultType::Return, "trash_login" }, - { R"("novikoff")", ResultType::Return, "novikoff" }, - { R"("trash_cat_link")", ResultType::Return, "trash_cat_link" }, - { R"("progs")", ResultType::Return, "progs" }, - { R"("trash_parent_link")", ResultType::Return, "trash_parent_link" }, - { R"("content")", ResultType::Return, "content" }, - { R"("trash_posted_parent")", ResultType::Return, "trash_posted_parent" }, - { R"("content.01.2016")", ResultType::Return, "content.01.2016" }, - { R"("trash_posted_cat")", ResultType::Return, "trash_posted_cat" }, - { R"("progs.01.2016")", ResultType::Return, "progs.01.2016" }, - { R"("trash_virus_count")", ResultType::Return, "trash_virus_count" }, - { R"("trash_is_android")", ResultType::Return, "trash_is_android" }, - { R"("trash_is_wp8")", ResultType::Return, "trash_is_wp8" }, - { R"("trash_is_ios")", ResultType::Return, "trash_is_ios" }, - { R"("trash_posted")", ResultType::Return, "trash_posted" }, - { R"("01.2016")", ResultType::Return, "01.2016" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("merchantId")", ResultType::Return, "merchantId" }, - { R"("13694_49246")", ResultType::Return, "13694_49246" }, - { R"("cps-source")", ResultType::Return, "cps-source" }, - { R"("wargaming")", ResultType::Return, "wargaming" }, - { R"("cps_provider")", ResultType::Return, "cps_provider" }, - { R"("default")", ResultType::Return, "default" }, - { R"("errorReason")", ResultType::Return, "errorReason" }, - { R"("no errors")", ResultType::Return, "no errors" }, - { R"("scid")", ResultType::Return, "scid" }, - { R"("isAuthPayment")", ResultType::Return, "isAuthPayment" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("rubric")", ResultType::Return, "rubric" }, - { R"("")", ResultType::Return, "" }, - { R"("rubric")", ResultType::Return, "rubric" }, - { R"("Мир")", ResultType::Return, "Мир" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("__ym")", ResultType::Return, "__ym" }, - { R"("ecommerce")", ResultType::Return, "ecommerce" }, - { R"("impressions")", ResultType::Return, "impressions" }, - { R"("id")", ResultType::Return, "id" }, - { R"("863813")", ResultType::Return, "863813" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Happy, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Happy, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("863839")", ResultType::Return, "863839" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("863847")", ResultType::Return, "863847" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911480")", ResultType::Return, "911480" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Puppy, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Puppy, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911484")", ResultType::Return, "911484" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Little bears, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Little bears, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911489")", ResultType::Return, "911489" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж")", ResultType::Return, "Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911496")", ResultType::Return, "911496" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Pretty, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Pretty, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911504")", ResultType::Return, "911504" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911508")", ResultType::Return, "911508" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Kittens, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Kittens, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911512")", ResultType::Return, "911512" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911516")", ResultType::Return, "911516" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911520")", ResultType::Return, "911520" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911524")", ResultType::Return, "911524" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911528")", ResultType::Return, "911528" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Turtle, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Turtle, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("888616")", ResultType::Return, "888616" }, - { R"("name")", ResultType::Return, "name" }, - { "\"3Д Футболка мужская \\\"Collorista\\\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж\"", ResultType::Return, "3Д Футболка мужская \"Collorista\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Одежда и обувь/Мужская одежда/Футболки/")", ResultType::Return, "/Одежда и обувь/Мужская одежда/Футболки/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("406.60")", ResultType::Return, "406.60" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("913361")", ResultType::Return, "913361" }, - { R"("name")", ResultType::Return, "name" }, - { R"("3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("470.00")", ResultType::Return, "470.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("913364")", ResultType::Return, "913364" }, - { R"("name")", ResultType::Return, "name" }, - { R"("3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("470.00")", ResultType::Return, "470.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("913367")", ResultType::Return, "913367" }, - { R"("name")", ResultType::Return, "name" }, - { R"("3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("470.00")", ResultType::Return, "470.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("913385")", ResultType::Return, "913385" }, - { R"("name")", ResultType::Return, "name" }, - { R"("3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("470.00")", ResultType::Return, "470.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("913391")", ResultType::Return, "913391" }, - { R"("name")", ResultType::Return, "name" }, - { R"("3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("470.00")", ResultType::Return, "470.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("usertype")", ResultType::Return, "usertype" }, - { R"("visitor")", ResultType::Return, "visitor" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("__ym")", ResultType::Return, "__ym" }, - { R"("ecommerce")", ResultType::Return, "ecommerce" }, - { R"("impressions")", ResultType::Return, "impressions" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("__ym")", ResultType::Return, "__ym" }, - { R"("ecommerce")", ResultType::Return, "ecommerce" }, - { R"("currencyCode")", ResultType::Return, "currencyCode" }, - { R"("RUR")", ResultType::Return, "RUR" }, - { R"("impressions")", ResultType::Return, "impressions" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Чайник электрический Mystery MEK-1627, белый")", ResultType::Return, "Чайник электрический Mystery MEK-1627, белый" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("Mystery")", ResultType::Return, "Mystery" }, - { R"("id")", ResultType::Return, "id" }, - { R"("187180")", ResultType::Return, "187180" }, - { R"("category")", ResultType::Return, "category" }, - { R"("Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery")", ResultType::Return, "Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("В наличии")", ResultType::Return, "В наличии" }, - { R"("price")", ResultType::Return, "price" }, - { R"("1630.00")", ResultType::Return, "1630.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("Карточка")", ResultType::Return, "Карточка" }, - { R"("position")", ResultType::Return, "position" }, - { R"("detail")", ResultType::Return, "detail" }, - { R"("actionField")", ResultType::Return, "actionField" }, - { R"("list")", ResultType::Return, "list" }, - { "\0\"", ResultType::Throw, "JSON: begin >= end." }, - { "\"/igrushki/konstruktory\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Творчество/Рисование/Инструменты и кра\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0t", ResultType::Throw, "JSON: begin >= end." }, - { "\"/Хозтовары/Хранение вещей и организа\xD1\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Хозтовары/Товары для стир\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"li\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/734859/samolet-radioupravlyaemyy-istrebitel-rabotaet-o\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/kosmetika-i-parfyum/parfyumeriya/mu\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/ko\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "", ResultType::Throw, "JSON: begin >= end." }, - { "\"/stroitelstvo-i-remont/stroit\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/av\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/s\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Строительство и ремонт/Строительный инструмент/Изм\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/avto/soputstvuy\0l", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/str\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Отвертка 2 в 1 \\\"TUNDRA basic\\\" 5х75 мм (+,-) \0\xFF", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/avtoinstrumen\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Мелкая бытовая техника/Мелки\xD0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Пряжа \\\"Бамбук стрейч\\0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Карандаш чёрнографитны\xD0\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0l", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/1071547/karandash-chernografitnyy-volshebstvo-nv-kruglyy-d-7-2mm-dl-176mm-plast-tuba/\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"ca\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"ca\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/1165424/chipbord-vyrubnoy-dlya-skrapbukinga-malyshi-mikki-maus-disney-bebi\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/posuda/kuhonnye-prinadlezhnosti-i-i\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Канцтовары/Ежедневники и блокн\xD0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/kanctovary/ezhednevniki-i-blok\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Стакан \xD0\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Набор бумаги для скрапбукинга \\\"Мои первый годик\\\": Микки Маус, Дисней бэби, 12 листов 29.5 х 29.5 см, 160\0\x80", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"c\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Органайзер для хранения аксессуаров, \0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"quantity\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Сменный блок для тетрадей на кольцах А5, 160 листов клетка, офсет \xE2\x84\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Сувениры/Ф\xD0\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"\0\"", ResultType::Return, "\0" }, - { "\"\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"va\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"ca\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"В \0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/letnie-tovary/z\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Посудомоечная машина Ha\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Крупная бытов\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Полочная акустическая система Magnat Needl\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"brand\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"pos\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"c\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"var\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Телевизоры и видеотехника/Всё для домашних кинотеатр\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Флеш-диск Transcend JetFlash 620 8GB (TS8GJF62\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Табурет Мег\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"variant\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Катал\xD0\0\"", ResultType::Return, "Катал\xD0\0" }, - { "\"К\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Полочная акустическая система Magnat Needl\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"brand\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"pos\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"c\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"17\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/igrushki/razvivayusc\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Ключница \\\"\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Игр\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Игрушки/Игрушки для девочек/Игровые модули дл\xD1\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Крупная бытовая техника/Стиральные машины/С фронт\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0 ", ResultType::Throw, "JSON: begin >= end." }, - { "\"Светодиодная лента SMD3528, 5 м. IP33, 60LED, зеленый, 4,8W/мет\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Сантехника/Мебель для ванных комнат/Стол\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0o", ResultType::Throw, "JSON: begin >= end." }, - { "\"/igrushki/konstruktory\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/posuda/kuhonnye-prinadlezhnosti-i-instrumenty/kuhonnye-pr\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Творчество/Рисование/Инструменты и кра\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0 ", ResultType::Throw, "JSON: begin >= end." }, - { "\"/Хозтовары/Хранение вещей и организа\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Хозтовары/Товары для стир\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"li\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/igrushki/igrus\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/734859/samolet-radioupravlyaemyy-istrebitel-rabotaet-o\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/kosmetika-i-parfyum/parfyumeriya/mu\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/ko\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/avto/avtomobilnyy\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/stroitelstvo-i-remont/stroit\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/av\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/s\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Строительство и ремонт/Строительный инструмент/Изм\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/avto/soputstvuy\0\"", ResultType::Return, "/avto/soputstvuy\0" }, - { "\"/str\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Отвертка 2 в 1 \\\"TUNDRA basic\\\" 5х75 мм (+,-) \0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/avtoinstrumen\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Чайник электрический Vitesse\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Мелкая бытовая техника/Мелки\xD0\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Пряжа \\\"Бамбук стрейч\\0о", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Карандаш чёрнографитны\xD0\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0\"", ResultType::Return, "/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0" }, - { "\"/1071547/karandash-chernografitnyy-volshebstvo-nv-kruglyy-d-7-2mm-dl-176mm-plast-tuba/\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"ca\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Подаро\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Средство для прочис\xD1\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"i\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/p\0\"", ResultType::Return, "/p\0" }, - { "\"/Сувениры/Магниты, н\xD0\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Дерев\xD0\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/prazdniki/svadba/svadebnaya-c\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Канцт\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Праздники/То\xD0\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"v\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Косметика \xD0\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Спорт и отдых/Настольные игры/Покер, руле\xD1\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"categ\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/retailr\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/retailrocket\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Ежедневник недат А5 140л кл,ляссе,обл пв\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/432809/ezhednevnik-organayzer-sredniy-s-remeshkom-na-knopke-v-oblozhke-kalkulyator-kalendar-do-\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/1165424/chipbord-vyrubnoy-dlya-skrapbukinga-malyshi-mikki-maus-disney-bebi\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/posuda/kuhonnye-prinadlezhnosti-i-i\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Канцтовары/Ежедневники и блокн\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/kanctovary/ezhednevniki-i-blok\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Стакан \xD0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Набор бумаги для скрапбукинга \\\"Мои первый годик\\\": Микки Маус, Дисней бэби, 12 листов 29.5 х 29.5 см, 160\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"c\0\"", ResultType::Return, "c\0" }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Вафельница Vitek WX-1102 FL")"sv, ResultType::Return, "Вафельница Vitek WX-1102 FL"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("184509")"sv, ResultType::Return, "184509"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("Все для детей/Детская техника/Vitek")"sv, ResultType::Return, "Все для детей/Детская техника/Vitek"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("В наличии")"sv, ResultType::Return, "В наличии"sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("2390.00")"sv, ResultType::Return, "2390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("Карточка")"sv, ResultType::Return, "Карточка"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("detail")"sv, ResultType::Return, "detail"sv }, + { R"("actionField")"sv, ResultType::Return, "actionField"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc")"sv, ResultType::Return, "http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc"sv }, + { R"("action")"sv, ResultType::Return, "action"sv }, + { R"("detail")"sv, ResultType::Return, "detail"sv }, + { R"("products")"sv, ResultType::Return, "products"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Вафельница Vitek WX-1102 FL")"sv, ResultType::Return, "Вафельница Vitek WX-1102 FL"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("184509")"sv, ResultType::Return, "184509"sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("2390.00")"sv, ResultType::Return, "2390.00"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("Vitek")"sv, ResultType::Return, "Vitek"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("Все для детей/Детская техника/Vitek")"sv, ResultType::Return, "Все для детей/Детская техника/Vitek"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("В наличии")"sv, ResultType::Return, "В наличии"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("isAuthorized")"sv, ResultType::Return, "isAuthorized"sv }, + { R"("isSubscriber")"sv, ResultType::Return, "isSubscriber"sv }, + { R"("postType")"sv, ResultType::Return, "postType"sv }, + { R"("Новости")"sv, ResultType::Return, "Новости"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("Электроплита GEFEST Брест ЭПНД 5140-01 0001")"sv, ResultType::Return, "Электроплита GEFEST Брест ЭПНД 5140-01 0001"sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("currencyCode")"sv, ResultType::Return, "currencyCode"sv }, + { R"("RUB")"sv, ResultType::Return, "RUB"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("trash_login")"sv, ResultType::Return, "trash_login"sv }, + { R"("novikoff")"sv, ResultType::Return, "novikoff"sv }, + { R"("trash_cat_link")"sv, ResultType::Return, "trash_cat_link"sv }, + { R"("progs")"sv, ResultType::Return, "progs"sv }, + { R"("trash_parent_link")"sv, ResultType::Return, "trash_parent_link"sv }, + { R"("content")"sv, ResultType::Return, "content"sv }, + { R"("trash_posted_parent")"sv, ResultType::Return, "trash_posted_parent"sv }, + { R"("content.01.2016")"sv, ResultType::Return, "content.01.2016"sv }, + { R"("trash_posted_cat")"sv, ResultType::Return, "trash_posted_cat"sv }, + { R"("progs.01.2016")"sv, ResultType::Return, "progs.01.2016"sv }, + { R"("trash_virus_count")"sv, ResultType::Return, "trash_virus_count"sv }, + { R"("trash_is_android")"sv, ResultType::Return, "trash_is_android"sv }, + { R"("trash_is_wp8")"sv, ResultType::Return, "trash_is_wp8"sv }, + { R"("trash_is_ios")"sv, ResultType::Return, "trash_is_ios"sv }, + { R"("trash_posted")"sv, ResultType::Return, "trash_posted"sv }, + { R"("01.2016")"sv, ResultType::Return, "01.2016"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("merchantId")"sv, ResultType::Return, "merchantId"sv }, + { R"("13694_49246")"sv, ResultType::Return, "13694_49246"sv }, + { R"("cps-source")"sv, ResultType::Return, "cps-source"sv }, + { R"("wargaming")"sv, ResultType::Return, "wargaming"sv }, + { R"("cps_provider")"sv, ResultType::Return, "cps_provider"sv }, + { R"("default")"sv, ResultType::Return, "default"sv }, + { R"("errorReason")"sv, ResultType::Return, "errorReason"sv }, + { R"("no errors")"sv, ResultType::Return, "no errors"sv }, + { R"("scid")"sv, ResultType::Return, "scid"sv }, + { R"("isAuthPayment")"sv, ResultType::Return, "isAuthPayment"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("rubric")"sv, ResultType::Return, "rubric"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("rubric")"sv, ResultType::Return, "rubric"sv }, + { R"("Мир")"sv, ResultType::Return, "Мир"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("__ym")"sv, ResultType::Return, "__ym"sv }, + { R"("ecommerce")"sv, ResultType::Return, "ecommerce"sv }, + { R"("impressions")"sv, ResultType::Return, "impressions"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("863813")"sv, ResultType::Return, "863813"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Happy, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Happy, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("863839")"sv, ResultType::Return, "863839"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("863847")"sv, ResultType::Return, "863847"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911480")"sv, ResultType::Return, "911480"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Puppy, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Puppy, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911484")"sv, ResultType::Return, "911484"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Little bears, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Little bears, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911489")"sv, ResultType::Return, "911489"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911496")"sv, ResultType::Return, "911496"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Pretty, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Pretty, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911504")"sv, ResultType::Return, "911504"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911508")"sv, ResultType::Return, "911508"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Kittens, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Kittens, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911512")"sv, ResultType::Return, "911512"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911516")"sv, ResultType::Return, "911516"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911520")"sv, ResultType::Return, "911520"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911524")"sv, ResultType::Return, "911524"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911528")"sv, ResultType::Return, "911528"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Turtle, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Turtle, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("888616")"sv, ResultType::Return, "888616"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { "\"3Д Футболка мужская \\\"Collorista\\\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж\""sv, ResultType::Return, "3Д Футболка мужская \"Collorista\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Одежда и обувь/Мужская одежда/Футболки/")"sv, ResultType::Return, "/Одежда и обувь/Мужская одежда/Футболки/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("406.60")"sv, ResultType::Return, "406.60"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("913361")"sv, ResultType::Return, "913361"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж")"sv, ResultType::Return, "3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("470.00")"sv, ResultType::Return, "470.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("913364")"sv, ResultType::Return, "913364"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж")"sv, ResultType::Return, "3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("470.00")"sv, ResultType::Return, "470.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("913367")"sv, ResultType::Return, "913367"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж")"sv, ResultType::Return, "3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("470.00")"sv, ResultType::Return, "470.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("913385")"sv, ResultType::Return, "913385"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж")"sv, ResultType::Return, "3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("470.00")"sv, ResultType::Return, "470.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("913391")"sv, ResultType::Return, "913391"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж")"sv, ResultType::Return, "3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("470.00")"sv, ResultType::Return, "470.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("usertype")"sv, ResultType::Return, "usertype"sv }, + { R"("visitor")"sv, ResultType::Return, "visitor"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("__ym")"sv, ResultType::Return, "__ym"sv }, + { R"("ecommerce")"sv, ResultType::Return, "ecommerce"sv }, + { R"("impressions")"sv, ResultType::Return, "impressions"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("__ym")"sv, ResultType::Return, "__ym"sv }, + { R"("ecommerce")"sv, ResultType::Return, "ecommerce"sv }, + { R"("currencyCode")"sv, ResultType::Return, "currencyCode"sv }, + { R"("RUR")"sv, ResultType::Return, "RUR"sv }, + { R"("impressions")"sv, ResultType::Return, "impressions"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Чайник электрический Mystery MEK-1627, белый")"sv, ResultType::Return, "Чайник электрический Mystery MEK-1627, белый"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("Mystery")"sv, ResultType::Return, "Mystery"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("187180")"sv, ResultType::Return, "187180"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery")"sv, ResultType::Return, "Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("В наличии")"sv, ResultType::Return, "В наличии"sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("1630.00")"sv, ResultType::Return, "1630.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("Карточка")"sv, ResultType::Return, "Карточка"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("detail")"sv, ResultType::Return, "detail"sv }, + { R"("actionField")"sv, ResultType::Return, "actionField"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { "\0\""sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"/igrushki/konstruktory\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Творчество/Рисование/Инструменты и кра\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0t"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\0t"sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"/Хозтовары/Хранение вещей и организа\xD1\0t"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Хозтовары/Товары для стир\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"li\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/734859/samolet-radioupravlyaemyy-istrebitel-rabotaet-o\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/kosmetika-i-parfyum/parfyumeriya/mu\0t"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/ko\0\x04"sv, ResultType::Throw, "JSON: begin >= end."sv }, + { ""sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"/stroitelstvo-i-remont/stroit\0t"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/av\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/s\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Строительство и ремонт/Строительный инструмент/Изм\0e"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/avto/soputstvuy\0l"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/str\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Отвертка 2 в 1 \\\"TUNDRA basic\\\" 5х75 мм (+,-) \0\xFF"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/avtoinstrumen\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Мелкая бытовая техника/Мелки\xD0\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Пряжа \\\"Бамбук стрейч\\0\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Карандаш чёрнографитны\xD0\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0l"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/1071547/karandash-chernografitnyy-volshebstvo-nv-kruglyy-d-7-2mm-dl-176mm-plast-tuba/\0e"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"ca\0e"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"ca\0e"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/1165424/chipbord-vyrubnoy-dlya-skrapbukinga-malyshi-mikki-maus-disney-bebi\0t"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/posuda/kuhonnye-prinadlezhnosti-i-i\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Канцтовары/Ежедневники и блокн\xD0\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/kanctovary/ezhednevniki-i-blok\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Стакан \xD0\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Набор бумаги для скрапбукинга \\\"Мои первый годик\\\": Микки Маус, Дисней бэби, 12 листов 29.5 х 29.5 см, 160\0\x80"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"c\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Органайзер для хранения аксессуаров, \0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"quantity\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Сменный блок для тетрадей на кольцах А5, 160 листов клетка, офсет \xE2\x84\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Сувениры/Ф\xD0\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"\0\""sv, ResultType::Return, "\0"sv }, + { "\"\0\x04"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"va\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"ca\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"В \0\x04"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/letnie-tovary/z\0\x04"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Посудомоечная машина Ha\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Крупная бытов\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Полочная акустическая система Magnat Needl\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"brand\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"pos\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"c\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"var\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Телевизоры и видеотехника/Всё для домашних кинотеатр\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Флеш-диск Transcend JetFlash 620 8GB (TS8GJF62\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Табурет Мег\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"variant\0\x04"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Катал\xD0\0\""sv, ResultType::Return, "Катал\xD0\0"sv }, + { "\"К\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Полочная акустическая система Magnat Needl\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"brand\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"pos\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"c\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"17\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/igrushki/razvivayusc\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Ключница \\\"\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Игр\xD1\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Игрушки/Игрушки для девочек/Игровые модули дл\xD1\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Крупная бытовая техника/Стиральные машины/С фронт\xD0\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\0 "sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"Светодиодная лента SMD3528, 5 м. IP33, 60LED, зеленый, 4,8W/мет\xD1\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Сантехника/Мебель для ванных комнат/Стол\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\0o"sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"/igrushki/konstruktory\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/posuda/kuhonnye-prinadlezhnosti-i-instrumenty/kuhonnye-pr\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Творчество/Рисование/Инструменты и кра\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\0 "sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"/Хозтовары/Хранение вещей и организа\xD1\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Хозтовары/Товары для стир\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"li\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/igrushki/igrus\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/734859/samolet-radioupravlyaemyy-istrebitel-rabotaet-o\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/kosmetika-i-parfyum/parfyumeriya/mu\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/ko\0\0"sv, ResultType::Throw, "JSON: begin >= end."sv }, + { "\"/avto/avtomobilnyy\0\0"sv, ResultType::Throw, "JSON: begin >= end."sv }, + { "\"/stroitelstvo-i-remont/stroit\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/av\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/s\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Строительство и ремонт/Строительный инструмент/Изм\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/avto/soputstvuy\0\""sv, ResultType::Return, "/avto/soputstvuy\0"sv }, + { "\"/str\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Отвертка 2 в 1 \\\"TUNDRA basic\\\" 5х75 мм (+,-) \0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/avtoinstrumen\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Чайник электрический Vitesse\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Мелкая бытовая техника/Мелки\xD0\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Пряжа \\\"Бамбук стрейч\\0о"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Карандаш чёрнографитны\xD0\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0\""sv, ResultType::Return, "/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0"sv }, + { "\"/1071547/karandash-chernografitnyy-volshebstvo-nv-kruglyy-d-7-2mm-dl-176mm-plast-tuba/\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"ca\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Подаро\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Средство для прочис\xD1\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"i\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/p\0\""sv, ResultType::Return, "/p\0"sv }, + { "\"/Сувениры/Магниты, н\xD0\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Дерев\xD0\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/prazdniki/svadba/svadebnaya-c\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Канцт\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Праздники/То\xD0\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"v\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Косметика \xD0\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Спорт и отдых/Настольные игры/Покер, руле\xD1\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"categ\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/retailr\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/retailrocket\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Ежедневник недат А5 140л кл,ляссе,обл пв\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/432809/ezhednevnik-organayzer-sredniy-s-remeshkom-na-knopke-v-oblozhke-kalkulyator-kalendar-do-\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/1165424/chipbord-vyrubnoy-dlya-skrapbukinga-malyshi-mikki-maus-disney-bebi\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/posuda/kuhonnye-prinadlezhnosti-i-i\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Канцтовары/Ежедневники и блокн\xD0\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/kanctovary/ezhednevniki-i-blok\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Стакан \xD0\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Набор бумаги для скрапбукинга \\\"Мои первый годик\\\": Микки Маус, Дисней бэби, 12 листов 29.5 х 29.5 см, 160\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"c\0\""sv, ResultType::Return, "c\0"sv }, }; for (auto i : boost::irange(0, 1/*00000*/)) @@ -641,15 +642,14 @@ TEST(JSONSuite, SimpleTest) { try { - JSON j(r.input, r.input + strlen(r.input)); + JSON j(r.input.data(), r.input.data() + r.input.size()); ASSERT_EQ(j.getString(), r.result); - ASSERT_TRUE(r.result_type == ResultType::Return); + ASSERT_EQ(r.result_type, ResultType::Return); } - catch (JSONException & e) + catch (const JSONException &) { - ASSERT_TRUE(r.result_type == ResultType::Throw); - ASSERT_EQ(e.message(), r.result); + ASSERT_EQ(r.result_type, ResultType::Throw); } } } From ffc39574f18cd77c50ab1c1c063aa5c5a199e91e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 02:04:01 +0300 Subject: [PATCH 029/233] Removed very old example (for jemalloc performance regression) --- base/common/tests/CMakeLists.txt | 4 --- base/common/tests/allocator.cpp | 47 -------------------------------- 2 files changed, 51 deletions(-) delete mode 100644 base/common/tests/allocator.cpp diff --git a/base/common/tests/CMakeLists.txt b/base/common/tests/CMakeLists.txt index 92be2f67c94..402a924baa9 100644 --- a/base/common/tests/CMakeLists.txt +++ b/base/common/tests/CMakeLists.txt @@ -1,12 +1,8 @@ include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) add_executable (local_date_time_comparison local_date_time_comparison.cpp) -add_executable (realloc-perf allocator.cpp) - -set(PLATFORM_LIBS ${CMAKE_DL_LIBS}) target_link_libraries (local_date_time_comparison PRIVATE common) -target_link_libraries (realloc-perf PRIVATE common) add_check(local_date_time_comparison) add_executable (dump_variable dump_variable.cpp) diff --git a/base/common/tests/allocator.cpp b/base/common/tests/allocator.cpp deleted file mode 100644 index 03f6228e0f5..00000000000 --- a/base/common/tests/allocator.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include -#include -#include -#include - - -void thread_func() -{ - for (size_t i = 0; i < 100; ++i) - { - size_t size = 4096; - - void * buf = malloc(size); - if (!buf) - abort(); - memset(buf, 0, size); - - while (size < 1048576) - { - size_t next_size = size * 4; - - void * new_buf = realloc(buf, next_size); - if (!new_buf) - abort(); - buf = new_buf; - - memset(reinterpret_cast(buf) + size, 0, next_size - size); - size = next_size; - } - - free(buf); - } -} - - -int main(int, char **) -{ - std::vector threads(16); - for (size_t i = 0; i < 1000; ++i) - { - for (auto & thread : threads) - thread = std::thread(thread_func); - for (auto & thread : threads) - thread.join(); - } - return 0; -} From 1096fc328ad5cb187bc0910272d3604e31da58fa Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 02:11:27 +0300 Subject: [PATCH 030/233] Make some tests automated --- base/common/tests/CMakeLists.txt | 7 ------- src/CMakeLists.txt | 2 ++ .../Common/tests/gtest_local_date_time_comparison.cpp | 8 ++++---- src/IO/tests/CMakeLists.txt | 2 -- src/Interpreters/tests/CMakeLists.txt | 1 - 5 files changed, 6 insertions(+), 14 deletions(-) rename base/common/tests/local_date_time_comparison.cpp => src/Common/tests/gtest_local_date_time_comparison.cpp (79%) diff --git a/base/common/tests/CMakeLists.txt b/base/common/tests/CMakeLists.txt index 402a924baa9..2a07a94055f 100644 --- a/base/common/tests/CMakeLists.txt +++ b/base/common/tests/CMakeLists.txt @@ -1,9 +1,2 @@ -include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) - -add_executable (local_date_time_comparison local_date_time_comparison.cpp) - -target_link_libraries (local_date_time_comparison PRIVATE common) -add_check(local_date_time_comparison) - add_executable (dump_variable dump_variable.cpp) target_link_libraries (dump_variable PRIVATE clickhouse_common_io) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a6a7d280479..82d84e42364 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -458,6 +458,8 @@ endif() dbms_target_link_libraries(PRIVATE _boost_context) +include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) + if (ENABLE_TESTS AND USE_GTEST) macro (grep_gtest_sources BASE_DIR DST_VAR) # Cold match files that are not in tests/ directories diff --git a/base/common/tests/local_date_time_comparison.cpp b/src/Common/tests/gtest_local_date_time_comparison.cpp similarity index 79% rename from base/common/tests/local_date_time_comparison.cpp rename to src/Common/tests/gtest_local_date_time_comparison.cpp index 5492ec31004..f75c2647100 100644 --- a/base/common/tests/local_date_time_comparison.cpp +++ b/src/Common/tests/gtest_local_date_time_comparison.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -16,14 +17,13 @@ void checkComparison() LocalDateTime a("2018-07-18 01:02:03"); LocalDateTime b("2018-07-18 01:02:03"); - if (a != b) - throw std::runtime_error("Test failed"); + EXPECT_EQ(a, b); + EXPECT_FALSE(a != b); } -int main(int, char **) +TEST(LocalDateTime, Comparison) { fillStackWithGarbage(); checkComparison(); - return 0; } diff --git a/src/IO/tests/CMakeLists.txt b/src/IO/tests/CMakeLists.txt index 79800d8339c..bcd0a8bba24 100644 --- a/src/IO/tests/CMakeLists.txt +++ b/src/IO/tests/CMakeLists.txt @@ -39,11 +39,9 @@ target_link_libraries (o_direct_and_dirty_pages PRIVATE clickhouse_common_io) add_executable (hashing_write_buffer hashing_write_buffer.cpp) target_link_libraries (hashing_write_buffer PRIVATE clickhouse_common_io) -add_check(hashing_write_buffer) add_executable (hashing_read_buffer hashing_read_buffer.cpp) target_link_libraries (hashing_read_buffer PRIVATE clickhouse_common_io) -add_check (hashing_read_buffer) add_executable (io_operators io_operators.cpp) target_link_libraries (io_operators PRIVATE clickhouse_common_io) diff --git a/src/Interpreters/tests/CMakeLists.txt b/src/Interpreters/tests/CMakeLists.txt index 1bc9d7fbacb..8905d2fe6e6 100644 --- a/src/Interpreters/tests/CMakeLists.txt +++ b/src/Interpreters/tests/CMakeLists.txt @@ -38,7 +38,6 @@ target_link_libraries (two_level_hash_map PRIVATE dbms) add_executable (in_join_subqueries_preprocessor in_join_subqueries_preprocessor.cpp) target_link_libraries (in_join_subqueries_preprocessor PRIVATE clickhouse_aggregate_functions dbms clickhouse_parsers) -add_check(in_join_subqueries_preprocessor) if (OS_LINUX) add_executable (internal_iotop internal_iotop.cpp) From 3eb3830dd8c409939223f5649c06f30f80de5363 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 02:13:49 +0300 Subject: [PATCH 031/233] Remove trash --- .../tests/gtest_global_register_functions.h.bak | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 src/Common/tests/gtest_global_register_functions.h.bak diff --git a/src/Common/tests/gtest_global_register_functions.h.bak b/src/Common/tests/gtest_global_register_functions.h.bak deleted file mode 100644 index 197ce5838b9..00000000000 --- a/src/Common/tests/gtest_global_register_functions.h.bak +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include - -struct RegisteredFunctionsState -{ - RegisteredFunctionsState() - { - DB::registerFunctions(); - } - - RegisteredFunctionsState(RegisteredFunctionsState &&) = default; -}; - -inline void tryRegisterFunctions() -{ - static RegisteredFunctionsState registered_functions_state; -} From 78cc70881647aea920bedebb45a74cd2134fd612 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 02:15:33 +0300 Subject: [PATCH 032/233] Fix bad whitespaces in test --- tests/queries/0_stateless/01691_DateTime64_clamp.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.sql b/tests/queries/0_stateless/01691_DateTime64_clamp.sql index 958de4edada..c77a66febb3 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.sql +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.sql @@ -11,7 +11,7 @@ SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; -- These are outsize of extended range and hence clamped -SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); -SELECT CAST(-1 * bitShiftLeft(toUInt64(1),35) AS DateTime64); -SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); -SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1), 35), 2); +SELECT CAST(-1 * bitShiftLeft(toUInt64(1), 35) AS DateTime64); +SELECT CAST(bitShiftLeft(toUInt64(1), 35) AS DateTime64); +SELECT toDateTime64(bitShiftLeft(toUInt64(1), 35), 2); From 4f08539754f4cc4065241fb00d9e4a3e7cefd567 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 03:01:08 +0300 Subject: [PATCH 033/233] Remove strange code --- base/common/DateLUTImpl.h | 44 ++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 8c2bbb3262d..4dee7eb5d55 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -109,6 +109,26 @@ private: return LUTIndex{(v / index.toUnderType()) & date_lut_mask}; } + /// Remainder of division in the sense of modular arithmetic: + /// the difference between x and the maximal divisable number not greater than x. + /// Example: -1 % 10 = 9, because -10 is the maximal number not greater than -1 that is divisable by 10. + /// Why do we need this: + /// - because the unix timestamp -1 is 1969-12-31 23:59:59 in UTC. + template + static constexpr inline auto mod(T x, U divisor) + { + /// This is the C++ way of modulo of division: + /// x % y is the number that: (x / y) * y + x % y == x + /// For example, -1 % 10 = -1 + /// Note that both variants are "correct" in the mathematical sense. They are just different operations. + auto res = x % divisor; + + if (unlikely(res < 0)) + res += divisor; + + return res; + } + public: /// The order of fields matters for alignment and sizeof. struct Values @@ -368,7 +388,7 @@ public: { const LUTIndex index = findIndex(t); - if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) + if (unlikely(index > DATE_LUT_MAX_DAY_NUM)) return t + offset_at_start_of_epoch; time_t res = t - lut[index].date; @@ -385,7 +405,7 @@ public: /// If it is overflow case, /// than limit number of hours to avoid insane results like 1970-01-01 89:28:15 - if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) + if (unlikely(index > DATE_LUT_MAX_DAY_NUM)) return static_cast((t + offset_at_start_of_epoch) / 3600) % 24; time_t time = t - lut[index].date; @@ -399,10 +419,10 @@ public: } /** Calculating offset from UTC in seconds. - * which means Using the same literal time of "t" to get the corresponding timestamp in UTC, - * then subtract the former from the latter to get the offset result. - * The boundaries when meets DST(daylight saving time) change should be handled very carefully. - */ + * which means Using the same literal time of "t" to get the corresponding timestamp in UTC, + * then subtract the former from the latter to get the offset result. + * The boundaries when meets DST(daylight saving time) change should be handled very carefully. + */ inline time_t timezoneOffset(time_t t) const { const LUTIndex index = findIndex(t); @@ -412,6 +432,7 @@ public: /// but we can figure out all the accumulated offsets from 1970-01-01 to that day just by get the whole difference between lut[].date, /// and then, we can directly subtract multiple 86400s to get the real DST offsets for the leap seconds is not considered now. time_t res = (lut[index].date - lut[daynum_offset_epoch].date) % 86400; + /// As so far to know, the maximal DST offset couldn't be more than 2 hours, so after the modulo operation the remainder /// will sits between [-offset --> 0 --> offset] which respectively corresponds to moving clock forward or backward. res = res > 43200 ? (86400 - res) : (0 - res); @@ -423,13 +444,6 @@ public: return res + offset_at_start_of_epoch; } - static inline time_t toSecondsSinceTheDayStart(time_t t) - { - t %= 86400; - t = (t < 0 ? t + 86400 : t); - - return t; - } /** Only for time zones with/when offset from UTC is multiple of five minutes. * This is true for all time zones: right now, all time zones have an offset that is multiple of 15 minutes. @@ -443,13 +457,13 @@ public: */ inline unsigned toSecond(time_t t) const { - return toSecondsSinceTheDayStart(t) % 60; + return mod(t, 60); } inline unsigned toMinute(time_t t) const { if (offset_is_whole_number_of_hours_everytime) - return (toSecondsSinceTheDayStart(t) / 60) % 60; + return mod((t / 60), 60); /// To consider the DST changing situation within this day. /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account From 7f1ae506fc3dcd145c3384db9ff065fd91c88419 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 03:11:40 +0300 Subject: [PATCH 034/233] The code is too complicated --- src/Functions/TransformDateTime64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/TransformDateTime64.h b/src/Functions/TransformDateTime64.h index 0a5e36cd2bd..e42c3155327 100644 --- a/src/Functions/TransformDateTime64.h +++ b/src/Functions/TransformDateTime64.h @@ -78,7 +78,7 @@ public: } } - template >>> + template >> inline auto execute(const T & t, Args && ... args) const { return wrapped_transform.execute(t, std::forward(args)...); From 9f3760e1e369839e3e0cd73a02e7e8325060323a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 03:18:48 +0300 Subject: [PATCH 035/233] Bad filename --- src/Common/tests/{gtest_DateLutImpl.cpp => gtest_DateLUTImpl.cpp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/Common/tests/{gtest_DateLutImpl.cpp => gtest_DateLUTImpl.cpp} (100%) diff --git a/src/Common/tests/gtest_DateLutImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp similarity index 100% rename from src/Common/tests/gtest_DateLutImpl.cpp rename to src/Common/tests/gtest_DateLUTImpl.cpp From 31010624d6e3729dc2715216cc60a905865c90d8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 03:35:23 +0300 Subject: [PATCH 036/233] Attempt to simplify code --- base/common/DateLUTImpl.h | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 4dee7eb5d55..eb34053ca6e 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -8,20 +8,23 @@ #include -#define DATE_LUT_MAX (0xFFFFFFFFU - 86400) -#define DATE_LUT_MAX_DAY_NUM (0xFFFFFFFFU / 86400) -/// Table size is bigger than DATE_LUT_MAX_DAY_NUM to fill all indices within UInt16 range: this allows to remove extra check. -#define DATE_LUT_SIZE 0x20000 #define DATE_LUT_MIN_YEAR 1925 /// 1925 since wast majority of timezones changed to 15-minute aligned offsets somewhere in 1924 or earlier. #define DATE_LUT_MAX_YEAR 2283 /// Last supported year (complete) #define DATE_LUT_YEARS (1 + DATE_LUT_MAX_YEAR - DATE_LUT_MIN_YEAR) /// Number of years in lookup table +#define DATE_LUT_SIZE 0x20000 + +#define DATE_LUT_MAX (0xFFFFFFFFU - 86400) +#define DATE_LUT_MAX_DAY_NUM 0xFFFF + + #if defined(__PPC__) #if !__clang__ #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif #endif + /// Flags for toYearWeek() function. enum class WeekModeFlag : UInt8 { @@ -208,17 +211,14 @@ private: /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. if (t >= lut[guess].date && t < lut[UInt32(guess + 1)].date) - return LUTIndex{guess}; + return LUTIndex(guess); /// Time zones that have offset 0 from UTC do daylight saving time change (if any) /// towards increasing UTC offset (example: British Standard Time). if (t >= lut[UInt32(guess + 1)].date) return LUTIndex(guess + 1); - if (lut[guess - 1].date <= t) - return LUTIndex(guess - 1); - - return LUTIndex(guess - 2); + return LUTIndex(guess - 1); } inline LUTIndex toLUTIndex(DayNum d) const @@ -388,9 +388,6 @@ public: { const LUTIndex index = findIndex(t); - if (unlikely(index > DATE_LUT_MAX_DAY_NUM)) - return t + offset_at_start_of_epoch; - time_t res = t - lut[index].date; if (res >= lut[index].time_at_offset_change()) @@ -403,11 +400,6 @@ public: { const LUTIndex index = findIndex(t); - /// If it is overflow case, - /// than limit number of hours to avoid insane results like 1970-01-01 89:28:15 - if (unlikely(index > DATE_LUT_MAX_DAY_NUM)) - return static_cast((t + offset_at_start_of_epoch) / 3600) % 24; - time_t time = t - lut[index].date; /// Data is cleaned to avoid possibility of underflow. From 6dc21cd09869da972fb9b87388d499fc2973a484 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 03:48:37 +0300 Subject: [PATCH 037/233] Remove discrepances --- src/Common/tests/gtest_DateLUTImpl.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 3a0da1ee1ee..0cb619e19f7 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -200,9 +200,9 @@ TEST(DateLUTTest, TimeValuesAtLeftBoderOfRange) EXPECT_EQ(lut.dateToString(time), "1970-01-01" /*std::string*/); } -TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) +TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOldLUT) { - // Value is at the right border of the OLD (small) LUT, and provides meaningful values where OLD LUT would provide garbage. + // Value is at the right border of the old (small) LUT, and provides meaningful values where old LUT would provide garbage. const DateLUTImpl & lut = DateLUT::instance("UTC"); const time_t time = 4294343873; // 2106-01-31T01:17:53 (Sunday) @@ -267,7 +267,7 @@ TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) class DateLUTWithTimeZone : public ::testing::TestWithParam {}; -TEST_P(DateLUTWithTimeZone, DISABLED_LoadLut) +TEST_P(DateLUTWithTimeZone, DISABLED_LoadLUT) { // There are some assumptions and assertions about TZ data made in DateLUTImpl which are verified upon loading, // to make sure that those assertions are true for all timezones we are going to load all of them one by one. From b9a8509f79b3aa96e46c05dc6be9f12eea5efbf5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 05:18:17 +0300 Subject: [PATCH 038/233] Adjustments --- base/common/DateLUTImpl.h | 5 ++--- src/Common/tests/gtest_DateLUTImpl.cpp | 10 +--------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index eb34053ca6e..050951b2409 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -402,7 +402,6 @@ public: time_t time = t - lut[index].date; - /// Data is cleaned to avoid possibility of underflow. if (time >= lut[index].time_at_offset_change()) time += lut[index].amount_of_offset_change(); @@ -454,8 +453,8 @@ public: inline unsigned toMinute(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) - return mod((t / 60), 60); + if (offset_is_whole_number_of_hours_everytime && t >= 0) + return (t / 60) % 60; /// To consider the DST changing situation within this day. /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 0cb619e19f7..91c1f40fdcb 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -267,7 +267,7 @@ TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOldLUT) class DateLUTWithTimeZone : public ::testing::TestWithParam {}; -TEST_P(DateLUTWithTimeZone, DISABLED_LoadLUT) +TEST_P(DateLUTWithTimeZone, LoadLUT) { // There are some assumptions and assertions about TZ data made in DateLUTImpl which are verified upon loading, // to make sure that those assertions are true for all timezones we are going to load all of them one by one. @@ -317,14 +317,6 @@ TEST_P(DateLUTWithTimeZone, getTimeZone) EXPECT_EQ(GetParam(), lut.getTimeZone()); } -TEST_P(DateLUTWithTimeZone, ZeroTime) -{ - const auto & lut = DateLUT::instance(GetParam()); - - EXPECT_EQ(0, lut.toDayNum(time_t{0})); - EXPECT_EQ(0, lut.toDayNum(DayNum{0})); - EXPECT_EQ(0, lut.toDayNum(ExtendedDayNum{0})); -} // Group of tests for timezones that have or had some time ago an offset which is not multiple of 15 minutes. INSTANTIATE_TEST_SUITE_P(ExoticTimezones, From ed492ccf5dba666e34b60d5ed045bab4e2b42bca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 05:36:36 +0300 Subject: [PATCH 039/233] Fix Africa/Juba --- base/common/DateLUTImpl.cpp | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index bf180acb835..71d17dbab2b 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -94,9 +94,10 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = 0; values.amount_of_offset_change_value = 0; - // TODO: this partially ignores fractional pre-epoch offsets, which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe/Minsk - // when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. - // https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 + /// TODO: This partially ignores fractional pre-epoch offsets, + /// which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe/Minsk + /// when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. + /// https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 if (start_of_day > 0 && start_of_day % 3600) offset_is_whole_number_of_hours_everytime = false; @@ -113,16 +114,28 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) /// Find a time (timestamp offset from beginning of day), /// when UTC offset was changed. Search is performed with 15-minute granularity, assuming it is enough. - time_t time_at_offset_change = 900; - while (time_at_offset_change < 86400) + time_t time_at_offset_change = 0; + + /// If offset was changed just at midnight. + if (utc_offset_at_beginning_of_day != cctz_time_zone.lookup( + std::chrono::system_clock::from_time_t(lut[i - 1].date - 1)).offset) { - auto utc_offset_at_current_time = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t( - lut[i - 1].date + time_at_offset_change)).offset; + /// time_at_offset_change is zero. + } + else + { + time_at_offset_change = 900; + while (time_at_offset_change < 86400) + { + auto utc_offset_at_current_time = cctz_time_zone.lookup( + std::chrono::system_clock::from_time_t( + lut[i - 1].date + time_at_offset_change)).offset; - if (utc_offset_at_current_time != utc_offset_at_beginning_of_day) - break; + if (utc_offset_at_current_time != utc_offset_at_beginning_of_day) + break; - time_at_offset_change += 900; + time_at_offset_change += 900; + } } lut[i - 1].time_at_offset_change_value = time_at_offset_change / Values::OffsetChangeFactor; From 9416f5901492c31d4a7ed2c6f9bf88760ffdddea Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 06:48:24 +0300 Subject: [PATCH 040/233] Some progress on tests --- src/Common/tests/gtest_DateLUTImpl.cpp | 46 ++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 91c1f40fdcb..95ac055e237 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -4,8 +4,11 @@ #include #include +#include + #include + /// For the expansion of gtest macros. #if defined(__clang__) #pragma clang diagnostic ignored "-Wused-but-marked-unused" @@ -25,14 +28,17 @@ cctz::civil_day YYYYMMDDToDay(unsigned value) value % 100); // day } -std::vector allTimezones() +std::vector allTimezones(bool with_weird_offsets = true) { std::vector result; const auto * timezone_name = auto_time_zones; while (*timezone_name) { - result.push_back(*timezone_name); + bool weird_offsets = (std::string_view(*timezone_name) == "Africa/Monrovia"); + + if (!weird_offsets || with_weird_offsets) + result.push_back(*timezone_name); ++timezone_name; } @@ -345,7 +351,7 @@ std::ostream & operator<<(std::ostream & ostr, const DateLUTImpl::Values & v) << "\n\t weekday : " << static_cast(v.day_of_week) << "\n\t days in month : " << static_cast(v.days_in_month) << "\n\t offset change : " << v.amount_of_offset_change() - << "\n\t offfset change at : " << v.time_at_offset_change() + << "\n\t offset change at : " << v.time_at_offset_change() << "\n}"; } @@ -387,6 +393,32 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) const auto tz_time = cctz::convert(std::chrono::system_clock::from_time_t(expected_time_t), tz); + /// Weird offset, not supported. + /// Example: Africa/Monrovia has offset UTC-0:44:30 in year 1970. + if (tz.lookup(std::chrono::system_clock::from_time_t(expected_time_t)).offset % 900) + continue; + + /// Unsupported timezone transitions - not in 15-minute time point or to different day. + /// Example: America/Goose_Bay decided to go back one hour at 00:01: + /// $ seq 1289097900 30 1289103600 | TZ=America/Goose_Bay LC_ALL=C xargs -I{} date -d @{} + /// Sat Nov 6 23:59:00 ADT 2010 + /// Sat Nov 6 23:59:30 ADT 2010 + /// Sun Nov 7 00:00:00 ADT 2010 + /// Sun Nov 7 00:00:30 ADT 2010 + /// Sat Nov 6 23:01:00 AST 2010 + /// Sat Nov 6 23:01:30 AST 2010 + cctz::time_zone::civil_transition transition{}; + if (tz.next_transition(std::chrono::system_clock::from_time_t(expected_time_t), &transition) + && transition.from.day() == tz_time.day() + && (transition.from.second() != 0 || transition.from.minute() % 900 != 0 + || (transition.from.day() != transition.to.day() + && (transition.from.hour() != 0 && transition.from.minute() != 0 && transition.from.second() != 0)))) + { + std::cerr << "Skipping " << timezone_name << " " << tz_time + << " because of unsupported timezone transition from " << transition.from << " to " << transition.to << "\n"; + continue; + } + EXPECT_EQ(tz_time.year(), lut.toYear(expected_time_t)); EXPECT_EQ(tz_time.month(), lut.toMonth(expected_time_t)); EXPECT_EQ(tz_time.day(), lut.toDayOfMonth(expected_time_t)); @@ -429,7 +461,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( - ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(allTimezones(false)), ::testing::ValuesIn(std::initializer_list{ // Values from tests/date_lut3.cpp {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19701231), 3191 /*53m 11s*/}, @@ -439,7 +471,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( - ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(allTimezones(false)), ::testing::ValuesIn(std::initializer_list{ // Values from tests/date_lut3.cpp {YYYYMMDDToDay(20100101), YYYYMMDDToDay(20101231), 3191 /*53m 11s*/}, @@ -459,7 +491,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( - ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(allTimezones(false)), ::testing::ValuesIn(std::initializer_list{ {YYYYMMDDToDay(19500101), YYYYMMDDToDay(19600101), 15 * 60}, {YYYYMMDDToDay(19300101), YYYYMMDDToDay(19350101), 11 * 15 * 60} @@ -469,7 +501,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( - ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(allTimezones(false)), ::testing::ValuesIn(std::initializer_list{ {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19700201), 15 * 60}, {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19701231), 11 * 13 * 17} From 66f495b6904c81f53b61e920e1a0f1d5fe4dd097 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 06:49:46 +0300 Subject: [PATCH 041/233] Fix build --- programs/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 9adca58b55a..6b322df5ffd 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -325,7 +325,7 @@ else () endif () if (ENABLE_TESTS AND USE_GTEST) - set (CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_libcommon unit_tests_dbms) + set (CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_dbms) add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_UNIT_TESTS_TARGETS}) add_dependencies(clickhouse-bundle clickhouse-tests) endif() From 7cc471f9b90c2adc6c32bb339d39c5ec9da5df7f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 04:22:12 +0300 Subject: [PATCH 042/233] Simplify code --- base/common/DateLUTImpl.cpp | 65 ++++++++++++++----------------------- 1 file changed, 24 insertions(+), 41 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 71d17dbab2b..e2bf60fff4e 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -69,7 +69,8 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) { cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date); - start_of_day = std::chrono::system_clock::to_time_t(lookup.pre); /// Ambiguity is possible. + /// Ambiguity is possible. + start_of_day = std::chrono::system_clock::to_time_t(lookup.pre); Values & values = lut[i]; values.year = date.year(); @@ -94,56 +95,38 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = 0; values.amount_of_offset_change_value = 0; - /// TODO: This partially ignores fractional pre-epoch offsets, + /// TODO: This partially ignores fractional offsets, /// which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe/Minsk /// when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. /// https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 if (start_of_day > 0 && start_of_day % 3600) offset_is_whole_number_of_hours_everytime = false; - /// If UTC offset was changed in previous day. - if (i != 0) + /// If UTC offset was changed this day. + /// Change in time zone without transition is possible, e.g. Moscow 1991 Sun, 31 Mar, 02:00 MSK to EEST + cctz::time_zone::civil_transition transition{}; + if (cctz_time_zone.next_transition(lookup.pre, &transition) + && transition.from.year() == date.year() + && transition.from.month() == date.month() + && transition.from.day() == date.day() + && transition.from != transition.to) { - auto amount_of_offset_change_at_prev_day = 86400 - (lut[i].date - lut[i - 1].date); - if (amount_of_offset_change_at_prev_day) - { - lut[i - 1].amount_of_offset_change_value = amount_of_offset_change_at_prev_day / Values::OffsetChangeFactor; + values.time_at_offset_change_value = (transition.from - cctz::civil_second(date)) / Values::OffsetChangeFactor; + values.amount_of_offset_change_value = (transition.to - transition.from) / Values::OffsetChangeFactor; - const auto utc_offset_at_beginning_of_day = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(lut[i - 1].date)).offset; +// std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n"; +// std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n"; - /// Find a time (timestamp offset from beginning of day), - /// when UTC offset was changed. Search is performed with 15-minute granularity, assuming it is enough. + /// We don't support too large changes. + if (values.amount_of_offset_change_value > 24 * 4) + values.amount_of_offset_change_value = 24 * 4; + else if (values.amount_of_offset_change_value < -24 * 4) + values.amount_of_offset_change_value = -24 * 4; - time_t time_at_offset_change = 0; - - /// If offset was changed just at midnight. - if (utc_offset_at_beginning_of_day != cctz_time_zone.lookup( - std::chrono::system_clock::from_time_t(lut[i - 1].date - 1)).offset) - { - /// time_at_offset_change is zero. - } - else - { - time_at_offset_change = 900; - while (time_at_offset_change < 86400) - { - auto utc_offset_at_current_time = cctz_time_zone.lookup( - std::chrono::system_clock::from_time_t( - lut[i - 1].date + time_at_offset_change)).offset; - - if (utc_offset_at_current_time != utc_offset_at_beginning_of_day) - break; - - time_at_offset_change += 900; - } - } - - lut[i - 1].time_at_offset_change_value = time_at_offset_change / Values::OffsetChangeFactor; - - /// We don't support cases when time change results in switching to previous day. - if (static_cast(lut[i - 1].time_at_offset_change()) + static_cast(lut[i - 1].amount_of_offset_change()) < 0) - lut[i - 1].time_at_offset_change_value = -lut[i - 1].amount_of_offset_change_value; - } + /// We don't support cases when time change results in switching to previous day. + /// Shift the point of time change later. + if (values.time_at_offset_change_value + values.amount_of_offset_change_value < 0) + values.time_at_offset_change_value = -values.amount_of_offset_change_value; } /// Going to next day. From 1722978c2b05feb15a1ae35e902d4ac31af70a92 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 04:45:12 +0300 Subject: [PATCH 043/233] Comments and corrections --- base/common/DateLUTImpl.cpp | 9 ++++++--- base/common/DateLUTImpl.h | 40 +++++++++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index e2bf60fff4e..0685b37365b 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -69,8 +69,11 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) { cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date); - /// Ambiguity is possible. - start_of_day = std::chrono::system_clock::to_time_t(lookup.pre); + /// Ambiguity is possible if time was changed backwards at the midnight + /// (or after midnight time has been changed to the previous day, for example two hours backwards at 01:00). + /// Then midnight appears twice. Usually time change happens exactly at 00:00. + /// Then we should use the second midnight as the start of the day. + start_of_day = std::chrono::system_clock::to_time_t(lookup.post); Values & values = lut[i]; values.year = date.year(); @@ -105,7 +108,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) /// If UTC offset was changed this day. /// Change in time zone without transition is possible, e.g. Moscow 1991 Sun, 31 Mar, 02:00 MSK to EEST cctz::time_zone::civil_transition transition{}; - if (cctz_time_zone.next_transition(lookup.pre, &transition) + if (cctz_time_zone.next_transition(lookup.post, &transition) && transition.from.year() == date.year() && transition.from.month() == date.month() && transition.from.day() == date.day() diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 050951b2409..d798cffab63 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -406,7 +406,9 @@ public: time += lut[index].amount_of_offset_change(); unsigned res = time / 3600; - return res <= 23 ? res : 0; + + /// In case time was changed backwards at the start of next day, we will repeat the hour 23. + return res <= 23 ? res : 23; } /** Calculating offset from UTC in seconds. @@ -456,28 +458,35 @@ public: if (offset_is_whole_number_of_hours_everytime && t >= 0) return (t / 60) % 60; - /// To consider the DST changing situation within this day. - /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account - LUTIndex index = findIndex(t); - UInt32 res = t - lut[index].date; - if (lut[index].amount_of_offset_change() != 0 && t >= lut[index].date + lut[index].time_at_offset_change()) - res += lut[index].amount_of_offset_change(); + /// To consider the DST changing situation within this day + /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account. - return res / 60 % 60; + LUTIndex index = findIndex(t); + UInt32 time = t - lut[index].date; + + if (time >= lut[index].time_at_offset_change()) + time += lut[index].amount_of_offset_change(); + + return time / 60 % 60; } + /// NOTE: These functions are wrong for negative time_t. + /// NOTE: Assuming timezone offset is a multiple of 15 minutes. inline time_t toStartOfMinute(time_t t) const { return t / 60 * 60; } inline time_t toStartOfFiveMinute(time_t t) const { return t / 300 * 300; } inline time_t toStartOfFifteenMinutes(time_t t) const { return t / 900 * 900; } + + /// NOTE: This most likely wrong for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate. inline time_t toStartOfTenMinutes(time_t t) const { return t / 600 * 600; } + /// NOTE: Assuming timezone transitions are multiple of hours. Lord Howe Island in Australia is a notable exception. inline time_t toStartOfHour(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) + if (offset_is_whole_number_of_hours_everytime && t >= 0) return t / 3600 * 3600; - UInt32 date = find(t).date; - return date + (UInt32(t) - date) / 3600 * 3600; + Int64 date = find(t).date; + return date + (t - date) / 3600 * 3600; } /** Number of calendar day since the beginning of UNIX epoch (1970-01-01 is zero) @@ -757,6 +766,7 @@ public: /// We count all hour-length intervals, unrelated to offset changes. inline time_t toRelativeHourNum(time_t t) const { + /// NOTE: This is also wrong for negative time_t. if (offset_is_whole_number_of_hours_everytime) return t / 3600; @@ -771,6 +781,7 @@ public: return toRelativeHourNum(lut[toLUTIndex(v)].date); } + /// NOTE: This is wrong for negative time_t. inline time_t toRelativeMinuteNum(time_t t) const { return t / 60; @@ -829,7 +840,10 @@ public: if (hours == 1) return toStartOfHour(t); UInt64 seconds = hours * 3600; + + /// NOTE: This is wrong for negative time_t. t = t / seconds * seconds; + if (offset_is_whole_number_of_hours_everytime) return t; return toStartOfHour(t); @@ -840,6 +854,8 @@ public: if (minutes == 1) return toStartOfMinute(t); UInt64 seconds = 60 * minutes; + + /// NOTE: This is wrong for negative time_t. return t / seconds * seconds; } @@ -847,6 +863,8 @@ public: { if (seconds == 1) return t; + + /// NOTE: This is wrong for negative time_t. return t / seconds * seconds; } From 790aa8697ce48257d913802f698ca886da11ec7b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 04:47:50 +0300 Subject: [PATCH 044/233] Fix typos --- base/common/DateLUTImpl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index d798cffab63..ec47fea60f6 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -113,8 +113,8 @@ private: } /// Remainder of division in the sense of modular arithmetic: - /// the difference between x and the maximal divisable number not greater than x. - /// Example: -1 % 10 = 9, because -10 is the maximal number not greater than -1 that is divisable by 10. + /// the difference between x and the maximal divisible number not greater than x. + /// Example: -1 % 10 = 9, because -10 is the maximal number not greater than -1 that is divisible by 10. /// Why do we need this: /// - because the unix timestamp -1 is 1969-12-31 23:59:59 in UTC. template From ecee81b714ec212b7a4c61c8dbb133d89ed9165c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 04:50:16 +0300 Subject: [PATCH 045/233] Fix clang-tidy --- base/common/DateLUTImpl.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 0685b37365b..46fc88aeb4d 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -46,17 +46,16 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) if (&inside_main) assert(inside_main); - cctz::time_zone cctz_time_zone; if (!cctz::load_time_zone(time_zone, &cctz_time_zone)) throw Poco::Exception("Cannot load time zone " + time_zone_); - const cctz::civil_day epoch{1970, 1, 1}; - const cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1}; + constexpr cctz::civil_day epoch{1970, 1, 1}; + constexpr cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1}; time_t start_of_day; - // Note: it's validated against all timezones in the system. - assert((epoch - lut_start) == daynum_offset_epoch); + /// Note: it's validated against all timezones in the system. + static_assert((epoch - lut_start) == daynum_offset_epoch); offset_at_start_of_epoch = cctz_time_zone.lookup(cctz_time_zone.lookup(epoch).pre).offset; offset_at_start_of_lut = cctz_time_zone.lookup(cctz_time_zone.lookup(lut_start).pre).offset; From 738d106874de5e61c943bbb022c1053fdbca10b0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 05:03:55 +0300 Subject: [PATCH 046/233] Support negative time_t in more functions --- base/common/DateLUTImpl.h | 60 +++++++++++++-------------------------- 1 file changed, 19 insertions(+), 41 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index ec47fea60f6..8da45fa6622 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -17,6 +17,10 @@ #define DATE_LUT_MAX (0xFFFFFFFFU - 86400) #define DATE_LUT_MAX_DAY_NUM 0xFFFF +/// A constant to add to time_t so every supported time point becomes non-negative and still has the same remainder of division by 3600. +/// If we treat "remainder of division" operation in the sense of modular arithmetic (not like in C++). +#define DATE_LUT_ADD (1970 - DATE_LUT_MIN_YEAR) * 366 * 86400 + #if defined(__PPC__) #if !__clang__ @@ -112,26 +116,6 @@ private: return LUTIndex{(v / index.toUnderType()) & date_lut_mask}; } - /// Remainder of division in the sense of modular arithmetic: - /// the difference between x and the maximal divisible number not greater than x. - /// Example: -1 % 10 = 9, because -10 is the maximal number not greater than -1 that is divisible by 10. - /// Why do we need this: - /// - because the unix timestamp -1 is 1969-12-31 23:59:59 in UTC. - template - static constexpr inline auto mod(T x, U divisor) - { - /// This is the C++ way of modulo of division: - /// x % y is the number that: (x / y) * y + x % y == x - /// For example, -1 % 10 = -1 - /// Note that both variants are "correct" in the mathematical sense. They are just different operations. - auto res = x % divisor; - - if (unlikely(res < 0)) - res += divisor; - - return res; - } - public: /// The order of fields matters for alignment and sizeof. struct Values @@ -450,13 +434,13 @@ public: */ inline unsigned toSecond(time_t t) const { - return mod(t, 60); + return (t + DATE_LUT_ADD) % 60; } inline unsigned toMinute(time_t t) const { - if (offset_is_whole_number_of_hours_everytime && t >= 0) - return (t / 60) % 60; + if (offset_is_whole_number_of_hours_everytime) + return ((t + DATE_LUT_ADD) / 60) % 60; /// To consider the DST changing situation within this day /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account. @@ -470,20 +454,19 @@ public: return time / 60 % 60; } - /// NOTE: These functions are wrong for negative time_t. /// NOTE: Assuming timezone offset is a multiple of 15 minutes. - inline time_t toStartOfMinute(time_t t) const { return t / 60 * 60; } - inline time_t toStartOfFiveMinute(time_t t) const { return t / 300 * 300; } - inline time_t toStartOfFifteenMinutes(time_t t) const { return t / 900 * 900; } + inline time_t toStartOfMinute(time_t t) const { return (t + DATE_LUT_ADD) / 60 * 60 - DATE_LUT_ADD; } + inline time_t toStartOfFiveMinute(time_t t) const { return (t + DATE_LUT_ADD) / 300 * 300 - DATE_LUT_ADD; } + inline time_t toStartOfFifteenMinutes(time_t t) const { return (t + DATE_LUT_ADD) / 900 * 900 - DATE_LUT_ADD; } /// NOTE: This most likely wrong for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate. - inline time_t toStartOfTenMinutes(time_t t) const { return t / 600 * 600; } + inline time_t toStartOfTenMinutes(time_t t) const { return (t + DATE_LUT_ADD) / 600 * 600 - DATE_LUT_ADD; } /// NOTE: Assuming timezone transitions are multiple of hours. Lord Howe Island in Australia is a notable exception. inline time_t toStartOfHour(time_t t) const { - if (offset_is_whole_number_of_hours_everytime && t >= 0) - return t / 3600 * 3600; + if (offset_is_whole_number_of_hours_everytime) + return (t + DATE_LUT_ADD) / 3600 * 3600 - DATE_LUT_ADD; Int64 date = find(t).date; return date + (t - date) / 3600 * 3600; @@ -766,13 +749,12 @@ public: /// We count all hour-length intervals, unrelated to offset changes. inline time_t toRelativeHourNum(time_t t) const { - /// NOTE: This is also wrong for negative time_t. if (offset_is_whole_number_of_hours_everytime) - return t / 3600; + return (t + DATE_LUT_ADD) / 3600 - (DATE_LUT_ADD / 3600); /// Assume that if offset was fractional, then the fraction is the same as at the beginning of epoch. /// NOTE This assumption is false for "Pacific/Pitcairn" and "Pacific/Kiritimati" time zones. - return (t + 86400 - offset_at_start_of_epoch) / 3600; + return (t + DATE_LUT_ADD + 86400 - offset_at_start_of_epoch) / 3600 - (DATE_LUT_ADD / 3600); } template @@ -781,10 +763,9 @@ public: return toRelativeHourNum(lut[toLUTIndex(v)].date); } - /// NOTE: This is wrong for negative time_t. inline time_t toRelativeMinuteNum(time_t t) const { - return t / 60; + return (t + DATE_LUT_ADD) / 60 - (DATE_LUT_ADD / 60); } template @@ -841,8 +822,7 @@ public: return toStartOfHour(t); UInt64 seconds = hours * 3600; - /// NOTE: This is wrong for negative time_t. - t = t / seconds * seconds; + t = (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; if (offset_is_whole_number_of_hours_everytime) return t; @@ -855,8 +835,7 @@ public: return toStartOfMinute(t); UInt64 seconds = 60 * minutes; - /// NOTE: This is wrong for negative time_t. - return t / seconds * seconds; + return (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; } inline time_t toStartOfSecondInterval(time_t t, UInt64 seconds) const @@ -864,8 +843,7 @@ public: if (seconds == 1) return t; - /// NOTE: This is wrong for negative time_t. - return t / seconds * seconds; + return (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; } inline LUTIndex makeLUTIndex(Int16 year, UInt8 month, UInt8 day_of_month) const From d276fac135aa2ea2866d867a73ac93657b5ef088 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 05:04:23 +0300 Subject: [PATCH 047/233] Support negative time_t in more functions --- base/common/DateLUTImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 8da45fa6622..2534f20838f 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -19,7 +19,7 @@ /// A constant to add to time_t so every supported time point becomes non-negative and still has the same remainder of division by 3600. /// If we treat "remainder of division" operation in the sense of modular arithmetic (not like in C++). -#define DATE_LUT_ADD (1970 - DATE_LUT_MIN_YEAR) * 366 * 86400 +#define DATE_LUT_ADD ((1970 - DATE_LUT_MIN_YEAR) * 366 * 86400) #if defined(__PPC__) From 7c8d17045cc806c5b1ca190d18510a79204525c0 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 5 Mar 2021 06:15:20 +0300 Subject: [PATCH 048/233] fix frame formatting error --- programs/client/Client.cpp | 30 ++++++++++++++++++++++-- src/Parsers/ASTWindowDefinition.cpp | 2 +- src/Parsers/ExpressionElementParsers.cpp | 7 ++---- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 3c27908741c..f213c81e298 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1317,7 +1317,10 @@ private: auto base_after_fuzz = fuzz_base->formatForErrorMessage(); - // Debug AST cloning errors. + // Check that the source AST didn't change after fuzzing. This + // helps debug AST cloning errors, where the cloned AST doesn't + // clone all its children, and erroneously points to some source + // child elements. if (base_before_fuzz != base_after_fuzz) { fmt::print(stderr, @@ -1334,7 +1337,7 @@ private: fmt::print(stderr, "IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent node doesn't implement clone() correctly."); - assert(false); + exit(1); } auto fuzzed_text = ast_to_process->formatForErrorMessage(); @@ -1344,6 +1347,29 @@ private: continue; } + // Check that the query is formatted properly and we can parse + // it back and format again and get the same result. Unfortunately + // we can't compare the ASTs, which would be more sensitive to + // errors. This double formatting check doesn't catch all errors, + // e.g. we can format query incorrectly, but to a valid SQL that + // we can then parse and format into the same SQL. + { + const auto * tmp_pos = fuzzed_text.c_str(); + auto parsed_formatted_query = parseQuery(tmp_pos, + tmp_pos + fuzzed_text.size(), + false /* allow_multi_statements */); + const auto formatted_twice + = parsed_formatted_query->formatForErrorMessage(); + + if (formatted_twice != fuzzed_text) + { + fmt::print(stderr, "The query formatting is broken. Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n", + formatted_twice, fuzzed_text); + + exit(1); + } + } + parsed_query = ast_to_process; query_to_send = parsed_query->formatForErrorMessage(); diff --git a/src/Parsers/ASTWindowDefinition.cpp b/src/Parsers/ASTWindowDefinition.cpp index dba2935e630..aee951fc1f3 100644 --- a/src/Parsers/ASTWindowDefinition.cpp +++ b/src/Parsers/ASTWindowDefinition.cpp @@ -81,7 +81,7 @@ void ASTWindowDefinition::formatImpl(const FormatSettings & settings, } else if (frame.end_type == WindowFrame::BoundaryType::Unbounded) { - settings.ostr << "UNBOUNDED PRECEDING"; + settings.ostr << "UNBOUNDED FOLLOWING"; } else { diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 7a426e7774d..da63ea6e658 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -533,6 +533,7 @@ static bool tryParseFrameDefinition(ASTWindowDefinition * node, IParser::Pos & p ParserKeyword keyword_groups("GROUPS"); ParserKeyword keyword_range("RANGE"); + node->frame.is_default = false; if (keyword_rows.ignore(pos, expected)) { node->frame.type = WindowFrame::FrameType::Rows; @@ -548,6 +549,7 @@ static bool tryParseFrameDefinition(ASTWindowDefinition * node, IParser::Pos & p else { /* No frame clause. */ + node->frame.is_default = true; return true; } @@ -699,11 +701,6 @@ static bool tryParseFrameDefinition(ASTWindowDefinition * node, IParser::Pos & p } } - if (!(node->frame == WindowFrame{})) - { - node->frame.is_default = false; - } - return true; } From 6b9d1f67a382d198f283a79260ffe485aaff9efb Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 5 Mar 2021 10:05:27 +0300 Subject: [PATCH 049/233] make NULL case consistent w/FieldVisitorToString --- src/Core/Field.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 93107d7bb2c..8aa79b0bbe1 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -452,7 +452,7 @@ template <> bool decimalLessOrEqual(DateTime64 x, DateTime64 y, UInt32 x_scale, inline void writeText(const Null &, WriteBuffer & buf) { - writeText(std::string("Null"), buf); + writeText(std::string("NULL"), buf); } String toString(const Field & x) From 70dfcdac8d1c90019674cc932e5e0e61ded563f8 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 5 Mar 2021 12:59:50 +0300 Subject: [PATCH 050/233] ignore some errors --- programs/client/Client.cpp | 48 +++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index f213c81e298..c4954fa51cc 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1354,19 +1354,45 @@ private: // e.g. we can format query incorrectly, but to a valid SQL that // we can then parse and format into the same SQL. { - const auto * tmp_pos = fuzzed_text.c_str(); - auto parsed_formatted_query = parseQuery(tmp_pos, - tmp_pos + fuzzed_text.size(), - false /* allow_multi_statements */); - const auto formatted_twice - = parsed_formatted_query->formatForErrorMessage(); - - if (formatted_twice != fuzzed_text) + ASTPtr parsed_formatted_query; + try { - fmt::print(stderr, "The query formatting is broken. Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n", - formatted_twice, fuzzed_text); + const auto * tmp_pos = fuzzed_text.c_str(); + parsed_formatted_query = parseQuery(tmp_pos, + tmp_pos + fuzzed_text.size(), + false /* allow_multi_statements */); + } + catch (Exception & e) + { + // Some complicated cases where we can generate the SQL + // which we can't parse: + // * first argument of lambda() replaced by fuzzer with + // something else, leading to constructs such as + // arrayMap((min(x) + 3) -> x + 1, ....) + // * internals of Enum replaced, leading to: + // Enum(equals(someFunction(y), 3)). + // We could filter them on case-by-case basis, but they + // are probably also helpful in that they test the parsing + // errors, so let's just ignore them in this check and + // send them to the server normally. + if (e.code() != ErrorCodes::SYNTAX_ERROR) + { + throw; + } + } - exit(1); + if (parsed_formatted_query) + { + const auto formatted_twice + = parsed_formatted_query->formatForErrorMessage(); + + if (formatted_twice != fuzzed_text) + { + fmt::print(stderr, "The query formatting is broken. Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n", + formatted_twice, fuzzed_text); + + exit(1); + } } } From 9ac39dcda9b1b6877634695f67cd7c3995e20dac Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 5 Mar 2021 15:36:19 +0300 Subject: [PATCH 051/233] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index adb2df570d7..8a1a14ee007 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -10,7 +10,7 @@ Deletes information about the `name` table from the server. The server stops kno Syntax: ``` sql -DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] +DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] ``` This does not delete the table’s data or metadata. On the next server launch, the server will read the metadata and find out about the table again. From 59b16ecb3c5b79133a09788ab83df3374054496d Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 5 Mar 2021 15:36:31 +0300 Subject: [PATCH 052/233] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 8a1a14ee007..f27f61a933a 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -5,7 +5,7 @@ toc_title: DETACH # DETACH Statement {#detach} -Deletes information about the `name` table from the server. The server stops knowing about the table’s existence. +Deletes information about the table or view from the server. The server stops knowing about their existence. Syntax: From e996e2be8eeb6021d6bc056c459b3cc24bce328f Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 5 Mar 2021 15:37:34 +0300 Subject: [PATCH 053/233] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index f27f61a933a..0bd4f730364 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -13,7 +13,15 @@ Syntax: DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] ``` -This does not delete the table’s data or metadata. On the next server launch, the server will read the metadata and find out about the table again. +Detaching does not delete the data or metadata for the table or view. If the table or view was not detached `PERMANENTLY`, on the next server launch the server will read the metadata and recall the table/view again. If the table or view was detached `PERMANENTLY`, there will be no automatic recall. + +Whether the table was detached permanently or not, in both cases you can reattach it using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). + +`ATTACH MATERIALIZED VIEW` doesn't work with short syntax (without `SELECT`), but you can attach it using the `ATTACH TABLE` query. + +Note that you can not detach permanently the table which is already detached (temporary). But you can attach it back and then detach permanently again. + +Also you can not [DROP](../../sql-reference/statements/drop.md#drop-table) the detached table, or [CREATE TABLE](../../sql-reference/statements/create/table.md) with the same name as detached permanently, or replace it with the other table with [RENAME TABLE](../../sql-reference/statements/rename.md) query. Similarly, a “detached” table can be re-attached using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). From 60cb84d41755a8c7e61074c8e586d18267c4145d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 5 Mar 2021 20:24:50 +0300 Subject: [PATCH 054/233] more debug info --- programs/client/Client.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index c4954fa51cc..c5b579f2046 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1390,6 +1390,9 @@ private: { fmt::print(stderr, "The query formatting is broken. Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n", formatted_twice, fuzzed_text); + fmt::print(stderr, "AST parsed back:\n'{}'\nSource AST:\n'{}'\n", + parsed_formatted_query->dumpTree(), + ast_to_process->dumpTree()); exit(1); } From 8d5d1b76fbc08a337bd4390dfc2e9c54883569d9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 00:36:29 +0300 Subject: [PATCH 055/233] Fix error --- base/common/DateLUTImpl.h | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 2534f20838f..6e8b424ab9d 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -5,7 +5,9 @@ #include "types.h" #include +#include #include +#include #define DATE_LUT_MIN_YEAR 1925 /// 1925 since wast majority of timezones changed to 15-minute aligned offsets somewhere in 1924 or earlier. @@ -231,6 +233,21 @@ private: return lut[toLUTIndex(v)]; } + template + static inline T roundDown(T x, Divisor divisor) + { + static_assert(std::is_integral_v && std::is_integral_v); + assert(divisor > 0); + + if (likely(x >= 0)) + return x / divisor * divisor; + + /// Integer division for negative numbers rounds them towards zero (up). + /// We will shift the number so it will be rounded towards -inf (down). + + return (x + 1 - divisor) / divisor * divisor; + } + public: const std::string & getTimeZone() const { return time_zone; } @@ -822,10 +839,12 @@ public: return toStartOfHour(t); UInt64 seconds = hours * 3600; - t = (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; + t = roundDown(t, seconds); if (offset_is_whole_number_of_hours_everytime) return t; + + /// TODO check if it's correct. return toStartOfHour(t); } @@ -833,9 +852,9 @@ public: { if (minutes == 1) return toStartOfMinute(t); - UInt64 seconds = 60 * minutes; - return (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; + UInt64 seconds = 60 * minutes; + return roundDown(t, seconds); } inline time_t toStartOfSecondInterval(time_t t, UInt64 seconds) const @@ -843,7 +862,7 @@ public: if (seconds == 1) return t; - return (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; + return roundDown(t, seconds); } inline LUTIndex makeLUTIndex(Int16 year, UInt8 month, UInt8 day_of_month) const From 0dbadc8d8eec1e23afdd635609aaf8fd612110a6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 01:35:23 +0300 Subject: [PATCH 056/233] Fix error --- base/common/DateLUTImpl.cpp | 28 ++++++++++++++++------------ base/common/DateLUTImpl.h | 19 +++++++++++++------ 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 46fc88aeb4d..a111a21d8fd 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -69,10 +69,18 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date); /// Ambiguity is possible if time was changed backwards at the midnight - /// (or after midnight time has been changed to the previous day, for example two hours backwards at 01:00). - /// Then midnight appears twice. Usually time change happens exactly at 00:00. - /// Then we should use the second midnight as the start of the day. - start_of_day = std::chrono::system_clock::to_time_t(lookup.post); + /// or after midnight time has been changed back to midnight, for example one hour backwards at 01:00 + /// or after midnight time has been changed to the previous day, for example two hours backwards at 01:00 + /// Then midnight appears twice. Usually time change happens exactly at 00:00 or 01:00. + + /// If transition did not involve previous day, we should use the first midnight as the start of the day, + /// otherwise it's better to use the second midnight. + + std::chrono::time_point start_of_day_time_point = lookup.trans < lookup.post + ? lookup.post /* Second midnight appears after transition, so there was a piece of previous day after transition */ + : lookup.pre; + + start_of_day = std::chrono::system_clock::to_time_t(start_of_day_time_point); Values & values = lut[i]; values.year = date.year(); @@ -97,17 +105,13 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = 0; values.amount_of_offset_change_value = 0; - /// TODO: This partially ignores fractional offsets, - /// which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe/Minsk - /// when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. - /// https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 - if (start_of_day > 0 && start_of_day % 3600) + if (offset_is_whole_number_of_hours_everytime && start_of_day > 0 && start_of_day % 3600) offset_is_whole_number_of_hours_everytime = false; /// If UTC offset was changed this day. /// Change in time zone without transition is possible, e.g. Moscow 1991 Sun, 31 Mar, 02:00 MSK to EEST cctz::time_zone::civil_transition transition{}; - if (cctz_time_zone.next_transition(lookup.post, &transition) + if (cctz_time_zone.next_transition(start_of_day_time_point - std::chrono::seconds(1), &transition) && transition.from.year() == date.year() && transition.from.month() == date.month() && transition.from.day() == date.day() @@ -116,8 +120,8 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = (transition.from - cctz::civil_second(date)) / Values::OffsetChangeFactor; values.amount_of_offset_change_value = (transition.to - transition.from) / Values::OffsetChangeFactor; -// std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n"; -// std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n"; + std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n"; + std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n"; /// We don't support too large changes. if (values.amount_of_offset_change_value > 24 * 4) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 6e8b424ab9d..e558a3b45ce 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -472,18 +472,25 @@ public: } /// NOTE: Assuming timezone offset is a multiple of 15 minutes. - inline time_t toStartOfMinute(time_t t) const { return (t + DATE_LUT_ADD) / 60 * 60 - DATE_LUT_ADD; } - inline time_t toStartOfFiveMinute(time_t t) const { return (t + DATE_LUT_ADD) / 300 * 300 - DATE_LUT_ADD; } - inline time_t toStartOfFifteenMinutes(time_t t) const { return (t + DATE_LUT_ADD) / 900 * 900 - DATE_LUT_ADD; } + inline time_t toStartOfMinute(time_t t) const { return roundDown(t, 60); } + inline time_t toStartOfFiveMinute(time_t t) const { return roundDown(t, 300); } + inline time_t toStartOfFifteenMinutes(time_t t) const { return roundDown(t, 900); } - /// NOTE: This most likely wrong for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate. - inline time_t toStartOfTenMinutes(time_t t) const { return (t + DATE_LUT_ADD) / 600 * 600 - DATE_LUT_ADD; } + inline time_t toStartOfTenMinutes(time_t t) const + { + if (offset_is_whole_number_of_hours_everytime) + return roundDown(t, 600); + + /// More complex logic is for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate. + Int64 date = find(t).date; + return date + (t - date) / 600 * 600; + } /// NOTE: Assuming timezone transitions are multiple of hours. Lord Howe Island in Australia is a notable exception. inline time_t toStartOfHour(time_t t) const { if (offset_is_whole_number_of_hours_everytime) - return (t + DATE_LUT_ADD) / 3600 * 3600 - DATE_LUT_ADD; + return roundDown(t, 3600); Int64 date = find(t).date; return date + (t - date) / 3600 * 3600; From e33fae76256d3c0e01e7e14f5c94a4cdd763caca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 01:35:49 +0300 Subject: [PATCH 057/233] Improve test --- src/Common/tests/gtest_DateLUTImpl.cpp | 39 ++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 95ac055e237..80cb73f06c8 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -407,22 +407,49 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) /// Sun Nov 7 00:00:30 ADT 2010 /// Sat Nov 6 23:01:00 AST 2010 /// Sat Nov 6 23:01:30 AST 2010 + + bool has_transition = false; cctz::time_zone::civil_transition transition{}; if (tz.next_transition(std::chrono::system_clock::from_time_t(expected_time_t), &transition) - && transition.from.day() == tz_time.day() - && (transition.from.second() != 0 || transition.from.minute() % 900 != 0 - || (transition.from.day() != transition.to.day() - && (transition.from.hour() != 0 && transition.from.minute() != 0 && transition.from.second() != 0)))) + && (transition.from.day() == tz_time.day() || transition.to.day() == tz_time.day())) + { + has_transition = true; + } + + if (has_transition && (transition.from.second() != 0 || transition.from.minute() % 900 != 0)) { std::cerr << "Skipping " << timezone_name << " " << tz_time - << " because of unsupported timezone transition from " << transition.from << " to " << transition.to << "\n"; + << " because of unsupported timezone transition from " << transition.from << " to " << transition.to + << " (not divisable by 15 minutes)\n"; + continue; + } + + /// Transition to previous day, but not from midnight. + if (has_transition && cctz::civil_day(transition.from) == cctz::civil_day(transition.to) + 1 + && transition.from != cctz::civil_day(transition.from)) + { + std::cerr << "Skipping " << timezone_name << " " << tz_time + << " because of unsupported timezone transition from " << transition.from << " to " << transition.to + << " (to previous day but not at midnight)\n"; + continue; + } + + /// To large transition. + if (has_transition + && cctz::civil_day(transition.from) != cctz::civil_day(transition.to) + && cctz::civil_day(transition.from) != cctz::civil_day(transition.to) + 1) + { + std::cerr << "Skipping " << timezone_name << " " << tz_time + << " because of unsupported timezone transition from " << transition.from << " to " << transition.to + << " (it is too large)\n"; continue; } EXPECT_EQ(tz_time.year(), lut.toYear(expected_time_t)); EXPECT_EQ(tz_time.month(), lut.toMonth(expected_time_t)); EXPECT_EQ(tz_time.day(), lut.toDayOfMonth(expected_time_t)); - EXPECT_EQ(static_cast(cctz::get_weekday(tz_time)) + 1, lut.toDayOfWeek(expected_time_t)); // tm.tm_wday Sunday is 0, while for DateLUTImpl it is 7 + /// tm.tm_wday Sunday is 0, while for DateLUTImpl it is 7 + EXPECT_EQ(static_cast(cctz::get_weekday(tz_time)) + 1, lut.toDayOfWeek(expected_time_t)); EXPECT_EQ(cctz::get_yearday(tz_time), lut.toDayOfYear(expected_time_t)); EXPECT_EQ(tz_time.hour(), lut.toHour(expected_time_t)); EXPECT_EQ(tz_time.minute(), lut.toMinute(expected_time_t)); From d406999e8ff03b91ed5f4dbd4cf0077fe32fbbd1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 01:36:48 +0300 Subject: [PATCH 058/233] Remove debug output --- base/common/DateLUTImpl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index a111a21d8fd..9e7a7eab7d3 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -120,8 +120,8 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = (transition.from - cctz::civil_second(date)) / Values::OffsetChangeFactor; values.amount_of_offset_change_value = (transition.to - transition.from) / Values::OffsetChangeFactor; - std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n"; - std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n"; +// std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n"; +// std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n"; /// We don't support too large changes. if (values.amount_of_offset_change_value > 24 * 4) From 8271cec093a71a319aac3a584f6c1501d9d2bdbe Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 02:36:24 +0300 Subject: [PATCH 059/233] Fix error --- base/common/DateLUTImpl.cpp | 4 +--- src/Common/tests/gtest_DateLUTImpl.cpp | 7 +++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 9e7a7eab7d3..6097763fb49 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -112,9 +112,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) /// Change in time zone without transition is possible, e.g. Moscow 1991 Sun, 31 Mar, 02:00 MSK to EEST cctz::time_zone::civil_transition transition{}; if (cctz_time_zone.next_transition(start_of_day_time_point - std::chrono::seconds(1), &transition) - && transition.from.year() == date.year() - && transition.from.month() == date.month() - && transition.from.day() == date.day() + && (cctz::civil_day(transition.from) == date || cctz::civil_day(transition.to) == date) && transition.from != transition.to) { values.time_at_offset_change_value = (transition.from - cctz::civil_second(date)) / Values::OffsetChangeFactor; diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 80cb73f06c8..7d3d38df645 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -416,11 +416,11 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) has_transition = true; } - if (has_transition && (transition.from.second() != 0 || transition.from.minute() % 900 != 0)) + if (has_transition && (transition.from.second() != 0 || transition.from.minute() % 15 != 0)) { std::cerr << "Skipping " << timezone_name << " " << tz_time << " because of unsupported timezone transition from " << transition.from << " to " << transition.to - << " (not divisable by 15 minutes)\n"; + << " (not divisible by 15 minutes)\n"; continue; } @@ -436,8 +436,7 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) /// To large transition. if (has_transition - && cctz::civil_day(transition.from) != cctz::civil_day(transition.to) - && cctz::civil_day(transition.from) != cctz::civil_day(transition.to) + 1) + && std::abs(transition.from - transition.to) > 3600 * 3) { std::cerr << "Skipping " << timezone_name << " " << tz_time << " because of unsupported timezone transition from " << transition.from << " to " << transition.to From 1bb62f578b75541689eec40971b3a10d8630ed1a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 04:10:38 +0300 Subject: [PATCH 060/233] Unit tests passing --- base/common/DateLUTImpl.cpp | 6 ++-- base/common/DateLUTImpl.h | 31 ++++++++-------- src/Common/tests/gtest_DateLUTImpl.cpp | 50 ++++++++++++-------------- 3 files changed, 42 insertions(+), 45 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 6097763fb49..e7faeb63760 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -59,7 +59,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) offset_at_start_of_epoch = cctz_time_zone.lookup(cctz_time_zone.lookup(epoch).pre).offset; offset_at_start_of_lut = cctz_time_zone.lookup(cctz_time_zone.lookup(lut_start).pre).offset; - offset_is_whole_number_of_hours_everytime = true; + offset_is_whole_number_of_hours_during_epoch = true; cctz::civil_day date = lut_start; @@ -105,8 +105,8 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = 0; values.amount_of_offset_change_value = 0; - if (offset_is_whole_number_of_hours_everytime && start_of_day > 0 && start_of_day % 3600) - offset_is_whole_number_of_hours_everytime = false; + if (offset_is_whole_number_of_hours_during_epoch && start_of_day > 0 && start_of_day % 3600) + offset_is_whole_number_of_hours_during_epoch = false; /// If UTC offset was changed this day. /// Change in time zone without transition is possible, e.g. Moscow 1991 Sun, 31 Mar, 02:00 MSK to EEST diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index e558a3b45ce..16abd3dfb0e 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -185,7 +185,7 @@ private: time_t offset_at_start_of_epoch; /// UTC offset at the beginning of the first supported year. time_t offset_at_start_of_lut; - bool offset_is_whole_number_of_hours_everytime; + bool offset_is_whole_number_of_hours_during_epoch; /// Time zone name. std::string time_zone; @@ -193,15 +193,19 @@ private: inline LUTIndex findIndex(time_t t) const { /// First guess. - const UInt32 guess = ((t / 86400) + daynum_offset_epoch) & date_lut_mask; + UInt32 guess = ((t / 86400) + daynum_offset_epoch) & date_lut_mask; + + /// For negative time_t the integer division was rounded up, so the guess is offset by one. + if (unlikely(t < 0)) + --guess; /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. - if (t >= lut[guess].date && t < lut[UInt32(guess + 1)].date) + if (t >= lut[guess].date && t < lut[guess + 1].date) return LUTIndex(guess); /// Time zones that have offset 0 from UTC do daylight saving time change (if any) /// towards increasing UTC offset (example: British Standard Time). - if (t >= lut[UInt32(guess + 1)].date) + if (t >= lut[guess + 1].date) return LUTIndex(guess + 1); return LUTIndex(guess - 1); @@ -253,7 +257,6 @@ public: // Methods only for unit-testing, it makes very little sense to use it from user code. auto getOffsetAtStartOfEpoch() const { return offset_at_start_of_epoch; } - auto getOffsetIsWholNumberOfHoursEveryWhere() const { return offset_is_whole_number_of_hours_everytime; } auto getTimeOffsetAtStartOfLUT() const { return offset_at_start_of_lut; } /// All functions below are thread-safe; arguments are not checked. @@ -456,8 +459,8 @@ public: inline unsigned toMinute(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) - return ((t + DATE_LUT_ADD) / 60) % 60; + if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) + return (t / 60) % 60; /// To consider the DST changing situation within this day /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account. @@ -478,8 +481,8 @@ public: inline time_t toStartOfTenMinutes(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) - return roundDown(t, 600); + if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) + return t / 600 * 600; /// More complex logic is for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate. Int64 date = find(t).date; @@ -489,8 +492,8 @@ public: /// NOTE: Assuming timezone transitions are multiple of hours. Lord Howe Island in Australia is a notable exception. inline time_t toStartOfHour(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) - return roundDown(t, 3600); + if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) + return t / 3600 * 3600; Int64 date = find(t).date; return date + (t - date) / 3600 * 3600; @@ -773,8 +776,8 @@ public: /// We count all hour-length intervals, unrelated to offset changes. inline time_t toRelativeHourNum(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) - return (t + DATE_LUT_ADD) / 3600 - (DATE_LUT_ADD / 3600); + if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) + return t / 3600; /// Assume that if offset was fractional, then the fraction is the same as at the beginning of epoch. /// NOTE This assumption is false for "Pacific/Pitcairn" and "Pacific/Kiritimati" time zones. @@ -848,7 +851,7 @@ public: t = roundDown(t, seconds); - if (offset_is_whole_number_of_hours_everytime) + if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) return t; /// TODO check if it's correct. diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 7d3d38df645..e18bd16e731 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -297,7 +297,6 @@ TEST_P(DateLUTWithTimeZone, VaidateTimeComponentsAroundEpoch) << "\n\tTimezone: " << timezone_name << "\n\ttimestamp: " << i << "\n\t offset at start of epoch : " << lut.getOffsetAtStartOfEpoch() - << "\n\t offset_is_whole_number_of_hours_everytime : " << lut.getOffsetIsWholNumberOfHoursEveryWhere() << "\n\t offset_at_start_of_lut : " << lut.getTimeOffsetAtStartOfLUT()); EXPECT_GE(24, lut.toHour(i)); @@ -336,7 +335,7 @@ INSTANTIATE_TEST_SUITE_P(ExoticTimezones, }) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimeZones, +INSTANTIATE_TEST_SUITE_P(AllTimeZones, DateLUTWithTimeZone, ::testing::ValuesIn(allTimezones()) ); @@ -391,11 +390,15 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) { SCOPED_TRACE(expected_time_t); - const auto tz_time = cctz::convert(std::chrono::system_clock::from_time_t(expected_time_t), tz); + const cctz::civil_second tz_time = cctz::convert(std::chrono::system_clock::from_time_t(expected_time_t), tz); /// Weird offset, not supported. /// Example: Africa/Monrovia has offset UTC-0:44:30 in year 1970. - if (tz.lookup(std::chrono::system_clock::from_time_t(expected_time_t)).offset % 900) + + auto timestamp_current_day_pre = std::chrono::system_clock::to_time_t(tz.lookup(cctz::civil_day(tz_time)).pre); + auto timestamp_current_day_post = std::chrono::system_clock::to_time_t(tz.lookup(cctz::civil_day(tz_time) + 1).post); + + if (timestamp_current_day_pre % 900 || timestamp_current_day_post % 900) continue; /// Unsupported timezone transitions - not in 15-minute time point or to different day. @@ -410,7 +413,7 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) bool has_transition = false; cctz::time_zone::civil_transition transition{}; - if (tz.next_transition(std::chrono::system_clock::from_time_t(expected_time_t), &transition) + if (tz.next_transition(std::chrono::system_clock::from_time_t(expected_time_t - 1), &transition) && (transition.from.day() == tz_time.day() || transition.to.day() == tz_time.day())) { has_transition = true; @@ -418,9 +421,9 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) if (has_transition && (transition.from.second() != 0 || transition.from.minute() % 15 != 0)) { - std::cerr << "Skipping " << timezone_name << " " << tz_time + /*std::cerr << "Skipping " << timezone_name << " " << tz_time << " because of unsupported timezone transition from " << transition.from << " to " << transition.to - << " (not divisible by 15 minutes)\n"; + << " (not divisible by 15 minutes)\n";*/ continue; } @@ -428,9 +431,9 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) if (has_transition && cctz::civil_day(transition.from) == cctz::civil_day(transition.to) + 1 && transition.from != cctz::civil_day(transition.from)) { - std::cerr << "Skipping " << timezone_name << " " << tz_time + /*std::cerr << "Skipping " << timezone_name << " " << tz_time << " because of unsupported timezone transition from " << transition.from << " to " << transition.to - << " (to previous day but not at midnight)\n"; + << " (to previous day but not at midnight)\n";*/ continue; } @@ -438,9 +441,9 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) if (has_transition && std::abs(transition.from - transition.to) > 3600 * 3) { - std::cerr << "Skipping " << timezone_name << " " << tz_time + /*std::cerr << "Skipping " << timezone_name << " " << tz_time << " because of unsupported timezone transition from " << transition.from << " to " << transition.to - << " (it is too large)\n"; + << " (it is too large)\n";*/ continue; } @@ -457,23 +460,14 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) const auto time_string = cctz::format("%E4Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(expected_time_t), tz); EXPECT_EQ(time_string, lut.timeToString(expected_time_t)); - // it makes sense to let test execute all checks above to simplify debugging, - // but once we've found a bad apple, no need to dig deeper. + /// It makes sense to let test execute all checks above to simplify debugging, + /// but once we've found a bad apple, no need to dig deeper. if (countFailures(*test_info->result()).total >= max_failures_per_case) break; } } -/** Next tests are disabled due to following reasons: - * 1. They are huge and take enormous amount of time to run - * 2. Current implementation of DateLUTImpl is inprecise and some cases fail and it seems impractical to try to fix those. - * 3. Many failures (~300) were fixed while refactoring, about ~40 remain the same and 3 new introduced: - * "Asia/Gaza" - * "Pacific/Enderbury" - * "Pacific/Kiritimati" - * So it would be tricky to skip knonw failures to allow all unit tests to pass. - */ -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, +INSTANTIATE_TEST_SUITE_P(AllTimezones_Year2010, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), @@ -484,7 +478,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, })) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, +INSTANTIATE_TEST_SUITE_P(AllTimezones_Year1970_WHOLE, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones(false)), @@ -494,7 +488,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, })) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, +INSTANTIATE_TEST_SUITE_P(AllTimezones_Year2010_WHOLE, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones(false)), @@ -504,7 +498,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, })) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, +INSTANTIATE_TEST_SUITE_P(AllTimezones_Year2020_WHOLE, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), @@ -514,7 +508,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, })) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, +INSTANTIATE_TEST_SUITE_P(AllTimezones_PreEpoch, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones(false)), @@ -524,7 +518,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, })) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970, +INSTANTIATE_TEST_SUITE_P(AllTimezones_Year1970, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones(false)), From 7052ecd446ef637d03cba1ac618a6a85d642dcb3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 09:30:52 +0300 Subject: [PATCH 061/233] Fix error --- base/common/DateLUTImpl.h | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 16abd3dfb0e..b75274e8569 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -442,19 +442,12 @@ public: } - /** Only for time zones with/when offset from UTC is multiple of five minutes. - * This is true for all time zones: right now, all time zones have an offset that is multiple of 15 minutes. - * - * "By 1929, most major countries had adopted hourly time zones. Nepal was the last - * country to adopt a standard offset, shifting slightly to UTC+5:45 in 1986." - * - https://en.wikipedia.org/wiki/Time_zone#Offsets_from_UTC - * - * Also please note, that unix timestamp doesn't count "leap seconds": - * each minute, with added or subtracted leap second, spans exactly 60 unix timestamps. - */ inline unsigned toSecond(time_t t) const { - return (t + DATE_LUT_ADD) % 60; + auto res = t % 60; + if (likely(res >= 0)) + return res; + return res + 60; } inline unsigned toMinute(time_t t) const From 5cf42e87be76f0f30771f63a155986077168eb1d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 09:31:01 +0300 Subject: [PATCH 062/233] Update test --- .../01702_toDateTime_from_string_clamping.reference | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference index 92639948fbc..644de54a6a2 100644 --- a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference +++ b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference @@ -2,8 +2,8 @@ SELECT toString(toDateTime('-922337203.6854775808', 1)); 1940-10-09 22:13:17.6 SELECT toString(toDateTime('9922337203.6854775808', 1)); -1925-07-26 00:46:43.6 +1925-07-26 23:46:43.6 SELECT toDateTime64(CAST('10000000000.1' AS Decimal64(1)), 1); -1928-01-11 00:46:40.1 +1928-01-11 23:46:40.1 SELECT toDateTime64(CAST('-10000000000.1' AS Decimal64(1)), 1); -2011-12-22 00:13:20.1 +2011-12-22 23:38:20.1 From d3b422a3360a8fa768bf910e74c8303486ac4858 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 10:10:15 +0300 Subject: [PATCH 063/233] Update test --- .../0_stateless/01699_timezoneOffset.reference | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/01699_timezoneOffset.reference b/tests/queries/0_stateless/01699_timezoneOffset.reference index 45f30314f5a..a1cc6391e6f 100644 --- a/tests/queries/0_stateless/01699_timezoneOffset.reference +++ b/tests/queries/0_stateless/01699_timezoneOffset.reference @@ -1,8 +1,8 @@ DST boundary test for Europe/Moscow: -0 1981-04-01 22:40:00 10800 355002000 -1 1981-04-01 22:50:00 10800 355002600 -2 1981-04-02 00:00:00 14400 355003200 -3 1981-04-02 00:10:00 14400 355003800 +0 1981-04-01 22:40:00 14400 354998400 +1 1981-04-01 22:50:00 14400 354999000 +2 1981-04-01 23:00:00 14400 354999600 +3 1981-04-01 23:10:00 14400 355000200 0 1981-09-30 23:00:00 14400 370724400 1 1981-09-30 23:10:00 14400 370725000 2 1981-09-30 23:20:00 14400 370725600 @@ -22,10 +22,10 @@ DST boundary test for Europe/Moscow: 16 1981-10-01 00:40:00 10800 370734000 17 1981-10-01 00:50:00 10800 370734600 DST boundary test for Asia/Tehran: -0 2020-03-21 22:40:00 12600 1584817800 -1 2020-03-21 22:50:00 12600 1584818400 -2 2020-03-22 00:00:00 16200 1584819000 -3 2020-03-22 00:10:00 16200 1584819600 +0 2020-03-21 22:40:00 16200 1584814200 +1 2020-03-21 22:50:00 16200 1584814800 +2 2020-03-21 23:00:00 16200 1584815400 +3 2020-03-21 23:10:00 16200 1584816000 0 2020-09-20 23:00:00 16200 1600626600 1 2020-09-20 23:10:00 16200 1600627200 2 2020-09-20 23:20:00 16200 1600627800 From 1200d9d9c5447a900a692f170af894f02d43147d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 10:10:37 +0300 Subject: [PATCH 064/233] Range checks and monotonicity --- base/common/DateLUTImpl.h | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index b75274e8569..6e968a0cd50 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -193,22 +193,28 @@ private: inline LUTIndex findIndex(time_t t) const { /// First guess. - UInt32 guess = ((t / 86400) + daynum_offset_epoch) & date_lut_mask; + Int64 guess = (t / 86400) + daynum_offset_epoch; /// For negative time_t the integer division was rounded up, so the guess is offset by one. if (unlikely(t < 0)) --guess; + if (guess < 0) + return LUTIndex(0); + if (guess >= DATE_LUT_SIZE) + return LUTIndex(DATE_LUT_SIZE - 1); + /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. - if (t >= lut[guess].date && t < lut[guess + 1].date) - return LUTIndex(guess); - /// Time zones that have offset 0 from UTC do daylight saving time change (if any) - /// towards increasing UTC offset (example: British Standard Time). - if (t >= lut[guess + 1].date) + if (t >= lut[guess].date) + { + if (guess + 1 >= DATE_LUT_SIZE || t < lut[guess + 1].date) + return LUTIndex(guess); + return LUTIndex(guess + 1); + } - return LUTIndex(guess - 1); + return LUTIndex(guess ? guess - 1 : 0); } inline LUTIndex toLUTIndex(DayNum d) const From 019ed517bc33892d107480ee30ddf53de8fd4fee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 03:05:06 +0300 Subject: [PATCH 065/233] Fix gcc warning --- src/Dictionaries/RangeHashedDictionary.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 9fb1a57a381..aec641c58fe 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -491,8 +491,9 @@ void RangeHashedDictionary::getIdsAndDates( start_dates.push_back(value.range.left); end_dates.push_back(value.range.right); - if (is_date && static_cast(end_dates.back()) > DATE_LUT_MAX_DAY_NUM) - end_dates.back() = 0; + if constexpr (std::numeric_limits::max() > DATE_LUT_MAX_DAY_NUM) /// Avoid warning about tautological comparison in next line. + if (is_date && static_cast(end_dates.back()) > DATE_LUT_MAX_DAY_NUM) + end_dates.back() = 0; } } } From 6b309dcc5c7da70bc9896ff3047afa1f7aff6955 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 03:14:39 +0300 Subject: [PATCH 066/233] Update tests --- tests/queries/0_stateless/00189_time_zones.reference | 7 ++++--- tests/queries/0_stateless/00189_time_zones.sql | 5 +++-- .../queries/0_stateless/01691_DateTime64_clamp.reference | 8 ++++---- .../01702_toDateTime_from_string_clamping.reference | 6 +++--- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/queries/0_stateless/00189_time_zones.reference b/tests/queries/0_stateless/00189_time_zones.reference index 664c30056de..e7e5a71782a 100644 --- a/tests/queries/0_stateless/00189_time_zones.reference +++ b/tests/queries/0_stateless/00189_time_zones.reference @@ -179,13 +179,13 @@ toRelativeYearNum 44 44 44 -44 +45 toRelativeMonthNum 536 536 536 537 -536 +537 toRelativeWeekNum 2335 2335 @@ -197,12 +197,13 @@ toRelativeDayNum 16343 16343 16344 -16343 +16344 toRelativeHourNum 392251 392251 392251 392251 +392252 toRelativeMinuteNum 23535110 23535110 diff --git a/tests/queries/0_stateless/00189_time_zones.sql b/tests/queries/0_stateless/00189_time_zones.sql index a0ef5b59517..36c7dfb402a 100644 --- a/tests/queries/0_stateless/00189_time_zones.sql +++ b/tests/queries/0_stateless/00189_time_zones.sql @@ -277,7 +277,8 @@ SELECT toRelativeDayNum(toDateTime(1412106600), 'Europe/Moscow') - toRelativeDay SELECT toRelativeDayNum(toDateTime(1412106600), 'Europe/Paris') - toRelativeDayNum(toDateTime(0), 'Europe/Paris'); SELECT toRelativeDayNum(toDateTime(1412106600), 'Europe/London') - toRelativeDayNum(toDateTime(0), 'Europe/London'); SELECT toRelativeDayNum(toDateTime(1412106600), 'Asia/Tokyo') - toRelativeDayNum(toDateTime(0), 'Asia/Tokyo'); -SELECT toRelativeDayNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelativeDayNum(toDateTime(0), 'Pacific/Pitcairn'); +-- NOTE: toRelativeDayNum(toDateTime(0), 'Pacific/Pitcairn') overflows from -1 to 65535 +SELECT toUInt16(toRelativeDayNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelativeDayNum(toDateTime(0), 'Pacific/Pitcairn')); /* toRelativeHourNum */ @@ -286,7 +287,7 @@ SELECT toRelativeHourNum(toDateTime(1412106600), 'Europe/Moscow') - toRelativeHo SELECT toRelativeHourNum(toDateTime(1412106600), 'Europe/Paris') - toRelativeHourNum(toDateTime(0), 'Europe/Paris'); SELECT toRelativeHourNum(toDateTime(1412106600), 'Europe/London') - toRelativeHourNum(toDateTime(0), 'Europe/London'); SELECT toRelativeHourNum(toDateTime(1412106600), 'Asia/Tokyo') - toRelativeHourNum(toDateTime(0), 'Asia/Tokyo'); --- known wrong result: SELECT toRelativeHourNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelativeHourNum(toDateTime(0), 'Pacific/Pitcairn'); +SELECT toRelativeHourNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelativeHourNum(toDateTime(0), 'Pacific/Pitcairn'); /* toRelativeMinuteNum */ diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.reference b/tests/queries/0_stateless/01691_DateTime64_clamp.reference index f29a9e2d1d5..881ab4feff8 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.reference +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.reference @@ -17,11 +17,11 @@ SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') 2106-02-07 09:28:16.00 SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; -- These are outsize of extended range and hence clamped -SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1), 35), 2); 1925-01-01 02:00:00.00 -SELECT CAST(-1 * bitShiftLeft(toUInt64(1),35) AS DateTime64); +SELECT CAST(-1 * bitShiftLeft(toUInt64(1), 35) AS DateTime64); 1925-01-01 02:00:00.000 -SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); +SELECT CAST(bitShiftLeft(toUInt64(1), 35) AS DateTime64); 2282-12-31 03:00:00.000 -SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); +SELECT toDateTime64(bitShiftLeft(toUInt64(1), 35), 2); 2282-12-31 03:00:00.00 diff --git a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference index 644de54a6a2..77da114be68 100644 --- a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference +++ b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference @@ -2,8 +2,8 @@ SELECT toString(toDateTime('-922337203.6854775808', 1)); 1940-10-09 22:13:17.6 SELECT toString(toDateTime('9922337203.6854775808', 1)); -1925-07-26 23:46:43.6 +2283-11-11 23:46:43.6 SELECT toDateTime64(CAST('10000000000.1' AS Decimal64(1)), 1); -1928-01-11 23:46:40.1 +2283-11-11 23:46:40.1 SELECT toDateTime64(CAST('-10000000000.1' AS Decimal64(1)), 1); -2011-12-22 23:38:20.1 +1925-01-01 23:09:20.1 From 460658aeabb8d9276bd3c5f620d09b4ca1fc601c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 03:18:55 +0300 Subject: [PATCH 067/233] Update tests --- src/Storages/StorageGenerateRandom.cpp | 2 +- .../01087_table_function_generate.reference | 60 +++++++++---------- .../01125_generate_random_qoega.reference | 2 +- .../01128_generate_random_nested.reference | 4 +- 4 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/Storages/StorageGenerateRandom.cpp b/src/Storages/StorageGenerateRandom.cpp index fd10691ecc4..f06daa3a2bd 100644 --- a/src/Storages/StorageGenerateRandom.cpp +++ b/src/Storages/StorageGenerateRandom.cpp @@ -215,7 +215,7 @@ ColumnPtr fillColumnWithRandomData( column->getData().resize(limit); for (size_t i = 0; i < limit; ++i) - column->getData()[i] = rng() % (DATE_LUT_MAX_DAY_NUM + 1); /// Slow + column->getData()[i] = rng() % (DATE_LUT_MAX_DAY_NUM + 1); return column; } diff --git a/tests/queries/0_stateless/01087_table_function_generate.reference b/tests/queries/0_stateless/01087_table_function_generate.reference index d7cc6b0a933..d8886945caa 100644 --- a/tests/queries/0_stateless/01087_table_function_generate.reference +++ b/tests/queries/0_stateless/01087_table_function_generate.reference @@ -1,14 +1,14 @@ UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 -2804162938822577320 -2776833771540858 3467776823 1163715250 31161 -2916 220 -117 -7885388429666205427 -1363628932535403038 484159052 -308788249 43346 13638 143 -105 -4357435422797280898 1355609803008819271 4126129912 -852056475 34184 9166 49 33 -5935810273536892891 -804738887697332962 3109335413 -80126721 47877 -31421 186 -77 -368066018677693974 -4927165984347126295 1015254922 2026080544 46037 -29626 240 108 -8124171311239967992 -1179703908046100129 1720727300 -138469036 33028 -12819 138 16 -15657812979985370729 -5733276247123822513 3254757884 -500590428 3829 30527 3 -81 -18371568619324220532 -6793779541583578394 1686821450 -455892108 43475 2284 252 -90 -821735343441964030 3148260644406230976 256251035 -885069056 11643 11455 176 90 -9558594037060121162 -2907172753635797124 4276198376 1947296644 45922 26632 97 43 +2804162938822577320 -2776833771540858 3467776823 1163715250 23903 -2916 220 -117 +7885388429666205427 -1363628932535403038 484159052 -308788249 44305 13638 143 -105 +4357435422797280898 1355609803008819271 4126129912 -852056475 58858 9166 49 33 +5935810273536892891 -804738887697332962 3109335413 -80126721 13655 -31421 186 -77 +368066018677693974 -4927165984347126295 1015254922 2026080544 21973 -29626 240 108 +8124171311239967992 -1179703908046100129 1720727300 -138469036 36175 -12819 138 16 +15657812979985370729 -5733276247123822513 3254757884 -500590428 13193 30527 3 -81 +18371568619324220532 -6793779541583578394 1686821450 -455892108 52282 2284 252 -90 +821735343441964030 3148260644406230976 256251035 -885069056 55255 11455 176 90 +9558594037060121162 -2907172753635797124 4276198376 1947296644 48701 26632 97 43 - Enum8(\'hello\' = 1, \'world\' = 5) hello @@ -47,16 +47,16 @@ h o - Date DateTime DateTime(\'Europe/Moscow\') -2077-09-17 1970-10-09 02:30:14 2074-08-12 11:31:27 -2005-11-19 2106-01-30 21:52:44 2097-05-25 07:54:35 -2007-02-24 2096-12-12 00:40:50 1988-08-10 11:16:31 -2019-06-30 2096-01-15 16:31:33 2063-10-20 08:48:17 -2039-01-16 2103-02-11 16:44:39 2036-10-09 04:29:10 -1994-11-03 1980-01-02 05:18:22 2055-12-23 12:33:52 -2083-08-20 2079-06-11 16:29:02 2000-12-05 17:46:24 -2030-06-25 2100-03-01 18:50:22 1993-03-25 01:19:12 -2087-03-16 2034-08-25 19:46:33 2045-12-10 16:47:40 -2006-04-30 2069-09-30 16:07:48 2084-08-26 03:33:12 +2113-06-12 1970-10-09 02:30:14 2074-08-12 11:31:27 +2103-11-03 2106-01-30 21:52:44 2097-05-25 07:54:35 +2008-03-16 2096-12-12 00:40:50 1988-08-10 11:16:31 +2126-11-26 2096-01-15 16:31:33 2063-10-20 08:48:17 +1991-02-02 2103-02-11 16:44:39 2036-10-09 04:29:10 +2096-11-03 1980-01-02 05:18:22 2055-12-23 12:33:52 +2024-12-16 2079-06-11 16:29:02 2000-12-05 17:46:24 +2085-04-07 2100-03-01 18:50:22 1993-03-25 01:19:12 +2135-05-30 2034-08-25 19:46:33 2045-12-10 16:47:40 +2094-12-18 2069-09-30 16:07:48 2084-08-26 03:33:12 - DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') 1978-06-07 23:50:57.320 2013-08-28 10:21:54.010758 1991-08-25 16:23:26.140215 @@ -225,14 +225,14 @@ RL,{Xs\\tw [114] -84125.1554 ('2023-06-06 06:55:06.492','bf9ab359-ef9f-ad11-7e6c-160368b1e5ea') [124] -114719.5228 ('2010-11-11 22:57:23.722','c1046ffb-3415-cc3a-509a-e0005856d7d7') - -[] 1900051923 { -189530.5846 h -5.6279699579452485e47 ('1980-08-29','2090-10-31 19:35:45','2038-07-15 05:22:51.805','63d9a12d-d1cf-1f3a-57c6-9bc6dddd0975') 8502 -[-102,-118] 392272782 Eb -14818.0200 o -2.664492247169164e59 ('2059-02-10','1994-07-16 00:40:02','2034-02-02 05:30:44.960','4fa09948-d32e-8903-63df-43ad759e43f7') DA61 -[-71] 775049089 \N -158115.1178 w 4.1323844687113747e-305 ('1997-02-15','2062-08-12 23:41:53','2074-02-13 10:29:40.749','c4a44dd7-d009-6f65-1494-9daedfa8a124') 83A7 -[-28,100] 3675466147 { -146685.1749 h 3.6676044396877755e142 ('1997-10-26','2002-06-26 03:33:41','2002-12-02 05:46:03.455','98714b2c-65e7-b5cb-a040-421e260c6d8d') 4B94 -[-23] 2514120753 (`u, -119659.6174 w 1.3231258347475906e34 ('2055-11-20','2080-03-28 08:11:25','2073-07-10 12:19:58.146','003b3b6b-088f-f941-aeb9-c26e0ee72b8e') 6B1F -[11,-36] 3308237300 \N 171205.1896 \N 5.634708707075817e195 ('2009-03-18','2041-11-11 13:19:44','2044-03-18 17:34:17.814','9e60f4cb-6e55-1deb-5ac4-d66a86a8886d') 1964 -[39] 1614362420 `4A8P 157144.0630 o -1.1843143253872814e-255 ('1991-04-27','2066-03-02 11:07:49','1997-10-22 20:14:13.755','97685503-2609-d2b9-981c-02fd75d106cb') A35B -[48,-120] 3848918261 1 Date: Mon, 8 Mar 2021 03:28:25 +0300 Subject: [PATCH 068/233] Try to enable long performance test --- tests/performance/date_time_long.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/performance/date_time_long.xml b/tests/performance/date_time_long.xml index 1229631a434..3a61a5992e5 100644 --- a/tests/performance/date_time_long.xml +++ b/tests/performance/date_time_long.xml @@ -1,5 +1,4 @@ - long datetime_transform From 095ee81a4e8769daf0805418150c61b22ad262e7 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 8 Mar 2021 19:40:05 +0300 Subject: [PATCH 069/233] Updated description --- docs/en/sql-reference/statements/detach.md | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 0bd4f730364..19ff8f10ad3 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -15,7 +15,7 @@ DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] Detaching does not delete the data or metadata for the table or view. If the table or view was not detached `PERMANENTLY`, on the next server launch the server will read the metadata and recall the table/view again. If the table or view was detached `PERMANENTLY`, there will be no automatic recall. -Whether the table was detached permanently or not, in both cases you can reattach it using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). +Whether the table was detached permanently or not, in both cases you can reattach it using the [ATTACH](../../sql-reference/statements/attach.md). System log tables can be also attached back (e.g. `query_log`, `text_log`, etc). Other system tables can't be reattached. On the next server launch the server will recall those tables again. `ATTACH MATERIALIZED VIEW` doesn't work with short syntax (without `SELECT`), but you can attach it using the `ATTACH TABLE` query. @@ -23,20 +23,4 @@ Note that you can not detach permanently the table which is already detached (te Also you can not [DROP](../../sql-reference/statements/drop.md#drop-table) the detached table, or [CREATE TABLE](../../sql-reference/statements/create/table.md) with the same name as detached permanently, or replace it with the other table with [RENAME TABLE](../../sql-reference/statements/rename.md) query. -Similarly, a “detached” table can be re-attached using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). - -## DETACH PERMANENTLY {#detach-permanently} - -Deletes information about `name` table or view from the server. Permanently detached tables won't automatically reappear after the server restart. - -Syntax: - -``` sql -DETACH TABLE/VIEW [IF EXISTS] [db.]name PERMAMENTLY [ON CLUSTER cluster] -``` - -This statement does not delete the table’s data or metadata. - -Permanently detached table or view can be reattached with [ATTACH](../../sql-reference/statements/attach.md) query and can be shown with [SHOW CREATE TABLE](../../sql-reference/statements/show.md#show-create-table) query. - [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From 61d40c3600ba6a1c6d6c0cf4919a3cdaebb3a31f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 9 Mar 2021 22:00:38 +0300 Subject: [PATCH 070/233] Fix optimize_skip_unused_shards for zero shards case v2: move check to the beginning of the StorageDistributed::read() --- src/Storages/StorageDistributed.cpp | 19 +++++++++++++++++++ ...e_skip_unused_shards_zero_shards.reference | 0 ...ptimize_skip_unused_shards_zero_shards.sql | 2 ++ .../queries/0_stateless/arcadia_skip_list.txt | 1 + 4 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.reference create mode 100644 tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 039cf63eca2..3a33e881611 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -48,6 +48,9 @@ #include #include +#include +#include + #include #include @@ -83,6 +86,7 @@ namespace DB namespace ErrorCodes { + extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; extern const int STORAGE_REQUIRES_PARAMETER; extern const int BAD_ARGUMENTS; @@ -532,6 +536,17 @@ void StorageDistributed::read( Block header = InterpreterSelectQuery(query_info.query, context, SelectQueryOptions(processed_stage)).getSampleBlock(); + /// Return directly (with correct header) if no shard to query. + if (query_info.cluster->getShardsInfo().empty()) + { + Pipe pipe(std::make_shared(header)); + auto read_from_pipe = std::make_unique(std::move(pipe)); + read_from_pipe->setStepDescription("Read from NullSource (Distributed)"); + query_plan.addStep(std::move(read_from_pipe)); + + return; + } + const Scalars & scalars = context.hasQueryContext() ? context.getQueryContext().getScalars() : Scalars{}; bool has_virtual_shard_num_column = std::find(column_names.begin(), column_names.end(), "_shard_num") != column_names.end(); @@ -546,6 +561,10 @@ void StorageDistributed::read( ClusterProxy::executeQuery(query_plan, select_stream_factory, log, modified_query_ast, context, query_info); + + /// This is a bug, it is possible only when there is no shards to query, and this is handled earlier. + if (!query_plan.isInitialized()) + throw Exception("Pipeline is not initialized", ErrorCodes::LOGICAL_ERROR); } diff --git a/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.reference b/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql b/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql new file mode 100644 index 00000000000..b95d640ca1a --- /dev/null +++ b/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql @@ -0,0 +1,2 @@ +create table dist_01756 (dummy UInt8) ENGINE = Distributed('test_cluster_two_shards', 'system', 'one', dummy); +select ignore(1), * from dist_01756 where 0 settings optimize_skip_unused_shards=1, force_optimize_skip_unused_shards=1 diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 6926f16e027..a85ddba4752 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -218,3 +218,4 @@ 01682_cache_dictionary_complex_key 01684_ssd_cache_dictionary_simple_key 01685_ssd_cache_dictionary_complex_key +01759_optimize_skip_unused_shards_zero_shards From 592dbb8a512622136aa90049e7d1543c80fb76bc Mon Sep 17 00:00:00 2001 From: George Date: Wed, 10 Mar 2021 12:56:13 +0300 Subject: [PATCH 071/233] Added example --- docs/en/sql-reference/statements/detach.md | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 19ff8f10ad3..f3e2f88ffc3 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -23,4 +23,27 @@ Note that you can not detach permanently the table which is already detached (te Also you can not [DROP](../../sql-reference/statements/drop.md#drop-table) the detached table, or [CREATE TABLE](../../sql-reference/statements/create/table.md) with the same name as detached permanently, or replace it with the other table with [RENAME TABLE](../../sql-reference/statements/rename.md) query. +**Example** + +Query: + +``` sql +CREATE TABLE test ENGINE = Log AS SELECT * FROM numbers(10); + +DETACH TABLE test; + +SELECT * FROM TEST; +``` + +Result: + +``` text +Ok. + +Ok. + +Received exception from server (version 21.3.1): +Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.TEST doesn't exist. +``` + [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From a8d78f2916cfe6f8ffa7ddaac0df677e4b3c817a Mon Sep 17 00:00:00 2001 From: George Date: Wed, 10 Mar 2021 13:53:42 +0300 Subject: [PATCH 072/233] translated column.md --- .../en/sql-reference/statements/alter/column.md | 2 +- .../ru/sql-reference/statements/alter/column.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index 16aa266ebf9..030f1dd92dd 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -191,7 +191,7 @@ Renames an existing column. Syntax: ```sql -ALTER TABLE table_name RENAME COLUMN column_name TO new_column_name; +ALTER TABLE table_name RENAME COLUMN column_name TO new_column_name ``` **Example** diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index 7a394e2f684..850b415e268 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -13,6 +13,7 @@ toc_title: "\u041c\u0430\u043d\u0438\u043f\u0443\u043b\u044f\u0446\u0438\u0438\u - [COMMENT COLUMN](#alter_comment-column) — добавляет комментарий к столбцу; - [MODIFY COLUMN](#alter_modify-column) — изменяет тип столбца, выражение для значения по умолчанию и TTL. - [MODIFY COLUMN REMOVE](#modify-remove) — удаляет какое-либо из свойств столбца. +- [RENAME COLUMN](#alter_rename-column) — переименовывает существующий столбец. Подробное описание для каждого действия приведено ниже. @@ -158,6 +159,22 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; - [REMOVE TTL](ttl.md). +## RENAME COLUMN {#alter_rename-column} + +Переименовывает существующий столбец. + +Синтаксис: + +```sql +ALTER TABLE table_name RENAME COLUMN column_name TO new_column_name +``` + +**Пример** + +```sql +ALTER TABLE table_with_ttl RENAME COLUMN column_ttl TO column_ttl_new; +``` + ## Ограничения запроса ALTER {#ogranicheniia-zaprosa-alter} Запрос `ALTER` позволяет создавать и удалять отдельные элементы (столбцы) вложенных структур данных, но не вложенные структуры данных целиком. Для добавления вложенной структуры данных, вы можете добавить столбцы с именем вида `name.nested_name` и типом `Array(T)` - вложенная структура данных полностью эквивалентна нескольким столбцам-массивам с именем, имеющим одинаковый префикс до точки. From 2da04f872cfe9e08761137199d0ee2feceb1fcc3 Mon Sep 17 00:00:00 2001 From: George Date: Wed, 10 Mar 2021 14:27:40 +0300 Subject: [PATCH 073/233] Updated and translated Index.md --- .../operations/external-authenticators/index.md | 2 ++ .../operations/external-authenticators/index.md | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 docs/ru/operations/external-authenticators/index.md diff --git a/docs/en/operations/external-authenticators/index.md b/docs/en/operations/external-authenticators/index.md index 95f80f192f5..fe4e6a42974 100644 --- a/docs/en/operations/external-authenticators/index.md +++ b/docs/en/operations/external-authenticators/index.md @@ -11,3 +11,5 @@ ClickHouse supports authenticating and managing users using external services. The following external authenticators and directories are supported: - [LDAP](./ldap.md#external-authenticators-ldap) [Authenticator](./ldap.md#ldap-external-authenticator) and [Directory](./ldap.md#ldap-external-user-directory) + +[Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/index.md) diff --git a/docs/ru/operations/external-authenticators/index.md b/docs/ru/operations/external-authenticators/index.md new file mode 100644 index 00000000000..db5c89a3d66 --- /dev/null +++ b/docs/ru/operations/external-authenticators/index.md @@ -0,0 +1,15 @@ +--- +toc_folder_title: \u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0430\u0443\u0442\u0435\u043d\u0442\u0438\u0444\u0438\u043a\u0430\u0442\u043e\u0440\u044b\u0020\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u0435\u0439\u0020\u0438\u0020\u043a\u0430\u0442\u0430\u043b\u043e\u0433\u0438 +toc_priority: 48 +toc_title: \u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435 +--- + +# Внешние аутентификаторы пользователей и каталоги {#external-authenticators} + +ClickHouse поддерживает аунтетификацию и управление пользователями внешними сервисами. + +Поддерживаются следующие внешние аутентификаторы и каталоги: + +- [LDAP](./ldap.md#external-authenticators-ldap) [Authenticator](./ldap.md#ldap-external-authenticator) и [Directory](./ldap.md#ldap-external-user-directory) + +[Original article](https://clickhouse.tech/docs/ru/operations/external-authenticators/index.md) From 1194d50e48b4d57956967b16c6bb37274c0ffb56 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 11 Mar 2021 15:16:52 +0300 Subject: [PATCH 074/233] Try to fix test_storage_s3: crash in WriteBufferFromS3 --- src/IO/WriteBufferFromS3.cpp | 9 ++++++++- src/Storages/StorageS3.cpp | 5 +++++ tests/integration/test_storage_s3/test.py | 4 ++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index a6ec60b295f..5edf01a940e 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -107,7 +107,14 @@ void WriteBufferFromS3::finalizeImpl() WriteBufferFromS3::~WriteBufferFromS3() { - finalizeImpl(); + try + { + finalizeImpl(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } } void WriteBufferFromS3::createMultipartUpload() diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index a31a7fa0944..e50eb1b4249 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -173,6 +173,11 @@ namespace writer->writePrefix(); } + void flush() override + { + writer->flush(); + } + void writeSuffix() override { writer->writeSuffix(); diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 3b4c56b524b..8baa1cd64b0 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -281,9 +281,9 @@ def test_put_get_with_globs(cluster): # Test multipart put. @pytest.mark.parametrize("maybe_auth,positive", [ - ("", True) + ("", True), # ("'minio','minio123',",True), Redirect with credentials not working with nginx. - # ("'wrongid','wrongkey',", False) ClickHouse crashes in some time after this test, local integration tests run fails. + ("'wrongid','wrongkey',", False), ]) def test_multipart_put(cluster, maybe_auth, positive): # type: (ClickHouseCluster) -> None From 38e1cb41cced59dcf8e0a457817ff7c9fd2480e2 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Mar 2021 17:04:59 +0300 Subject: [PATCH 075/233] done --- src/IO/Progress.h | 8 ++++---- src/Interpreters/executeQuery.cpp | 4 ++-- .../Formats/Impl/ParallelFormattingOutputFormat.h | 3 +++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/IO/Progress.h b/src/IO/Progress.h index a3efb96db98..64bf3a404af 100644 --- a/src/IO/Progress.h +++ b/src/IO/Progress.h @@ -64,12 +64,12 @@ struct Progress std::atomic written_rows {0}; std::atomic written_bytes {0}; - Progress() {} + Progress() = default; Progress(size_t read_rows_, size_t read_bytes_, size_t total_rows_to_read_ = 0) : read_rows(read_rows_), read_bytes(read_bytes_), total_rows_to_read(total_rows_to_read_) {} - Progress(ReadProgress read_progress) + explicit Progress(ReadProgress read_progress) : read_rows(read_progress.read_rows), read_bytes(read_progress.read_bytes), total_rows_to_read(read_progress.total_rows_to_read) {} - Progress(WriteProgress write_progress) + explicit Progress(WriteProgress write_progress) : written_rows(write_progress.written_rows), written_bytes(write_progress.written_bytes) {} void read(ReadBuffer & in, UInt64 server_revision); @@ -86,7 +86,7 @@ struct Progress written_rows += rhs.written_rows; written_bytes += rhs.written_bytes; - return rhs.read_rows || rhs.written_rows ? true : false; + return rhs.read_rows || rhs.written_rows; } void reset() diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 1a0aa031d6f..a5c21405ff1 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -1014,7 +1014,7 @@ void executeQuery( ? getIdentifierName(ast_query_with_output->format) : context.getDefaultFormat(); - auto out = context.getOutputStream(format_name, *out_buf, streams.in->getHeader()); + auto out = context.getOutputStreamParallelIfPossible(format_name, *out_buf, streams.in->getHeader()); /// Save previous progress callback if any. TODO Do it more conveniently. auto previous_progress_callback = context.getProgressCallback(); @@ -1059,7 +1059,7 @@ void executeQuery( return std::make_shared(header); }); - auto out = context.getOutputFormat(format_name, *out_buf, pipeline.getHeader()); + auto out = context.getOutputFormatParallelIfPossible(format_name, *out_buf, pipeline.getHeader()); out->setAutoFlush(); /// Save previous progress callback if any. TODO Do it more conveniently. diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 7e7c44a8aae..2efc369e178 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -101,6 +101,9 @@ public: finishAndWait(); } + /// There are no formats which support parallel formatting and progress writing at the same time + void onProgress(const Progress &) override {} + protected: void consume(Chunk chunk) override final { From 138dedf2df59a1728bd76874d78cd2d3875146e9 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 24 Feb 2021 20:04:37 +0300 Subject: [PATCH 076/233] done --- src/Processors/Formats/IRowInputFormat.h | 1 + .../Formats/Impl/CSVRowInputFormat.cpp | 5 ++- .../Formats/Impl/ParallelParsingInputFormat.h | 3 ++ .../Impl/TabSeparatedRowInputFormat.cpp | 5 +-- ...0161_parallel_parsing_with_names.reference | 8 +++++ .../00161_parallel_parsing_with_names.sh | 31 +++++++++++++++++++ 6 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 tests/queries/1_stateful/00161_parallel_parsing_with_names.reference create mode 100755 tests/queries/1_stateful/00161_parallel_parsing_with_names.sh diff --git a/src/Processors/Formats/IRowInputFormat.h b/src/Processors/Formats/IRowInputFormat.h index b7863704062..b99c3789383 100644 --- a/src/Processors/Formats/IRowInputFormat.h +++ b/src/Processors/Formats/IRowInputFormat.h @@ -53,6 +53,7 @@ public: void resetParser() override; protected: + friend class ParallelParsingInputFormat; /** Read next row and append it to the columns. * If no more rows - return false. */ diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index f7f08411dfa..ba3d449de97 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -8,6 +8,8 @@ #include +#include + namespace DB { @@ -155,7 +157,7 @@ void CSVRowInputFormat::readPrefix() size_t num_columns = data_types.size(); const auto & header = getPort().getHeader(); - if (with_names) + if (with_names && getCurrentUnitNumber() == 0) { /// This CSV file has a header row with column names. Depending on the /// settings, use it or skip it. @@ -492,6 +494,7 @@ static std::pair fileSegmentationEngineCSVImpl(ReadBuffer & in, DB void registerFileSegmentationEngineCSV(FormatFactory & factory) { factory.registerFileSegmentationEngine("CSV", &fileSegmentationEngineCSVImpl); + factory.registerFileSegmentationEngine("CSVWithNames", &fileSegmentationEngineCSVImpl); } } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 9dda2dfe55d..606842242bc 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB { @@ -97,6 +98,8 @@ public: segmentator_thread = ThreadFromGlobalPool( &ParallelParsingInputFormat::segmentatorThreadFunction, this, CurrentThread::getGroup()); + + LOG_DEBUG(&Poco::Logger::get("ParallelParsingInputFormat"), "Parallel parsing is used"); } ~ParallelParsingInputFormat() override diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 96b01a5bd9b..5ce612e6e73 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -136,7 +136,7 @@ void TabSeparatedRowInputFormat::readPrefix() skipBOMIfExists(in); } - if (with_names) + if (with_names && getCurrentUnitNumber() == 0) { if (format_settings.with_names_use_header) { @@ -463,9 +463,10 @@ static std::pair fileSegmentationEngineTabSeparatedImpl(ReadBuffer void registerFileSegmentationEngineTabSeparated(FormatFactory & factory) { // We can use the same segmentation engine for TSKV. - for (const auto * name : {"TabSeparated", "TSV", "TSKV"}) + for (const std::string & name : {"TabSeparated", "TSV", "TSKV"}) { factory.registerFileSegmentationEngine(name, &fileSegmentationEngineTabSeparatedImpl); + factory.registerFileSegmentationEngine(name + "WithNames", &fileSegmentationEngineTabSeparatedImpl); } } diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference b/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference new file mode 100644 index 00000000000..4e0017f2d78 --- /dev/null +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference @@ -0,0 +1,8 @@ +TSVWithNames, false +50000 +TSVWithNames, true +50000 +CSVWithNames, false +50000 +CSVWithNames, true +50000 diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh new file mode 100755 index 00000000000..79a2d5c22c5 --- /dev/null +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +FORMATS=('TSVWithNames' 'CSVWithNames') +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" + +for format in "${FORMATS[@]}" +do + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(a DateTime, b String, c FixedString(16)) ENGINE=Memory()" + + echo "$format, false"; + $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ + "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ + $CLICKHOUSE_CLIENT --input_format_parallel_parsing=false -q "INSERT INTO parsing_with_names FORMAT $format" + + $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" + $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" + + + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(a DateTime, b String, c FixedString(16)) ENGINE=Memory()" + echo "$format, true"; + $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ + "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ + $CLICKHOUSE_CLIENT --input_format_parallel_parsing=true -q "INSERT INTO parsing_with_names FORMAT $format" + + $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" + $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" +done \ No newline at end of file From 61fdeb58972c0b166710e9f862d327f2e10213d1 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 24 Feb 2021 20:12:22 +0300 Subject: [PATCH 077/233] better --- src/Processors/Formats/IRowInputFormat.h | 1 - src/Processors/Formats/Impl/CSVRowInputFormat.cpp | 4 ++-- src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Processors/Formats/IRowInputFormat.h b/src/Processors/Formats/IRowInputFormat.h index b99c3789383..b7863704062 100644 --- a/src/Processors/Formats/IRowInputFormat.h +++ b/src/Processors/Formats/IRowInputFormat.h @@ -53,7 +53,6 @@ public: void resetParser() override; protected: - friend class ParallelParsingInputFormat; /** Read next row and append it to the columns. * If no more rows - return false. */ diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index ba3d449de97..0f7ca7b9e79 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -8,8 +8,6 @@ #include -#include - namespace DB { @@ -157,6 +155,8 @@ void CSVRowInputFormat::readPrefix() size_t num_columns = data_types.size(); const auto & header = getPort().getHeader(); + /// This is a bit of abstraction leakage, but we have almost the same code in other places. + /// Thus, we check if this InputFormat is working with the "real" beggining of the data in case of parallel parsing. if (with_names && getCurrentUnitNumber() == 0) { /// This CSV file has a header row with column names. Depending on the diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 5ce612e6e73..00926b6e99b 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -136,6 +136,8 @@ void TabSeparatedRowInputFormat::readPrefix() skipBOMIfExists(in); } + /// This is a bit of abstraction leakage, but we have almost the same code in other places. + /// Thus, we check if this InputFormat is working with the "real" beggining of the data in case of parallel parsing. if (with_names && getCurrentUnitNumber() == 0) { if (format_settings.with_names_use_header) From 8b30bb0fb540c1c8db386c0d1f3ca0f6e3a60f1c Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 24 Feb 2021 21:03:02 +0300 Subject: [PATCH 078/233] style --- src/Processors/Formats/Impl/CSVRowInputFormat.cpp | 2 +- src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index 0f7ca7b9e79..6610a7a0d82 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -156,7 +156,7 @@ void CSVRowInputFormat::readPrefix() const auto & header = getPort().getHeader(); /// This is a bit of abstraction leakage, but we have almost the same code in other places. - /// Thus, we check if this InputFormat is working with the "real" beggining of the data in case of parallel parsing. + /// Thus, we check if this InputFormat is working with the "real" beginning of the data in case of parallel parsing. if (with_names && getCurrentUnitNumber() == 0) { /// This CSV file has a header row with column names. Depending on the diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 00926b6e99b..7dcfd4e930d 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -137,7 +137,7 @@ void TabSeparatedRowInputFormat::readPrefix() } /// This is a bit of abstraction leakage, but we have almost the same code in other places. - /// Thus, we check if this InputFormat is working with the "real" beggining of the data in case of parallel parsing. + /// Thus, we check if this InputFormat is working with the "real" beginning of the data in case of parallel parsing. if (with_names && getCurrentUnitNumber() == 0) { if (format_settings.with_names_use_header) From 91a418b3a9682a4817948264d6cc57a08a117de4 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 24 Feb 2021 21:09:31 +0300 Subject: [PATCH 079/233] better --- src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 7dcfd4e930d..510cf67065f 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -465,10 +465,9 @@ static std::pair fileSegmentationEngineTabSeparatedImpl(ReadBuffer void registerFileSegmentationEngineTabSeparated(FormatFactory & factory) { // We can use the same segmentation engine for TSKV. - for (const std::string & name : {"TabSeparated", "TSV", "TSKV"}) + for (const std::string & name : {"TabSeparated", "TSV", "TSKV", "TabSeparatedWithNames", "TSVWithNames"}) { factory.registerFileSegmentationEngine(name, &fileSegmentationEngineTabSeparatedImpl); - factory.registerFileSegmentationEngine(name + "WithNames", &fileSegmentationEngineTabSeparatedImpl); } } From eab35bfc7f6c18c322c291904ae0514b35a3a61a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 28 Feb 2021 02:37:28 +0300 Subject: [PATCH 080/233] Update ParallelParsingInputFormat.h --- src/Processors/Formats/Impl/ParallelParsingInputFormat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 606842242bc..f1a290619fa 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -99,7 +99,7 @@ public: segmentator_thread = ThreadFromGlobalPool( &ParallelParsingInputFormat::segmentatorThreadFunction, this, CurrentThread::getGroup()); - LOG_DEBUG(&Poco::Logger::get("ParallelParsingInputFormat"), "Parallel parsing is used"); + LOG_TRACE(&Poco::Logger::get("ParallelParsingInputFormat"), "Parallel parsing is used"); } ~ParallelParsingInputFormat() override From 369c9da1611a97bc0c0ebf37a8fdba29dd47dc08 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Mon, 1 Mar 2021 22:58:55 +0300 Subject: [PATCH 081/233] better --- src/Processors/Formats/IInputFormat.cpp | 1 + src/Processors/Formats/IInputFormat.h | 25 ++++++++ .../Formats/Impl/CSVRowInputFormat.cpp | 50 ++++++++-------- .../Formats/Impl/CSVRowInputFormat.h | 10 ---- .../Impl/ParallelParsingInputFormat.cpp | 9 +++ .../Formats/Impl/ParallelParsingInputFormat.h | 3 + .../Impl/TabSeparatedRowInputFormat.cpp | 58 +++++++++---------- .../Formats/Impl/TabSeparatedRowInputFormat.h | 4 -- src/Processors/ISource.h | 2 +- .../00161_parallel_parsing_with_names.sh | 3 +- 10 files changed, 95 insertions(+), 70 deletions(-) diff --git a/src/Processors/Formats/IInputFormat.cpp b/src/Processors/Formats/IInputFormat.cpp index 0fbc78ea8c0..069d25564b1 100644 --- a/src/Processors/Formats/IInputFormat.cpp +++ b/src/Processors/Formats/IInputFormat.cpp @@ -13,6 +13,7 @@ namespace ErrorCodes IInputFormat::IInputFormat(Block header, ReadBuffer & in_) : ISource(std::move(header)), in(in_) { + column_mapping = std::make_shared(); } void IInputFormat::resetParser() diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index e1537aff6c5..b8ee4d438df 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -2,9 +2,26 @@ #include +#include + namespace DB { +/// Used to pass info from header between different InputFormats in ParallelParsing +struct ColumnMapping +{ + /// Maps indexes of columns in the input file to indexes of table columns + using OptionalIndexes = std::vector>; + OptionalIndexes column_indexes_for_input_fields; + + /// Tracks which columns we have read in a single read() call. + /// For columns that are never read, it is initialized to false when we + /// read the file header, and never changed afterwards. + /// For other columns, it is updated on each read() call. + std::vector read_columns; +}; + +using ColumnMappingPtr = std::shared_ptr; class ReadBuffer; @@ -39,9 +56,17 @@ public: return none; } + /// Must be called from ParallelParsingInputFormat after readSuffix + ColumnMappingPtr getColumnMapping() const { return column_mapping; } + /// Must be called from ParallelParsingInputFormat before readPrefix + void setColumnMapping(ColumnMappingPtr column_mapping_ ) { column_mapping = column_mapping_; } + size_t getCurrentUnitNumber() const { return current_unit_number; } void setCurrentUnitNumber(size_t current_unit_number_) { current_unit_number = current_unit_number_; } +protected: + ColumnMappingPtr column_mapping{}; + private: /// Number of currently parsed chunk (if parallel parsing is enabled) size_t current_unit_number = 0; diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index 6610a7a0d82..efb8ad93d72 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -55,13 +55,13 @@ void CSVRowInputFormat::addInputColumn(const String & column_name) { if (format_settings.skip_unknown_fields) { - column_indexes_for_input_fields.push_back(std::nullopt); + column_mapping->column_indexes_for_input_fields.push_back(std::nullopt); return; } throw Exception( "Unknown field found in CSV header: '" + column_name + "' " + - "at position " + std::to_string(column_indexes_for_input_fields.size()) + + "at position " + std::to_string(column_mapping->column_indexes_for_input_fields.size()) + "\nSet the 'input_format_skip_unknown_fields' parameter explicitly to ignore and proceed", ErrorCodes::INCORRECT_DATA ); @@ -69,11 +69,11 @@ void CSVRowInputFormat::addInputColumn(const String & column_name) const auto column_index = column_it->second; - if (read_columns[column_index]) + if (column_mapping->read_columns[column_index]) throw Exception("Duplicate field found while parsing CSV header: " + column_name, ErrorCodes::INCORRECT_DATA); - read_columns[column_index] = true; - column_indexes_for_input_fields.emplace_back(column_index); + column_mapping->read_columns[column_index] = true; + column_mapping->column_indexes_for_input_fields.emplace_back(column_index); } static void skipEndOfLine(ReadBuffer & in) @@ -165,7 +165,7 @@ void CSVRowInputFormat::readPrefix() { /// Look at the file header to see which columns we have there. /// The missing columns are filled with defaults. - read_columns.assign(header.columns(), false); + column_mapping->read_columns.assign(header.columns(), false); do { String column_name; @@ -179,7 +179,7 @@ void CSVRowInputFormat::readPrefix() skipDelimiter(in, format_settings.csv.delimiter, true); - for (auto read_column : read_columns) + for (auto read_column : column_mapping->read_columns) { if (!read_column) { @@ -196,12 +196,12 @@ void CSVRowInputFormat::readPrefix() /// The default: map each column of the file to the column of the table with /// the same index. - read_columns.assign(header.columns(), true); - column_indexes_for_input_fields.resize(header.columns()); + column_mapping->read_columns.assign(header.columns(), true); + column_mapping->column_indexes_for_input_fields.resize(header.columns()); - for (size_t i = 0; i < column_indexes_for_input_fields.size(); ++i) + for (size_t i = 0; i < column_mapping->column_indexes_for_input_fields.size(); ++i) { - column_indexes_for_input_fields[i] = i; + column_mapping->column_indexes_for_input_fields[i] = i; } } @@ -218,12 +218,12 @@ bool CSVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ext /// it doesn't have to check it. bool have_default_columns = have_always_default_columns; - ext.read_columns.assign(read_columns.size(), true); + ext.read_columns.assign(column_mapping->read_columns.size(), true); const auto delimiter = format_settings.csv.delimiter; - for (size_t file_column = 0; file_column < column_indexes_for_input_fields.size(); ++file_column) + for (size_t file_column = 0; file_column < column_mapping->column_indexes_for_input_fields.size(); ++file_column) { - const auto & table_column = column_indexes_for_input_fields[file_column]; - const bool is_last_file_column = file_column + 1 == column_indexes_for_input_fields.size(); + const auto & table_column = column_mapping->column_indexes_for_input_fields[file_column]; + const bool is_last_file_column = file_column + 1 == column_mapping->column_indexes_for_input_fields.size(); if (table_column) { @@ -245,9 +245,9 @@ bool CSVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ext if (have_default_columns) { - for (size_t i = 0; i < read_columns.size(); i++) + for (size_t i = 0; i < column_mapping->read_columns.size(); i++) { - if (!read_columns[i]) + if (!column_mapping->read_columns[i]) { /// The column value for this row is going to be overwritten /// with default by the caller, but the general assumption is @@ -268,7 +268,7 @@ bool CSVRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & columns, { const char delimiter = format_settings.csv.delimiter; - for (size_t file_column = 0; file_column < column_indexes_for_input_fields.size(); ++file_column) + for (size_t file_column = 0; file_column < column_mapping->column_indexes_for_input_fields.size(); ++file_column) { if (file_column == 0 && in.eof()) { @@ -277,10 +277,10 @@ bool CSVRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & columns, } skipWhitespacesAndTabs(in); - if (column_indexes_for_input_fields[file_column].has_value()) + if (column_mapping->column_indexes_for_input_fields[file_column].has_value()) { const auto & header = getPort().getHeader(); - size_t col_idx = column_indexes_for_input_fields[file_column].value(); + size_t col_idx = column_mapping->column_indexes_for_input_fields[file_column].value(); if (!deserializeFieldAndPrintDiagnosticInfo(header.getByPosition(col_idx).name, data_types[col_idx], *columns[col_idx], out, file_column)) return false; @@ -296,7 +296,7 @@ bool CSVRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & columns, skipWhitespacesAndTabs(in); /// Delimiters - if (file_column + 1 == column_indexes_for_input_fields.size()) + if (file_column + 1 == column_mapping->column_indexes_for_input_fields.size()) { if (in.eof()) return false; @@ -358,9 +358,9 @@ void CSVRowInputFormat::syncAfterError() void CSVRowInputFormat::tryDeserializeField(const DataTypePtr & type, IColumn & column, size_t file_column) { - if (column_indexes_for_input_fields[file_column]) + if (column_mapping->column_indexes_for_input_fields[file_column]) { - const bool is_last_file_column = file_column + 1 == column_indexes_for_input_fields.size(); + const bool is_last_file_column = file_column + 1 == column_mapping->column_indexes_for_input_fields.size(); readField(column, type, is_last_file_column); } else @@ -406,8 +406,8 @@ bool CSVRowInputFormat::readField(IColumn & column, const DataTypePtr & type, bo void CSVRowInputFormat::resetParser() { RowInputFormatWithDiagnosticInfo::resetParser(); - column_indexes_for_input_fields.clear(); - read_columns.clear(); + column_mapping->column_indexes_for_input_fields.clear(); + column_mapping->read_columns.clear(); have_always_default_columns = false; } diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.h b/src/Processors/Formats/Impl/CSVRowInputFormat.h index c884eb6c3db..3cdafd8ec8d 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.h +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.h @@ -38,16 +38,6 @@ private: using IndexesMap = std::unordered_map; IndexesMap column_indexes_by_names; - /// Maps indexes of columns in the input file to indexes of table columns - using OptionalIndexes = std::vector>; - OptionalIndexes column_indexes_for_input_fields; - - /// Tracks which columns we have read in a single read() call. - /// For columns that are never read, it is initialized to false when we - /// read the file header, and never changed afterwards. - /// For other columns, it is updated on each read() call. - std::vector read_columns; - /// Whether we have any columns that are not read from file at all, /// and must be always initialized with defaults. bool have_always_default_columns = false; diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index d1660b53019..ce041f6636b 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -89,6 +89,11 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa unit.chunk_ext.chunk.clear(); unit.chunk_ext.block_missing_values.clear(); + /// Propagate column_mapping to other parsers. + /// Note: column_mapping is used only for *WithNames types + if (current_ticket_number != 0) + input_format->setColumnMapping(column_mapping); + // We don't know how many blocks will be. So we have to read them all // until an empty block occurred. Chunk chunk; @@ -100,6 +105,10 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa unit.chunk_ext.block_missing_values.emplace_back(parser.getMissingValues()); } + /// Extract column_mapping from first parser to propage it to others + if (current_ticket_number == 0) + column_mapping = input_format->getColumnMapping(); + // We suppose we will get at least some blocks for a non-empty buffer, // except at the end of file. Also see a matching assert in readImpl(). assert(unit.is_last || !unit.chunk_ext.chunk.empty() || parsing_finished); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index f1a290619fa..49bb69faeac 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -253,6 +253,9 @@ private: { parserThreadFunction(group, ticket_number); }); + /// We have to wait here to possibly extract ColumnMappingPtr from the first parser. + if (ticket_number == 0) + pool.wait(); } void finishAndWait() diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 510cf67065f..04c6b4c3ee0 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -62,19 +62,19 @@ TabSeparatedRowInputFormat::TabSeparatedRowInputFormat(const Block & header_, Re column_indexes_by_names.emplace(column_info.name, i); } - column_indexes_for_input_fields.reserve(num_columns); - read_columns.assign(num_columns, false); + column_mapping->column_indexes_for_input_fields.reserve(num_columns); + column_mapping->read_columns.assign(num_columns, false); } void TabSeparatedRowInputFormat::setupAllColumnsByTableSchema() { const auto & header = getPort().getHeader(); - read_columns.assign(header.columns(), true); - column_indexes_for_input_fields.resize(header.columns()); + column_mapping->read_columns.assign(header.columns(), true); + column_mapping->column_indexes_for_input_fields.resize(header.columns()); - for (size_t i = 0; i < column_indexes_for_input_fields.size(); ++i) - column_indexes_for_input_fields[i] = i; + for (size_t i = 0; i < column_mapping->column_indexes_for_input_fields.size(); ++i) + column_mapping->column_indexes_for_input_fields[i] = i; } @@ -85,13 +85,13 @@ void TabSeparatedRowInputFormat::addInputColumn(const String & column_name) { if (format_settings.skip_unknown_fields) { - column_indexes_for_input_fields.push_back(std::nullopt); + column_mapping->column_indexes_for_input_fields.push_back(std::nullopt); return; } throw Exception( "Unknown field found in TSV header: '" + column_name + "' " + - "at position " + std::to_string(column_indexes_for_input_fields.size()) + + "at position " + std::to_string(column_mapping->column_indexes_for_input_fields.size()) + "\nSet the 'input_format_skip_unknown_fields' parameter explicitly to ignore and proceed", ErrorCodes::INCORRECT_DATA ); @@ -99,11 +99,11 @@ void TabSeparatedRowInputFormat::addInputColumn(const String & column_name) const auto column_index = column_it->second; - if (read_columns[column_index]) + if (column_mapping->read_columns[column_index]) throw Exception("Duplicate field found while parsing TSV header: " + column_name, ErrorCodes::INCORRECT_DATA); - read_columns[column_index] = true; - column_indexes_for_input_fields.emplace_back(column_index); + column_mapping->read_columns[column_index] = true; + column_mapping->column_indexes_for_input_fields.emplace_back(column_index); } @@ -113,8 +113,8 @@ void TabSeparatedRowInputFormat::fillUnreadColumnsWithDefaults(MutableColumns & if (unlikely(row_num == 1)) { columns_to_fill_with_default_values.clear(); - for (size_t index = 0; index < read_columns.size(); ++index) - if (read_columns[index] == 0) + for (size_t index = 0; index < column_mapping->read_columns.size(); ++index) + if (column_mapping->read_columns[index] == 0) columns_to_fill_with_default_values.push_back(index); } @@ -167,7 +167,7 @@ void TabSeparatedRowInputFormat::readPrefix() else { setupAllColumnsByTableSchema(); - skipTSVRow(in, column_indexes_for_input_fields.size()); + skipTSVRow(in, column_mapping->column_indexes_for_input_fields.size()); } } else @@ -175,7 +175,7 @@ void TabSeparatedRowInputFormat::readPrefix() if (with_types) { - skipTSVRow(in, column_indexes_for_input_fields.size()); + skipTSVRow(in, column_mapping->column_indexes_for_input_fields.size()); } } @@ -187,11 +187,11 @@ bool TabSeparatedRowInputFormat::readRow(MutableColumns & columns, RowReadExtens updateDiagnosticInfo(); - ext.read_columns.assign(read_columns.size(), true); - for (size_t file_column = 0; file_column < column_indexes_for_input_fields.size(); ++file_column) + ext.read_columns.assign(column_mapping->read_columns.size(), true); + for (size_t file_column = 0; file_column < column_mapping->column_indexes_for_input_fields.size(); ++file_column) { - const auto & column_index = column_indexes_for_input_fields[file_column]; - const bool is_last_file_column = file_column + 1 == column_indexes_for_input_fields.size(); + const auto & column_index = column_mapping->column_indexes_for_input_fields[file_column]; + const bool is_last_file_column = file_column + 1 == column_mapping->column_indexes_for_input_fields.size(); if (column_index) { const auto & type = data_types[*column_index]; @@ -204,7 +204,7 @@ bool TabSeparatedRowInputFormat::readRow(MutableColumns & columns, RowReadExtens } /// skip separators - if (file_column + 1 < column_indexes_for_input_fields.size()) + if (file_column + 1 < column_mapping->column_indexes_for_input_fields.size()) { assertChar('\t', in); } @@ -240,7 +240,7 @@ bool TabSeparatedRowInputFormat::readField(IColumn & column, const DataTypePtr & bool TabSeparatedRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & columns, WriteBuffer & out) { - for (size_t file_column = 0; file_column < column_indexes_for_input_fields.size(); ++file_column) + for (size_t file_column = 0; file_column < column_mapping->column_indexes_for_input_fields.size(); ++file_column) { if (file_column == 0 && in.eof()) { @@ -248,10 +248,10 @@ bool TabSeparatedRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & return false; } - if (column_indexes_for_input_fields[file_column].has_value()) + if (column_mapping->column_indexes_for_input_fields[file_column].has_value()) { const auto & header = getPort().getHeader(); - size_t col_idx = column_indexes_for_input_fields[file_column].value(); + size_t col_idx = column_mapping->column_indexes_for_input_fields[file_column].value(); if (!deserializeFieldAndPrintDiagnosticInfo(header.getByPosition(col_idx).name, data_types[col_idx], *columns[col_idx], out, file_column)) return false; @@ -266,7 +266,7 @@ bool TabSeparatedRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & } /// Delimiters - if (file_column + 1 == column_indexes_for_input_fields.size()) + if (file_column + 1 == column_mapping->column_indexes_for_input_fields.size()) { if (!in.eof()) { @@ -332,7 +332,7 @@ bool TabSeparatedRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & void TabSeparatedRowInputFormat::tryDeserializeField(const DataTypePtr & type, IColumn & column, size_t file_column) { - if (column_indexes_for_input_fields[file_column]) + if (column_mapping->column_indexes_for_input_fields[file_column]) { // check null value for type is not nullable. don't cross buffer bound for simplicity, so maybe missing some case if (!type->isNullable() && !in.eof()) @@ -351,7 +351,7 @@ void TabSeparatedRowInputFormat::tryDeserializeField(const DataTypePtr & type, I } } } - const bool is_last_file_column = file_column + 1 == column_indexes_for_input_fields.size(); + const bool is_last_file_column = file_column + 1 == column_mapping->column_indexes_for_input_fields.size(); readField(column, type, is_last_file_column); } else @@ -370,8 +370,8 @@ void TabSeparatedRowInputFormat::resetParser() { RowInputFormatWithDiagnosticInfo::resetParser(); const auto & sample = getPort().getHeader(); - read_columns.assign(sample.columns(), false); - column_indexes_for_input_fields.clear(); + column_mapping->read_columns.assign(sample.columns(), false); + column_mapping->column_indexes_for_input_fields.clear(); columns_to_fill_with_default_values.clear(); } @@ -465,7 +465,7 @@ static std::pair fileSegmentationEngineTabSeparatedImpl(ReadBuffer void registerFileSegmentationEngineTabSeparated(FormatFactory & factory) { // We can use the same segmentation engine for TSKV. - for (const std::string & name : {"TabSeparated", "TSV", "TSKV", "TabSeparatedWithNames", "TSVWithNames"}) + for (const auto & name : {"TabSeparated", "TSV", "TSKV", "TabSeparatedWithNames", "TSVWithNames"}) { factory.registerFileSegmentationEngine(name, &fileSegmentationEngineTabSeparatedImpl); } diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h index 0141d87403a..db70b4d3fea 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h @@ -41,10 +41,6 @@ private: using IndexesMap = std::unordered_map; IndexesMap column_indexes_by_names; - using OptionalIndexes = std::vector>; - OptionalIndexes column_indexes_for_input_fields; - - std::vector read_columns; std::vector columns_to_fill_with_default_values; void addInputColumn(const String & column_name); diff --git a/src/Processors/ISource.h b/src/Processors/ISource.h index b7e2b5dce8e..db91c0c5bce 100644 --- a/src/Processors/ISource.h +++ b/src/Processors/ISource.h @@ -19,7 +19,7 @@ protected: virtual std::optional tryGenerate(); public: - ISource(Block header); + explicit ISource(Block header); Status prepare() override; void work() override; diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh index 79a2d5c22c5..ab7706e100f 100755 --- a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh @@ -9,7 +9,8 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" for format in "${FORMATS[@]}" do - $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(a DateTime, b String, c FixedString(16)) ENGINE=Memory()" + # Columns are permuted + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime, b String) ENGINE=Memory()" echo "$format, false"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ From 3a7d48f3a4c8c1a708ace4b0d9800e5ba3d1b8e6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 2 Mar 2021 01:32:11 +0300 Subject: [PATCH 082/233] style and tsan fix --- src/Processors/Formats/IInputFormat.h | 4 +++- src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp | 6 +++++- src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index b8ee4d438df..a297f487318 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -10,6 +10,8 @@ namespace DB /// Used to pass info from header between different InputFormats in ParallelParsing struct ColumnMapping { + /// Non-atomic because only read access in possible + bool is_set; /// Maps indexes of columns in the input file to indexes of table columns using OptionalIndexes = std::vector>; OptionalIndexes column_indexes_for_input_fields; @@ -59,7 +61,7 @@ public: /// Must be called from ParallelParsingInputFormat after readSuffix ColumnMappingPtr getColumnMapping() const { return column_mapping; } /// Must be called from ParallelParsingInputFormat before readPrefix - void setColumnMapping(ColumnMappingPtr column_mapping_ ) { column_mapping = column_mapping_; } + void setColumnMapping(ColumnMappingPtr column_mapping_) { column_mapping = column_mapping_; } size_t getCurrentUnitNumber() const { return current_unit_number; } void setCurrentUnitNumber(size_t current_unit_number_) { current_unit_number = current_unit_number_; } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index ce041f6636b..4c2b9df304b 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -92,7 +92,11 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa /// Propagate column_mapping to other parsers. /// Note: column_mapping is used only for *WithNames types if (current_ticket_number != 0) + { + column_mapping->is_set = true; input_format->setColumnMapping(column_mapping); + } + // We don't know how many blocks will be. So we have to read them all // until an empty block occurred. @@ -105,7 +109,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa unit.chunk_ext.block_missing_values.emplace_back(parser.getMissingValues()); } - /// Extract column_mapping from first parser to propage it to others + /// Extract column_mapping from first parser to propagate it to others if (current_ticket_number == 0) column_mapping = input_format->getColumnMapping(); diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 04c6b4c3ee0..ffb1b96f70e 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -170,7 +170,7 @@ void TabSeparatedRowInputFormat::readPrefix() skipTSVRow(in, column_mapping->column_indexes_for_input_fields.size()); } } - else + else if (!column_mapping->is_set) setupAllColumnsByTableSchema(); if (with_types) From 3372dd7b6a69d67abfa658c8518dfe9a9e56d03d Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Mar 2021 16:31:19 +0300 Subject: [PATCH 083/233] fix stupid bug --- src/Processors/Formats/IInputFormat.h | 3 ++- .../Formats/Impl/CSVRowInputFormat.cpp | 22 ++++++++++--------- .../Formats/Impl/CSVRowInputFormat.h | 1 + .../Impl/ParallelParsingInputFormat.cpp | 7 +++--- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index a297f487318..95910bf51e5 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -10,7 +10,8 @@ namespace DB /// Used to pass info from header between different InputFormats in ParallelParsing struct ColumnMapping { - /// Non-atomic because only read access in possible + /// Non-atomic because there is strict `happens-before` between read and write access + /// See InputFormatParallelParsing bool is_set; /// Maps indexes of columns in the input file to indexes of table columns using OptionalIndexes = std::vector>; diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index efb8ad93d72..4cec07f38dc 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -145,6 +145,16 @@ static void skipRow(ReadBuffer & in, const FormatSettings::CSV & settings, size_ } } +void CSVRowInputFormat::setupAllColumnsByTableSchema() +{ + const auto & header = getPort().getHeader(); + column_mapping->read_columns.assign(header.columns(), true); + column_mapping->column_indexes_for_input_fields.resize(header.columns()); + + for (size_t i = 0; i < column_mapping->column_indexes_for_input_fields.size(); ++i) + column_mapping->column_indexes_for_input_fields[i] = i; +} + void CSVRowInputFormat::readPrefix() { @@ -193,16 +203,8 @@ void CSVRowInputFormat::readPrefix() else skipRow(in, format_settings.csv, num_columns); } - - /// The default: map each column of the file to the column of the table with - /// the same index. - column_mapping->read_columns.assign(header.columns(), true); - column_mapping->column_indexes_for_input_fields.resize(header.columns()); - - for (size_t i = 0; i < column_mapping->column_indexes_for_input_fields.size(); ++i) - { - column_mapping->column_indexes_for_input_fields[i] = i; - } + else if (!column_mapping->is_set) + setupAllColumnsByTableSchema(); } diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.h b/src/Processors/Formats/Impl/CSVRowInputFormat.h index 3cdafd8ec8d..86e41cf0a43 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.h +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.h @@ -44,6 +44,7 @@ private: void addInputColumn(const String & column_name); + void setupAllColumnsByTableSchema(); bool parseRowAndPrintDiagnosticInfo(MutableColumns & columns, WriteBuffer & out) override; void tryDeserializeField(const DataTypePtr & type, IColumn & column, size_t file_column) override; bool isGarbageAfterField(size_t, ReadBuffer::Position pos) override diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 4c2b9df304b..a1d115c734c 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -92,11 +92,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa /// Propagate column_mapping to other parsers. /// Note: column_mapping is used only for *WithNames types if (current_ticket_number != 0) - { - column_mapping->is_set = true; input_format->setColumnMapping(column_mapping); - } - // We don't know how many blocks will be. So we have to read them all // until an empty block occurred. @@ -111,7 +107,10 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa /// Extract column_mapping from first parser to propagate it to others if (current_ticket_number == 0) + { column_mapping = input_format->getColumnMapping(); + column_mapping->is_set = true; + } // We suppose we will get at least some blocks for a non-empty buffer, // except at the end of file. Also see a matching assert in readImpl(). From e7a1398def3dfc553a932af908a4bbe6defba8ba Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 9 Mar 2021 22:02:56 +0300 Subject: [PATCH 084/233] try fix tests --- .../Formats/Impl/ParallelParsingInputFormat.cpp | 1 + .../Formats/Impl/ParallelParsingInputFormat.h | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index a1d115c734c..1ead36b99eb 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -110,6 +110,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa { column_mapping = input_format->getColumnMapping(); column_mapping->is_set = true; + first_parser_finished.Notify(); } // We suppose we will get at least some blocks for a non-empty buffer, diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 49bb69faeac..f42cd3eadd0 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -12,6 +12,18 @@ #include #include +/// I don't know why, but clang warns about static annotations +/// error: macro name is a reserved identifier [-Werror,-Wreserved-id-macro] +/// #define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreserved-id-macro" +#endif +#include +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + namespace DB { @@ -202,6 +214,8 @@ private: std::condition_variable reader_condvar; std::condition_variable segmentator_condvar; + absl::Notification first_parser_finished; + std::atomic parsing_finished{false}; /// There are multiple "parsers", that's why we use thread pool. @@ -255,7 +269,7 @@ private: }); /// We have to wait here to possibly extract ColumnMappingPtr from the first parser. if (ticket_number == 0) - pool.wait(); + first_parser_finished.WaitForNotification(); } void finishAndWait() From f3ee1290745d8a5fdf00ae0ab5fa4e9572f9295c Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 10 Mar 2021 00:22:04 +0300 Subject: [PATCH 085/233] fix grpc --- src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 1ead36b99eb..0029a7e1084 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -123,6 +123,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa } catch (...) { + first_parser_finished.Notify(); onBackgroundException(unit.offset); } } From 761b369fa2212dbe4f58fd67622dad0e03c12630 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 10 Mar 2021 01:20:38 +0300 Subject: [PATCH 086/233] move to poco::event --- .../Formats/Impl/ParallelParsingInputFormat.cpp | 4 ++-- .../Formats/Impl/ParallelParsingInputFormat.h | 17 +++-------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 0029a7e1084..1055342cbea 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -110,7 +110,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa { column_mapping = input_format->getColumnMapping(); column_mapping->is_set = true; - first_parser_finished.Notify(); + first_parser_finished.set(); } // We suppose we will get at least some blocks for a non-empty buffer, @@ -123,7 +123,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa } catch (...) { - first_parser_finished.Notify(); + first_parser_finished.set(); onBackgroundException(unit.offset); } } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index f42cd3eadd0..559507055b9 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -11,18 +11,7 @@ #include #include #include - -/// I don't know why, but clang warns about static annotations -/// error: macro name is a reserved identifier [-Werror,-Wreserved-id-macro] -/// #define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wreserved-id-macro" -#endif -#include -#ifdef __clang__ -#pragma clang diagnostic pop -#endif +#include namespace DB { @@ -214,7 +203,7 @@ private: std::condition_variable reader_condvar; std::condition_variable segmentator_condvar; - absl::Notification first_parser_finished; + Poco::Event first_parser_finished; std::atomic parsing_finished{false}; @@ -269,7 +258,7 @@ private: }); /// We have to wait here to possibly extract ColumnMappingPtr from the first parser. if (ticket_number == 0) - first_parser_finished.WaitForNotification(); + first_parser_finished.wait(); } void finishAndWait() From b434a2f0bdff85f679bd53b7b9ecbe12a83eabfb Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Mar 2021 02:15:16 +0300 Subject: [PATCH 087/233] disable parallel parsing for grpc tests --- .../Formats/Impl/ParallelParsingInputFormat.cpp | 4 +--- tests/integration/test_grpc_protocol/configs/users.xml | 8 ++++++++ tests/integration/test_grpc_protocol/test.py | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_grpc_protocol/configs/users.xml diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 1055342cbea..1ad913a1a59 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -123,7 +123,6 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa } catch (...) { - first_parser_finished.set(); onBackgroundException(unit.offset); } } @@ -131,8 +130,6 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa void ParallelParsingInputFormat::onBackgroundException(size_t offset) { - tryLogCurrentException(__PRETTY_FUNCTION__); - std::unique_lock lock(mutex); if (!background_exception) { @@ -143,6 +140,7 @@ void ParallelParsingInputFormat::onBackgroundException(size_t offset) } tryLogCurrentException(__PRETTY_FUNCTION__); parsing_finished = true; + first_parser_finished.set(); reader_condvar.notify_all(); segmentator_condvar.notify_all(); } diff --git a/tests/integration/test_grpc_protocol/configs/users.xml b/tests/integration/test_grpc_protocol/configs/users.xml new file mode 100644 index 00000000000..2ae1a397fe5 --- /dev/null +++ b/tests/integration/test_grpc_protocol/configs/users.xml @@ -0,0 +1,8 @@ + + + + + 0 + + + diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index d8604276281..594879427ca 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -27,7 +27,7 @@ import clickhouse_grpc_pb2_grpc config_dir = os.path.join(SCRIPT_DIR, './configs') cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/grpc_config.xml']) +node = cluster.add_instance('node', main_configs=['configs/grpc_config.xml'], user_configs=["configs/users.xml"]) grpc_port = 9100 main_channel = None From 87eb2592deb9dca4eb39fa424493db4bf9d9e403 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Mar 2021 18:10:06 +0300 Subject: [PATCH 088/233] add input_format_skip_unknown_fields --- .../1_stateful/00161_parallel_parsing_with_names.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh index ab7706e100f..ad7b83c55a4 100755 --- a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh @@ -14,8 +14,8 @@ do echo "$format, false"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ - "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ - $CLICKHOUSE_CLIENT --input_format_parallel_parsing=false -q "INSERT INTO parsing_with_names FORMAT $format" + "SELECT URLRegions as d, ClientEventTime as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ + $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=false -q "INSERT INTO parsing_with_names FORMAT $format" $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" @@ -24,8 +24,8 @@ do $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(a DateTime, b String, c FixedString(16)) ENGINE=Memory()" echo "$format, true"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ - "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ - $CLICKHOUSE_CLIENT --input_format_parallel_parsing=true -q "INSERT INTO parsing_with_names FORMAT $format" + "SELECT URLRegions as d, ClientEventTime as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ + $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=true -q "INSERT INTO parsing_with_names FORMAT $format" $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" From 8e68edebea30779f0e8856dc2e5f000d3bec6387 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Mar 2021 18:21:58 +0300 Subject: [PATCH 089/233] better test --- .../00161_parallel_parsing_with_names.reference | 8 ++++---- .../1_stateful/00161_parallel_parsing_with_names.sh | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference b/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference index 4e0017f2d78..fb0ba75c148 100644 --- a/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference @@ -1,8 +1,8 @@ TSVWithNames, false -50000 +29caf86494f169d6339f6c5610b20731 - TSVWithNames, true -50000 +29caf86494f169d6339f6c5610b20731 - CSVWithNames, false -50000 +29caf86494f169d6339f6c5610b20731 - CSVWithNames, true -50000 +29caf86494f169d6339f6c5610b20731 - diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh index ad7b83c55a4..ca9984900e1 100755 --- a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh @@ -17,16 +17,16 @@ do "SELECT URLRegions as d, ClientEventTime as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=false -q "INSERT INTO parsing_with_names FORMAT $format" - $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" + $CLICKHOUSE_CLIENT -q "SELECT * FROM parsing_with_names;" | md5sum $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" - $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(a DateTime, b String, c FixedString(16)) ENGINE=Memory()" + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime, b String) ENGINE=Memory()" echo "$format, true"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ "SELECT URLRegions as d, ClientEventTime as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=true -q "INSERT INTO parsing_with_names FORMAT $format" - $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" + $CLICKHOUSE_CLIENT -q "SELECT * FROM parsing_with_names;" | md5sum $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" done \ No newline at end of file From 65f90f2ce9ea9e9d4076f06c58ddd981c82cc098 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 11 Mar 2021 21:52:10 +0300 Subject: [PATCH 090/233] Fix distributed requests cancellation with async_socket_for_remote=1 Before this patch for distributed queries, that requires cancellation (simple select from multiple shards with limit, i.e. `select * from remote('127.{2,3}', system.numbers) limit 100`) it is very easy to trigger the situation when remote shard is in the middle of sending Data block while the initiator already send Cancel and expecting some new packet, but it will receive not new packet, but part of the Data block that was in the middle of sending before cancellation, and this will lead to some various errors, like: - Unknown packet X from server Y - Unexpected packet from server Y - and a lot more... Fix this, by correctly waiting for the pending packet before cancellation. It is not very easy to write a test, since localhost is too fast. Also note, that it is not possible to get these errors with hedged requests (use_hedged_requests=1) since handle fibers correctly. But it had been disabled by default for 21.3 in #21534, while async_socket_for_remote is enabled by default. --- .../RemoteQueryExecutorReadContext.cpp | 15 +++++++++++---- src/DataStreams/RemoteQueryExecutorReadContext.h | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/DataStreams/RemoteQueryExecutorReadContext.cpp b/src/DataStreams/RemoteQueryExecutorReadContext.cpp index c2a65f02d08..11cc2dcd8e4 100644 --- a/src/DataStreams/RemoteQueryExecutorReadContext.cpp +++ b/src/DataStreams/RemoteQueryExecutorReadContext.cpp @@ -104,11 +104,11 @@ void RemoteQueryExecutorReadContext::setConnectionFD(int fd, const Poco::Timespa connection_fd_description = fd_description; } -bool RemoteQueryExecutorReadContext::checkTimeout() const +bool RemoteQueryExecutorReadContext::checkTimeout(bool blocking) const { try { - return checkTimeoutImpl(); + return checkTimeoutImpl(blocking); } catch (DB::Exception & e) { @@ -118,13 +118,13 @@ bool RemoteQueryExecutorReadContext::checkTimeout() const } } -bool RemoteQueryExecutorReadContext::checkTimeoutImpl() const +bool RemoteQueryExecutorReadContext::checkTimeoutImpl(bool blocking) const { /// Wait for epoll will not block if it was polled externally. epoll_event events[3]; events[0].data.fd = events[1].data.fd = events[2].data.fd = -1; - int num_events = epoll.getManyReady(3, events,/* blocking = */ false); + int num_events = epoll.getManyReady(3, events, blocking); bool is_socket_ready = false; bool is_pipe_alarmed = false; @@ -184,9 +184,16 @@ bool RemoteQueryExecutorReadContext::resumeRoutine() void RemoteQueryExecutorReadContext::cancel() { std::lock_guard guard(fiber_lock); + /// It is safe to just destroy fiber - we are not in the process of reading from socket. boost::context::fiber to_destroy = std::move(fiber); + while (is_read_in_progress.load(std::memory_order_relaxed)) + { + checkTimeout(/* blocking= */ true); + to_destroy = std::move(to_destroy).resume(); + } + /// Send something to pipe to cancel executor waiting. uint64_t buf = 0; while (-1 == write(pipe_fd[1], &buf, sizeof(buf))) diff --git a/src/DataStreams/RemoteQueryExecutorReadContext.h b/src/DataStreams/RemoteQueryExecutorReadContext.h index cb6421f78d0..5fbe52469cd 100644 --- a/src/DataStreams/RemoteQueryExecutorReadContext.h +++ b/src/DataStreams/RemoteQueryExecutorReadContext.h @@ -54,8 +54,8 @@ public: explicit RemoteQueryExecutorReadContext(IConnections & connections_); ~RemoteQueryExecutorReadContext(); - bool checkTimeout() const; - bool checkTimeoutImpl() const; + bool checkTimeout(bool blocking = false) const; + bool checkTimeoutImpl(bool blocking) const; void setConnectionFD(int fd, const Poco::Timespan & timeout = 0, const std::string & fd_description = ""); void setTimer() const; From e4b4665ff556f7da153dd0bfb8afe210e1fc7d24 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 12 Mar 2021 13:58:03 +0300 Subject: [PATCH 091/233] better --- programs/client/Client.cpp | 8 ++++---- .../Formats/Impl/ParallelFormattingOutputFormat.h | 9 +++++++++ src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp | 2 ++ .../00416_pocopatch_progress_in_http_headers.sh | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 3c27908741c..c878a3071c4 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -2096,10 +2096,10 @@ private: current_format = "Vertical"; /// It is not clear how to write progress with parallel formatting. It may increase code complexity significantly. - if (!need_render_progress) - block_out_stream = context.getOutputStreamParallelIfPossible(current_format, *out_buf, block); - else - block_out_stream = context.getOutputStream(current_format, *out_buf, block); + // if (!need_render_progress) + // block_out_stream = context.getOutputStreamParallelIfPossible(current_format, *out_buf, block); + // else + block_out_stream = context.getOutputStream(current_format, *out_buf, block); block_out_stream->writePrefix(); } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 2efc369e178..3fcd1f0aadf 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -6,6 +6,9 @@ #include #include #include +#include "IO/ReadBuffer.h" +#include "IO/ReadBufferFromString.h" +#include "IO/WriteBufferFromString.h" #include #include #include @@ -104,6 +107,12 @@ public: /// There are no formats which support parallel formatting and progress writing at the same time void onProgress(const Progress &) override {} + String getContentType() const override + { + WriteBufferFromOwnString buffer; + return internal_formatter_creator(buffer)->getContentType(); + } + protected: void consume(Chunk chunk) override final { diff --git a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp index 355af038da9..7ded716b34e 100644 --- a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp +++ b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp @@ -149,6 +149,8 @@ void WriteBufferFromHTTPServerResponse::onProgress(const Progress & progress) { std::lock_guard lock(mutex); + std::cout << StackTrace().toString() << std::endl; + /// Cannot add new headers if body was started to send. if (headers_finished_sending) return; diff --git a/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh b/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh index 5d9cd12e4bf..6e9814cbca8 100755 --- a/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh +++ b/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&max_block_size=5&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT max(number) FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' # This test will fail with external poco (progress not supported) -${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' +${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&output_format_parallel_formatting=0" -d 'SELECT number FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&enable_http_compression=1" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d # 'send_progress_in_http_headers' is false by default From 8b27da2b8168ba5369b8ae530019282b2c06d223 Mon Sep 17 00:00:00 2001 From: George Date: Fri, 12 Mar 2021 19:00:41 +0300 Subject: [PATCH 092/233] first draft --- .../external-authenticators/ldap.md | 71 ++++---- .../external-authenticators/ldap.md | 154 ++++++++++++++++++ 2 files changed, 184 insertions(+), 41 deletions(-) create mode 100644 docs/ru/operations/external-authenticators/ldap.md diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index 36a13227852..523a4ff2993 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -35,30 +35,27 @@ To define LDAP server you must add `ldap_servers` section to the `config.xml`. F Note, that you can define multiple LDAP servers inside the `ldap_servers` section using distinct names. -Parameters: +**Parameters** -- `host` - LDAP server hostname or IP, this parameter is mandatory and cannot be empty. -- `port` - LDAP server port, default is `636` if `enable_tls` is set to `true`, `389` otherwise. -- `bind_dn` - template used to construct the DN to bind to. - - The resulting DN will be constructed by replacing all `{user_name}` substrings of the - template with the actual user name during each authentication attempt. -- `verification_cooldown` - a period of time, in seconds, after a successful bind attempt, - during which the user will be assumed to be successfully authenticated for all consecutive - requests without contacting the LDAP server. +- `host` — LDAP server hostname or IP, this parameter is mandatory and cannot be empty. +- `port` — LDAP server port, default is `636` if `enable_tls` is set to `true`, `389` otherwise. +- `bind_dn` — template used to construct the DN to bind to. + - The resulting DN will be constructed by replacing all `{user_name}` substrings of the template with the actual user name during each authentication attempt. +- `verification_cooldown` — a period of time, in seconds, after a successful bind attempt, during which the user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server. - Specify `0` (the default) to disable caching and force contacting the LDAP server for each authentication request. -- `enable_tls` - flag to trigger use of secure connection to the LDAP server. +- `enable_tls` — flag to trigger use of secure connection to the LDAP server. - Specify `no` for plain text `ldap://` protocol (not recommended). - Specify `yes` for LDAP over SSL/TLS `ldaps://` protocol (recommended, the default). - Specify `starttls` for legacy StartTLS protocol (plain text `ldap://` protocol, upgraded to TLS). -- `tls_minimum_protocol_version` - the minimum protocol version of SSL/TLS. +- `tls_minimum_protocol_version` — the minimum protocol version of SSL/TLS. - Accepted values are: `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, `tls1.2` (the default). -- `tls_require_cert` - SSL/TLS peer certificate verification behavior. +- `tls_require_cert` — SSL/TLS peer certificate verification behavior. - Accepted values are: `never`, `allow`, `try`, `demand` (the default). -- `tls_cert_file` - path to certificate file. -- `tls_key_file` - path to certificate key file. -- `tls_ca_cert_file` - path to CA certificate file. -- `tls_ca_cert_dir` - path to the directory containing CA certificates. -- `tls_cipher_suite` - allowed cipher suite (in OpenSSL notation). +- `tls_cert_file` — path to certificate file. +- `tls_key_file` — path to certificate key file. +- `tls_ca_cert_file` — path to CA certificate file. +- `tls_ca_cert_dir` — path to the directory containing CA certificates. +- `tls_cipher_suite` — allowed cipher suite (in OpenSSL notation). ## LDAP External Authenticator {#ldap-external-authenticator} @@ -87,9 +84,10 @@ Note, that user `my_user` refers to `my_ldap_server`. This LDAP server must be c When SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled in ClickHouse, users that are authenticated by LDAP servers can also be created using the [CRATE USER](../../sql-reference/statements/create/user.md#create-user-statement) statement. +Query: ```sql -CREATE USER my_user IDENTIFIED WITH ldap_server BY 'my_ldap_server' +CREATE USER my_user IDENTIFIED WITH ldap_server BY 'my_ldap_server'; ``` ## LDAP Exernal User Directory {#ldap-external-user-directory} @@ -123,34 +121,25 @@ Example (goes into `config.xml`): ``` -Note that `my_ldap_server` referred in the `ldap` section inside the `user_directories` section must be a previously -defined LDAP server that is configured in the `config.xml` (see [LDAP Server Definition](#ldap-server-definition)). +Note that `my_ldap_server` referred in the `ldap` section inside the `user_directories` section must be a previously defined LDAP server that is configured in the `config.xml` (see [LDAP Server Definition](#ldap-server-definition)). Parameters: -- `server` - one of LDAP server names defined in the `ldap_servers` config section above. +- `server` — one of LDAP server names defined in the `ldap_servers` config section above. This parameter is mandatory and cannot be empty. -- `roles` - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. - - If no roles are specified here or assigned during role mapping (below), user will not be able - to perform any actions after authentication. -- `role_mapping` - section with LDAP search parameters and mapping rules. - - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` - and the name of the logged in user. For each entry found during that search, the value of the specified - attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, - and the rest of the value becomes the name of a local role defined in ClickHouse, - which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. +- `roles` — section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. + - If no roles are specified here or assigned during role mapping (below), user will not be able to perform any actions after authentication. +- `role_mapping` — section with LDAP search parameters and mapping rules. + - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` and the name of the logged in user. For each entry found during that search, the value of the specified attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. - There can be multiple `role_mapping` sections defined inside the same `ldap` section. All of them will be applied. - - `base_dn` - template used to construct the base DN for the LDAP search. - - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` - substrings of the template with the actual user name and bind DN during each LDAP search. - - `scope` - scope of the LDAP search. + - `base_dn` — template used to construct the base DN for the LDAP search. + - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during each LDAP search. + - `scope` — scope of the LDAP search. - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default). - - `search_filter` - template used to construct the search filter for the LDAP search. - - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` - substrings of the template with the actual user name, bind DN, and base DN during each LDAP search. + - `search_filter` — template used to construct the search filter for the LDAP search. + - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, and base DN during each LDAP search. - Note, that the special characters must be escaped properly in XML. - - `attribute` - attribute name whose values will be returned by the LDAP search. - - `prefix` - prefix, that will be expected to be in front of each string in the original - list of strings returned by the LDAP search. Prefix will be removed from the original - strings and resulting strings will be treated as local role names. Empty, by default. + - `attribute` — attribute name whose values will be returned by the LDAP search. + - `prefix` — prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated as local role names. Empty, by default. +[Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap.md) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md new file mode 100644 index 00000000000..fe364c69f05 --- /dev/null +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -0,0 +1,154 @@ +# LDAP {#external-authenticators-ldap} + +Для аутентификации пользователей ClickHouse можно использовать сервер LDAP. Для этого есть два разных подхода: + +- использовать LDAP как внешний аутентификатор для существующих пользователей, которые определены в `users.xml` или в локальных путях управления контролем +- использовать LDAP как внешний пользовательский каталог и разрешить аутентификацию локально неопределенных пользователей, если они есть на LDAP сервере + +Для этих обоих подходов необходимо определить в ClickHouse конфиге внутренне названный LDAP сервер, чтобы другие части конфига могли ссылаться на него. + +## Определение LDAP сервера {#ldap-server-definition} + +Чтобы определить LDAP сервер, необходимо добавить секцию `ldap_servers` в `config.xml`. Например: + +```xml + + + + + localhost + 636 + uid={user_name},ou=users,dc=example,dc=com + 300 + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + + +``` + +Обратите внимание, что можно определить несколько LDAP серверов внутри секции `ldap_servers` используя различные имена. + +**Параметры** + +- `host` — LDAP server hostname or IP, this parameter is mandatory and cannot be empty. имя хоста сервера LDAP или его IP. Этот параметр обязательный и не может быть пустым. +- `port` — порт сервера LDAP. По-умолчанию: при значение `true` настройки `enable_tls` — `636`, иначе `389`. +- `bind_dn` — шаблон для создания DN для привязки. + - конечный DN будет создан заменой всех подстрок `{user_name}` шаблона на настоящее имя пользователя при каждой попытке аутентификации. +- `verification_cooldown` — a period of time, in seconds, after a successful bind attempt, during which the user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server. + - Specify `0` (the default) to disable caching and force contacting the LDAP server for each authentication request. +- `enable_tls` — flag to trigger use of secure connection to the LDAP server. + - Specify `no` for plain text `ldap://` protocol (not recommended). + - Specify `yes` for LDAP over SSL/TLS `ldaps://` protocol (recommended, the default). + - Specify `starttls` for legacy StartTLS protocol (plain text `ldap://` protocol, upgraded to TLS). +- `tls_minimum_protocol_version` — the minimum protocol version of SSL/TLS. + - Accepted values are: `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, `tls1.2` (the default). +- `tls_require_cert` — SSL/TLS peer certificate verification behavior. + - Accepted values are: `never`, `allow`, `try`, `demand` (the default). +- `tls_cert_file` — path to certificate file. +- `tls_key_file` — path to certificate key file. +- `tls_ca_cert_file` — path to CA certificate file. +- `tls_ca_cert_dir` — path to the directory containing CA certificates. +- `tls_cipher_suite` — allowed cipher suite (in OpenSSL notation). + +## LDAP External Authenticator {#ldap-external-authenticator} + +A remote LDAP server can be used as a method for verifying passwords for locally defined users (users defined in `users.xml` or in local access control paths). In order to achieve this, specify previously defined LDAP server name instead of `password` or similar sections in the user definition. + +At each login attempt, ClickHouse will try to "bind" to the specified DN defined by the `bind_dn` parameter in the [LDAP server definition](#ldap-server-definition) using the provided credentials, and if successful, the user will be considered authenticated. This is often called a "simple bind" method. + +For example, + +```xml + + + + + + + + my_ldap_server + + + + +``` + +Note, that user `my_user` refers to `my_ldap_server`. This LDAP server must be configured in the main `config.xml` file as described previously. + +When SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled in ClickHouse, users that are authenticated by LDAP servers can also be created using the [CRATE USER](../../sql-reference/statements/create/user.md#create-user-statement) statement. + + +```sql +CREATE USER my_user IDENTIFIED WITH ldap_server BY 'my_ldap_server' +``` + +## LDAP Exernal User Directory {#ldap-external-user-directory} + +In addition to the locally defined users, a remote LDAP server can be used as a source of user definitions. In order to achieve this, specify previously defined LDAP server name (see [LDAP Server Definition](#ldap-server-definition)) in the `ldap` section inside the `users_directories` section of the `config.xml` file. + +At each login attempt, ClickHouse will try to find the user definition locally and authenticate it as usual, but if the user is not defined, ClickHouse will assume it exists in the external LDAP directory, and will try to "bind" to the specified DN at the LDAP server using the provided credentials. If successful, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. Additionally, LDAP "search" can be performed and results can be transformed and treated as role names and then be assigned to the user if the `role_mapping` section is also configured. All this implies that the SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled and roles are created using the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. + +Example (goes into `config.xml`): + +```xml + + + + + + my_ldap_server + + + + + + ou=groups,dc=example,dc=com + subtree + (&(objectClass=groupOfNames)(member={bind_dn})) + cn + clickhouse_ + + + + +``` + +Note that `my_ldap_server` referred in the `ldap` section inside the `user_directories` section must be a previously +defined LDAP server that is configured in the `config.xml` (see [LDAP Server Definition](#ldap-server-definition)). + +Parameters: + +- `server` - one of LDAP server names defined in the `ldap_servers` config section above. + This parameter is mandatory and cannot be empty. +- `roles` - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. + - If no roles are specified here or assigned during role mapping (below), user will not be able + to perform any actions after authentication. +- `role_mapping` - section with LDAP search parameters and mapping rules. + - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` + and the name of the logged in user. For each entry found during that search, the value of the specified + attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, + and the rest of the value becomes the name of a local role defined in ClickHouse, + which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. + - There can be multiple `role_mapping` sections defined inside the same `ldap` section. All of them will be applied. + - `base_dn` - template used to construct the base DN for the LDAP search. + - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` + substrings of the template with the actual user name and bind DN during each LDAP search. + - `scope` - scope of the LDAP search. + - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default). + - `search_filter` - template used to construct the search filter for the LDAP search. + - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` + substrings of the template with the actual user name, bind DN, and base DN during each LDAP search. + - Note, that the special characters must be escaped properly in XML. + - `attribute` - attribute name whose values will be returned by the LDAP search. + - `prefix` - prefix, that will be expected to be in front of each string in the original + list of strings returned by the LDAP search. Prefix will be removed from the original + strings and resulting strings will be treated as local role names. Empty, by default. + +[Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap.md) From b3f8ac796c24dfebb0eacfbb0c51d0c0ca094750 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 12 Mar 2021 19:09:55 +0300 Subject: [PATCH 093/233] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index f3e2f88ffc3..6da49c4d3d4 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -5,7 +5,7 @@ toc_title: DETACH # DETACH Statement {#detach} -Deletes information about the table or view from the server. The server stops knowing about their existence. +Deletes information about the table or materialized view from the server. The server stops knowing about their existence. Syntax: From ed87521fcfc3c6a8bc81eb105d0a2e1e9167a7f2 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 12 Mar 2021 19:10:01 +0300 Subject: [PATCH 094/233] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 6da49c4d3d4..47490f3e0f9 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -13,7 +13,7 @@ Syntax: DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] ``` -Detaching does not delete the data or metadata for the table or view. If the table or view was not detached `PERMANENTLY`, on the next server launch the server will read the metadata and recall the table/view again. If the table or view was detached `PERMANENTLY`, there will be no automatic recall. +Detaching does not delete the data or metadata for the table or materialized view. If the table or view was not detached `PERMANENTLY`, on the next server launch the server will read the metadata and recall the table/view again. If the table or view was detached `PERMANENTLY`, there will be no automatic recall. Whether the table was detached permanently or not, in both cases you can reattach it using the [ATTACH](../../sql-reference/statements/attach.md). System log tables can be also attached back (e.g. `query_log`, `text_log`, etc). Other system tables can't be reattached. On the next server launch the server will recall those tables again. From ca5e20a57dc576cb1a03d5a952b76732ed6c6309 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 12 Mar 2021 19:11:27 +0300 Subject: [PATCH 095/233] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 47490f3e0f9..ae97d716a5e 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -25,25 +25,10 @@ Also you can not [DROP](../../sql-reference/statements/drop.md#drop-table) the d **Example** -Query: +Creating a table: ``` sql CREATE TABLE test ENGINE = Log AS SELECT * FROM numbers(10); - -DETACH TABLE test; - -SELECT * FROM TEST; -``` - -Result: - -``` text -Ok. - -Ok. - -Received exception from server (version 21.3.1): -Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.TEST doesn't exist. -``` +SELECT * FROM test; [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From 53fd7cb8a840719af7a4316b216d931a5a1806b3 Mon Sep 17 00:00:00 2001 From: George Date: Fri, 12 Mar 2021 19:14:40 +0300 Subject: [PATCH 096/233] Updated example --- docs/en/sql-reference/statements/detach.md | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index ae97d716a5e..cb0d7cf7b66 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -27,8 +27,44 @@ Also you can not [DROP](../../sql-reference/statements/drop.md#drop-table) the d Creating a table: +Query: + ``` sql CREATE TABLE test ENGINE = Log AS SELECT * FROM numbers(10); SELECT * FROM test; +``` + +Result: + +``` text +┌─number─┐ +│ 0 │ +│ 1 │ +│ 2 │ +│ 3 │ +│ 4 │ +│ 5 │ +│ 6 │ +│ 7 │ +│ 8 │ +│ 9 │ +└────────┘ +``` + +Detaching the table: + +Query: + +``` sql +DETACH TABLE test; +SELECT * FROM test; +``` + +Result: + +``` text +Received exception from server (version 21.4.1): +Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.test doesn't exist. +``` [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From f89247d7bd6d790d5d59679add762b082e905329 Mon Sep 17 00:00:00 2001 From: George Date: Fri, 12 Mar 2021 19:28:04 +0300 Subject: [PATCH 097/233] updated attach.md --- docs/en/sql-reference/statements/attach.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/attach.md b/docs/en/sql-reference/statements/attach.md index 035441ef5f1..ffb577a8839 100644 --- a/docs/en/sql-reference/statements/attach.md +++ b/docs/en/sql-reference/statements/attach.md @@ -17,4 +17,8 @@ If the table was previously detached ([DETACH](../../sql-reference/statements/de ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] ``` -This query is used when starting the server. The server stores table metadata as files with `ATTACH` queries, which it simply runs at launch (with the exception of system tables, which are explicitly created on the server). +This query is used when starting the server. The server stores table metadata as files with `ATTACH` queries, which it simply runs at launch (with the exception of some system tables, which are explicitly created on the server). + +If the table was detached permanently, it won't be reattached at the server start, so you need to use `ATTACH` query explicitly. + +[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/attach/) From 34915d031145cfab1ca6f5f365c59b4e3acc77f3 Mon Sep 17 00:00:00 2001 From: Michael Monashev Date: Fri, 12 Mar 2021 21:13:20 +0300 Subject: [PATCH 098/233] Fix ORDER BY syntax --- docs/ru/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index bc74b2592b9..0615613533b 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -517,7 +517,7 @@ CREATE TABLE table_for_aggregation y Int ) ENGINE = MergeTree -ORDER BY k1, k2 +ORDER BY (k1, k2) TTL d + INTERVAL 1 MONTH GROUP BY k1, k2 SET x = max(x), y = min(y); ``` From 508953ca006f29e82d8428edd540492f165b3c96 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 12 Mar 2021 23:51:53 +0300 Subject: [PATCH 099/233] more debug info --- programs/client/Client.cpp | 47 +++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index c5b579f2046..da99541a3e7 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1255,6 +1255,29 @@ private: return true; } + // Prints changed settings to stderr. Useful for debugging fuzzing failures. + void printChangedSettings() const + { + const auto & changes = context.getSettingsRef().changes(); + if (!changes.empty()) + { + fmt::print(stderr, "Changed settings: "); + for (size_t i = 0; i < changes.size(); ++i) + { + if (i) + { + fmt::print(stderr, ", "); + } + fmt::print(stderr, "{} = '{}'", changes[i].name, + toString(changes[i].value)); + } + fmt::print(stderr, "\n"); + } + else + { + fmt::print(stderr, "No changed settings.\n"); + } + } /// Returns false when server is not available. bool processWithFuzzing(const String & text) @@ -1323,6 +1346,8 @@ private: // child elements. if (base_before_fuzz != base_after_fuzz) { + printChangedSettings(); + fmt::print(stderr, "Base before fuzz: {}\n" "Base after fuzz: {}\n", @@ -1388,6 +1413,8 @@ private: if (formatted_twice != fuzzed_text) { + printChangedSettings(); + fmt::print(stderr, "The query formatting is broken. Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n", formatted_twice, fuzzed_text); fmt::print(stderr, "AST parsed back:\n'{}'\nSource AST:\n'{}'\n", @@ -1433,25 +1460,7 @@ private: // Print the changed settings because they might be needed to // reproduce the error. - const auto & changes = context.getSettingsRef().changes(); - if (!changes.empty()) - { - fmt::print(stderr, "Changed settings: "); - for (size_t i = 0; i < changes.size(); ++i) - { - if (i) - { - fmt::print(stderr, ", "); - } - fmt::print(stderr, "{} = '{}'", changes[i].name, - toString(changes[i].value)); - } - fmt::print(stderr, "\n"); - } - else - { - fmt::print(stderr, "No changed settings.\n"); - } + printChangedSettings(); return false; } From 57d1dc7a6cdc6f87c9f64f91debf47355c6a0e32 Mon Sep 17 00:00:00 2001 From: George Date: Sun, 14 Mar 2021 00:33:55 +0300 Subject: [PATCH 100/233] Work-in-progress --- .../external-authenticators/ldap.md | 55 ++++----- .../external-authenticators/ldap.md | 108 +++++++++--------- 2 files changed, 79 insertions(+), 84 deletions(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index 523a4ff2993..cb8aa07dc41 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -2,10 +2,10 @@ LDAP server can be used to authenticate ClickHouse users. There are two different approaches for doing this: -- use LDAP as an external authenticator for existing users, which are defined in `users.xml` or in local access control paths -- use LDAP as an external user directory and allow locally undefined users to be authenticated if they exist on the LDAP server +- Use LDAP as an external authenticator for existing users, which are defined in `users.xml` or in local access control paths. +- Use LDAP as an external user directory and allow locally undefined users to be authenticated if they exist on the LDAP server. -For both of these approaches, an internally named LDAP server must be defined in the ClickHouse config so that other parts of config are able to refer to it. +For both of these approaches, an internally named LDAP server must be defined in the ClickHouse config so that other parts of the config can refer to it. ## LDAP Server Definition {#ldap-server-definition} @@ -43,27 +43,27 @@ Note, that you can define multiple LDAP servers inside the `ldap_servers` sectio - The resulting DN will be constructed by replacing all `{user_name}` substrings of the template with the actual user name during each authentication attempt. - `verification_cooldown` — a period of time, in seconds, after a successful bind attempt, during which the user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server. - Specify `0` (the default) to disable caching and force contacting the LDAP server for each authentication request. -- `enable_tls` — flag to trigger use of secure connection to the LDAP server. +- `enable_tls` — a flag to trigger the use of the secure connection to the LDAP server. - Specify `no` for plain text `ldap://` protocol (not recommended). - Specify `yes` for LDAP over SSL/TLS `ldaps://` protocol (recommended, the default). - Specify `starttls` for legacy StartTLS protocol (plain text `ldap://` protocol, upgraded to TLS). -- `tls_minimum_protocol_version` — the minimum protocol version of SSL/TLS. +- `tls_minimum_protocol_version` — The minimum protocol version of SSL/TLS. - Accepted values are: `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, `tls1.2` (the default). - `tls_require_cert` — SSL/TLS peer certificate verification behavior. - Accepted values are: `never`, `allow`, `try`, `demand` (the default). -- `tls_cert_file` — path to certificate file. -- `tls_key_file` — path to certificate key file. -- `tls_ca_cert_file` — path to CA certificate file. -- `tls_ca_cert_dir` — path to the directory containing CA certificates. -- `tls_cipher_suite` — allowed cipher suite (in OpenSSL notation). +- `tls_cert_file` — Path to certificate file. +- `tls_key_file` — Path to certificate key file. +- `tls_ca_cert_file` — Path to CA certificate file. +- `tls_ca_cert_dir` — Path to the directory containing CA certificates. +- `tls_cipher_suite` — Allowed cipher suite (in OpenSSL notation). ## LDAP External Authenticator {#ldap-external-authenticator} -A remote LDAP server can be used as a method for verifying passwords for locally defined users (users defined in `users.xml` or in local access control paths). In order to achieve this, specify previously defined LDAP server name instead of `password` or similar sections in the user definition. +A remote LDAP server can be used as a method for verifying passwords for locally defined users (users defined in `users.xml` or in local access control paths). To achieve this, specify previously defined LDAP server name instead of `password` or similar sections in the user definition. -At each login attempt, ClickHouse will try to "bind" to the specified DN defined by the `bind_dn` parameter in the [LDAP server definition](#ldap-server-definition) using the provided credentials, and if successful, the user will be considered authenticated. This is often called a "simple bind" method. +At each login attempt, ClickHouse tries to "bind" to the specified DN defined by the `bind_dn` parameter in the [LDAP server definition](#ldap-server-definition) using the provided credentials, and if successful, the user is considered authenticated. This is often called a "simple bind" method. -For example, +**Example** ```xml @@ -82,7 +82,7 @@ For example, Note, that user `my_user` refers to `my_ldap_server`. This LDAP server must be configured in the main `config.xml` file as described previously. -When SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled in ClickHouse, users that are authenticated by LDAP servers can also be created using the [CRATE USER](../../sql-reference/statements/create/user.md#create-user-statement) statement. +When SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled, users that are authenticated by LDAP servers can also be created using the [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement) statement. Query: @@ -92,11 +92,13 @@ CREATE USER my_user IDENTIFIED WITH ldap_server BY 'my_ldap_server'; ## LDAP Exernal User Directory {#ldap-external-user-directory} -In addition to the locally defined users, a remote LDAP server can be used as a source of user definitions. In order to achieve this, specify previously defined LDAP server name (see [LDAP Server Definition](#ldap-server-definition)) in the `ldap` section inside the `users_directories` section of the `config.xml` file. +In addition to the locally defined users, a remote LDAP server can be used as a source of user definitions. To achieve this, specify previously defined LDAP server name (see [LDAP Server Definition](#ldap-server-definition)) in the `ldap` section inside the `users_directories` section of the `config.xml` file. -At each login attempt, ClickHouse will try to find the user definition locally and authenticate it as usual, but if the user is not defined, ClickHouse will assume it exists in the external LDAP directory, and will try to "bind" to the specified DN at the LDAP server using the provided credentials. If successful, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. Additionally, LDAP "search" can be performed and results can be transformed and treated as role names and then be assigned to the user if the `role_mapping` section is also configured. All this implies that the SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled and roles are created using the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. +At each login attempt, ClickHouse tries to find the user definition locally and authenticate it as usual. If the user is not defined, ClickHouse will assume the definition exists in the external LDAP directory and will try to "bind" to the specified DN at the LDAP server using the provided credentials. If successful, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. Additionally, LDAP "search" can be performed and results can be transformed and treated as role names and then be assigned to the user if the `role_mapping` section is also configured. All this implies that the SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled and roles are created using the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. -Example (goes into `config.xml`): +**Example** + +Goes into `config.xml`. ```xml @@ -123,23 +125,22 @@ Example (goes into `config.xml`): Note that `my_ldap_server` referred in the `ldap` section inside the `user_directories` section must be a previously defined LDAP server that is configured in the `config.xml` (see [LDAP Server Definition](#ldap-server-definition)). -Parameters: +**Parameters** -- `server` — one of LDAP server names defined in the `ldap_servers` config section above. - This parameter is mandatory and cannot be empty. -- `roles` — section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. +- `server` — One of LDAP server names defined in the `ldap_servers` config section above. This parameter is mandatory and cannot be empty. Одно из имен +- `roles` — Section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. - If no roles are specified here or assigned during role mapping (below), user will not be able to perform any actions after authentication. -- `role_mapping` — section with LDAP search parameters and mapping rules. +- `role_mapping` — Section with LDAP search parameters and mapping rules. - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` and the name of the logged in user. For each entry found during that search, the value of the specified attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. - There can be multiple `role_mapping` sections defined inside the same `ldap` section. All of them will be applied. - - `base_dn` — template used to construct the base DN for the LDAP search. + - `base_dn` — Template used to construct the base DN for the LDAP search. - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during each LDAP search. - - `scope` — scope of the LDAP search. + - `scope` — Scope of the LDAP search. - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default). - - `search_filter` — template used to construct the search filter for the LDAP search. + - `search_filter` — Template used to construct the search filter for the LDAP search. - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, and base DN during each LDAP search. - Note, that the special characters must be escaped properly in XML. - - `attribute` — attribute name whose values will be returned by the LDAP search. - - `prefix` — prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated as local role names. Empty, by default. + - `attribute` — Attribute name whose values will be returned by the LDAP search. + - `prefix` — Prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated as local role names. Empty, by default. [Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap.md) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index fe364c69f05..3d71ec1eba3 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -1,11 +1,11 @@ # LDAP {#external-authenticators-ldap} -Для аутентификации пользователей ClickHouse можно использовать сервер LDAP. Для этого есть два разных подхода: +Для аутентификации пользователей ClickHouse можно использовать сервер LDAP. Можно использовать два подхода: -- использовать LDAP как внешний аутентификатор для существующих пользователей, которые определены в `users.xml` или в локальных путях управления контролем -- использовать LDAP как внешний пользовательский каталог и разрешить аутентификацию локально неопределенных пользователей, если они есть на LDAP сервере +- Использовать LDAP как внешний аутентификатор для существующих пользователей, которые определены в `users.xml` или в локальных путях управления контролем. +- Использовать LDAP как внешний пользовательский каталог и разрешить аутентификацию локально неопределенных пользователей, если они есть на LDAP сервере. -Для этих обоих подходов необходимо определить в ClickHouse конфиге внутренне названный LDAP сервер, чтобы другие части конфига могли ссылаться на него. +Для обоих подходов необходимо определить в конфиге ClickHouse внутренне названный LDAP сервер, чтобы другие части конфига могли ссылаться на него. ## Определение LDAP сервера {#ldap-server-definition} @@ -37,33 +37,33 @@ **Параметры** -- `host` — LDAP server hostname or IP, this parameter is mandatory and cannot be empty. имя хоста сервера LDAP или его IP. Этот параметр обязательный и не может быть пустым. -- `port` — порт сервера LDAP. По-умолчанию: при значение `true` настройки `enable_tls` — `636`, иначе `389`. +- `host` — имя хоста сервера LDAP или его IP. Этот параметр обязательный и не может быть пустым. +- `port` — порт сервера LDAP. По-умолчанию: при значении `true` настройки `enable_tls` — `636`, иначе `389`. - `bind_dn` — шаблон для создания DN для привязки. - - конечный DN будет создан заменой всех подстрок `{user_name}` шаблона на настоящее имя пользователя при каждой попытке аутентификации. -- `verification_cooldown` — a period of time, in seconds, after a successful bind attempt, during which the user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server. - - Specify `0` (the default) to disable caching and force contacting the LDAP server for each authentication request. -- `enable_tls` — flag to trigger use of secure connection to the LDAP server. - - Specify `no` for plain text `ldap://` protocol (not recommended). - - Specify `yes` for LDAP over SSL/TLS `ldaps://` protocol (recommended, the default). - - Specify `starttls` for legacy StartTLS protocol (plain text `ldap://` protocol, upgraded to TLS). -- `tls_minimum_protocol_version` — the minimum protocol version of SSL/TLS. - - Accepted values are: `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, `tls1.2` (the default). -- `tls_require_cert` — SSL/TLS peer certificate verification behavior. - - Accepted values are: `never`, `allow`, `try`, `demand` (the default). -- `tls_cert_file` — path to certificate file. -- `tls_key_file` — path to certificate key file. -- `tls_ca_cert_file` — path to CA certificate file. -- `tls_ca_cert_dir` — path to the directory containing CA certificates. -- `tls_cipher_suite` — allowed cipher suite (in OpenSSL notation). + - конечный DN будет создан заменой всех подстрок `{user_name}` шаблона на фактическое имя пользователя при каждой попытке аутентификации. +- `verification_cooldown` — промежуток времени (в секундах) после успешной попытки привязки, в течение которого пользователь будет считаться успешно аутентифицированным без с сервером LDAP для всех последующих запросов. + - Укажите `0` (по-умолчанию), чтобы отключить кеширования и заставить связываться с сервером LDAP для каждого запроса аутетификации. +- `enable_tls` — флаг, включающий использование защищенного соединения с сервером LDAP. + - Укажите `no` для текстового `ldap://` протокола (не рекомендовано). + - Укажите `yes` для LDAP через SSL/TLS `ldaps://` протокола (рекомендовано, используется по-умолчанию). + - Укажите `starttls` для устаревшего StartTLS протокола (текстовый `ldap://` протокол, модернизированный до TLS). +- `tls_minimum_protocol_version` — минимальная версия протокола SSL/TLS. + - Принимаемые значения: `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, `tls1.2` (по-умолчанию). +- `tls_require_cert` — поведение при проверке сертификата SSL/TLS. + - Принимаемые значения: `never`, `allow`, `try`, `demand` (по-умолчанию). +- `tls_cert_file` — путь до файла сертификата. +- `tls_key_file` — путь к файлу ключа сертификата. +- `tls_ca_cert_file` — путь к файлу ЦС сертификата. +- `tls_ca_cert_dir` — путь к каталогу, содержащая сертификаты ЦС. +- `tls_cipher_suite` — разрешить набор шифров (в нотации OpenSSL). -## LDAP External Authenticator {#ldap-external-authenticator} +## LDAP внешний аутентификатор {#ldap-external-authenticator} -A remote LDAP server can be used as a method for verifying passwords for locally defined users (users defined in `users.xml` or in local access control paths). In order to achieve this, specify previously defined LDAP server name instead of `password` or similar sections in the user definition. +Удаленный сервер LDAP можно использовать как метод верификации паролей локально определенных пользователей (пользователей, которые определены в `users.xml` или в локальных путях управления контролем). Для этого укажите имя определенного до этого сервера LDAP вместо `password` или другой похожей секции в определении пользователя. -At each login attempt, ClickHouse will try to "bind" to the specified DN defined by the `bind_dn` parameter in the [LDAP server definition](#ldap-server-definition) using the provided credentials, and if successful, the user will be considered authenticated. This is often called a "simple bind" method. +При каждой попытке авторизации, ClickHouse пытается "привязаться" к DN, указанному в [определение LDAP сервера](#ldap-server-definition) параметром `bind_dn`, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается аутентифицированным. Обычно это называют методом "простой привязки". -For example, +**Например** ```xml @@ -80,22 +80,25 @@ For example, ``` -Note, that user `my_user` refers to `my_ldap_server`. This LDAP server must be configured in the main `config.xml` file as described previously. +Обратите внимание, что пользователь `my_user` ссылается на `my_ldap_server`. Этот LDAP сервер должен быть настроен в основном файле `config.xml`, как это было описано ранее. -When SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled in ClickHouse, users that are authenticated by LDAP servers can also be created using the [CRATE USER](../../sql-reference/statements/create/user.md#create-user-statement) statement. +При включенном SQL-ориентированным [Управлением доступом](../access-rights.md#access-control) пользователи, аутентифицированные LDAP серверами, могут также быть созданы выражением [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement). +Запрос: ```sql -CREATE USER my_user IDENTIFIED WITH ldap_server BY 'my_ldap_server' +CREATE USER my_user IDENTIFIED WITH ldap_server BY 'my_ldap_server'; ``` -## LDAP Exernal User Directory {#ldap-external-user-directory} +## Внешний пользовательский каталог LDAP {#ldap-external-user-directory} -In addition to the locally defined users, a remote LDAP server can be used as a source of user definitions. In order to achieve this, specify previously defined LDAP server name (see [LDAP Server Definition](#ldap-server-definition)) in the `ldap` section inside the `users_directories` section of the `config.xml` file. +В добавок к локально определенным пользователям, удаленный LDAP сервер может быть использован как источник определения пользователей. Для этого укажите имя определенного до этого сервера LDAP (см. [Определение LDAP сервера](#ldap-server-definition)) в секции `ldap` внутри секции `users_directories` файла `config.xml`. -At each login attempt, ClickHouse will try to find the user definition locally and authenticate it as usual, but if the user is not defined, ClickHouse will assume it exists in the external LDAP directory, and will try to "bind" to the specified DN at the LDAP server using the provided credentials. If successful, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. Additionally, LDAP "search" can be performed and results can be transformed and treated as role names and then be assigned to the user if the `role_mapping` section is also configured. All this implies that the SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled and roles are created using the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. +При каждой попытке авторизации, ClicHouse пытается локально найти определение пользователя и авторизовать его как обычно. Если определение не будет найдено, ClickHouse предполагает, что оно находится во внешнем LDAP каталоге, и попытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю будут присвоены роли из списка, указанного в секции `roles`. Кроме того, может быть выполнен LDAP поиск, а его результаты могут быть преобразованы в имена ролей и присвоены пользователям, если была настроена секция `role_mapping`. Все это работает при условии, что SQL-ориентированное [Управлением доступом](../access-rights.md#access-control) включено, а роли созданы выражением[CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). -Example (goes into `config.xml`): +**Пример** + +В `config.xml`. ```xml @@ -120,35 +123,26 @@ Example (goes into `config.xml`): ``` -Note that `my_ldap_server` referred in the `ldap` section inside the `user_directories` section must be a previously -defined LDAP server that is configured in the `config.xml` (see [LDAP Server Definition](#ldap-server-definition)). +Обратите внимание, что `my_ldap_server`, указанный в секции `ldap` внутри секции `user_directories`, должен быть настроен в файле `config.xml`, как это было описано ранее. (см. [Определение LDAP сервера](#ldap-server-definition)). -Parameters: +**Параметры** -- `server` - one of LDAP server names defined in the `ldap_servers` config section above. +- `server` — One of LDAP server names defined in the `ldap_servers` config section above. This parameter is mandatory and cannot be empty. -- `roles` - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. - - If no roles are specified here or assigned during role mapping (below), user will not be able - to perform any actions after authentication. -- `role_mapping` - section with LDAP search parameters and mapping rules. - - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` - and the name of the logged in user. For each entry found during that search, the value of the specified - attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, - and the rest of the value becomes the name of a local role defined in ClickHouse, - which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. +- `roles` — Section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. + - If no roles are specified here or assigned during role mapping (below), user will not be able to perform any actions after authentication. +- `role_mapping` — Section with LDAP search parameters and mapping rules. + - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` and the name of the logged in user. For each entry found during that search, the value of the specified attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. - There can be multiple `role_mapping` sections defined inside the same `ldap` section. All of them will be applied. - - `base_dn` - template used to construct the base DN for the LDAP search. - - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` - substrings of the template with the actual user name and bind DN during each LDAP search. - - `scope` - scope of the LDAP search. + - `base_dn` — Template used to construct the base DN for the LDAP search. + - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during each LDAP search. + - `scope` — Scope of the LDAP search. - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default). - - `search_filter` - template used to construct the search filter for the LDAP search. - - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` - substrings of the template with the actual user name, bind DN, and base DN during each LDAP search. + - `search_filter` — Template used to construct the search filter for the LDAP search. + - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, and base DN during each LDAP search. - Note, that the special characters must be escaped properly in XML. - - `attribute` - attribute name whose values will be returned by the LDAP search. - - `prefix` - prefix, that will be expected to be in front of each string in the original - list of strings returned by the LDAP search. Prefix will be removed from the original - strings and resulting strings will be treated as local role names. Empty, by default. + - `attribute` — Attribute name whose values will be returned by the LDAP search. + - `prefix` — Prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated as local role names. Empty, by default. + [Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap.md) From 8eea6a87e991eec1a196ea5291b64ec8293ab583 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 14 Mar 2021 08:35:07 +0000 Subject: [PATCH 101/233] Support replicas priority for postgres dictionary source --- .../compose/docker_compose_postgres.yml | 7 ++ src/Common/ErrorCodes.cpp | 1 + .../PostgreSQL/DatabasePostgreSQL.cpp | 2 +- .../PostgreSQLDictionarySource.cpp | 31 +++---- src/Dictionaries/PostgreSQLDictionarySource.h | 6 +- .../PostgreSQL/PostgreSQLConnection.cpp | 44 +++++++++- .../PostgreSQL/PostgreSQLConnection.h | 18 ++-- .../PostgreSQLReplicaConnection.cpp | 77 +++++++++++++++++ .../PostgreSQL/PostgreSQLReplicaConnection.h | 35 ++++++++ .../configs/dictionaries/postgres_dict.xml | 83 +++++++++++++++++++ .../configs/postgres_dict.xml | 37 --------- .../test_dictionaries_postgresql/test.py | 76 +++++++++++++---- 12 files changed, 332 insertions(+), 85 deletions(-) create mode 100644 src/Storages/PostgreSQL/PostgreSQLReplicaConnection.cpp create mode 100644 src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h create mode 100644 tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml delete mode 100644 tests/integration/test_dictionaries_postgresql/configs/postgres_dict.xml diff --git a/docker/test/integration/runner/compose/docker_compose_postgres.yml b/docker/test/integration/runner/compose/docker_compose_postgres.yml index 5657352e1b3..58ed97251fb 100644 --- a/docker/test/integration/runner/compose/docker_compose_postgres.yml +++ b/docker/test/integration/runner/compose/docker_compose_postgres.yml @@ -11,3 +11,10 @@ services: default: aliases: - postgre-sql.local + postgres2: + image: postgres + restart: always + environment: + POSTGRES_PASSWORD: mysecretpassword + ports: + - 5441:5432 diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index f44cfd938d6..3a0fccc5358 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -546,6 +546,7 @@ M(577, INVALID_SHARD_ID) \ M(578, INVALID_FORMAT_INSERT_QUERY_WITH_DATA) \ \ + M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ M(1001, STD_EXCEPTION) \ diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 722b9c64edb..511cf8d847c 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -168,7 +168,7 @@ StoragePtr DatabasePostgreSQL::fetchTable(const String & table_name, const Conte return StoragePtr{}; auto storage = StoragePostgreSQL::create( - StorageID(database_name, table_name), table_name, std::make_shared(connection->conn_str()), + StorageID(database_name, table_name), table_name, std::make_shared(*connection), ColumnsDescription{*columns}, ConstraintsDescription{}, context); if (cache_tables) diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index aa852404750..5b71ad61120 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include "readInvalidateQuery.h" #endif @@ -29,11 +28,10 @@ PostgreSQLDictionarySource::PostgreSQLDictionarySource( const DictionaryStructure & dict_struct_, const Poco::Util::AbstractConfiguration & config_, const std::string & config_prefix, - PostgreSQLConnectionPtr connection_, const Block & sample_block_) : dict_struct{dict_struct_} , sample_block(sample_block_) - , connection(std::move(connection_)) + , connection(std::make_shared(config_, config_prefix)) , log(&Poco::Logger::get("PostgreSQLDictionarySource")) , db(config_.getString(fmt::format("{}.db", config_prefix), "")) , table(config_.getString(fmt::format("{}.table", config_prefix), "")) @@ -50,7 +48,7 @@ PostgreSQLDictionarySource::PostgreSQLDictionarySource( PostgreSQLDictionarySource::PostgreSQLDictionarySource(const PostgreSQLDictionarySource & other) : dict_struct(other.dict_struct) , sample_block(other.sample_block) - , connection(std::make_shared(other.connection->conn_str())) + , connection(other.connection) , log(&Poco::Logger::get("PostgreSQLDictionarySource")) , db(other.db) , table(other.table) @@ -68,8 +66,7 @@ PostgreSQLDictionarySource::PostgreSQLDictionarySource(const PostgreSQLDictionar BlockInputStreamPtr PostgreSQLDictionarySource::loadAll() { LOG_TRACE(log, load_all_query); - return std::make_shared( - connection->conn(), load_all_query, sample_block, max_block_size); + return loadBase(load_all_query); } @@ -77,23 +74,28 @@ BlockInputStreamPtr PostgreSQLDictionarySource::loadUpdatedAll() { auto load_update_query = getUpdateFieldAndDate(); LOG_TRACE(log, load_update_query); - return std::make_shared(connection->conn(), load_update_query, sample_block, max_block_size); + return loadBase(load_update_query); } BlockInputStreamPtr PostgreSQLDictionarySource::loadIds(const std::vector & ids) { const auto query = query_builder.composeLoadIdsQuery(ids); - return std::make_shared(connection->conn(), query, sample_block, max_block_size); + return loadBase(query); } BlockInputStreamPtr PostgreSQLDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) { const auto query = query_builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::AND_OR_CHAIN); - return std::make_shared(connection->conn(), query, sample_block, max_block_size); + return loadBase(query); } +BlockInputStreamPtr PostgreSQLDictionarySource::loadBase(const String & query) +{ + return std::make_shared(connection->get(), query, sample_block, max_block_size); +} + bool PostgreSQLDictionarySource::isModified() const { if (!invalidate_query.empty()) @@ -112,7 +114,7 @@ std::string PostgreSQLDictionarySource::doInvalidateQuery(const std::string & re Block invalidate_sample_block; ColumnPtr column(ColumnString::create()); invalidate_sample_block.insert(ColumnWithTypeAndName(column, std::make_shared(), "Sample Block")); - PostgreSQLBlockInputStream block_input_stream(connection->conn(), request, invalidate_sample_block, 1); + PostgreSQLBlockInputStream block_input_stream(connection->get(), request, invalidate_sample_block, 1); return readInvalidateQuery(block_input_stream); } @@ -172,15 +174,8 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory) { #if USE_LIBPQXX const auto config_prefix = root_config_prefix + ".postgresql"; - auto connection = std::make_shared( - config.getString(fmt::format("{}.db", config_prefix), ""), - config.getString(fmt::format("{}.host", config_prefix), ""), - config.getUInt(fmt::format("{}.port", config_prefix), 0), - config.getString(fmt::format("{}.user", config_prefix), ""), - config.getString(fmt::format("{}.password", config_prefix), "")); - return std::make_unique( - dict_struct, config, config_prefix, connection, sample_block); + dict_struct, config, config_prefix, sample_block); #else (void)dict_struct; (void)config; diff --git a/src/Dictionaries/PostgreSQLDictionarySource.h b/src/Dictionaries/PostgreSQLDictionarySource.h index a826ff15f4f..dd2d35db83a 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.h +++ b/src/Dictionaries/PostgreSQLDictionarySource.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include @@ -26,7 +26,6 @@ public: const DictionaryStructure & dict_struct_, const Poco::Util::AbstractConfiguration & config_, const std::string & config_prefix, - PostgreSQLConnectionPtr connection_, const Block & sample_block_); /// copy-constructor is provided in order to support cloneability @@ -48,10 +47,11 @@ public: private: std::string getUpdateFieldAndDate(); std::string doInvalidateQuery(const std::string & request) const; + BlockInputStreamPtr loadBase(const String & query); const DictionaryStructure dict_struct; Block sample_block; - PostgreSQLConnectionPtr connection; + PostgreSQLReplicaConnectionPtr connection; Poco::Logger * log; const std::string db; diff --git a/src/Storages/PostgreSQL/PostgreSQLConnection.cpp b/src/Storages/PostgreSQL/PostgreSQLConnection.cpp index 668550ec721..58eb7192eb9 100644 --- a/src/Storages/PostgreSQL/PostgreSQLConnection.cpp +++ b/src/Storages/PostgreSQL/PostgreSQLConnection.cpp @@ -6,23 +6,63 @@ #include #include #include +#include namespace DB { +PostgreSQLConnection::PostgreSQLConnection(std::string dbname, std::string host, UInt16 port, std::string user, std::string password) +{ + address = host + ':' + std::to_string(port); + connection_str = formatConnectionString(std::move(dbname), std::move(host), port, std::move(user), std::move(password)); +} + + +PostgreSQLConnection::PostgreSQLConnection(const PostgreSQLConnection & other) + : connection_str(other.connection_str) + , address(other.address) +{ +} + + PostgreSQLConnection::ConnectionPtr PostgreSQLConnection::conn() { - checkUpdateConnection(); + connect(); return connection; } -void PostgreSQLConnection::checkUpdateConnection() + +void PostgreSQLConnection::connect() { if (!connection || !connection->is_open()) connection = std::make_unique(connection_str); } + +bool PostgreSQLConnection::tryConnect() +{ + try + { + connect(); + } + catch (const pqxx::broken_connection & pqxx_error) + { + LOG_ERROR( + &Poco::Logger::get("PostgreSQLConnection"), + "Unable to setup connection to {}, reason: {}", + getAddress(), pqxx_error.what()); + return false; + } + catch (...) + { + throw; + } + + return true; +} + + std::string PostgreSQLConnection::formatConnectionString( std::string dbname, std::string host, UInt16 port, std::string user, std::string password) { diff --git a/src/Storages/PostgreSQL/PostgreSQLConnection.h b/src/Storages/PostgreSQL/PostgreSQLConnection.h index ae79a3436e0..f23308ddef9 100644 --- a/src/Storages/PostgreSQL/PostgreSQLConnection.h +++ b/src/Storages/PostgreSQL/PostgreSQLConnection.h @@ -16,29 +16,31 @@ namespace DB /// Connection is not made until actually used. class PostgreSQLConnection { +public: using ConnectionPtr = std::shared_ptr; -public: - PostgreSQLConnection(std::string dbname, std::string host, UInt16 port, std::string user, std::string password) - : connection_str(formatConnectionString(std::move(dbname), std::move(host), port, std::move(user), std::move(password))) {} + PostgreSQLConnection(std::string dbname, std::string host, UInt16 port, std::string user, std::string password); - PostgreSQLConnection(const std::string & connection_str_) : connection_str(connection_str_) {} + PostgreSQLConnection(const PostgreSQLConnection & other); - PostgreSQLConnection(const PostgreSQLConnection &) = delete; PostgreSQLConnection operator =(const PostgreSQLConnection &) = delete; + bool tryConnect(); + ConnectionPtr conn(); + const std::string & getAddress() { return address; } + std::string & conn_str() { return connection_str; } private: - ConnectionPtr connection; - std::string connection_str; + void connect(); static std::string formatConnectionString( std::string dbname, std::string host, UInt16 port, std::string user, std::string password); - void checkUpdateConnection(); + ConnectionPtr connection; + std::string connection_str, address; }; using PostgreSQLConnectionPtr = std::shared_ptr; diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.cpp b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.cpp new file mode 100644 index 00000000000..0c1efc16e05 --- /dev/null +++ b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.cpp @@ -0,0 +1,77 @@ +#include "PostgreSQLReplicaConnection.h" +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int POSTGRESQL_CONNECTION_FAILURE; +} + + +PostgreSQLReplicaConnection::PostgreSQLReplicaConnection( + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + const size_t num_retries_) + : log(&Poco::Logger::get("PostgreSQLConnection")) + , num_retries(num_retries_) +{ + auto db = config.getString(config_prefix + ".db", ""); + auto host = config.getString(config_prefix + ".host", ""); + auto port = config.getUInt(config_prefix + ".port", 0); + auto user = config.getString(config_prefix + ".user", ""); + auto password = config.getString(config_prefix + ".password", ""); + + if (config.has(config_prefix + ".replica")) + { + Poco::Util::AbstractConfiguration::Keys config_keys; + config.keys(config_prefix, config_keys); + + for (const auto & config_key : config_keys) + { + if (config_key.starts_with("replica")) + { + std::string replica_name = config_prefix + "." + config_key; + size_t priority = config.getInt(replica_name + ".priority", 0); + + auto replica_host = config.getString(replica_name + ".host", host); + auto replica_port = config.getUInt(replica_name + ".port", port); + auto replica_user = config.getString(replica_name + ".user", user); + auto replica_password = config.getString(replica_name + ".password", password); + + replicas[priority] = std::make_shared(db, replica_host, replica_port, replica_user, replica_password); + } + } + } + else + { + replicas[0] = std::make_shared(db, host, port, user, password); + } +} + + +PostgreSQLReplicaConnection::PostgreSQLReplicaConnection(const PostgreSQLReplicaConnection & other) + : log(&Poco::Logger::get("PostgreSQLConnection")) + , replicas(other.replicas) + , num_retries(other.num_retries) +{ +} + + +PostgreSQLConnection::ConnectionPtr PostgreSQLReplicaConnection::get() +{ + for (size_t i = 0; i < num_retries; ++i) + { + for (auto & replica : replicas) + { + if (replica.second->tryConnect()) + return replica.second->conn(); + } + } + + throw Exception(ErrorCodes::POSTGRESQL_CONNECTION_FAILURE, "Unable to connect to any of the replicas"); +} + +} diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h new file mode 100644 index 00000000000..9b5dec5e9a5 --- /dev/null +++ b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h @@ -0,0 +1,35 @@ +#include "PostgreSQLConnection.h" +#include +#include +#include + +namespace DB +{ + +class PostgreSQLReplicaConnection +{ + +public: + static constexpr inline auto POSTGRESQL_CONNECTION_DEFAULT_RETRIES_NUM = 5; + + PostgreSQLReplicaConnection( + const Poco::Util::AbstractConfiguration & config, + const String & config_name, + const size_t num_retries = POSTGRESQL_CONNECTION_DEFAULT_RETRIES_NUM); + + PostgreSQLReplicaConnection(const PostgreSQLReplicaConnection & other); + + PostgreSQLConnection::ConnectionPtr get(); + + +private: + using ReplicasByPriority = std::map; + + Poco::Logger * log; + ReplicasByPriority replicas; + size_t num_retries; +}; + +using PostgreSQLReplicaConnectionPtr = std::shared_ptr; + +} diff --git a/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml b/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml new file mode 100644 index 00000000000..4ee07d0972a --- /dev/null +++ b/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml @@ -0,0 +1,83 @@ + + + + dict0 + + + clickhouse + postgres1 + 5432 + postgres + mysecretpassword + test0
+ SELECT value FROM test0 WHERE id = 0 + + + + + + + + id + UInt32 + + + id + UInt32 + + + + value + UInt32 + + + + 1 + + + dict1 + + + clickhouse + postgres + mysecretpassword + test1
+ + postgres1 + 3 + 5432 + + + postgres2 + 5433 + 1 + + + postgres2 + 5432 + 2 + + + + + + + + + id + UInt32 + + + id + UInt32 + + + + value + UInt32 + + + + 1 + + diff --git a/tests/integration/test_dictionaries_postgresql/configs/postgres_dict.xml b/tests/integration/test_dictionaries_postgresql/configs/postgres_dict.xml deleted file mode 100644 index 2572930a798..00000000000 --- a/tests/integration/test_dictionaries_postgresql/configs/postgres_dict.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - dict0 - - - clickhouse - postgres1 - 5432 - postgres - mysecretpassword - test0
- SELECT value FROM test0 WHERE id = 0 - - - - - - - - id - UInt32 - - - id - UInt32 - - - - value - UInt32 - - - - 1 - - diff --git a/tests/integration/test_dictionaries_postgresql/test.py b/tests/integration/test_dictionaries_postgresql/test.py index b83c00409af..1ca3c89a5ee 100644 --- a/tests/integration/test_dictionaries_postgresql/test.py +++ b/tests/integration/test_dictionaries_postgresql/test.py @@ -6,7 +6,10 @@ from helpers.cluster import ClickHouseCluster from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/config.xml', 'configs/postgres_dict.xml', 'configs/log_conf.xml'], with_postgres=True) +node1 = cluster.add_instance('node1', main_configs=[ + 'configs/config.xml', + 'configs/dictionaries/postgres_dict.xml', + 'configs/log_conf.xml'], with_postgres=True) postgres_dict_table_template = """ CREATE TABLE IF NOT EXISTS {} ( @@ -18,11 +21,12 @@ click_dict_table_template = """ ) ENGINE = Dictionary({}) """ -def get_postgres_conn(database=False): +def get_postgres_conn(port=5432, database=False): if database == True: - conn_string = "host='localhost' dbname='clickhouse' user='postgres' password='mysecretpassword'" + conn_string = "host='localhost' port={} dbname='clickhouse' user='postgres' password='mysecretpassword'".format(port) else: - conn_string = "host='localhost' user='postgres' password='mysecretpassword'" + conn_string = "host='localhost' port={} user='postgres' password='mysecretpassword'".format(port) + conn = psycopg2.connect(conn_string) conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) conn.autocommit = True @@ -32,15 +36,13 @@ def create_postgres_db(conn, name): cursor = conn.cursor() cursor.execute("CREATE DATABASE {}".format(name)) -def create_postgres_table(conn, table_name): - cursor = conn.cursor() +def create_postgres_table(cursor, table_name): cursor.execute(postgres_dict_table_template.format(table_name)) -def create_and_fill_postgres_table(table_name): - conn = get_postgres_conn(True) - create_postgres_table(conn, table_name) +def create_and_fill_postgres_table(cursor, table_name, host='postgres1', port=5432): + create_postgres_table(cursor, table_name) # Fill postgres table using clickhouse postgres table function and check - table_func = '''postgresql('postgres1:5432', 'clickhouse', '{}', 'postgres', 'mysecretpassword')'''.format(table_name) + table_func = '''postgresql('{}:{}', 'clickhouse', '{}', 'postgres', 'mysecretpassword')'''.format(host, port, table_name) node1.query('''INSERT INTO TABLE FUNCTION {} SELECT number, number from numbers(10000) '''.format(table_func, table_name)) result = node1.query("SELECT count() FROM {}".format(table_func)) @@ -54,10 +56,16 @@ def create_dict(table_name, index=0): def started_cluster(): try: cluster.start() - postgres_conn = get_postgres_conn() node1.query("CREATE DATABASE IF NOT EXISTS test") - print("postgres connected") + + postgres_conn = get_postgres_conn(port=5432) + print("postgres1 connected") create_postgres_db(postgres_conn, 'clickhouse') + + postgres_conn = get_postgres_conn(port=5441) + print("postgres2 connected") + create_postgres_db(postgres_conn, 'clickhouse') + yield cluster finally: @@ -65,10 +73,10 @@ def started_cluster(): def test_load_dictionaries(started_cluster): - conn = get_postgres_conn(True) + conn = get_postgres_conn(database=True) cursor = conn.cursor() table_name = 'test0' - create_and_fill_postgres_table(table_name) + create_and_fill_postgres_table(cursor, table_name) create_dict(table_name) dict_name = 'dict0' @@ -76,14 +84,17 @@ def test_load_dictionaries(started_cluster): assert node1.query("SELECT count() FROM `test`.`dict_table_{}`".format(table_name)).rstrip() == '10000' assert node1.query("SELECT dictGetUInt32('{}', 'id', toUInt64(0))".format(dict_name)) == '0\n' assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(9999))".format(dict_name)) == '9999\n' + cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) + node1.query("DROP TABLE IF EXISTS {}".format(table_name)) + node1.query("DROP DICTIONARY IF EXISTS {}".format(dict_name)) def test_invalidate_query(started_cluster): - conn = get_postgres_conn(True) + conn = get_postgres_conn(database=True) cursor = conn.cursor() table_name = 'test0' - create_and_fill_postgres_table(table_name) + create_and_fill_postgres_table(cursor, table_name) # invalidate query: SELECT value FROM test0 WHERE id = 0 dict_name = 'dict0' @@ -112,6 +123,39 @@ def test_invalidate_query(started_cluster): assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name)) == '2\n' assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(1))".format(dict_name)) == '2\n' + node1.query("DROP TABLE IF EXISTS {}".format(table_name)) + node1.query("DROP DICTIONARY IF EXISTS {}".format(dict_name)) + + +def test_dictionary_with_replicas(started_cluster): + conn1 = get_postgres_conn(port=5432, database=True) + cursor1 = conn1.cursor() + conn2 = get_postgres_conn(port=5441, database=True) + cursor2 = conn2.cursor() + + create_postgres_table(cursor1, 'test1') + create_postgres_table(cursor2, 'test1') + + cursor1.execute('INSERT INTO test1 select i, i from generate_series(0, 99) as t(i);'); + cursor2.execute('INSERT INTO test1 select i, i from generate_series(100, 199) as t(i);'); + + create_dict('test1', 1) + result = node1.query("SELECT * FROM `test`.`dict_table_test1` ORDER BY id") + + # priority 0 - non running port + assert node1.contains_in_log('Unable to setup connection to postgres2:5433*') + + # priority 1 - postgres2, table contains rows with values 100-200 + # priority 2 - postgres1, table contains rows with values 0-100 + expected = node1.query("SELECT number, number FROM numbers(100, 100)") + assert(result == expected) + + cursor1.execute("DROP TABLE IF EXISTS test1") + cursor2.execute("DROP TABLE IF EXISTS test1") + + node1.query("DROP TABLE IF EXISTS test1") + node1.query("DROP DICTIONARY IF EXISTS dict1") + if __name__ == '__main__': cluster.start() From 797063ae0fae95c3a79e8ccab1d3b19bee8e843d Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 14 Mar 2021 10:35:10 +0000 Subject: [PATCH 102/233] Support non-default table schema for postgres storage/table-function --- src/Storages/StoragePostgreSQL.cpp | 20 ++++++++++++------ src/Storages/StoragePostgreSQL.h | 4 +++- .../TableFunctionPostgreSQL.cpp | 16 +++++++++----- src/TableFunctions/TableFunctionPostgreSQL.h | 2 +- .../test_storage_postgresql/test.py | 21 +++++++++++++++++++ 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 78ec8c34e41..3e122ed2fc7 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -45,9 +45,11 @@ StoragePostgreSQL::StoragePostgreSQL( PostgreSQLConnectionPtr connection_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, - const Context & context_) + const Context & context_, + const String & remote_table_schema_) : IStorage(table_id_) , remote_table_name(remote_table_name_) + , remote_table_schema(remote_table_schema_) , global_context(context_) , connection(std::move(connection_)) { @@ -69,9 +71,11 @@ Pipe StoragePostgreSQL::read( { metadata_snapshot->check(column_names_, getVirtuals(), getStorageID()); + /// Connection is already made to the needed database, so it should not be present in the query; + /// remote_table_schema is empty if it is not specified, will access only table_name. String query = transformQueryForExternalDatabase( query_info_, metadata_snapshot->getColumns().getOrdinary(), - IdentifierQuotingStyle::DoubleQuotes, "", remote_table_name, context_); + IdentifierQuotingStyle::DoubleQuotes, remote_table_schema, remote_table_name, context_); Block sample_block; for (const String & column_name : column_names_) @@ -293,9 +297,9 @@ void registerStoragePostgreSQL(StorageFactory & factory) { ASTs & engine_args = args.engine_args; - if (engine_args.size() != 5) - throw Exception("Storage PostgreSQL requires 5 parameters: " - "PostgreSQL('host:port', 'database', 'table', 'username', 'password'.", + if (engine_args.size() < 5 || engine_args.size() > 6) + throw Exception("Storage PostgreSQL requires 5-6 parameters: " + "PostgreSQL('host:port', 'database', 'table', 'username', 'password' [, 'schema']", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); for (auto & engine_arg : engine_args) @@ -304,6 +308,10 @@ void registerStoragePostgreSQL(StorageFactory & factory) auto parsed_host_port = parseAddress(engine_args[0]->as().value.safeGet(), 5432); const String & remote_table = engine_args[2]->as().value.safeGet(); + String remote_table_schema; + if (engine_args.size() == 6) + remote_table_schema = engine_args[5]->as().value.safeGet(); + auto connection = std::make_shared( engine_args[1]->as().value.safeGet(), parsed_host_port.first, @@ -312,7 +320,7 @@ void registerStoragePostgreSQL(StorageFactory & factory) engine_args[4]->as().value.safeGet()); return StoragePostgreSQL::create( - args.table_id, remote_table, connection, args.columns, args.constraints, args.context); + args.table_id, remote_table, connection, args.columns, args.constraints, args.context, remote_table_schema); }, { .source_access_type = AccessType::POSTGRES, diff --git a/src/Storages/StoragePostgreSQL.h b/src/Storages/StoragePostgreSQL.h index 8aebae5896b..0d574c9e98e 100644 --- a/src/Storages/StoragePostgreSQL.h +++ b/src/Storages/StoragePostgreSQL.h @@ -28,7 +28,8 @@ public: PostgreSQLConnectionPtr connection_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, - const Context & context_); + const Context & context_, + const std::string & remote_table_schema_ = ""); String getName() const override { return "PostgreSQL"; } @@ -47,6 +48,7 @@ private: friend class PostgreSQLBlockOutputStream; String remote_table_name; + String remote_table_schema; Context global_context; PostgreSQLConnectionPtr connection; }; diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index eefdff1fa87..0e3f1c5da24 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -29,7 +29,7 @@ StoragePtr TableFunctionPostgreSQL::executeImpl(const ASTPtr & /*ast_function*/, auto columns = getActualTableStructure(context); auto result = std::make_shared( StorageID(getDatabaseName(), table_name), remote_table_name, - connection, columns, ConstraintsDescription{}, context); + connection, columns, ConstraintsDescription{}, context, remote_table_schema); result->startup(); return result; @@ -39,7 +39,10 @@ StoragePtr TableFunctionPostgreSQL::executeImpl(const ASTPtr & /*ast_function*/, ColumnsDescription TableFunctionPostgreSQL::getActualTableStructure(const Context & context) const { const bool use_nulls = context.getSettingsRef().external_table_functions_use_nulls; - auto columns = fetchPostgreSQLTableStructure(connection->conn(), remote_table_name, use_nulls); + auto columns = fetchPostgreSQLTableStructure( + connection->conn(), + remote_table_schema.empty() ? remote_table_name : remote_table_schema + '.' + remote_table_name, + use_nulls); return ColumnsDescription{*columns}; } @@ -54,9 +57,9 @@ void TableFunctionPostgreSQL::parseArguments(const ASTPtr & ast_function, const ASTs & args = func_args.arguments->children; - if (args.size() != 5) - throw Exception("Table function 'PostgreSQL' requires 5 parameters: " - "PostgreSQL('host:port', 'database', 'table', 'user', 'password').", + if (args.size() < 5 || args.size() > 6) + throw Exception("Table function 'PostgreSQL' requires 5-6 parameters: " + "PostgreSQL('host:port', 'database', 'table', 'user', 'password', [, 'schema']).", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); for (auto & arg : args) @@ -65,6 +68,9 @@ void TableFunctionPostgreSQL::parseArguments(const ASTPtr & ast_function, const auto parsed_host_port = parseAddress(args[0]->as().value.safeGet(), 5432); remote_table_name = args[2]->as().value.safeGet(); + if (args.size() == 6) + remote_table_schema = args[5]->as().value.safeGet(); + connection = std::make_shared( args[1]->as().value.safeGet(), parsed_host_port.first, diff --git a/src/TableFunctions/TableFunctionPostgreSQL.h b/src/TableFunctions/TableFunctionPostgreSQL.h index e625cbd9bf6..92e061e18ca 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.h +++ b/src/TableFunctions/TableFunctionPostgreSQL.h @@ -30,7 +30,7 @@ private: void parseArguments(const ASTPtr & ast_function, const Context & context) override; String connection_str; - String remote_table_name; + String remote_table_name, remote_table_schema; PostgreSQLConnectionPtr connection; }; diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index cee495438a2..58f3233bacc 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -132,6 +132,27 @@ def test_postgres_conversions(started_cluster): assert(result == expected) +def test_non_default_scema(started_cluster): + conn = get_postgres_conn(True) + cursor = conn.cursor() + cursor.execute('CREATE SCHEMA test_schema') + cursor.execute('CREATE TABLE test_schema.test_table (a integer)') + cursor.execute('INSERT INTO test_schema.test_table SELECT i FROM generate_series(0, 99) as t(i)') + + node1.query(''' + CREATE TABLE test_pg_table_schema (a UInt32) + ENGINE PostgreSQL('postgres1:5432', 'clickhouse', 'test_table', 'postgres', 'mysecretpassword', 'test_schema'); + ''') + + result = node1.query('SELECT * FROM test_pg_table_schema') + expected = node1.query('SELECT number FROM numbers(100)') + assert(result == expected) + + table_function = '''postgresql('postgres1:5432', 'clickhouse', 'test_table', 'postgres', 'mysecretpassword', 'test_schema')''' + result = node1.query('SELECT * FROM {}'.format(table_function)) + assert(result == expected) + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") From 6e69299191e10588500b32510dc0f1e9ac4d1471 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 14 Mar 2021 10:43:23 +0000 Subject: [PATCH 103/233] Fix style --- src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h index 9b5dec5e9a5..289183d8451 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h +++ b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h @@ -1,3 +1,5 @@ +#pragma once + #include "PostgreSQLConnection.h" #include #include From 4be01d927dcb81fc824f3a9b6367456809432029 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Sun, 14 Mar 2021 23:57:31 +0300 Subject: [PATCH 104/233] Bump CI From f092d22a253f2c471f28358d81a77dc75385936f Mon Sep 17 00:00:00 2001 From: George Date: Mon, 15 Mar 2021 00:25:21 +0300 Subject: [PATCH 105/233] Updated description --- .../external-authenticators/ldap.md | 14 ++--- .../external-authenticators/ldap.md | 54 +++++++++---------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index cb8aa07dc41..98d6e18b72e 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -39,11 +39,11 @@ Note, that you can define multiple LDAP servers inside the `ldap_servers` sectio - `host` — LDAP server hostname or IP, this parameter is mandatory and cannot be empty. - `port` — LDAP server port, default is `636` if `enable_tls` is set to `true`, `389` otherwise. -- `bind_dn` — template used to construct the DN to bind to. +- `bind_dn` — Template used to construct the DN to bind to. - The resulting DN will be constructed by replacing all `{user_name}` substrings of the template with the actual user name during each authentication attempt. -- `verification_cooldown` — a period of time, in seconds, after a successful bind attempt, during which the user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server. +- `verification_cooldown` — A period of time, in seconds, after a successful bind attempt, during which the user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server. - Specify `0` (the default) to disable caching and force contacting the LDAP server for each authentication request. -- `enable_tls` — a flag to trigger the use of the secure connection to the LDAP server. +- `enable_tls` — A flag to trigger the use of the secure connection to the LDAP server. - Specify `no` for plain text `ldap://` protocol (not recommended). - Specify `yes` for LDAP over SSL/TLS `ldaps://` protocol (recommended, the default). - Specify `starttls` for legacy StartTLS protocol (plain text `ldap://` protocol, upgraded to TLS). @@ -127,20 +127,20 @@ Note that `my_ldap_server` referred in the `ldap` section inside the `user_direc **Parameters** -- `server` — One of LDAP server names defined in the `ldap_servers` config section above. This parameter is mandatory and cannot be empty. Одно из имен +- `server` — One of LDAP server names defined in the `ldap_servers` config section above. This parameter is mandatory and cannot be empty. - `roles` — Section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. - If no roles are specified here or assigned during role mapping (below), user will not be able to perform any actions after authentication. - `role_mapping` — Section with LDAP search parameters and mapping rules. - - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` and the name of the logged in user. For each entry found during that search, the value of the specified attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. + - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` and the name of the logged-in user. For each entry found during that search, the value of the specified attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. - There can be multiple `role_mapping` sections defined inside the same `ldap` section. All of them will be applied. - `base_dn` — Template used to construct the base DN for the LDAP search. - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during each LDAP search. - `scope` — Scope of the LDAP search. - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default). - `search_filter` — Template used to construct the search filter for the LDAP search. - - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, and base DN during each LDAP search. + - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}` and `{base_dn}` substrings of the template with the actual user name, bind DN and base DN during each LDAP search. - Note, that the special characters must be escaped properly in XML. - `attribute` — Attribute name whose values will be returned by the LDAP search. - - `prefix` — Prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated as local role names. Empty, by default. + - `prefix` — Prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. The prefix will be removed from the original strings and the resulting strings will be treated as local role names. Empty by default. [Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap.md) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 3d71ec1eba3..f44c0ff2120 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -1,11 +1,11 @@ # LDAP {#external-authenticators-ldap} -Для аутентификации пользователей ClickHouse можно использовать сервер LDAP. Можно использовать два подхода: +Для аутентификации пользователей ClickHouse можно использовать сервер LDAP. Существует два подхода: - Использовать LDAP как внешний аутентификатор для существующих пользователей, которые определены в `users.xml` или в локальных путях управления контролем. - Использовать LDAP как внешний пользовательский каталог и разрешить аутентификацию локально неопределенных пользователей, если они есть на LDAP сервере. -Для обоих подходов необходимо определить в конфиге ClickHouse внутренне названный LDAP сервер, чтобы другие части конфига могли ссылаться на него. +Для обоих подходов необходимо определить в конфиге ClickHouse с внутренним именем LDAP сервер, чтобы другие части конфига могли ссылаться на него. ## Определение LDAP сервера {#ldap-server-definition} @@ -33,16 +33,16 @@ ``` -Обратите внимание, что можно определить несколько LDAP серверов внутри секции `ldap_servers` используя различные имена. +Обратите внимание, что можно определить несколько LDAP серверов внутри секции `ldap_servers`, используя различные имена. **Параметры** -- `host` — имя хоста сервера LDAP или его IP. Этот параметр обязательный и не может быть пустым. -- `port` — порт сервера LDAP. По-умолчанию: при значении `true` настройки `enable_tls` — `636`, иначе `389`. +- `host` — имя хоста сервера LDAP или его IP. Этот параметр обязательный и не может быть оставлен пустым. +- `port` — порт сервера LDAP. По-умолчанию: `636` при значении `true` настройки `enable_tls`, иначе `389`. - `bind_dn` — шаблон для создания DN для привязки. - - конечный DN будет создан заменой всех подстрок `{user_name}` шаблона на фактическое имя пользователя при каждой попытке аутентификации. -- `verification_cooldown` — промежуток времени (в секундах) после успешной попытки привязки, в течение которого пользователь будет считаться успешно аутентифицированным без с сервером LDAP для всех последующих запросов. - - Укажите `0` (по-умолчанию), чтобы отключить кеширования и заставить связываться с сервером LDAP для каждого запроса аутетификации. + - Конечный DN будет создан заменой всех подстрок `{user_name}` шаблона на фактическое имя пользователя при каждой попытке аутентификации. +- `verification_cooldown` — промежуток времени (в секундах) после успешной попытки привязки, в течение которого пользователь будет считаться успешно аутентифицированным, и сможет совершать запросы без контакта с серверов LDAP. + - Укажите `0` (по-умолчанию), чтобы отключить кеширование и заставить связываться с сервером LDAP для каждого запроса аутентификации. - `enable_tls` — флаг, включающий использование защищенного соединения с сервером LDAP. - Укажите `no` для текстового `ldap://` протокола (не рекомендовано). - Укажите `yes` для LDAP через SSL/TLS `ldaps://` протокола (рекомендовано, используется по-умолчанию). @@ -51,7 +51,7 @@ - Принимаемые значения: `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, `tls1.2` (по-умолчанию). - `tls_require_cert` — поведение при проверке сертификата SSL/TLS. - Принимаемые значения: `never`, `allow`, `try`, `demand` (по-умолчанию). -- `tls_cert_file` — путь до файла сертификата. +- `tls_cert_file` — путь к файлу сертификата. - `tls_key_file` — путь к файлу ключа сертификата. - `tls_ca_cert_file` — путь к файлу ЦС сертификата. - `tls_ca_cert_dir` — путь к каталогу, содержащая сертификаты ЦС. @@ -61,7 +61,7 @@ Удаленный сервер LDAP можно использовать как метод верификации паролей локально определенных пользователей (пользователей, которые определены в `users.xml` или в локальных путях управления контролем). Для этого укажите имя определенного до этого сервера LDAP вместо `password` или другой похожей секции в определении пользователя. -При каждой попытке авторизации, ClickHouse пытается "привязаться" к DN, указанному в [определение LDAP сервера](#ldap-server-definition) параметром `bind_dn`, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается аутентифицированным. Обычно это называют методом "простой привязки". +При каждой попытке авторизации, ClickHouse пытается "привязаться" к DN, указанному в [определении LDAP сервера](#ldap-server-definition) параметром `bind_dn`, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается аутентифицированным. Обычно это называют методом "простой привязки". **Например** @@ -94,7 +94,7 @@ CREATE USER my_user IDENTIFIED WITH ldap_server BY 'my_ldap_server'; В добавок к локально определенным пользователям, удаленный LDAP сервер может быть использован как источник определения пользователей. Для этого укажите имя определенного до этого сервера LDAP (см. [Определение LDAP сервера](#ldap-server-definition)) в секции `ldap` внутри секции `users_directories` файла `config.xml`. -При каждой попытке авторизации, ClicHouse пытается локально найти определение пользователя и авторизовать его как обычно. Если определение не будет найдено, ClickHouse предполагает, что оно находится во внешнем LDAP каталоге, и попытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю будут присвоены роли из списка, указанного в секции `roles`. Кроме того, может быть выполнен LDAP поиск, а его результаты могут быть преобразованы в имена ролей и присвоены пользователям, если была настроена секция `role_mapping`. Все это работает при условии, что SQL-ориентированное [Управлением доступом](../access-rights.md#access-control) включено, а роли созданы выражением[CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). +При каждой попытке авторизации, ClicHouse пытается локально найти определение пользователя и авторизовать его как обычно. Если определение не будет найдено, ClickHouse предполагает, что оно находится во внешнем LDAP каталоге, и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю будут присвоены роли из списка, указанного в секции `roles`. Кроме того, может быть выполнен LDAP поиск, а его результаты могут быть преобразованы в имена ролей и присвоены пользователям, если была настроена секция `role_mapping`. Все это работает при условии, что SQL-ориентированное [Управлением доступом](../access-rights.md#access-control) включено, а роли созданы выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). **Пример** @@ -127,22 +127,20 @@ CREATE USER my_user IDENTIFIED WITH ldap_server BY 'my_ldap_server'; **Параметры** -- `server` — One of LDAP server names defined in the `ldap_servers` config section above. - This parameter is mandatory and cannot be empty. -- `roles` — Section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. - - If no roles are specified here or assigned during role mapping (below), user will not be able to perform any actions after authentication. -- `role_mapping` — Section with LDAP search parameters and mapping rules. - - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` and the name of the logged in user. For each entry found during that search, the value of the specified attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. - - There can be multiple `role_mapping` sections defined inside the same `ldap` section. All of them will be applied. - - `base_dn` — Template used to construct the base DN for the LDAP search. - - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during each LDAP search. - - `scope` — Scope of the LDAP search. - - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default). - - `search_filter` — Template used to construct the search filter for the LDAP search. - - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, and base DN during each LDAP search. - - Note, that the special characters must be escaped properly in XML. - - `attribute` — Attribute name whose values will be returned by the LDAP search. - - `prefix` — Prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated as local role names. Empty, by default. - +- `server` — одно из имен сервера LDAP, определенных в секции конфига `ldap_servers выше. Этот параметр обязательный и не может быть оставлен пустым. +- `roles` — секция со списком локально определенных ролей, которые будут присвоены каждому пользователю, полученному от сервера LDAP. + - Если роли не указаны здесь или в секции `role_mapping` (ниже), пользователь не сможет выполнять никаких операций после аутентификации. +- `role_mapping` — секция c параметрами LDAP поиска и правилами отображения. + - При аутентификации пользователя, пока еще связанного с LDAP, производится LDAP поиск с помощью `search_filter` и имени этого пользователя. Для каждой записи, найденной в ходе поиска, выделяется значение указанного атрибута. У каждого атрибута, имеющего указанный префикс, удаляется этот префикс, а остальная часть значения становится именем локальной роли, определенной в ClickHouse, причем предполагается, что эта роль была создана выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) до этого. + - Внутри одной секции `ldap` может быть несколько секций `role_mapping`. Все они будут применены. + - `base_dn` — шаблон, который используется для создания базового DN для LDAP поиска. + - конечный DN будет создан заменой всех подстрок `{user_name}` и `{bind_dn}` шаблона на фактическое имя пользователя и DN привязки соответственно при каждом LDAP поиске. + - `scope` — Область LDAP поиска. + - Принимаемые значения: `base`, `one_level`, `children`, `subtree` (по-умолчанию). + - `search_filter` — шаблон, который используется для создания фильтра для каждого LDAP поиска. + - Конечный фильтр будет создан заменой всех подстрок `{user_name}`, `{bind_dn}` и `{base_dn}` шаблона на фактическое имя пользователя, DN привязи и базовый DN при соответственно каждом LDAP поиске. + - Обратите внимание, что специальные символы должны быть правильно экранированы в XML. + - `attribute` — имя атрибута, значение которого будет возвращаться LDAP поиском. + - `prefix` — префикс, который, как предполагается, будет находиться перед началом каждой строки в исходном списке строк, возвращаемых LDAP поиском. Префикс будет удален из исходных строк, а сами они будут рассматриваться как имена локальных ролей. По-умолчанию пусто. [Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap.md) From 6402b2c33c45b3e04d81515baa05ffdae1cdce4b Mon Sep 17 00:00:00 2001 From: George Date: Mon, 15 Mar 2021 00:39:24 +0300 Subject: [PATCH 106/233] Small fixes --- docs/en/operations/external-authenticators/ldap.md | 2 +- docs/ru/operations/external-authenticators/ldap.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index 98d6e18b72e..4c2748d6141 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -87,7 +87,7 @@ When SQL-driven [Access Control and Account Management](../access-rights.md#acce Query: ```sql -CREATE USER my_user IDENTIFIED WITH ldap_server BY 'my_ldap_server'; +CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; ``` ## LDAP Exernal User Directory {#ldap-external-user-directory} diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index f44c0ff2120..f13c3b99def 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -55,7 +55,7 @@ - `tls_key_file` — путь к файлу ключа сертификата. - `tls_ca_cert_file` — путь к файлу ЦС сертификата. - `tls_ca_cert_dir` — путь к каталогу, содержащая сертификаты ЦС. -- `tls_cipher_suite` — разрешить набор шифров (в нотации OpenSSL). +- `tls_cipher_suite` — разрешенный набор шифров (в нотации OpenSSL). ## LDAP внешний аутентификатор {#ldap-external-authenticator} @@ -87,7 +87,7 @@ Запрос: ```sql -CREATE USER my_user IDENTIFIED WITH ldap_server BY 'my_ldap_server'; +CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; ``` ## Внешний пользовательский каталог LDAP {#ldap-external-user-directory} From 832f041e23034a40a383bd4199318dd64fb2bdfb Mon Sep 17 00:00:00 2001 From: George Date: Mon, 15 Mar 2021 00:44:36 +0300 Subject: [PATCH 107/233] Fixed toc --- docs/ru/operations/external-authenticators/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/operations/external-authenticators/index.md b/docs/ru/operations/external-authenticators/index.md index db5c89a3d66..ffe705dffb2 100644 --- a/docs/ru/operations/external-authenticators/index.md +++ b/docs/ru/operations/external-authenticators/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: \u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0430\u0443\u0442\u0435\u043d\u0442\u0438\u0444\u0438\u043a\u0430\u0442\u043e\u0440\u044b\u0020\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u0435\u0439\u0020\u0438\u0020\u043a\u0430\u0442\u0430\u043b\u043e\u0433\u0438 +toc_folder_title: "\u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0430\u0443\u0442\u0435\u043d\u0442\u0438\u0444\u0438\u043a\u0430\u0442\u043e\u0440\u044b\u0020\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u0435\u0439\u0020\u0438\u0020\u043a\u0430\u0442\u0430\u043b\u043e\u0433\u0438" toc_priority: 48 -toc_title: \u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" --- # Внешние аутентификаторы пользователей и каталоги {#external-authenticators} From 0a27c814901c17a906bc66ba7829f854eff592c9 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 15 Mar 2021 00:46:02 +0300 Subject: [PATCH 108/233] fixed index.md --- docs/ru/operations/external-authenticators/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/index.md b/docs/ru/operations/external-authenticators/index.md index ffe705dffb2..22ecb9bc2d7 100644 --- a/docs/ru/operations/external-authenticators/index.md +++ b/docs/ru/operations/external-authenticators/index.md @@ -10,6 +10,6 @@ ClickHouse поддерживает аунтетификацию и управл Поддерживаются следующие внешние аутентификаторы и каталоги: -- [LDAP](./ldap.md#external-authenticators-ldap) [Authenticator](./ldap.md#ldap-external-authenticator) и [Directory](./ldap.md#ldap-external-user-directory) +- [LDAP](./ldap.md#external-authenticators-ldap) [аутентификатор](./ldap.md#ldap-external-authenticator) и [каталог](./ldap.md#ldap-external-user-directory) [Original article](https://clickhouse.tech/docs/ru/operations/external-authenticators/index.md) From 8e86067d6da3107c073030e788e9cbd66eabd922 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 15 Mar 2021 01:05:58 +0300 Subject: [PATCH 109/233] Some fixes --- .../operations/external-authenticators/ldap.md | 8 +++++--- .../operations/external-authenticators/index.md | 2 +- .../operations/external-authenticators/ldap.md | 16 +++++++++------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index ebad4f2dbe8..e528e2a7c07 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -9,7 +9,9 @@ For both of these approaches, an internally named LDAP server must be defined in ## LDAP Server Definition {#ldap-server-definition} -To define LDAP server you must add `ldap_servers` section to the `config.xml`. For example, +To define LDAP server you must add `ldap_servers` section to the `config.xml`. + +**Example** ```xml @@ -87,9 +89,9 @@ When SQL-driven [Access Control and Account Management](../access-rights.md#acce Query: - +```sql CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; - +``` ## LDAP Exernal User Directory {#ldap-external-user-directory} diff --git a/docs/ru/operations/external-authenticators/index.md b/docs/ru/operations/external-authenticators/index.md index 22ecb9bc2d7..6b75e864fb8 100644 --- a/docs/ru/operations/external-authenticators/index.md +++ b/docs/ru/operations/external-authenticators/index.md @@ -12,4 +12,4 @@ ClickHouse поддерживает аунтетификацию и управл - [LDAP](./ldap.md#external-authenticators-ldap) [аутентификатор](./ldap.md#ldap-external-authenticator) и [каталог](./ldap.md#ldap-external-user-directory) -[Original article](https://clickhouse.tech/docs/ru/operations/external-authenticators/index.md) +[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/external-authenticators/index.md) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index f13c3b99def..7f901898a99 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -5,11 +5,13 @@ - Использовать LDAP как внешний аутентификатор для существующих пользователей, которые определены в `users.xml` или в локальных путях управления контролем. - Использовать LDAP как внешний пользовательский каталог и разрешить аутентификацию локально неопределенных пользователей, если они есть на LDAP сервере. -Для обоих подходов необходимо определить в конфиге ClickHouse с внутренним именем LDAP сервер, чтобы другие части конфига могли ссылаться на него. +Для обоих подходов необходимо определить в конфиге ClickHouse LDAP сервер с внутренним именем, чтобы другие части конфига могли ссылаться на него. ## Определение LDAP сервера {#ldap-server-definition} -Чтобы определить LDAP сервер, необходимо добавить секцию `ldap_servers` в `config.xml`. Например: +Чтобы определить LDAP сервер, необходимо добавить секцию `ldap_servers` в `config.xml`. + +**Пример** ```xml @@ -57,13 +59,13 @@ - `tls_ca_cert_dir` — путь к каталогу, содержащая сертификаты ЦС. - `tls_cipher_suite` — разрешенный набор шифров (в нотации OpenSSL). -## LDAP внешний аутентификатор {#ldap-external-authenticator} +## Внешний аутентификатор LDAP {#ldap-external-authenticator} Удаленный сервер LDAP можно использовать как метод верификации паролей локально определенных пользователей (пользователей, которые определены в `users.xml` или в локальных путях управления контролем). Для этого укажите имя определенного до этого сервера LDAP вместо `password` или другой похожей секции в определении пользователя. При каждой попытке авторизации, ClickHouse пытается "привязаться" к DN, указанному в [определении LDAP сервера](#ldap-server-definition) параметром `bind_dn`, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается аутентифицированным. Обычно это называют методом "простой привязки". -**Например** +**Пример** ```xml @@ -94,7 +96,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; В добавок к локально определенным пользователям, удаленный LDAP сервер может быть использован как источник определения пользователей. Для этого укажите имя определенного до этого сервера LDAP (см. [Определение LDAP сервера](#ldap-server-definition)) в секции `ldap` внутри секции `users_directories` файла `config.xml`. -При каждой попытке авторизации, ClicHouse пытается локально найти определение пользователя и авторизовать его как обычно. Если определение не будет найдено, ClickHouse предполагает, что оно находится во внешнем LDAP каталоге, и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю будут присвоены роли из списка, указанного в секции `roles`. Кроме того, может быть выполнен LDAP поиск, а его результаты могут быть преобразованы в имена ролей и присвоены пользователям, если была настроена секция `role_mapping`. Все это работает при условии, что SQL-ориентированное [Управлением доступом](../access-rights.md#access-control) включено, а роли созданы выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). +При каждой попытке авторизации ClicHouse пытается локально найти определение пользователя и авторизовать его как обычно. Если определение не будет найдено, ClickHouse предполагает, что оно находится во внешнем LDAP каталоге, и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю будут присвоены роли из списка, указанного в секции `roles`. Кроме того, может быть выполнен LDAP поиск, а его результаты могут быть преобразованы в имена ролей и присвоены пользователям, если была настроена секция `role_mapping`. Все это работает при условии, что SQL-ориентированное [Управлением доступом](../access-rights.md#access-control) включено, а роли созданы выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). **Пример** @@ -127,7 +129,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; **Параметры** -- `server` — одно из имен сервера LDAP, определенных в секции конфига `ldap_servers выше. Этот параметр обязательный и не может быть оставлен пустым. +- `server` — одно из имен сервера LDAP, определенного в секции конфига `ldap_servers` выше. Этот параметр обязательный и не может быть оставлен пустым. - `roles` — секция со списком локально определенных ролей, которые будут присвоены каждому пользователю, полученному от сервера LDAP. - Если роли не указаны здесь или в секции `role_mapping` (ниже), пользователь не сможет выполнять никаких операций после аутентификации. - `role_mapping` — секция c параметрами LDAP поиска и правилами отображения. @@ -143,4 +145,4 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; - `attribute` — имя атрибута, значение которого будет возвращаться LDAP поиском. - `prefix` — префикс, который, как предполагается, будет находиться перед началом каждой строки в исходном списке строк, возвращаемых LDAP поиском. Префикс будет удален из исходных строк, а сами они будут рассматриваться как имена локальных ролей. По-умолчанию пусто. -[Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap.md) +[Оригинальная статья](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap.md) From 5b7ef512f58f5697af0820a5c470a9c7c06b03d4 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 15 Mar 2021 09:14:30 +0300 Subject: [PATCH 110/233] Update PostgreSQLReplicaConnection.h --- src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h index 289183d8451..9465d4a119b 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h +++ b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h @@ -16,8 +16,8 @@ public: PostgreSQLReplicaConnection( const Poco::Util::AbstractConfiguration & config, - const String & config_name, - const size_t num_retries = POSTGRESQL_CONNECTION_DEFAULT_RETRIES_NUM); + const String & config_prefix, + const size_t num_retries_ = POSTGRESQL_CONNECTION_DEFAULT_RETRIES_NUM); PostgreSQLReplicaConnection(const PostgreSQLReplicaConnection & other); From 69b8ded5786ce341e842aec7d1967d6b6661761a Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Mar 2021 11:22:15 +0300 Subject: [PATCH 111/233] Fix alter modify for decimal columns when type size doesn't change --- src/Storages/StorageReplicatedMergeTree.cpp | 74 +++++++++---------- .../01761_alter_decimal_zookeeper.reference | 9 +++ .../01761_alter_decimal_zookeeper.sql | 31 ++++++++ 3 files changed, 75 insertions(+), 39 deletions(-) create mode 100644 tests/queries/0_stateless/01761_alter_decimal_zookeeper.reference create mode 100644 tests/queries/0_stateless/01761_alter_decimal_zookeeper.sql diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d08c5b6ad7c..391d685be78 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -907,8 +907,7 @@ void StorageReplicatedMergeTree::setTableStructure( StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); - if (new_columns != new_metadata.columns) - new_metadata.columns = new_columns; + new_metadata.columns = new_columns; if (!metadata_diff.empty()) { @@ -976,46 +975,43 @@ void StorageReplicatedMergeTree::setTableStructure( } /// Changes in columns may affect following metadata fields - if (new_metadata.columns != old_metadata.columns) + new_metadata.column_ttls_by_name.clear(); + for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs()) { - new_metadata.column_ttls_by_name.clear(); - for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs()) - { - auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, global_context, new_metadata.primary_key); - new_metadata.column_ttls_by_name[name] = new_ttl_entry; - } - - if (new_metadata.partition_key.definition_ast != nullptr) - new_metadata.partition_key.recalculateWithNewColumns(new_metadata.columns, global_context); - - if (!metadata_diff.sorting_key_changed) /// otherwise already updated - new_metadata.sorting_key.recalculateWithNewColumns(new_metadata.columns, global_context); - - /// Primary key is special, it exists even if not defined - if (new_metadata.primary_key.definition_ast != nullptr) - { - new_metadata.primary_key.recalculateWithNewColumns(new_metadata.columns, global_context); - } - else - { - new_metadata.primary_key = KeyDescription::getKeyFromAST(new_metadata.sorting_key.definition_ast, new_metadata.columns, global_context); - new_metadata.primary_key.definition_ast = nullptr; - } - - if (!metadata_diff.sampling_expression_changed && new_metadata.sampling_key.definition_ast != nullptr) - new_metadata.sampling_key.recalculateWithNewColumns(new_metadata.columns, global_context); - - if (!metadata_diff.skip_indices_changed) /// otherwise already updated - { - for (auto & index : new_metadata.secondary_indices) - index.recalculateWithNewColumns(new_metadata.columns, global_context); - } - - if (!metadata_diff.ttl_table_changed && new_metadata.table_ttl.definition_ast != nullptr) - new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( - new_metadata.table_ttl.definition_ast, new_metadata.columns, global_context, new_metadata.primary_key); + auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, global_context, new_metadata.primary_key); + new_metadata.column_ttls_by_name[name] = new_ttl_entry; } + if (new_metadata.partition_key.definition_ast != nullptr) + new_metadata.partition_key.recalculateWithNewColumns(new_metadata.columns, global_context); + + if (!metadata_diff.sorting_key_changed) /// otherwise already updated + new_metadata.sorting_key.recalculateWithNewColumns(new_metadata.columns, global_context); + + /// Primary key is special, it exists even if not defined + if (new_metadata.primary_key.definition_ast != nullptr) + { + new_metadata.primary_key.recalculateWithNewColumns(new_metadata.columns, global_context); + } + else + { + new_metadata.primary_key = KeyDescription::getKeyFromAST(new_metadata.sorting_key.definition_ast, new_metadata.columns, global_context); + new_metadata.primary_key.definition_ast = nullptr; + } + + if (!metadata_diff.sampling_expression_changed && new_metadata.sampling_key.definition_ast != nullptr) + new_metadata.sampling_key.recalculateWithNewColumns(new_metadata.columns, global_context); + + if (!metadata_diff.skip_indices_changed) /// otherwise already updated + { + for (auto & index : new_metadata.secondary_indices) + index.recalculateWithNewColumns(new_metadata.columns, global_context); + } + + if (!metadata_diff.ttl_table_changed && new_metadata.table_ttl.definition_ast != nullptr) + new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( + new_metadata.table_ttl.definition_ast, new_metadata.columns, global_context, new_metadata.primary_key); + /// Even if the primary/sorting/partition keys didn't change we must reinitialize it /// because primary/partition key column types might have changed. checkTTLExpressions(new_metadata, old_metadata); diff --git a/tests/queries/0_stateless/01761_alter_decimal_zookeeper.reference b/tests/queries/0_stateless/01761_alter_decimal_zookeeper.reference new file mode 100644 index 00000000000..5dcc95fd7b7 --- /dev/null +++ b/tests/queries/0_stateless/01761_alter_decimal_zookeeper.reference @@ -0,0 +1,9 @@ +1 5.00000000 +2 6.00000000 +CREATE TABLE default.test_alter_decimal\n(\n `n` UInt64,\n `d` Decimal(18, 8)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/01761_alter_decimal_zookeeper\', \'r1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +1 5.00000000 +2 6.00000000 +CREATE TABLE default.test_alter_decimal\n(\n `n` UInt64,\n `d` Decimal(18, 8)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/01761_alter_decimal_zookeeper\', \'r1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +1 5.00000000 +2 6.00000000 +3 7.00000000 diff --git a/tests/queries/0_stateless/01761_alter_decimal_zookeeper.sql b/tests/queries/0_stateless/01761_alter_decimal_zookeeper.sql new file mode 100644 index 00000000000..01766f0d6c2 --- /dev/null +++ b/tests/queries/0_stateless/01761_alter_decimal_zookeeper.sql @@ -0,0 +1,31 @@ +DROP TABLE IF EXISTS test_alter_decimal; + +CREATE TABLE test_alter_decimal +(n UInt64, d Decimal(15, 8)) +ENGINE = ReplicatedMergeTree('/clickhouse/01761_alter_decimal_zookeeper', 'r1') +ORDER BY tuple(); + +INSERT INTO test_alter_decimal VALUES (1, toDecimal32(5, 5)); + +INSERT INTO test_alter_decimal VALUES (2, toDecimal32(6, 6)); + +SELECT * FROM test_alter_decimal ORDER BY n; + +ALTER TABLE test_alter_decimal MODIFY COLUMN d Decimal(18, 8); + +SHOW CREATE TABLE test_alter_decimal; + +SELECT * FROM test_alter_decimal ORDER BY n; + +DETACH TABLE test_alter_decimal; +ATTACH TABLE test_alter_decimal; + +SHOW CREATE TABLE test_alter_decimal; + +INSERT INTO test_alter_decimal VALUES (3, toDecimal32(7, 7)); + +OPTIMIZE TABLE test_alter_decimal FINAL; + +SELECT * FROM test_alter_decimal ORDER BY n; + +DROP TABLE IF EXISTS test_alter_decimal; From caccc6da3787d4f328149ccae528328c165d4810 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Mar 2021 12:01:26 +0300 Subject: [PATCH 112/233] Add missing tests --- .../test_optimize_on_insert/__init__.py | 1 + .../test_optimize_on_insert/test.py | 48 +++++++++++++++++++ ...560_optimize_on_insert_zookeeper.reference | 1 + .../01560_optimize_on_insert_zookeeper.sql | 36 ++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 tests/integration/test_optimize_on_insert/__init__.py create mode 100644 tests/integration/test_optimize_on_insert/test.py create mode 100644 tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.reference create mode 100644 tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.sql diff --git a/tests/integration/test_optimize_on_insert/__init__.py b/tests/integration/test_optimize_on_insert/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_optimize_on_insert/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_optimize_on_insert/test.py b/tests/integration/test_optimize_on_insert/test.py new file mode 100644 index 00000000000..da4e20edf0c --- /dev/null +++ b/tests/integration/test_optimize_on_insert/test.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', with_zookeeper=True) +node2 = cluster.add_instance('node2', with_zookeeper=True) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + + +def get_data_files_for_table(node, table_name): + raw_output = node.exec_in_container(["bash", "-c", "ls /var/lib/clickhouse/data/default/{}".format(table_name)]) + return raw_output.strip().split("\n") + +def test_empty_parts_optimize(start_cluster): + for n, node in enumerate([node1, node2]): + node.query(""" + CREATE TABLE empty (key UInt32, val UInt32, date Datetime) + ENGINE=ReplicatedSummingMergeTree('/clickhouse/01560_optimize_on_insert', '{}', val) + PARTITION BY date ORDER BY key; + """.format(n+1)) + + node1.query("INSERT INTO empty VALUES (1, 1, '2020-01-01'), (1, 1, '2020-01-01'), (1, -2, '2020-01-01')") + + node2.query("SYSTEM SYNC REPLICA empty", timeout=15) + + assert node1.query("SELECT * FROM empty") == "" + assert node2.query("SELECT * FROM empty") == "" + + # No other tmp files exists + assert set(get_data_files_for_table(node1, "empty")) == {"detached", "format_version.txt"} + assert set(get_data_files_for_table(node2, "empty")) == {"detached", "format_version.txt"} + + node1.query("INSERT INTO empty VALUES (1, 1, '2020-02-01'), (1, 1, '2020-02-01'), (1, -2, '2020-02-01')", settings={"insert_quorum": 2}) + + assert node1.query("SELECT * FROM empty") == "" + assert node2.query("SELECT * FROM empty") == "" diff --git a/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.reference b/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.reference new file mode 100644 index 00000000000..e89c6201fb7 --- /dev/null +++ b/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.reference @@ -0,0 +1 @@ +Check creating empty parts diff --git a/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.sql b/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.sql new file mode 100644 index 00000000000..a98818b2195 --- /dev/null +++ b/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.sql @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS empty1; +DROP TABLE IF EXISTS empty2; + +SELECT 'Check creating empty parts'; + +CREATE TABLE empty1 (key UInt32, val UInt32, date Datetime) +ENGINE=ReplicatedSummingMergeTree('/clickhouse/01560_optimize_on_insert', '1', val) +PARTITION BY date ORDER BY key; + +CREATE TABLE empty2 (key UInt32, val UInt32, date Datetime) +ENGINE=ReplicatedSummingMergeTree('/clickhouse/01560_optimize_on_insert', '2', val) +PARTITION BY date ORDER BY key; + +INSERT INTO empty2 VALUES (1, 1, '2020-01-01'), (1, 1, '2020-01-01'), (1, -2, '2020-01-01'); + +SYSTEM SYNC REPLICA empty1; + +SELECT * FROM empty1 ORDER BY key; +SELECT * FROM empty2 ORDER BY key; + +SELECT table, partition, active FROM system.parts where table = 'empty1' and database=currentDatabase() and active = 1; +SELECT table, partition, active FROM system.parts where table = 'empty2' and database=currentDatabase() and active = 1; + +DETACH table empty1; +DETACH table empty2; +ATTACH table empty1; +ATTACH table empty2; + +SELECT * FROM empty1 ORDER BY key; +SELECT * FROM empty2 ORDER BY key; + +SELECT table, partition, active FROM system.parts where table = 'empty1' and database=currentDatabase() and active = 1; +SELECT table, partition, active FROM system.parts where table = 'empty2' and database=currentDatabase() and active = 1; + +DROP TABLE IF EXISTS empty1; +DROP TABLE IF EXISTS empty2; From 35e0d0f672321d8ab2120dd873f562a39d6310e6 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Mon, 15 Mar 2021 14:17:16 +0300 Subject: [PATCH 113/233] Reverted S3 connection pools. --- .../mergetree-family/mergetree.md | 2 -- .../mergetree-family/mergetree.md | 2 -- src/IO/S3/PocoHTTPClient.cpp | 26 +++++++++++-------- src/IO/S3/PocoHTTPClient.h | 3 +-- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 52d9111dc90..70cf7f2212e 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -748,7 +748,6 @@ Configuration markup: 10000 5000 - 100 10 1000 /var/lib/clickhouse/disks/s3/ @@ -771,7 +770,6 @@ Optional parameters: - `proxy` — Proxy configuration for S3 endpoint. Each `uri` element inside `proxy` block should contain a proxy URL. - `connect_timeout_ms` — Socket connect timeout in milliseconds. Default value is `10 seconds`. - `request_timeout_ms` — Request timeout in milliseconds. Default value is `5 seconds`. -- `max_connections` — S3 connections pool size. Default value is `100`. - `retry_attempts` — Number of retry attempts in case of failed request. Default value is `10`. - `min_bytes_for_seek` — Minimal number of bytes to use seek operation instead of sequential read. Default value is `1 Mb`. - `metadata_path` — Path on local FS to store metadata files for S3. Default value is `/var/lib/clickhouse/disks//`. diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index bc74b2592b9..b10087e0697 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -733,7 +733,6 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' 10000 5000 - 100 10 1000 /var/lib/clickhouse/disks/s3/ @@ -758,7 +757,6 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' - `proxy` — конфигурация прокси-сервера для конечной точки S3. Каждый элемент `uri` внутри блока `proxy` должен содержать URL прокси-сервера. - `connect_timeout_ms` — таймаут подключения к сокету в миллисекундах. Значение по умолчанию: 10 секунд. - `request_timeout_ms` — таймаут выполнения запроса в миллисекундах. Значение по умолчанию: 5 секунд. -- `max_connections` — размер пула соединений S3. Значение по умолчанию: `100`. - `retry_attempts` — число попыток выполнения запроса в случае возникновения ошибки. Значение по умолчанию: `10`. - `min_bytes_for_seek` — минимальное количество байтов, которые используются для операций поиска вместо последовательного чтения. Значение по умолчанию: 1 МБайт. - `metadata_path` — путь к локальному файловому хранилищу для хранения файлов с метаданными для S3. Значение по умолчанию: `/var/lib/clickhouse/disks//`. diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 471044dd08c..c31c12a1899 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -86,7 +86,6 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & clientConfigu )) , remote_host_filter(clientConfiguration.remote_host_filter) , s3_max_redirects(clientConfiguration.s3_max_redirects) - , max_connections(clientConfiguration.maxConnections) { } @@ -156,19 +155,24 @@ void PocoHTTPClient::makeRequestInternal( for (unsigned int attempt = 0; attempt <= s3_max_redirects; ++attempt) { Poco::URI target_uri(uri); - Poco::URI proxy_uri; - - auto request_configuration = per_request_configuration(request); - if (!request_configuration.proxyHost.empty()) - { - proxy_uri.setScheme(Aws::Http::SchemeMapper::ToString(request_configuration.proxyScheme)); - proxy_uri.setHost(request_configuration.proxyHost); - proxy_uri.setPort(request_configuration.proxyPort); - } /// Reverse proxy can replace host header with resolved ip address instead of host name. /// This can lead to request signature difference on S3 side. - auto session = makePooledHTTPSession(target_uri, proxy_uri, timeouts, max_connections, false); + auto session = makeHTTPSession(target_uri, timeouts, false); + + auto request_configuration = per_request_configuration(request); + + if (!request_configuration.proxyHost.empty()) + { + bool use_tunnel = request_configuration.proxyScheme == Aws::Http::Scheme::HTTP && target_uri.getScheme() == "https"; + + session->setProxy( + request_configuration.proxyHost, + request_configuration.proxyPort, + Aws::Http::SchemeMapper::ToString(request_configuration.proxyScheme), + use_tunnel + ); + } Poco::Net::HTTPRequest poco_request(Poco::Net::HTTPRequest::HTTP_1_1); diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 918943a413c..da6c4dd5985 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -40,7 +40,7 @@ private: class PocoHTTPResponse : public Aws::Http::Standard::StandardHttpResponse { public: - using SessionPtr = PooledHTTPSessionPtr; + using SessionPtr = HTTPSessionPtr; PocoHTTPResponse(const std::shared_ptr request) : Aws::Http::Standard::StandardHttpResponse(request) @@ -91,7 +91,6 @@ private: ConnectionTimeouts timeouts; const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects; - unsigned int max_connections; }; } From 307b89f6b0101268507a284ffa35f6a2682a569d Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Mar 2021 14:20:11 +0300 Subject: [PATCH 114/233] fix skip list --- tests/queries/skip_list.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 974ef48ef3c..5dc0aef0081 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -111,6 +111,7 @@ "memory_tracking", "memory_usage", "live_view", + "01761_alter_decimal_zookeeper", "01720_type_map_and_casts", "01413_alter_update_supertype", "01149_zookeeper_mutation_stuck_after_replace_partition", @@ -753,6 +754,7 @@ "01700_system_zookeeper_path_in", "01715_background_checker_blather_zookeeper", "01747_alter_partition_key_enum_zookeeper", + "01761_alter_decimal_zookeeper", "attach", "ddl_dictionaries", "dictionary", From 31ec24255e74098aa8e39723cfd14781770d9ce9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Mar 2021 14:24:52 +0300 Subject: [PATCH 115/233] Fix skip list --- tests/queries/skip_list.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 974ef48ef3c..90564d354e5 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -111,6 +111,7 @@ "memory_tracking", "memory_usage", "live_view", + "01560_optimize_on_insert_zookeeper", "01720_type_map_and_casts", "01413_alter_update_supertype", "01149_zookeeper_mutation_stuck_after_replace_partition", @@ -736,6 +737,7 @@ "01530_drop_database_atomic_sync", "01541_max_memory_usage_for_user_long", "01542_dictionary_load_exception_race", + "01560_optimize_on_insert_zookeeper", "01575_disable_detach_table_of_dictionary", "01593_concurrent_alter_mutations_kill", "01593_concurrent_alter_mutations_kill_many_replicas", From c2b398b39a5bada456e2c8d0d2fd84b7c1290a8a Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 15 Mar 2021 13:25:22 +0200 Subject: [PATCH 116/233] Documnentation on OPTIMIZE DEDUPLICATE BY expression. --- docs/en/sql-reference/statements/optimize.md | 66 ++++++++++++++++++- .../sql-reference/statements/select/index.md | 3 + 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 9b16a12d2e2..ea1fac90466 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -6,7 +6,7 @@ toc_title: OPTIMIZE # OPTIMIZE Statement {#misc_operations-optimize} ``` sql -OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE] +OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE [BY expression]] ``` This query tries to initialize an unscheduled merge of data parts for tables with a table engine from the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) family. @@ -18,7 +18,69 @@ When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engin - If `OPTIMIZE` doesn’t perform a merge for any reason, it doesn’t notify the client. To enable notifications, use the [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) setting. - If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](../../sql-reference/statements/alter/index.md#alter-how-to-specify-part-expr). - If you specify `FINAL`, optimization is performed even when all the data is already in one part. Also merge is forced even if concurrent merges are performed. -- If you specify `DEDUPLICATE`, then completely identical rows will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine. +- If you specify `DEDUPLICATE`, then completely identical rows (unless by-clause is specified) will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine. + + +### BY expression {#by-expression} + +If you want to perform deduplication on custom set of columns rather than on all, you can specify list of columns explicitly or use any combination of [`*`](../../../sql-reference/statements/select/index.md#asterisk), [`COLUMNS`](../../../sql-reference/statements/select/index.md#columns-expression) or [`EXCEPT`](../../../sql-reference/statements/select/index.md#except-modifier) expressions. The explictly written or implicitly expanded list of columns must include all columns specified in row ordering expression (both primary and sorting keys) and partitioning expression (partitioning key). + +Note that `*` behaves just like in `SELECT`: `MATERIALIZED`, and `ALIAS` columns are not used for expansion. +Also, it is an error to specify empty list of columns, or write an expression that results in an empty list of columns, or deduplicate by an ALIAS column. + +``` sql +OPTIMIZE TABLE table DEDUPLICATE; -- the old one +OPTIMIZE TABLE table DEDUPLICATE BY *; -- not the same as the old one, excludes MATERIALIZED columns (see the note above) +OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT colX; +OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT (colX, colY); +OPTIMIZE TABLE table DEDUPLICATE BY col1,col2,col3; +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex'); +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT colX; +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT (colX, colY); +``` + +**Example:** + +A silly synthetic table. +``` sql +CREATE TABLE example ( + primary_key Int32, + secondary_key Int32, + value UInt32, + partition_key UInt32, + materialized_value UInt32 MATERIALIZED 12345, + aliased_value UInt32 ALIAS 2, + PRIMARY KEY primary_key +) ENGINE=MergeTree +PARTITION BY partition_key +ORDER BY (primary_key, secondary_key); +``` + +``` sql +-- The 'old' deduplicate, all columns are taken into account, i.e. row is removed only if all values in all columns are equal to corresponding values in previous row. +OPTIMIZE TABLE example FINAL DEDUPLICATE; +``` + +``` sql +-- Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED`: `primary_key`, `secondary_key`, `value`, `partition_key`, and `materialized_value` columns. +OPTIMIZE TABLE example FINAL DEDUPLICATE BY *; +``` + +``` sql +-- Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED` and explicitly not `materialized_value`: `primary_key`, `secondary_key`, `value`, and `partition_key` columns. +OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT materialized_value; +``` + +``` sql +-- Deduplicate explicitly by `primary_key`, `secondary_key`, and `partition_key` columns. +OPTIMIZE TABLE example FINAL DEDUPLICATE BY primary_key, secondary_key, partition_key; +``` + +``` sql +-- Deduplicate by any column matching a regex: `primary_key`, `secondary_key`, and `partition_key` columns. +OPTIMIZE TABLE example FINAL DEDUPLICATE BY COLUMNS('.*_key'); +``` + !!! warning "Warning" `OPTIMIZE` can’t fix the “Too many parts” error. diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index e99ebef838c..ada4699c224 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -57,6 +57,9 @@ Specifics of each optional clause are covered in separate sections, which are li If you want to include all columns in the result, use the asterisk (`*`) symbol. For example, `SELECT * FROM ...`. + +### COLUMNS expression {#columns-expression} + To match some columns in the result with a [re2](https://en.wikipedia.org/wiki/RE2_(software)) regular expression, you can use the `COLUMNS` expression. ``` sql From 10a885c9b2e90f37a896d02b74e5d8f9474ea8a7 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Mar 2021 14:36:01 +0300 Subject: [PATCH 117/233] return back formatting in client --- programs/client/Client.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index c878a3071c4..3c27908741c 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -2096,10 +2096,10 @@ private: current_format = "Vertical"; /// It is not clear how to write progress with parallel formatting. It may increase code complexity significantly. - // if (!need_render_progress) - // block_out_stream = context.getOutputStreamParallelIfPossible(current_format, *out_buf, block); - // else - block_out_stream = context.getOutputStream(current_format, *out_buf, block); + if (!need_render_progress) + block_out_stream = context.getOutputStreamParallelIfPossible(current_format, *out_buf, block); + else + block_out_stream = context.getOutputStream(current_format, *out_buf, block); block_out_stream->writePrefix(); } From 588f3ee11ec5cd3f753ceda50fca06650b04a1e9 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Mar 2021 14:43:57 +0300 Subject: [PATCH 118/233] better[2] --- src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h | 2 -- src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 3fcd1f0aadf..584aa364d27 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -6,8 +6,6 @@ #include #include #include -#include "IO/ReadBuffer.h" -#include "IO/ReadBufferFromString.h" #include "IO/WriteBufferFromString.h" #include #include diff --git a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp index 7ded716b34e..355af038da9 100644 --- a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp +++ b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp @@ -149,8 +149,6 @@ void WriteBufferFromHTTPServerResponse::onProgress(const Progress & progress) { std::lock_guard lock(mutex); - std::cout << StackTrace().toString() << std::endl; - /// Cannot add new headers if body was started to send. if (headers_finished_sending) return; From 1f92c8ce581bfb8c820a7220a63ae005ed13316c Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Mar 2021 14:51:24 +0300 Subject: [PATCH 119/233] parallel formatting everywhere --- programs/obfuscator/Obfuscator.cpp | 2 +- programs/odbc-bridge/MainHandler.cpp | 2 +- src/Dictionaries/HTTPDictionarySource.cpp | 4 ++-- src/Storages/HDFS/StorageHDFS.cpp | 2 +- src/Storages/StorageS3.cpp | 2 +- src/Storages/StorageURL.cpp | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index 950db4e4f05..3ccbfd44357 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -1180,7 +1180,7 @@ try file_in.seek(0, SEEK_SET); BlockInputStreamPtr input = context.getInputFormat(input_format, file_in, header, max_block_size); - BlockOutputStreamPtr output = context.getOutputStream(output_format, file_out, header); + BlockOutputStreamPtr output = context.getOutputStreamParallelIfPossible(output_format, file_out, header); if (processed_rows + source_rows > limit) input = std::make_shared(input, limit - processed_rows, 0); diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 4fcc9deea6a..079fc371ab4 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -176,7 +176,7 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse std::string query = params.get("query"); LOG_TRACE(log, "Query: {}", query); - BlockOutputStreamPtr writer = FormatFactory::instance().getOutputStream(format, out, *sample_block, context); + BlockOutputStreamPtr writer = FormatFactory::instance().getOutputStreamParallelIfPossible(format, out, *sample_block, context); auto pool = getPool(connection_string); ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size); copyData(inp, *writer); diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index ddcac117e58..62bf478afc4 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -136,7 +136,7 @@ BlockInputStreamPtr HTTPDictionarySource::loadIds(const std::vector & id ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [block, this](std::ostream & ostr) { WriteBufferFromOStream out_buffer(ostr); - auto output_stream = context.getOutputStream(format, out_buffer, sample_block); + auto output_stream = context.getOutputStreamParallelIfPossible(format, out_buffer, sample_block); formatBlock(output_stream, block); }; @@ -157,7 +157,7 @@ BlockInputStreamPtr HTTPDictionarySource::loadKeys(const Columns & key_columns, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [block, this](std::ostream & ostr) { WriteBufferFromOStream out_buffer(ostr); - auto output_stream = context.getOutputStream(format, out_buffer, sample_block); + auto output_stream = context.getOutputStreamParallelIfPossible(format, out_buffer, sample_block); formatBlock(output_stream, block); }; diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index f7afd4a497d..e26d3375c33 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -183,7 +183,7 @@ public: : sample_block(sample_block_) { write_buf = wrapWriteBufferWithCompressionMethod(std::make_unique(uri, context.getGlobalContext().getConfigRef()), compression_method, 3); - writer = FormatFactory::instance().getOutputStream(format, *write_buf, sample_block, context); + writer = FormatFactory::instance().getOutputStreamParallelIfPossible(format, *write_buf, sample_block, context); } Block getHeader() const override diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index a31a7fa0944..1cbbe14d09f 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -155,7 +155,7 @@ namespace { write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique(client, bucket, key, min_upload_part_size, max_single_part_upload_size), compression_method, 3); - writer = FormatFactory::instance().getOutputStream(format, *write_buf, sample_block, context); + writer = FormatFactory::instance().getOutputStreamParallelIfPossible(format, *write_buf, sample_block, context); } Block getHeader() const override diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index ca984f9ece9..8b16a08b957 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -155,7 +155,7 @@ StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri, write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, timeouts), compression_method, 3); - writer = FormatFactory::instance().getOutputStream(format, *write_buf, sample_block, + writer = FormatFactory::instance().getOutputStreamParallelIfPossible(format, *write_buf, sample_block, context, {} /* write callback */, format_settings); } From c92d8c776cb650f96d3303e1982d04cdfae7ee56 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Mar 2021 15:14:58 +0300 Subject: [PATCH 120/233] try to avoid race in grpc server --- src/Server/GRPCServer.cpp | 6 ++++-- tests/integration/test_grpc_protocol/configs/users.xml | 8 -------- tests/integration/test_grpc_protocol/test.py | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) delete mode 100644 tests/integration/test_grpc_protocol/configs/users.xml diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index ede9bbff063..52a2c106488 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -783,8 +783,6 @@ namespace if (!io.out) return; - initializeBlockInputStream(io.out->getHeader()); - bool has_data_to_insert = (insert_query && insert_query->data) || !query_info.input_data().empty() || query_info.next_query_info(); if (!has_data_to_insert) @@ -795,6 +793,10 @@ namespace throw Exception("No data to insert", ErrorCodes::NO_DATA_TO_INSERT); } + /// This is significant, because parallel parsing may be used. + /// So we mustn't touch the input stream from other thread. + initializeBlockInputStream(io.out->getHeader()); + block_input_stream->readPrefix(); io.out->writePrefix(); diff --git a/tests/integration/test_grpc_protocol/configs/users.xml b/tests/integration/test_grpc_protocol/configs/users.xml deleted file mode 100644 index 2ae1a397fe5..00000000000 --- a/tests/integration/test_grpc_protocol/configs/users.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - 0 - - - diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index 594879427ca..d8604276281 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -27,7 +27,7 @@ import clickhouse_grpc_pb2_grpc config_dir = os.path.join(SCRIPT_DIR, './configs') cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/grpc_config.xml'], user_configs=["configs/users.xml"]) +node = cluster.add_instance('node', main_configs=['configs/grpc_config.xml']) grpc_port = 9100 main_channel = None From 23a1c634602c94fa010c238088c0c0706d4d2b80 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Mar 2021 17:40:51 +0300 Subject: [PATCH 121/233] Update StorageURL.cpp --- src/Storages/StorageURL.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 8b16a08b957..ca984f9ece9 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -155,7 +155,7 @@ StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri, write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, timeouts), compression_method, 3); - writer = FormatFactory::instance().getOutputStreamParallelIfPossible(format, *write_buf, sample_block, + writer = FormatFactory::instance().getOutputStream(format, *write_buf, sample_block, context, {} /* write callback */, format_settings); } From 4e35cf55098ea8df3f68542f46eaed4420f76838 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 15 Mar 2021 19:26:44 +0300 Subject: [PATCH 122/233] Update src/Storages/StoragePostgreSQL.cpp Co-authored-by: tavplubix --- src/Storages/StoragePostgreSQL.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 3e122ed2fc7..ac1a5569293 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -298,7 +298,7 @@ void registerStoragePostgreSQL(StorageFactory & factory) ASTs & engine_args = args.engine_args; if (engine_args.size() < 5 || engine_args.size() > 6) - throw Exception("Storage PostgreSQL requires 5-6 parameters: " + throw Exception("Storage PostgreSQL requires from 5 to 6 parameters: " "PostgreSQL('host:port', 'database', 'table', 'username', 'password' [, 'schema']", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); From dec1e9eb6fb5c9402bd41ffd5db6f691cdb5dc9f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 15 Mar 2021 19:40:48 +0300 Subject: [PATCH 123/233] make the fuzzer use sources from the CI --- docker/test/fuzzer/run-fuzzer.sh | 53 ++++++++++++++------------------ 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 9af401238a3..22fe4220aaf 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -14,35 +14,31 @@ BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-11_debug_none_bundled_unsplitted function clone { -( + # The download() function is dependent on CI binaries anyway, so we can take + # the repo from the CI as well. For local runs, start directly from the "fuzz" + # stage. rm -rf ch ||: mkdir ch cd ch - - git init - git remote add origin https://github.com/ClickHouse/ClickHouse - - # Network is unreliable. GitHub neither. - for _ in {1..100}; do git fetch --depth=100 origin "$SHA_TO_TEST" && break; sleep 1; done - # Used to obtain the list of modified or added tests - for _ in {1..100}; do git fetch --depth=100 origin master && break; sleep 1; done - - # If not master, try to fetch pull/.../{head,merge} - if [ "$PR_TO_TEST" != "0" ] - then - for _ in {1..100}; do git fetch --depth=100 origin "refs/pull/$PR_TO_TEST/*:refs/heads/pull/$PR_TO_TEST/*" && break; sleep 1; done - fi - - git checkout "$SHA_TO_TEST" -) + wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/repo/clickhouse_no_subs.tar.gz" + tar -xvf clickhouse_no_subs.tar.gz + tree ||: + ls -lath ||: } function download { - wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse" + wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse" & + wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/repo/ci-changed-files.txt" & + wait + chmod +x clickhouse ln -s ./clickhouse ./clickhouse-server ln -s ./clickhouse ./clickhouse-client + + + # clickhouse-server is in the current dir + export PATH="$PWD:$PATH" } function configure @@ -77,22 +73,19 @@ function watchdog function fuzz { # Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests. - cd ch - NEW_TESTS=$(git diff --name-only "$(git merge-base origin/master "$SHA_TO_TEST"~)" "$SHA_TO_TEST" | grep -P 'tests/queries/0_stateless/.*\.sql' | sed -r -e 's!^!ch/!' | sort -R) - cd .. + # Don't overwrite the NEW_TESTS_OPT so that it can be set from the environment. + NEW_TESTS="$(grep -P 'tests/queries/0_stateless/.*\.sql' ci-changed-files.txt | sed -r -e 's!^!ch/!' | sort -R)" if [[ -n "$NEW_TESTS" ]] then - NEW_TESTS_OPT="--interleave-queries-file ${NEW_TESTS}" - else - NEW_TESTS_OPT="" + NEW_TESTS_OPT="${NEW_TESTS_OPT:---interleave-queries-file ${NEW_TESTS}}" fi - ./clickhouse-server --config-file db/config.xml -- --path db 2>&1 | tail -100000 > server.log & + clickhouse-server --config-file db/config.xml -- --path db 2>&1 | tail -100000 > server.log & server_pid=$! kill -0 $server_pid - while ! ./clickhouse-client --query "select 1" && kill -0 $server_pid ; do echo . ; sleep 1 ; done - ./clickhouse-client --query "select 1" + while ! clickhouse-client --query "select 1" && kill -0 $server_pid ; do echo . ; sleep 1 ; done + clickhouse-client --query "select 1" kill -0 $server_pid echo Server started @@ -111,14 +104,14 @@ continue # SC2012: Use find instead of ls to better handle non-alphanumeric filenames. They are all alphanumeric. # SC2046: Quote this to prevent word splitting. Actually I need word splitting. # shellcheck disable=SC2012,SC2046 - ./clickhouse-client --query-fuzzer-runs=1000 --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) $NEW_TESTS_OPT \ + clickhouse-client --query-fuzzer-runs=1000 --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) $NEW_TESTS_OPT \ > >(tail -n 100000 > fuzzer.log) \ 2>&1 \ || fuzzer_exit_code=$? echo "Fuzzer exit code is $fuzzer_exit_code" - ./clickhouse-client --query "select elapsed, query from system.processes" ||: + clickhouse-client --query "select elapsed, query from system.processes" ||: killall clickhouse-server ||: for _ in {1..10} do From 9e8ebb5e2bbf9a740d269c4e251280aec4578125 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Mon, 15 Mar 2021 20:07:36 +0300 Subject: [PATCH 124/233] Update TableFunctionPostgreSQL.cpp --- src/TableFunctions/TableFunctionPostgreSQL.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index 0e3f1c5da24..0cf7c36b5ae 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -58,7 +58,7 @@ void TableFunctionPostgreSQL::parseArguments(const ASTPtr & ast_function, const ASTs & args = func_args.arguments->children; if (args.size() < 5 || args.size() > 6) - throw Exception("Table function 'PostgreSQL' requires 5-6 parameters: " + throw Exception("Table function 'PostgreSQL' requires from 5 to 6 parameters: " "PostgreSQL('host:port', 'database', 'table', 'user', 'password', [, 'schema']).", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); From badd5165dac3e864efeb863da3324a57ebabc6ee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 20:36:49 +0300 Subject: [PATCH 125/233] Fix UBSan report in rounding to years intervals --- base/common/DateLUTImpl.h | 9 ++++++++- .../0_stateless/01761_round_year_bounds.reference | 0 tests/queries/0_stateless/01761_round_year_bounds.sql | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01761_round_year_bounds.reference create mode 100644 tests/queries/0_stateless/01761_round_year_bounds.sql diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 6e968a0cd50..fc30ee9c6a0 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -807,7 +807,14 @@ public: return toFirstDayNumOfYear(v); const LUTIndex i = toLUTIndex(v); - return toDayNum(years_lut[lut[i].year / years * years - DATE_LUT_MIN_YEAR]); + + UInt16 year = lut[i].year / years * years; + + /// For example, rounding down 1925 to 100 years will be 1900, but it's less than min supported year. + if (unlikely(year < DATE_LUT_MIN_YEAR)) + year = DATE_LUT_MIN_YEAR; + + return toDayNum(years_lut[year - DATE_LUT_MIN_YEAR]); } inline ExtendedDayNum toStartOfQuarterInterval(ExtendedDayNum d, UInt64 quarters) const diff --git a/tests/queries/0_stateless/01761_round_year_bounds.reference b/tests/queries/0_stateless/01761_round_year_bounds.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01761_round_year_bounds.sql b/tests/queries/0_stateless/01761_round_year_bounds.sql new file mode 100644 index 00000000000..fed12c55568 --- /dev/null +++ b/tests/queries/0_stateless/01761_round_year_bounds.sql @@ -0,0 +1 @@ +SELECT toStartOfInterval(toDateTime(-9223372036854775808), toIntervalYear(100), 'Europe/Moscow') FORMAT Null; From 845f4afbf4c7287a586b42874150f7b6ef039b80 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 21:09:05 +0300 Subject: [PATCH 126/233] Fix parsing of pre-epoch time --- base/common/DateLUTImpl.h | 7 +------ .../01762_datetime64_extended_parsing.reference | 1 + .../0_stateless/01762_datetime64_extended_parsing.sql | 1 + 3 files changed, 3 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/01762_datetime64_extended_parsing.reference create mode 100644 tests/queries/0_stateless/01762_datetime64_extended_parsing.sql diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index fc30ee9c6a0..867862ad51e 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -913,12 +913,7 @@ public: if (time_offset >= lut[index].time_at_offset_change()) time_offset -= lut[index].amount_of_offset_change(); - UInt32 res = lut[index].date + time_offset; - - if (unlikely(res > DATE_LUT_MAX)) - return 0; - - return res; + return lut[index].date + time_offset; } template diff --git a/tests/queries/0_stateless/01762_datetime64_extended_parsing.reference b/tests/queries/0_stateless/01762_datetime64_extended_parsing.reference new file mode 100644 index 00000000000..531b6f8bf13 --- /dev/null +++ b/tests/queries/0_stateless/01762_datetime64_extended_parsing.reference @@ -0,0 +1 @@ +1925-01-02 03:04:05.678901 diff --git a/tests/queries/0_stateless/01762_datetime64_extended_parsing.sql b/tests/queries/0_stateless/01762_datetime64_extended_parsing.sql new file mode 100644 index 00000000000..a7ad447b215 --- /dev/null +++ b/tests/queries/0_stateless/01762_datetime64_extended_parsing.sql @@ -0,0 +1 @@ +SELECT toDateTime64('1925-01-02 03:04:05.678901', 6); From 60f125c2b5b9c002e12238ff73f80f6814448699 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 15 Mar 2021 21:36:42 +0300 Subject: [PATCH 127/233] translation draft --- docs/ru/sql-reference/statements/attach.md | 5 +- docs/ru/sql-reference/statements/detach.md | 61 ++++++++++++++++++++-- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/docs/ru/sql-reference/statements/attach.md b/docs/ru/sql-reference/statements/attach.md index 259ab893e63..be5b0b6d44a 100644 --- a/docs/ru/sql-reference/statements/attach.md +++ b/docs/ru/sql-reference/statements/attach.md @@ -10,14 +10,15 @@ toc_title: ATTACH - вместо слова `CREATE` используется слово `ATTACH`; - запрос не создаёт данные на диске, а предполагает, что данные уже лежат в соответствующих местах, и всего лишь добавляет информацию о таблице на сервер. После выполнения запроса `ATTACH` сервер будет знать о существовании таблицы. -Если таблица перед этим была отсоединена (`DETACH`), т.е. её структура известна, можно использовать сокращенную форму записи без определения структуры. +Если таблица перед этим была откреплена ([DETACH](../../sql-reference/statements/detach.md)), т.е. её структура известна, можно использовать сокращенную форму записи без определения структуры. ``` sql ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] ``` -Этот запрос используется при старте сервера. Сервер хранит метаданные таблиц в виде файлов с запросами `ATTACH`, которые он просто исполняет при запуске (за исключением системных таблиц, которые явно создаются на сервере). +Этот запрос используется при старте сервера. Сервер хранит метаданные таблиц в виде файлов с запросами `ATTACH`, которые он просто исполняет при запуске (за исключением некоторых системных таблиц, которые явно создаются на сервере). +Если таблица была откреплена перманентно, она не будет прикреплена обратно во время старта сервера, так что нужно явно использовать запрос `ATTACH`, чтобы прикрепить ее. [Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/attach/) diff --git a/docs/ru/sql-reference/statements/detach.md b/docs/ru/sql-reference/statements/detach.md index 00d0a4b20c6..bec8f4c5ff7 100644 --- a/docs/ru/sql-reference/statements/detach.md +++ b/docs/ru/sql-reference/statements/detach.md @@ -5,15 +5,66 @@ toc_title: DETACH # DETACH {#detach-statement} -Удаляет из сервера информацию о таблице name. Сервер перестаёт знать о существовании таблицы. +Удаляет из сервера информацию о таблице или материализованном представлении. Сервер перестаёт знать о существовании таблицы. + +Синтаксис: ``` sql -DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] +DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] ``` -Но ни данные, ни метаданные таблицы не удаляются. При следующем запуске сервера, сервер прочитает метаданные и снова узнает о таблице. -Также, «отцепленную» таблицу можно прицепить заново запросом `ATTACH` (за исключением системных таблиц, для которых метаданные не хранятся). +Но ни данные, ни метаданные таблицы или материализованного представления не удаляются. При следующем запуске сервера, если не было использовано `PERMANENTLY`, сервер прочитает метаданные и снова узнает о таблице/представлении. Если таблица или представление были откреплено перманентно, сервер не прикрепит их обратно автоматически. -Запроса `DETACH DATABASE` нет. +Независимо от того, каким способом таблица была откреплена, ее можно прикрепить обратно с помощью запроса [ATTACH](../../sql-reference/statements/attach.md). Системные log таблицы также могут быть прикреплены обратно (к примеру `query_log`, `text_log` и др.) Другие системные таблицы не могут быть прикреплены обратно, но на следующем запуске сервер снова вспомнит об этих таблицах. + +`ATTACH MATERIALIZED VIEW` не может быть использован с кратким синтаксисом (без `SELECT`), но можно прикрепить представление с помощью запроса `ATTACH TABLE`. + +Обратите внимание, что нельзя перманентно открепить таблицу, которая уже временно откреплена. Для этого ее сначала надо прикрепить обратно, а затем снова открепить перманентно. + +Также нельзя использовать [DROP](../../sql-reference/statements/drop.md#drop-table) с открепленной таблицей или создавать таблицу с помощью [CREATE TABLE](../../sql-reference/statements/create/table.md) с таким же именем, как уже открепленная таблица. Еще нельзя заменить открепленную таблицу другой с помощью запроса [RENAME TABLE](../../sql-reference/statements/rename.md). + +**Пример** + +Создание таблицы: + +Запрос: + +``` sql +CREATE TABLE test ENGINE = Log AS SELECT * FROM numbers(10); +SELECT * FROM test; +``` + +Результат: + +``` text +┌─number─┐ +│ 0 │ +│ 1 │ +│ 2 │ +│ 3 │ +│ 4 │ +│ 5 │ +│ 6 │ +│ 7 │ +│ 8 │ +│ 9 │ +└────────┘ +``` + +Открепление таблицы: + +Запрос: + +``` sql +DETACH TABLE test; +SELECT * FROM test; +``` + +Результат: + +``` text +Received exception from server (version 21.4.1): +Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.test doesn't exist. +``` [Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/detach/) From a78b234a6d62412edbdc21f187495a240c13ea63 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 15 Mar 2021 21:45:57 +0300 Subject: [PATCH 128/233] fixes --- docker/test/fuzzer/run-fuzzer.sh | 10 +++++++--- docker/test/performance-comparison/compare.sh | 4 +++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 22fe4220aaf..6858e838850 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -4,7 +4,9 @@ set -eux set -o pipefail trap "exit" INT TERM -trap 'kill $(jobs -pr) ||:' EXIT +# The watchdog is in the separate process group, so we have to kill it separately +# if the script terminates earlier. +trap 'kill $(jobs -pr) ${watchdog_pid:-} ||:' EXIT stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -20,7 +22,7 @@ function clone rm -rf ch ||: mkdir ch cd ch - wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/repo/clickhouse_no_subs.tar.gz" + wget -nv -nd -c "https://clickhouse-test-reports.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/repo/clickhouse_no_subs.tar.gz" tar -xvf clickhouse_no_subs.tar.gz tree ||: ls -lath ||: @@ -29,7 +31,7 @@ function clone function download { wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse" & - wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/repo/ci-changed-files.txt" & + wget -nv -nd -c "https://clickhouse-test-reports.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/repo/ci-changed-files.txt" & wait chmod +x clickhouse @@ -78,6 +80,8 @@ function fuzz if [[ -n "$NEW_TESTS" ]] then NEW_TESTS_OPT="${NEW_TESTS_OPT:---interleave-queries-file ${NEW_TESTS}}" + else + NEW_TESTS_OPT="${NEW_TESTS_OPT:-}" fi clickhouse-server --config-file db/config.xml -- --path db 2>&1 | tail -100000 > server.log & diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 9a0d8093a55..1505fc77691 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -2,7 +2,9 @@ set -exu set -o pipefail trap "exit" INT TERM -trap 'kill $(jobs -pr) ||:' EXIT +# The watchdog is in the separate process group, so we have to kill it separately +# if the script terminates earlier. +trap 'kill $(jobs -pr) ${watchdog_pid:-} ||:' EXIT stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" From 671395e8c81e79f079334650112737eca6e3f6c7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 22:23:27 +0300 Subject: [PATCH 129/233] Most likely improve performance --- base/common/DateLUTImpl.h | 145 +++++++++++++----- base/common/LocalDate.h | 20 --- base/common/LocalDateTime.h | 57 ++----- programs/git-import/git-import.cpp | 2 +- .../PostgreSQLBlockInputStream.cpp | 15 +- .../ClickHouseDictionarySource.cpp | 5 +- src/Dictionaries/MySQLDictionarySource.cpp | 7 +- .../PostgreSQLDictionarySource.cpp | 5 +- src/Dictionaries/RedisBlockInputStream.cpp | 7 +- src/Dictionaries/XDBCDictionarySource.cpp | 5 +- src/Formats/MySQLBlockInputStream.cpp | 8 +- src/IO/WriteHelpers.h | 12 +- .../MergeTree/MergeTreeMutationEntry.cpp | 4 +- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 4 +- .../ReplicatedMergeTreeMutationEntry.cpp | 4 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 16 files changed, 163 insertions(+), 139 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 867862ad51e..6bac9bd5126 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -944,16 +944,70 @@ public: } + struct DateComponents + { + uint16_t year; + uint8_t month; + uint8_t day; + }; + + struct TimeComponents + { + uint8_t hour; + uint8_t minute; + uint8_t second; + }; + + struct DateTimeComponents + { + DateComponents date; + TimeComponents time; + }; + + inline DateComponents toDateComponents(time_t t) const + { + const Values & values = getValues(t); + return { values.year, values.month, values.day_of_month }; + } + + inline DateTimeComponents toDateTimeComponents(time_t t) const + { + const LUTIndex index = findIndex(t); + const Values & values = lut[index]; + + DateTimeComponents res; + + res.date.year = values.year; + res.date.month = values.month; + res.date.day = values.day_of_month; + + time_t time = t - values.date; + if (time >= values.time_at_offset_change()) + time += values.amount_of_offset_change(); + + res.time.second = time % 60; + res.time.minute = time / 60 % 60; + res.time.hour = time / 3600; + + /// In case time was changed backwards at the start of next day, we will repeat the hour 23. + if (unlikely(res.time.hour > 23)) + res.time.hour = 23; + + return res; + } + + inline UInt64 toNumYYYYMMDDhhmmss(time_t t) const { - const Values & values = find(t); + DateTimeComponents components = toDateTimeComponents(t); + return - toSecond(t) - + toMinute(t) * 100 - + toHour(t) * 10000 - + UInt64(values.day_of_month) * 1000000 - + UInt64(values.month) * 100000000 - + UInt64(values.year) * 10000000000; + components.time.second + + components.time.minute * 100 + + components.time.hour * 10000 + + UInt64(components.date.day) * 1000000 + + UInt64(components.date.month) * 100000000 + + UInt64(components.date.year) * 10000000000; } inline time_t YYYYMMDDhhmmssToTime(UInt64 num) const @@ -972,16 +1026,19 @@ public: inline NO_SANITIZE_UNDEFINED time_t addDays(time_t t, Int64 delta) const { - LUTIndex index = findIndex(t); - time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); + const LUTIndex index = findIndex(t); + const Values & values = lut[index]; - index += delta; - index &= date_lut_mask; + time_t time = t - values.date; + if (time >= values.time_at_offset_change()) + time += values.amount_of_offset_change(); - if (time_offset >= lut[index].time_at_offset_change()) - time_offset -= lut[index].amount_of_offset_change(); + const LUTIndex new_index = index + delta; - return lut[index].date + time_offset; + if (time >= lut[new_index].time_at_offset_change()) + time -= lut[new_index].amount_of_offset_change(); + + return lut[new_index].date + time; } inline NO_SANITIZE_UNDEFINED time_t addWeeks(time_t t, Int64 delta) const @@ -1033,12 +1090,17 @@ public: { const auto result_day = addMonthsIndex(t, delta); - time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); + const LUTIndex index = findIndex(t); + const Values & values = lut[index]; - if (time_offset >= lut[result_day].time_at_offset_change()) - time_offset -= lut[result_day].amount_of_offset_change(); + time_t time = t - values.date; + if (time >= values.time_at_offset_change()) + time += values.amount_of_offset_change(); - return lut[result_day].date + time_offset; + if (time >= lut[result_day].time_at_offset_change()) + time -= lut[result_day].amount_of_offset_change(); + + return lut[result_day].date + time; } inline ExtendedDayNum NO_SANITIZE_UNDEFINED addMonths(ExtendedDayNum d, Int64 delta) const @@ -1077,12 +1139,17 @@ public: { auto result_day = addYearsIndex(t, delta); - time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); + const LUTIndex index = findIndex(t); + const Values & values = lut[index]; - if (time_offset >= lut[result_day].time_at_offset_change()) - time_offset -= lut[result_day].amount_of_offset_change(); + time_t time = t - values.date; + if (time >= values.time_at_offset_change()) + time += values.amount_of_offset_change(); - return lut[result_day].date + time_offset; + if (time >= lut[result_day].time_at_offset_change()) + time -= lut[result_day].amount_of_offset_change(); + + return lut[result_day].date + time; } inline ExtendedDayNum addYears(ExtendedDayNum d, Int64 delta) const @@ -1093,29 +1160,25 @@ public: inline std::string timeToString(time_t t) const { - const Values & values = getValues(t); + DateTimeComponents components = toDateTimeComponents(t); std::string s {"0000-00-00 00:00:00"}; - s[0] += values.year / 1000; - s[1] += (values.year / 100) % 10; - s[2] += (values.year / 10) % 10; - s[3] += values.year % 10; - s[5] += values.month / 10; - s[6] += values.month % 10; - s[8] += values.day_of_month / 10; - s[9] += values.day_of_month % 10; + s[0] += components.date.year / 1000; + s[1] += (components.date.year / 100) % 10; + s[2] += (components.date.year / 10) % 10; + s[3] += components.date.year % 10; + s[5] += components.date.month / 10; + s[6] += components.date.month % 10; + s[8] += components.date.day / 10; + s[9] += components.date.day % 10; - auto hour = toHour(t); - auto minute = toMinute(t); - auto second = toSecond(t); - - s[11] += hour / 10; - s[12] += hour % 10; - s[14] += minute / 10; - s[15] += minute % 10; - s[17] += second / 10; - s[18] += second % 10; + s[11] += components.time.hour / 10; + s[12] += components.time.hour % 10; + s[14] += components.time.minute / 10; + s[15] += components.time.minute % 10; + s[17] += components.time.second / 10; + s[18] += components.time.second % 10; return s; } diff --git a/base/common/LocalDate.h b/base/common/LocalDate.h index 7e1260c1385..b1e6eeb907c 100644 --- a/base/common/LocalDate.h +++ b/base/common/LocalDate.h @@ -92,17 +92,6 @@ public: LocalDate(const LocalDate &) noexcept = default; LocalDate & operator= (const LocalDate &) noexcept = default; - LocalDate & operator= (time_t time) - { - init(time); - return *this; - } - - operator time_t() const - { - return DateLUT::instance().makeDate(m_year, m_month, m_day); - } - DayNum getDayNum() const { const auto & lut = DateLUT::instance(); @@ -167,12 +156,3 @@ public: }; static_assert(sizeof(LocalDate) == 4); - - -namespace std -{ -inline string to_string(const LocalDate & date) -{ - return date.toString(); -} -} diff --git a/base/common/LocalDateTime.h b/base/common/LocalDateTime.h index 0e237789bd1..4c2cf0e637d 100644 --- a/base/common/LocalDateTime.h +++ b/base/common/LocalDateTime.h @@ -29,29 +29,16 @@ private: /// NOTE We may use attribute packed instead, but it is less portable. unsigned char pad = 0; - void init(time_t time) + void init(time_t time, const DateLUTImpl & time_zone) { - if (unlikely(time > DATE_LUT_MAX || time == 0)) - { - m_year = 0; - m_month = 0; - m_day = 0; - m_hour = 0; - m_minute = 0; - m_second = 0; + DateLUTImpl::DateTimeComponents components = time_zone.toDateTimeComponents(time); - return; - } - - const auto & date_lut = DateLUT::instance(); - const auto & values = date_lut.getValues(time); - - m_year = values.year; - m_month = values.month; - m_day = values.day_of_month; - m_hour = date_lut.toHour(time); - m_minute = date_lut.toMinute(time); - m_second = date_lut.toSecond(time); + m_year = components.date.year; + m_month = components.date.month; + m_day = components.date.day; + m_hour = components.time.hour; + m_minute = components.time.minute; + m_second = components.time.second; (void)pad; /// Suppress unused private field warning. } @@ -73,9 +60,9 @@ private: } public: - explicit LocalDateTime(time_t time) + explicit LocalDateTime(time_t time, const DateLUTImpl & time_zone = DateLUT::instance()) { - init(time); + init(time, time_zone); } LocalDateTime(unsigned short year_, unsigned char month_, unsigned char day_, @@ -104,19 +91,6 @@ public: LocalDateTime(const LocalDateTime &) noexcept = default; LocalDateTime & operator= (const LocalDateTime &) noexcept = default; - LocalDateTime & operator= (time_t time) - { - init(time); - return *this; - } - - operator time_t() const - { - return m_year == 0 - ? 0 - : DateLUT::instance().makeDateTime(m_year, m_month, m_day, m_hour, m_minute, m_second); - } - unsigned short year() const { return m_year; } unsigned char month() const { return m_month; } unsigned char day() const { return m_day; } @@ -167,14 +141,3 @@ public: }; static_assert(sizeof(LocalDateTime) == 8); - - -namespace std -{ -inline string to_string(const LocalDateTime & datetime) -{ - stringstream str; - str << datetime; - return str.str(); -} -} diff --git a/programs/git-import/git-import.cpp b/programs/git-import/git-import.cpp index ae8b55e2aff..b07435dcf78 100644 --- a/programs/git-import/git-import.cpp +++ b/programs/git-import/git-import.cpp @@ -1064,7 +1064,7 @@ void processCommit( time_t commit_time; readText(commit_time, in); - commit.time = commit_time; + commit.time = LocalDateTime(commit_time); assertChar('\0', in); readNullTerminated(commit.author, in); std::string parent_hash; diff --git a/src/DataStreams/PostgreSQLBlockInputStream.cpp b/src/DataStreams/PostgreSQLBlockInputStream.cpp index da6a83fb930..4cf2d942885 100644 --- a/src/DataStreams/PostgreSQLBlockInputStream.cpp +++ b/src/DataStreams/PostgreSQLBlockInputStream.cpp @@ -160,8 +160,13 @@ void PostgreSQLBlockInputStream::insertValue(IColumn & column, std::string_view assert_cast(column).insertValue(UInt16{LocalDate{std::string(value)}.getDayNum()}); break; case ValueType::vtDateTime: - assert_cast(column).insertValue(time_t{LocalDateTime{std::string(value)}}); + { + ReadBufferFromString in(value); + time_t time = 0; + readDateTimeText(time, in); + assert_cast(column).insertValue(time); break; + } case ValueType::vtDateTime64:[[fallthrough]]; case ValueType::vtDecimal32: [[fallthrough]]; case ValueType::vtDecimal64: [[fallthrough]]; @@ -257,7 +262,13 @@ void PostgreSQLBlockInputStream::prepareArrayInfo(size_t column_idx, const DataT else if (which.isDate()) parser = [](std::string & field) -> Field { return UInt16{LocalDate{field}.getDayNum()}; }; else if (which.isDateTime()) - parser = [](std::string & field) -> Field { return time_t{LocalDateTime{field}}; }; + parser = [](std::string & field) -> Field + { + ReadBufferFromString in(field); + time_t time = 0; + readDateTimeText(time, in); + return time; + }; else if (which.isDecimal32()) parser = [nested](std::string & field) -> Field { diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index f4c17884afa..5e69df2c8a0 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -118,10 +118,9 @@ std::string ClickHouseDictionarySource::getUpdateFieldAndDate() { if (update_time != std::chrono::system_clock::from_time_t(0)) { - auto tmp_time = update_time; + time_t hr_time = std::chrono::system_clock::to_time_t(update_time) - 1; + std::string str_time = DateLUT::instance().timeToString(hr_time); update_time = std::chrono::system_clock::now(); - time_t hr_time = std::chrono::system_clock::to_time_t(tmp_time) - 1; - std::string str_time = std::to_string(LocalDateTime(hr_time)); return query_builder.composeUpdateQuery(update_field, str_time); } else diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 572080aee1e..a1d0cbe7ffc 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -107,10 +107,9 @@ std::string MySQLDictionarySource::getUpdateFieldAndDate() { if (update_time != std::chrono::system_clock::from_time_t(0)) { - auto tmp_time = update_time; + time_t hr_time = std::chrono::system_clock::to_time_t(update_time) - 1; + std::string str_time = DateLUT::instance().timeToString(hr_time); update_time = std::chrono::system_clock::now(); - time_t hr_time = std::chrono::system_clock::to_time_t(tmp_time) - 1; - std::string str_time = std::to_string(LocalDateTime(hr_time)); return query_builder.composeUpdateQuery(update_field, str_time); } else @@ -262,7 +261,7 @@ LocalDateTime MySQLDictionarySource::getLastModification(mysqlxx::Pool::Entry & if (!update_time_value.isNull()) { modification_time = update_time_value.getDateTime(); - LOG_TRACE(log, "Got modification time: {}", modification_time); + LOG_TRACE(log, "Got modification time: {}", update_time_value.getString()); } /// fetch remaining rows to avoid "commands out of sync" error diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index aa852404750..93a57383380 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -127,10 +127,9 @@ std::string PostgreSQLDictionarySource::getUpdateFieldAndDate() { if (update_time != std::chrono::system_clock::from_time_t(0)) { - auto tmp_time = update_time; + time_t hr_time = std::chrono::system_clock::to_time_t(update_time) - 1; + std::string str_time = DateLUT::instance().timeToString(hr_time); update_time = std::chrono::system_clock::now(); - time_t hr_time = std::chrono::system_clock::to_time_t(tmp_time) - 1; - std::string str_time = std::to_string(LocalDateTime(hr_time)); return query_builder.composeUpdateQuery(update_field, str_time); } else diff --git a/src/Dictionaries/RedisBlockInputStream.cpp b/src/Dictionaries/RedisBlockInputStream.cpp index a5514d14155..6a2072f7f6a 100644 --- a/src/Dictionaries/RedisBlockInputStream.cpp +++ b/src/Dictionaries/RedisBlockInputStream.cpp @@ -99,8 +99,13 @@ namespace DB assert_cast(column).insertValue(parse(string_value).getDayNum()); break; case ValueType::vtDateTime: - assert_cast(column).insertValue(static_cast(parse(string_value))); + { + ReadBufferFromString in(string_value); + time_t time = 0; + readDateTimeText(time, in); + assert_cast(column).insertValue(time); break; + } case ValueType::vtUUID: assert_cast(column).insertValue(parse(string_value)); break; diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index 3615f72605f..37d54f3549c 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -156,10 +156,9 @@ std::string XDBCDictionarySource::getUpdateFieldAndDate() { if (update_time != std::chrono::system_clock::from_time_t(0)) { - auto tmp_time = update_time; + time_t hr_time = std::chrono::system_clock::to_time_t(update_time) - 1; + std::string str_time = DateLUT::instance().timeToString(hr_time); update_time = std::chrono::system_clock::now(); - time_t hr_time = std::chrono::system_clock::to_time_t(tmp_time) - 1; - std::string str_time = std::to_string(LocalDateTime(hr_time)); return query_builder.composeUpdateQuery(update_field, str_time); } else diff --git a/src/Formats/MySQLBlockInputStream.cpp b/src/Formats/MySQLBlockInputStream.cpp index 87df0c1f4b1..bf7cf266e91 100644 --- a/src/Formats/MySQLBlockInputStream.cpp +++ b/src/Formats/MySQLBlockInputStream.cpp @@ -11,6 +11,7 @@ # include # include # include +# include # include # include # include @@ -97,8 +98,13 @@ namespace assert_cast(column).insertValue(UInt16(value.getDate().getDayNum())); break; case ValueType::vtDateTime: - assert_cast(column).insertValue(UInt32(value.getDateTime())); + { + ReadBufferFromString in(value); + time_t time = 0; + readDateTimeText(time, in); + assert_cast(column).insertValue(time); break; + } case ValueType::vtUUID: assert_cast(column).insert(parse(value.data(), value.size())); break; diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 72c6b69114f..b9497b6f87e 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -821,24 +821,18 @@ inline void writeDateTimeText(const LocalDateTime & datetime, WriteBuffer & buf) template inline void writeDateTimeText(time_t datetime, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - const auto & values = time_zone.getValues(datetime); - writeDateTimeText( - LocalDateTime(values.year, values.month, values.day_of_month, - time_zone.toHour(datetime), time_zone.toMinute(datetime), time_zone.toSecond(datetime)), buf); + writeDateTimeText(LocalDateTime(datetime, time_zone), buf); } /// In the format YYYY-MM-DD HH:MM:SS.NNNNNNNNN, according to the specified time zone. template -inline void writeDateTimeText(DateTime64 datetime64, UInt32 scale, WriteBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void writeDateTimeText(DateTime64 datetime64, UInt32 scale, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { static constexpr UInt32 MaxScale = DecimalUtils::max_precision; scale = scale > MaxScale ? MaxScale : scale; auto components = DecimalUtils::split(datetime64, scale); - const auto & values = date_lut.getValues(components.whole); - writeDateTimeText( - LocalDateTime(values.year, values.month, values.day_of_month, - date_lut.toHour(components.whole), date_lut.toMinute(components.whole), date_lut.toSecond(components.whole)), buf); + writeDateTimeText(LocalDateTime(components.whole, time_zone), buf); if (scale > 0) { diff --git a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp index 44c4b3c4d10..49c4e93eb1d 100644 --- a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp +++ b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp @@ -75,7 +75,9 @@ MergeTreeMutationEntry::MergeTreeMutationEntry(DiskPtr disk_, const String & pat LocalDateTime create_time_dt; *buf >> "create time: " >> create_time_dt >> "\n"; - create_time = create_time_dt; + create_time = DateLUT::instance().makeDateTime( + create_time_dt.year(), create_time_dt.month(), create_time_dt.day(), + create_time_dt.hour(), create_time_dt.minute(), create_time_dt.second()); *buf >> "commands: "; commands.readText(*buf); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index 9a9f25fd470..07c64d9c95c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -162,7 +162,9 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) { LocalDateTime create_time_dt; in >> "create_time: " >> create_time_dt >> "\n"; - create_time = create_time_dt; + create_time = DateLUT::instance().makeDateTime( + create_time_dt.year(), create_time_dt.month(), create_time_dt.day(), + create_time_dt.hour(), create_time_dt.minute(), create_time_dt.second()); } in >> "source replica: " >> source_replica >> "\n"; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp index b2299b2cbbd..c617befe9c4 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp @@ -37,7 +37,9 @@ void ReplicatedMergeTreeMutationEntry::readText(ReadBuffer & in) LocalDateTime create_time_dt; in >> "create time: " >> create_time_dt >> "\n"; - create_time = create_time_dt; + create_time = DateLUT::instance().makeDateTime( + create_time_dt.year(), create_time_dt.month(), create_time_dt.day(), + create_time_dt.hour(), create_time_dt.minute(), create_time_dt.second()); in >> "source replica: " >> source_replica >> "\n"; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 3490e9918c5..a8de89a8bb0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1450,7 +1450,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) { LOG_INFO(log, "Will try to fetch part {} until '{}' because this part assigned to recompression merge. " "Source replica {} will try to merge this part first", entry.new_part_name, - LocalDateTime(entry.create_time + storage_settings_ptr->try_fetch_recompressed_part_timeout.totalSeconds()), entry.source_replica); + DateLUT::instance().timeToString(entry.create_time + storage_settings_ptr->try_fetch_recompressed_part_timeout.totalSeconds()), entry.source_replica); return false; } From e485a27dc1b86077481e7b74b08272070ddcfe40 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 15 Mar 2021 22:45:45 +0300 Subject: [PATCH 130/233] some fixes --- programs/client/Client.cpp | 13 +++++++++---- programs/client/QueryFuzzer.cpp | 9 +++++++++ src/Common/FieldVisitors.cpp | 12 +++++++++++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 91fa36e9e92..527fec17c63 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1413,13 +1413,18 @@ private: if (formatted_twice != fuzzed_text) { + fmt::print(stderr, "The query formatting is broken.\n"); + printChangedSettings(); - fmt::print(stderr, "The query formatting is broken. Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n", + fmt::print(stderr, "Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n", formatted_twice, fuzzed_text); - fmt::print(stderr, "AST parsed back:\n'{}'\nSource AST:\n'{}'\n", - parsed_formatted_query->dumpTree(), - ast_to_process->dumpTree()); + fmt::print(stderr, "In more detail:\n"); + fmt::print(stderr, "AST-1:\n'{}'\n", ast_to_process->dumpTree()); + fmt::print(stderr, "Text-1 (AST-1 formatted):\n'{}'\n", fuzzed_text); + fmt::print(stderr, "AST-2 (Text-1 parsed):\n'{}'\n", parsed_formatted_query->dumpTree()); + fmt::print(stderr, "Text-2 (AST-2 formatted):\n'{}'\n", formatted_twice); + fmt::print(stderr, "Text-1 must be equal to Text-2, but it is not.\n"); exit(1); } diff --git a/programs/client/QueryFuzzer.cpp b/programs/client/QueryFuzzer.cpp index 8d8d8daaf39..0c8dc0731f9 100644 --- a/programs/client/QueryFuzzer.cpp +++ b/programs/client/QueryFuzzer.cpp @@ -570,6 +570,15 @@ void QueryFuzzer::addColumnLike(const ASTPtr ast) } const auto name = ast->formatForErrorMessage(); + if (name == "Null") + { + // The `Null` identifier from FORMAT Null clause. We don't quote it + // properly when formatting the AST, and while the resulting query + // technically works, it has non-standard case for Null (the standard + // is NULL), so it breaks the query formatting idempotence check. + // Just plug this particular case for now. + return; + } if (name.size() < 200) { column_like_map.insert({name, ast}); diff --git a/src/Common/FieldVisitors.cpp b/src/Common/FieldVisitors.cpp index dae153bd8d2..62f04816032 100644 --- a/src/Common/FieldVisitors.cpp +++ b/src/Common/FieldVisitors.cpp @@ -180,7 +180,17 @@ String FieldVisitorToString::operator() (const Tuple & x) const { WriteBufferFromOwnString wb; - wb << '('; + // For single-element tuples we must use the explicit tuple() function, + // or they will be parsed back as plain literals. + if (x.size() > 1) + { + wb << '('; + } + else + { + wb << "tuple("; + } + for (auto it = x.begin(); it != x.end(); ++it) { if (it != x.begin()) From f48bf2aaba9f38a02a830991922e1cd97fe9c0f5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 23:29:35 +0300 Subject: [PATCH 131/233] Fix issue --- base/common/DateLUTImpl.h | 15 ++++++++++++--- ...1702_toDateTime_from_string_clamping.reference | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 6bac9bd5126..98b767ccbcc 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -985,9 +985,18 @@ public: if (time >= values.time_at_offset_change()) time += values.amount_of_offset_change(); - res.time.second = time % 60; - res.time.minute = time / 60 % 60; - res.time.hour = time / 3600; + if (unlikely(time < 0)) + { + res.time.second = 0; + res.time.minute = 0; + res.time.hour = 0; + } + else + { + res.time.second = time % 60; + res.time.minute = time / 60 % 60; + res.time.hour = time / 3600; + } /// In case time was changed backwards at the start of next day, we will repeat the hour 23. if (unlikely(res.time.hour > 23)) diff --git a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference index 77da114be68..7e8307d66a6 100644 --- a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference +++ b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference @@ -6,4 +6,4 @@ SELECT toString(toDateTime('9922337203.6854775808', 1)); SELECT toDateTime64(CAST('10000000000.1' AS Decimal64(1)), 1); 2283-11-11 23:46:40.1 SELECT toDateTime64(CAST('-10000000000.1' AS Decimal64(1)), 1); -1925-01-01 23:09:20.1 +1925-01-01 00:00:00.1 From 31d7f10a94c5c9505c8ca11b3cee311a313e9948 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 23:34:42 +0300 Subject: [PATCH 132/233] Remove bad code --- src/DataTypes/DataTypeDateTime.cpp | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/DataTypes/DataTypeDateTime.cpp b/src/DataTypes/DataTypeDateTime.cpp index d2bbb4a1efa..d8ae7f688ae 100644 --- a/src/DataTypes/DataTypeDateTime.cpp +++ b/src/DataTypes/DataTypeDateTime.cpp @@ -12,10 +12,14 @@ #include #include +namespace DB +{ + namespace { -using namespace DB; -inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone) + +inline void readTextHelper( + time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone) { switch (settings.date_time_input_format) { @@ -27,16 +31,16 @@ inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings & setti return; } } + } -namespace DB -{ TimezoneMixin::TimezoneMixin(const String & time_zone_name) : has_explicit_time_zone(!time_zone_name.empty()), time_zone(DateLUT::instance(time_zone_name)), utc_time_zone(DateLUT::instance("UTC")) -{} +{ +} DataTypeDateTime::DataTypeDateTime(const String & time_zone_name) : TimezoneMixin(time_zone_name) @@ -45,7 +49,8 @@ DataTypeDateTime::DataTypeDateTime(const String & time_zone_name) DataTypeDateTime::DataTypeDateTime(const TimezoneMixin & time_zone_) : TimezoneMixin(time_zone_) -{} +{ +} String DataTypeDateTime::doGetName() const { @@ -87,7 +92,7 @@ void DataTypeDateTime::deserializeWholeText(IColumn & column, ReadBuffer & istr, void DataTypeDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { time_t x; - ::readText(x, istr, settings, time_zone, utc_time_zone); + readTextHelper(x, istr, settings, time_zone, utc_time_zone); assert_cast(column).getData().push_back(x); } @@ -103,7 +108,7 @@ void DataTypeDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr time_t x; if (checkChar('\'', istr)) /// Cases: '2017-08-31 18:36:48' or '1504193808' { - ::readText(x, istr, settings, time_zone, utc_time_zone); + readTextHelper(x, istr, settings, time_zone, utc_time_zone); assertChar('\'', istr); } else /// Just 1504193808 or 01504193808 @@ -125,7 +130,7 @@ void DataTypeDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr, time_t x; if (checkChar('"', istr)) { - ::readText(x, istr, settings, time_zone, utc_time_zone); + readTextHelper(x, istr, settings, time_zone, utc_time_zone); assertChar('"', istr); } else @@ -154,7 +159,7 @@ void DataTypeDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, c if (maybe_quote == '\'' || maybe_quote == '\"') ++istr.position(); - ::readText(x, istr, settings, time_zone, utc_time_zone); + readTextHelper(x, istr, settings, time_zone, utc_time_zone); if (maybe_quote == '\'' || maybe_quote == '\"') assertChar(maybe_quote, istr); From 3f67f4f47b79e101b8fc70e7501a5bc07c0a3c03 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 23:40:33 +0300 Subject: [PATCH 133/233] Saturation for DateTime --- src/DataStreams/PostgreSQLBlockInputStream.cpp | 2 ++ src/DataTypes/DataTypeDateTime.cpp | 11 +++++++++++ src/Dictionaries/RedisBlockInputStream.cpp | 2 ++ src/Formats/MySQLBlockInputStream.cpp | 2 ++ src/Formats/ProtobufSerializer.cpp | 2 ++ src/Functions/FunctionsConversion.h | 8 +++++--- 6 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/DataStreams/PostgreSQLBlockInputStream.cpp b/src/DataStreams/PostgreSQLBlockInputStream.cpp index 4cf2d942885..8350dc86849 100644 --- a/src/DataStreams/PostgreSQLBlockInputStream.cpp +++ b/src/DataStreams/PostgreSQLBlockInputStream.cpp @@ -164,6 +164,8 @@ void PostgreSQLBlockInputStream::insertValue(IColumn & column, std::string_view ReadBufferFromString in(value); time_t time = 0; readDateTimeText(time, in); + if (time < 0) + time = 0; assert_cast(column).insertValue(time); break; } diff --git a/src/DataTypes/DataTypeDateTime.cpp b/src/DataTypes/DataTypeDateTime.cpp index d8ae7f688ae..09dcb5f3e2e 100644 --- a/src/DataTypes/DataTypeDateTime.cpp +++ b/src/DataTypes/DataTypeDateTime.cpp @@ -93,6 +93,8 @@ void DataTypeDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & ist { time_t x; readTextHelper(x, istr, settings, time_zone, utc_time_zone); + if (x < 0) + x = 0; assert_cast(column).getData().push_back(x); } @@ -115,6 +117,8 @@ void DataTypeDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr { readIntText(x, istr); } + if (x < 0) + x = 0; assert_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. } @@ -137,6 +141,10 @@ void DataTypeDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr, { readIntText(x, istr); } + + if (x < 0) + x = 0; + assert_cast(column).getData().push_back(x); } @@ -164,6 +172,9 @@ void DataTypeDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, c if (maybe_quote == '\'' || maybe_quote == '\"') assertChar(maybe_quote, istr); + if (x < 0) + x = 0; + assert_cast(column).getData().push_back(x); } diff --git a/src/Dictionaries/RedisBlockInputStream.cpp b/src/Dictionaries/RedisBlockInputStream.cpp index 6a2072f7f6a..39cc2d610df 100644 --- a/src/Dictionaries/RedisBlockInputStream.cpp +++ b/src/Dictionaries/RedisBlockInputStream.cpp @@ -103,6 +103,8 @@ namespace DB ReadBufferFromString in(string_value); time_t time = 0; readDateTimeText(time, in); + if (time < 0) + time = 0; assert_cast(column).insertValue(time); break; } diff --git a/src/Formats/MySQLBlockInputStream.cpp b/src/Formats/MySQLBlockInputStream.cpp index bf7cf266e91..29cf749de3b 100644 --- a/src/Formats/MySQLBlockInputStream.cpp +++ b/src/Formats/MySQLBlockInputStream.cpp @@ -102,6 +102,8 @@ namespace ReadBufferFromString in(value); time_t time = 0; readDateTimeText(time, in); + if (time < 0) + time = 0; assert_cast(column).insertValue(time); break; } diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index 2ab8e11a854..f99888975bc 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -1486,6 +1486,8 @@ namespace ReadBufferFromString buf{str}; time_t tm = 0; readDateTimeText(tm, buf); + if (tm < 0) + tm = 0; return tm; } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index fb3c2787c96..ef74f7778b1 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -769,9 +769,11 @@ inline void parseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb template <> inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone) { - time_t tmp = 0; - readDateTimeText(tmp, rb, *time_zone); - x = tmp; + time_t time = 0; + readDateTimeText(time, rb, *time_zone); + if (time < 0) + time = 0; + x = time; } From c1a2745bfbdf54906ede3411c02b6f320107d052 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 00:04:03 +0300 Subject: [PATCH 134/233] Add one more variant to memcpy benchmark --- base/glibc-compatibility/memcpy/memcpy.h | 2 +- utils/memcpy-bench/memcpy-bench.cpp | 172 ++++++++++++++++++++++- 2 files changed, 172 insertions(+), 2 deletions(-) diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index f9f81bcb0fe..211d144cecb 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -178,7 +178,7 @@ tail: size -= padding; } - /// Aligned unrolled copy. We will use all available SSE registers. + /// Aligned unrolled copy. We will use half of available SSE registers. /// It's not possible to have both src and dst aligned. /// So, we will use aligned stores and unaligned loads. __m128i c0, c1, c2, c3, c4, c5, c6, c7; diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp index cd769640017..5c664a76fe2 100644 --- a/utils/memcpy-bench/memcpy-bench.cpp +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -35,7 +35,7 @@ void NO_INLINE loop(uint8_t * dst, uint8_t * src, size_t size, F && chunk_size_d size -= bytes_to_copy; /// Execute at least one SSE instruction as a penalty after running AVX code. - __asm__ volatile ("pxor %%xmm7, %%xmm7" ::: "xmm7"); + __asm__ __volatile__ ("pxor %%xmm15, %%xmm15" ::: "xmm15"); } } @@ -385,6 +385,7 @@ void memcpy_my_medium_avx(uint8_t * __restrict & __restrict dst, const uint8_t * bool have_avx = true; + static uint8_t * memcpy_my(uint8_t * __restrict dst, const uint8_t * __restrict src, size_t size) { uint8_t * ret = dst; @@ -560,6 +561,174 @@ tail: return ret; } + +static uint8_t * memcpy_my2(uint8_t * __restrict dst, const uint8_t * __restrict src, size_t size) +{ + uint8_t * ret = dst; + + if (size <= 16) + { + if (size >= 8) + { + __builtin_memcpy(dst + size - 8, src + size - 8, 8); + __builtin_memcpy(dst, src, 8); + } + else if (size >= 4) + { + __builtin_memcpy(dst + size - 4, src + size - 4, 4); + __builtin_memcpy(dst, src, 4); + } + else if (size >= 2) + { + __builtin_memcpy(dst + size - 2, src + size - 2, 2); + __builtin_memcpy(dst, src, 2); + } + else if (size >= 1) + { + *dst = *src; + } + } + else + { + if (size <= 128) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); + + while (size > 16) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); + dst += 16; + src += 16; + size -= 16; + } + } + else if (size < 30000 || !have_avx) + { + /// Align destination to 16 bytes boundary. + size_t padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + + if (padding > 0) + { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + /// Aligned unrolled copy. + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + while (size >= 128) + { + c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); + c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); + c2 = _mm_loadu_si128(reinterpret_cast(src) + 2); + c3 = _mm_loadu_si128(reinterpret_cast(src) + 3); + c4 = _mm_loadu_si128(reinterpret_cast(src) + 4); + c5 = _mm_loadu_si128(reinterpret_cast(src) + 5); + c6 = _mm_loadu_si128(reinterpret_cast(src) + 6); + c7 = _mm_loadu_si128(reinterpret_cast(src) + 7); + src += 128; + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7); + dst += 128; + + size -= 128; + } + + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); + + while (size > 16) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); + dst += 16; + src += 16; + size -= 16; + } + } + else + { + size_t padding = (32 - (reinterpret_cast(dst) & 31)) & 31; + + if (padding > 0) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "memory"); + + dst += padding; + src += padding; + size -= padding; + } + + while (size >= 256) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups 0x20(%[s]), %%ymm1\n" + "vmovups 0x40(%[s]), %%ymm2\n" + "vmovups 0x60(%[s]), %%ymm3\n" + "vmovups 0x80(%[s]), %%ymm4\n" + "vmovups 0xa0(%[s]), %%ymm5\n" + "vmovups 0xc0(%[s]), %%ymm6\n" + "vmovups 0xe0(%[s]), %%ymm7\n" + "add $0x100,%[s]\n" + "vmovaps %%ymm0, (%[d])\n" + "vmovaps %%ymm1, 0x20(%[d])\n" + "vmovaps %%ymm2, 0x40(%[d])\n" + "vmovaps %%ymm3, 0x60(%[d])\n" + "vmovaps %%ymm4, 0x80(%[d])\n" + "vmovaps %%ymm5, 0xa0(%[d])\n" + "vmovaps %%ymm6, 0xc0(%[d])\n" + "vmovaps %%ymm7, 0xe0(%[d])\n" + "add $0x100, %[d]\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "memory"); + + size -= 256; + } + + __asm__( + "vmovups -0x20(%[s],%[size],1), %%ymm0\n" + "vmovups %%ymm0, -0x20(%[d],%[size],1)\n" + : [d]"+r"(dst), [s]"+r"(src) + : [size]"r"(size) + : "ymm0", "memory"); + + while (size > 32) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "memory"); + + dst += 32; + src += 32; + size -= 32; + } + + __asm__ __volatile__ ("vzeroupper" + ::: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", + "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15"); + } + } + + return ret; +} + extern "C" void * __memcpy_erms(void * __restrict destination, const void * __restrict source, size_t size); extern "C" void * __memcpy_sse2_unaligned(void * __restrict destination, const void * __restrict source, size_t size); extern "C" void * __memcpy_ssse3(void * __restrict destination, const void * __restrict source, size_t size); @@ -592,6 +761,7 @@ uint64_t dispatchMemcpyVariants(size_t memcpy_variant, uint8_t * dst, uint8_t * VARIANT(10, memcpy_fast_sse) VARIANT(11, memcpy_fast_avx) VARIANT(12, memcpy_my) + VARIANT(13, memcpy_my2) VARIANT(21, __memcpy_erms) VARIANT(22, __memcpy_sse2_unaligned) From b9ae9c9cd2f55e494a2dd0dfa181e6406a2be936 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 00:05:30 +0300 Subject: [PATCH 135/233] Add one more variant to memcpy benchmark --- utils/memcpy-bench/memcpy-bench.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp index 5c664a76fe2..704ea04a184 100644 --- a/utils/memcpy-bench/memcpy-bench.cpp +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -816,7 +816,7 @@ int main(int argc, char ** argv) for size in 4096 16384 50000 65536 100000 1000000 10000000 100000000; do for threads in 1 2 4 $(($(nproc) / 2)) $(nproc); do for distribution in 1 2 3 4 5; do - for variant in {1..12} {21..29}; do + for variant in {1..13} {21..29}; do for i in {1..10}; do ./memcpy-bench --tsv --size $size --variant $variant --threads $threads --distribution $distribution; done; From fe371d9cfceb5b97f324565351d6f75ced34fbc1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 01:13:27 +0300 Subject: [PATCH 136/233] Minor modifications --- utils/memcpy-bench/memcpy-bench.cpp | 320 +++++++++++++++++----------- 1 file changed, 191 insertions(+), 129 deletions(-) diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp index 704ea04a184..b607c45370d 100644 --- a/utils/memcpy-bench/memcpy-bench.cpp +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -588,142 +588,203 @@ static uint8_t * memcpy_my2(uint8_t * __restrict dst, const uint8_t * __restrict *dst = *src; } } + else if (size <= 128) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); + + while (size > 16) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); + dst += 16; + src += 16; + size -= 16; + } + } + else if (size < 30000 || !have_avx) + { + /// Align destination to 16 bytes boundary. + size_t padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + + if (padding > 0) + { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + /// Aligned unrolled copy. + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + while (size >= 128) + { + c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); + c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); + c2 = _mm_loadu_si128(reinterpret_cast(src) + 2); + c3 = _mm_loadu_si128(reinterpret_cast(src) + 3); + c4 = _mm_loadu_si128(reinterpret_cast(src) + 4); + c5 = _mm_loadu_si128(reinterpret_cast(src) + 5); + c6 = _mm_loadu_si128(reinterpret_cast(src) + 6); + c7 = _mm_loadu_si128(reinterpret_cast(src) + 7); + src += 128; + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7); + dst += 128; + + size -= 128; + } + + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); + + while (size > 16) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); + dst += 16; + src += 16; + size -= 16; + } + } else { - if (size <= 128) + size_t padding = (32 - (reinterpret_cast(dst) & 31)) & 31; + + if (padding > 0) { - _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); - - while (size > 16) - { - _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); - dst += 16; - src += 16; - size -= 16; - } - } - else if (size < 30000 || !have_avx) - { - /// Align destination to 16 bytes boundary. - size_t padding = (16 - (reinterpret_cast(dst) & 15)) & 15; - - if (padding > 0) - { - __m128i head = _mm_loadu_si128(reinterpret_cast(src)); - _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); - dst += padding; - src += padding; - size -= padding; - } - - /// Aligned unrolled copy. - __m128i c0, c1, c2, c3, c4, c5, c6, c7; - - while (size >= 128) - { - c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); - c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); - c2 = _mm_loadu_si128(reinterpret_cast(src) + 2); - c3 = _mm_loadu_si128(reinterpret_cast(src) + 3); - c4 = _mm_loadu_si128(reinterpret_cast(src) + 4); - c5 = _mm_loadu_si128(reinterpret_cast(src) + 5); - c6 = _mm_loadu_si128(reinterpret_cast(src) + 6); - c7 = _mm_loadu_si128(reinterpret_cast(src) + 7); - src += 128; - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7); - dst += 128; - - size -= 128; - } - - _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); - - while (size > 16) - { - _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); - dst += 16; - src += 16; - size -= 16; - } - } - else - { - size_t padding = (32 - (reinterpret_cast(dst) & 31)) & 31; - - if (padding > 0) - { - __asm__( - "vmovups (%[s]), %%ymm0\n" - "vmovups %%ymm0, (%[d])\n" - : [d]"+r"(dst), [s]"+r"(src) - : - : "ymm0", "memory"); - - dst += padding; - src += padding; - size -= padding; - } - - while (size >= 256) - { - __asm__( - "vmovups (%[s]), %%ymm0\n" - "vmovups 0x20(%[s]), %%ymm1\n" - "vmovups 0x40(%[s]), %%ymm2\n" - "vmovups 0x60(%[s]), %%ymm3\n" - "vmovups 0x80(%[s]), %%ymm4\n" - "vmovups 0xa0(%[s]), %%ymm5\n" - "vmovups 0xc0(%[s]), %%ymm6\n" - "vmovups 0xe0(%[s]), %%ymm7\n" - "add $0x100,%[s]\n" - "vmovaps %%ymm0, (%[d])\n" - "vmovaps %%ymm1, 0x20(%[d])\n" - "vmovaps %%ymm2, 0x40(%[d])\n" - "vmovaps %%ymm3, 0x60(%[d])\n" - "vmovaps %%ymm4, 0x80(%[d])\n" - "vmovaps %%ymm5, 0xa0(%[d])\n" - "vmovaps %%ymm6, 0xc0(%[d])\n" - "vmovaps %%ymm7, 0xe0(%[d])\n" - "add $0x100, %[d]\n" - : [d]"+r"(dst), [s]"+r"(src) - : - : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "memory"); - - size -= 256; - } - __asm__( - "vmovups -0x20(%[s],%[size],1), %%ymm0\n" - "vmovups %%ymm0, -0x20(%[d],%[size],1)\n" + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" : [d]"+r"(dst), [s]"+r"(src) - : [size]"r"(size) + : : "ymm0", "memory"); - while (size > 32) - { - __asm__( - "vmovups (%[s]), %%ymm0\n" - "vmovups %%ymm0, (%[d])\n" - : [d]"+r"(dst), [s]"+r"(src) - : - : "ymm0", "memory"); - - dst += 32; - src += 32; - size -= 32; - } - - __asm__ __volatile__ ("vzeroupper" - ::: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", - "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15"); + dst += padding; + src += padding; + size -= padding; } + + while (size >= 512) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups 0x20(%[s]), %%ymm1\n" + "vmovups 0x40(%[s]), %%ymm2\n" + "vmovups 0x60(%[s]), %%ymm3\n" + "vmovups 0x80(%[s]), %%ymm4\n" + "vmovups 0xa0(%[s]), %%ymm5\n" + "vmovups 0xc0(%[s]), %%ymm6\n" + "vmovups 0xe0(%[s]), %%ymm7\n" + "vmovups 0x100(%[s]), %%ymm8\n" + "vmovups 0x120(%[s]), %%ymm9\n" + "vmovups 0x140(%[s]), %%ymm10\n" + "vmovups 0x160(%[s]), %%ymm11\n" + "vmovups 0x180(%[s]), %%ymm12\n" + "vmovups 0x1a0(%[s]), %%ymm13\n" + "vmovups 0x1c0(%[s]), %%ymm14\n" + "vmovups 0x1e0(%[s]), %%ymm15\n" + "add $0x200, %[s]\n" + "sub $0x200, %[size]\n" + "vmovaps %%ymm0, (%[d])\n" + "vmovaps %%ymm1, 0x20(%[d])\n" + "vmovaps %%ymm2, 0x40(%[d])\n" + "vmovaps %%ymm3, 0x60(%[d])\n" + "vmovaps %%ymm4, 0x80(%[d])\n" + "vmovaps %%ymm5, 0xa0(%[d])\n" + "vmovaps %%ymm6, 0xc0(%[d])\n" + "vmovaps %%ymm7, 0xe0(%[d])\n" + "vmovaps %%ymm8, 0x100(%[d])\n" + "vmovaps %%ymm9, 0x120(%[d])\n" + "vmovaps %%ymm10, 0x140(%[d])\n" + "vmovaps %%ymm11, 0x160(%[d])\n" + "vmovaps %%ymm12, 0x180(%[d])\n" + "vmovaps %%ymm13, 0x1a0(%[d])\n" + "vmovaps %%ymm14, 0x1c0(%[d])\n" + "vmovaps %%ymm15, 0x1e0(%[d])\n" + "add $0x200, %[d]\n" + : [d]"+r"(dst), [s]"+r"(src), [size]"+r"(size) + : + : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", + "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15", + "memory"); + } + + /*while (size >= 256) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups 0x20(%[s]), %%ymm1\n" + "vmovups 0x40(%[s]), %%ymm2\n" + "vmovups 0x60(%[s]), %%ymm3\n" + "vmovups 0x80(%[s]), %%ymm4\n" + "vmovups 0xa0(%[s]), %%ymm5\n" + "vmovups 0xc0(%[s]), %%ymm6\n" + "vmovups 0xe0(%[s]), %%ymm7\n" + "add $0x100,%[s]\n" + "vmovaps %%ymm0, (%[d])\n" + "vmovaps %%ymm1, 0x20(%[d])\n" + "vmovaps %%ymm2, 0x40(%[d])\n" + "vmovaps %%ymm3, 0x60(%[d])\n" + "vmovaps %%ymm4, 0x80(%[d])\n" + "vmovaps %%ymm5, 0xa0(%[d])\n" + "vmovaps %%ymm6, 0xc0(%[d])\n" + "vmovaps %%ymm7, 0xe0(%[d])\n" + "add $0x100, %[d]\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "memory"); + + size -= 256; + }*/ + + /*while (size > 128) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups 0x20(%[s]), %%ymm1\n" + "vmovups 0x40(%[s]), %%ymm2\n" + "vmovups 0x60(%[s]), %%ymm3\n" + "add $0x80, %[s]\n" + "sub $0x80, %[size]\n" + "vmovaps %%ymm0, (%[d])\n" + "vmovaps %%ymm1, 0x20(%[d])\n" + "vmovaps %%ymm2, 0x40(%[d])\n" + "vmovaps %%ymm3, 0x60(%[d])\n" + "add $0x80, %[d]\n" + : [d]"+r"(dst), [s]"+r"(src), [size]"+r"(size) + : + : "ymm0", "ymm1", "ymm2", "ymm3", "memory"); + }*/ + + __asm__( + "vmovups -0x20(%[s],%[size],1), %%ymm0\n" + "vmovups %%ymm0, -0x20(%[d],%[size],1)\n" + : [d]"+r"(dst), [s]"+r"(src) + : [size]"r"(size) + : "ymm0", "memory"); + + while (size > 32) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "memory"); + + dst += 32; + src += 32; + size -= 32; + } + + __asm__ __volatile__ ("vzeroupper" + ::: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", + "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15"); } return ret; @@ -893,7 +954,8 @@ clickhouse-local --structure ' } else { - std::cout << ": processed in " << (elapsed_ns / 1e9) << " sec, " << (size * iterations * 1.0 / elapsed_ns) << " GB/sec\n"; + std::cout << ": " << num_threads << " threads, " << "size: " << size << ", distribution " << generator_variant + << ", processed in " << (elapsed_ns / 1e9) << " sec, " << (size * iterations * 1.0 / elapsed_ns) << " GB/sec\n"; } return 0; From 3c16ea0bc80153400a0d24b881ec5951bc021d80 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 16 Mar 2021 01:56:27 +0300 Subject: [PATCH 137/233] fix incorrect number of rows for Chunks with no columns in PartialSortingTransform --- .../Transforms/PartialSortingTransform.cpp | 7 ++++--- src/Processors/Transforms/WindowTransform.cpp | 17 +++++++++++------ .../01591_window_functions.reference | 5 +++++ .../0_stateless/01591_window_functions.sql | 4 ++++ 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/Processors/Transforms/PartialSortingTransform.cpp b/src/Processors/Transforms/PartialSortingTransform.cpp index 2fd0a64ee92..10ce2c4b773 100644 --- a/src/Processors/Transforms/PartialSortingTransform.cpp +++ b/src/Processors/Transforms/PartialSortingTransform.cpp @@ -91,8 +91,10 @@ size_t getFilterMask(const ColumnRawPtrs & lhs, const ColumnRawPtrs & rhs, size_ void PartialSortingTransform::transform(Chunk & chunk) { + const auto rows_num = chunk.getNumRows(); + if (read_rows) - read_rows->add(chunk.getNumRows()); + read_rows->add(rows_num); auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()); @@ -101,7 +103,6 @@ void PartialSortingTransform::transform(Chunk & chunk) */ if (!threshold_block_columns.empty()) { - UInt64 rows_num = block.rows(); auto block_columns = extractColumns(block, description); size_t result_size_hint = getFilterMask( @@ -134,7 +135,7 @@ void PartialSortingTransform::transform(Chunk & chunk) } } - chunk.setColumns(block.getColumns(), block.rows()); + chunk.setColumns(block.getColumns(), rows_num); } } diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index 0013e0061e2..1fc51bd4112 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -881,12 +881,13 @@ void WindowTransform::appendChunk(Chunk & chunk) assert(chunk.hasRows()); blocks.push_back({}); auto & block = blocks.back(); + // Use the number of rows from the Chunk, because it is correct even in + // the case where the Chunk has no columns. Not sure if this actually + // happens, because even in the case of `count() over ()` we have a dummy + // input column. + block.rows = chunk.getNumRows(); block.input_columns = chunk.detachColumns(); - // Even in case of `count() over ()` we should have a dummy input column. - // Not sure how reliable this is... - block.rows = block.input_columns[0]->size(); - for (auto & ws : workspaces) { // Aggregate functions can't work with constant columns, so we have to @@ -1109,9 +1110,7 @@ IProcessor::Status WindowTransform::prepare() if (output.canPush()) { // Output the ready block. -// fmt::print(stderr, "output block {}\n", next_output_block_number); const auto i = next_output_block_number - first_block_number; - ++next_output_block_number; auto & block = blocks[i]; auto columns = block.input_columns; for (auto & res : block.output_columns) @@ -1120,6 +1119,12 @@ IProcessor::Status WindowTransform::prepare() } output_data.chunk.setColumns(columns, block.rows); +// fmt::print(stderr, "output block {} as chunk '{}'\n", +// next_output_block_number, +// output_data.chunk.dumpStructure()); + + ++next_output_block_number; + output.pushData(std::move(output_data)); } diff --git a/tests/queries/0_stateless/01591_window_functions.reference b/tests/queries/0_stateless/01591_window_functions.reference index d2543f0db75..d83808ce37d 100644 --- a/tests/queries/0_stateless/01591_window_functions.reference +++ b/tests/queries/0_stateless/01591_window_functions.reference @@ -993,3 +993,8 @@ order by number 7 6 8 8 7 9 9 8 9 +-- In this case, we had a problem with PartialSortingTransform returning zero-row +-- chunks for input chunks w/o columns. +select count() over () from numbers(4) where number < 2; +2 +2 diff --git a/tests/queries/0_stateless/01591_window_functions.sql b/tests/queries/0_stateless/01591_window_functions.sql index 03bd8371e23..14b08ad2875 100644 --- a/tests/queries/0_stateless/01591_window_functions.sql +++ b/tests/queries/0_stateless/01591_window_functions.sql @@ -345,3 +345,7 @@ from numbers(10) window w as (order by number range between 1 preceding and 1 following) order by number ; + +-- In this case, we had a problem with PartialSortingTransform returning zero-row +-- chunks for input chunks w/o columns. +select count() over () from numbers(4) where number < 2; From 6403198c8491c5c4cba88f93327f62159df506ff Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 16 Mar 2021 02:18:50 +0300 Subject: [PATCH 138/233] check formatting only for the queries we can execute --- programs/client/Client.cpp | 120 +++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 59 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 527fec17c63..4a61662c238 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1372,65 +1372,6 @@ private: continue; } - // Check that the query is formatted properly and we can parse - // it back and format again and get the same result. Unfortunately - // we can't compare the ASTs, which would be more sensitive to - // errors. This double formatting check doesn't catch all errors, - // e.g. we can format query incorrectly, but to a valid SQL that - // we can then parse and format into the same SQL. - { - ASTPtr parsed_formatted_query; - try - { - const auto * tmp_pos = fuzzed_text.c_str(); - parsed_formatted_query = parseQuery(tmp_pos, - tmp_pos + fuzzed_text.size(), - false /* allow_multi_statements */); - } - catch (Exception & e) - { - // Some complicated cases where we can generate the SQL - // which we can't parse: - // * first argument of lambda() replaced by fuzzer with - // something else, leading to constructs such as - // arrayMap((min(x) + 3) -> x + 1, ....) - // * internals of Enum replaced, leading to: - // Enum(equals(someFunction(y), 3)). - // We could filter them on case-by-case basis, but they - // are probably also helpful in that they test the parsing - // errors, so let's just ignore them in this check and - // send them to the server normally. - if (e.code() != ErrorCodes::SYNTAX_ERROR) - { - throw; - } - } - - if (parsed_formatted_query) - { - const auto formatted_twice - = parsed_formatted_query->formatForErrorMessage(); - - if (formatted_twice != fuzzed_text) - { - fmt::print(stderr, "The query formatting is broken.\n"); - - printChangedSettings(); - - fmt::print(stderr, "Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n", - formatted_twice, fuzzed_text); - fmt::print(stderr, "In more detail:\n"); - fmt::print(stderr, "AST-1:\n'{}'\n", ast_to_process->dumpTree()); - fmt::print(stderr, "Text-1 (AST-1 formatted):\n'{}'\n", fuzzed_text); - fmt::print(stderr, "AST-2 (Text-1 parsed):\n'{}'\n", parsed_formatted_query->dumpTree()); - fmt::print(stderr, "Text-2 (AST-2 formatted):\n'{}'\n", formatted_twice); - fmt::print(stderr, "Text-1 must be equal to Text-2, but it is not.\n"); - - exit(1); - } - } - } - parsed_query = ast_to_process; query_to_send = parsed_query->formatForErrorMessage(); @@ -1470,6 +1411,67 @@ private: return false; } + // Check that after the query is formatted, we can parse it back, + // format again and get the same result. Unfortunately, we can't + // compare the ASTs, which would be more sensitive to errors. This + // double formatting check doesn't catch all errors, e.g. we can + // format query incorrectly, but to a valid SQL that we can then + // parse and format into the same SQL. + // There are some complicated cases where we can generate the SQL + // which we can't parse: + // * first argument of lambda() replaced by fuzzer with + // something else, leading to constructs such as + // arrayMap((min(x) + 3) -> x + 1, ....) + // * internals of Enum replaced, leading to: + // Enum(equals(someFunction(y), 3)). + // And there are even the cases when we can parse the query, but + // it's logically incorrect and its formatting is a mess, such as + // when `lambda()` function gets substituted into a wrong place. + // To avoid dealing with these cases, run the check only for the + // queries we were able to successfully execute. + if (!have_error) + { + ASTPtr parsed_formatted_query; + try + { + const auto * tmp_pos = query_to_send.c_str(); + parsed_formatted_query = parseQuery(tmp_pos, + tmp_pos + query_to_send.size(), + false /* allow_multi_statements */); + } + catch (Exception & e) + { + if (e.code() != ErrorCodes::SYNTAX_ERROR) + { + throw; + } + } + + if (parsed_formatted_query) + { + const auto formatted_twice + = parsed_formatted_query->formatForErrorMessage(); + + if (formatted_twice != query_to_send) + { + fmt::print(stderr, "The query formatting is broken.\n"); + + printChangedSettings(); + + fmt::print(stderr, "Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n", + formatted_twice, query_to_send); + fmt::print(stderr, "In more detail:\n"); + fmt::print(stderr, "AST-1:\n'{}'\n", parsed_query->dumpTree()); + fmt::print(stderr, "Text-1 (AST-1 formatted):\n'{}'\n", query_to_send); + fmt::print(stderr, "AST-2 (Text-1 parsed):\n'{}'\n", parsed_formatted_query->dumpTree()); + fmt::print(stderr, "Text-2 (AST-2 formatted):\n'{}'\n", formatted_twice); + fmt::print(stderr, "Text-1 must be equal to Text-2, but it is not.\n"); + + exit(1); + } + } + } + // The server is still alive so we're going to continue fuzzing. // Determine what we're going to use as the starting AST. if (have_error) From 2328d568278113505fc4ce5e4ef42163059d4d2c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 16 Mar 2021 04:19:33 +0300 Subject: [PATCH 139/233] fix --- src/Processors/Transforms/PartialSortingTransform.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Processors/Transforms/PartialSortingTransform.cpp b/src/Processors/Transforms/PartialSortingTransform.cpp index 10ce2c4b773..33ff639f10d 100644 --- a/src/Processors/Transforms/PartialSortingTransform.cpp +++ b/src/Processors/Transforms/PartialSortingTransform.cpp @@ -91,7 +91,7 @@ size_t getFilterMask(const ColumnRawPtrs & lhs, const ColumnRawPtrs & rhs, size_ void PartialSortingTransform::transform(Chunk & chunk) { - const auto rows_num = chunk.getNumRows(); + auto rows_num = chunk.getNumRows(); if (read_rows) read_rows->add(rows_num); @@ -117,13 +117,15 @@ void PartialSortingTransform::transform(Chunk & chunk) { for (auto & column : block) column.column = column.column->filter(filter, result_size_hint); + + rows_num = block.rows(); } } sortBlock(block, description, limit); /// Check if we can use this block for optimization. - if (min_limit_for_partial_sort_optimization <= limit && limit <= block.rows()) + if (min_limit_for_partial_sort_optimization <= limit && limit <= rows_num) { auto block_columns = extractColumns(block, description); @@ -135,6 +137,8 @@ void PartialSortingTransform::transform(Chunk & chunk) } } + assert(block.columns() == 0 || block.rows() == rows_num); + chunk.setColumns(block.getColumns(), rows_num); } From 54f124aae499d55779632d0f0a2641a4ec41a0c0 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 15 Mar 2021 07:55:59 +0300 Subject: [PATCH 140/233] Cleanup 00626_replace_partition_from_table_zookeeper --- .../00626_replace_partition_from_table_zookeeper.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index 5aa445858db..f5f667d084b 100755 --- a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -12,19 +12,22 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function query_with_retry { - retry=0 + local query="$1" && shift + + local retry=0 until [ $retry -ge 5 ] do - result=$($CLICKHOUSE_CLIENT $2 --query="$1" 2>&1) + local result + result="$($CLICKHOUSE_CLIENT "$@" --query="$query" 2>&1)" if [ "$?" == 0 ]; then echo -n "$result" return else - retry=$(($retry + 1)) + retry=$((retry + 1)) sleep 3 fi done - echo "Query '$1' failed with '$result'" + echo "Query '$query' failed with '$result'" } $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS src;" @@ -139,7 +142,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE src;" $CLICKHOUSE_CLIENT --query="SELECT count(), sum(d), uniqExact(_part) FROM dst_r1;" $CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r1;" -query_with_retry "OPTIMIZE TABLE dst_r1 PARTITION 1;" "--replication_alter_partitions_sync=0 --optimize_throw_if_noop=1" +query_with_retry "OPTIMIZE TABLE dst_r1 PARTITION 1;" --replication_alter_partitions_sync=0 --optimize_throw_if_noop=1 $CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r1;" $CLICKHOUSE_CLIENT --query="SELECT count(), sum(d), uniqExact(_part) FROM dst_r1;" From e6b9740a6957ecf94ce92090e233d8441147c7d7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 15 Mar 2021 08:29:05 +0300 Subject: [PATCH 141/233] Add $CLICKHOUSE_TEST_ZOOKEEPER_PREFIX (for *.sh stateless tests) --- tests/queries/shell_config.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index d20b5669cc5..5b942a95d02 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -5,6 +5,13 @@ export ASAN_OPTIONS=detect_odr_violation=0 export CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:="test"} export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL:="warning"} + +# Unique zookeeper path (based on test name and current database) to avoid overlaps +export CLICKHOUSE_TEST_PATH="${BASH_SOURCE[1]}" +CLICKHOUSE_TEST_NAME="$(basename "$CLICKHOUSE_TEST_PATH" .sh)" +export CLICKHOUSE_TEST_NAME +export CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_NAME}_${CLICKHOUSE_DATABASE}" + [ -v CLICKHOUSE_CONFIG_CLIENT ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} " [ -v CLICKHOUSE_HOST ] && CLICKHOUSE_CLIENT_OPT0+=" --host=${CLICKHOUSE_HOST} " [ -v CLICKHOUSE_PORT_TCP ] && CLICKHOUSE_CLIENT_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} " From 6c2622d4519fee3207426582c8cdafd85b204b1b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 15 Mar 2021 08:14:28 +0300 Subject: [PATCH 142/233] Add style check for ReplicatedMergeTree path --- utils/check-style/check-style | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index f8926a9af2f..db6b33a569b 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -97,6 +97,36 @@ for test_case in "${tests_with_query_log[@]}"; do grep -qE current_database.*currentDatabase "$test_case" || echo "Queries to system.query_log/system.query_thread_log does not have current_database = currentDatabase() condition in $test_case" done +# Queries with ReplicatedMergeTree +# NOTE: it is not that accuate, but at least something. +tests_with_replicated_merge_tree=( $( + find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' | + grep -vP $EXCLUDE_DIRS | + xargs grep --with-filename -e ReplicatedMergeTree | cut -d: -f1 | sort -u +) ) +for test_case in "${tests_with_replicated_merge_tree[@]}"; do + case "$test_case" in + *.sh) + test_case_zk_prefix="\$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX" + grep -q -e "ReplicatedMergeTree.*$test_case_zk_prefix" "$test_case" || echo "ReplicatedMergeTree should contain '$test_case_zk_prefix' in zookeeper path to avoid overlaps ($test_case)" + ;; + *.sql) + # NOTE: *.sql is not supported because it is not possible right now, because: + # - ReplicatedMergeTree supports only ASTLiteral for zookeeper path + # (and adding support of other nodes, with evaluating them are not that easy, due to zk_prefix is "optional") + # - Hence concat(currentDatabase(), 'foo') + # - Also params cannot be used, because the are wrapped with CAST() + # + # But hopefully they will not be a problem + # (since they do not do any "stressing" and overlap probability should be lower). + ;; + *.py) + # Right now there is not such tests anyway + echo "No ReplicatedMergeTree style check for *.py ($test_case)" + ;; + esac +done + # All the submodules should be from https://github.com/ find $ROOT_PATH -name '.gitmodules' | while read i; do grep -F 'url = ' $i | grep -v -F 'https://github.com/' && echo 'All the submodules should be from https://github.com/'; done From 05a8c73eb948ad2e225af3e69ba7c493d2ae4b8a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 15 Mar 2021 07:51:28 +0300 Subject: [PATCH 143/233] Do not overlap zookeeper path for ReplicatedMergeTree in stateless *.sh tests Found with stress tests for 00626_replace_partition_from_table_zookeeper [1]: 2021.03.15 00:59:48.200106 [ 27417 ] {0f47dbeb-938a-4560-8408-a7cc7b3bafb8} ContextAccess (default): Access granted: CREATE TABLE ON test_31.dst_r1 ... 2021.03.15 00:59:48.403227 [ 27417 ] {0f47dbeb-938a-4560-8408-a7cc7b3bafb8} test_31.dst_r1: This table /clickhouse/test_00626/dst_1 is already created, will add new replica 2021.03.15 00:59:48.736450 [ 83006 ] {b2db1355-3ec3-4e3a-9c79-f93f27c6e658} ContextAccess (default): Access granted: CREATE TABLE ON test_31.dst_r2 ... 2021.03.15 00:59:48.851768 [ 83006 ] {b2db1355-3ec3-4e3a-9c79-f93f27c6e658} test_31.dst_r2: This table /clickhouse/test_00626/dst_1 is already created, will add new replica ... 2021.03.15 00:59:48.919059 [ 366 ] {} test_31.dst_r2 (ReplicatedMergeTreeQueue): Loading queue from /clickhouse/test_00626/dst_1/replicas/2/queue 2021.03.15 00:59:48.919948 [ 366 ] {} test_31.dst_r2 (ReplicatedMergeTreeQueue): Having 3 queue entries to load, 0 entries already loaded. 2021.03.15 00:59:48.921833 [ 366 ] {} test_31.dst_r2 (ReplicatedMergeTreeQueue): Loaded queue ... 2021.03.15 00:59:51.904230 [ 246952 ] {59753eea-3896-45ca-8625-fdaa094ee9ef} ContextAccess (default): Access granted: SYSTEM SYNC REPLICA ON test_31.dst_r2 ... 2021.03.15 01:04:51.913683 [ 246952 ] {59753eea-3896-45ca-8625-fdaa094ee9ef} InterpreterSystemQuery: SYNC REPLICA test_31.dst_r2: Timed out! [1]: https://clickhouse-test-reports.s3.yandex.net/21716/402bf77783cbda48a9ee1b748bfce3c52ef8fe11/stress_test_(memory)/test_run.txt.out.log But the problem is more generic, so fix all tests. --- .../00029_test_zookeeper_optimize_exception.sh | 3 +-- ...ear_column_in_partition_concurrent_zookeeper.sh | 4 ++-- ...00626_replace_partition_from_table_zookeeper.sh | 4 ++-- ...terialized_view_and_too_many_parts_zookeeper.sh | 10 +++++----- ...licated_mutations_default_database_zookeeper.sh | 2 +- .../00652_replicated_mutations_zookeeper.sh | 8 ++++---- ...00715_fetch_merged_or_mutated_part_zookeeper.sh | 4 ++-- .../00834_kill_mutation_replicated_zookeeper.sh | 4 ++-- .../00953_zookeeper_suetin_deduplication_bug.sh | 14 +++++++------- .../00975_indices_mutation_replicated_zookeeper.sh | 4 ++-- .../00992_system_parts_race_condition_zookeeper.sh | 4 ++-- ...3_system_parts_race_condition_drop_zookeeper.sh | 2 +- .../01013_sync_replica_timeout_zookeeper.sh | 5 ++--- ...ns_with_nondeterministic_functions_zookeeper.sh | 5 ++--- .../01034_move_partition_from_table_zookeeper.sh | 12 ++++++------ ...ncurrent_move_partition_from_table_zookeeper.sh | 4 ++-- ..._zookeeper_system_mutations_with_parts_names.sh | 2 +- .../01076_parallel_alter_replicated_zookeeper.sh | 2 +- .../01079_bad_alters_zookeeper.reference | 4 ++-- .../0_stateless/01079_bad_alters_zookeeper.sh | 2 +- ...079_parallel_alter_add_drop_column_zookeeper.sh | 2 +- .../01079_parallel_alter_detach_table_zookeeper.sh | 2 +- .../01079_parallel_alter_modify_zookeeper.sh | 2 +- .../01103_optimize_drop_race_zookeeper.sh | 2 +- ...8_restart_replicas_rename_deadlock_zookeeper.sh | 2 +- .../0_stateless/01192_rename_database_zookeeper.sh | 2 +- .../01213_alter_rename_column_zookeeper.reference | 4 ++-- .../01213_alter_rename_column_zookeeper.sh | 2 +- .../01305_replica_create_drop_zookeeper.sh | 4 ++-- .../01307_multiple_leaders_zookeeper.sh | 2 +- .../01318_long_unsuccessful_mutation_zookeeper.sh | 2 +- .../01320_create_sync_race_condition_zookeeper.sh | 2 +- ...01338_long_select_and_alter_zookeeper.reference | 2 +- .../01338_long_select_and_alter_zookeeper.sh | 2 +- ...396_inactive_replica_cleanup_nodes_zookeeper.sh | 12 ++++++------ .../01414_mutations_and_errors_zookeeper.sh | 2 +- .../01417_freeze_partition_verbose_zookeeper.sh | 2 +- .../0_stateless/01459_manual_write_to_replicas.sh | 2 +- .../01459_manual_write_to_replicas_quorum.sh | 2 +- .../01508_race_condition_rename_clear_zookeeper.sh | 2 +- .../01509_check_many_parallel_quorum_inserts.sh | 2 +- .../01509_check_parallel_quorum_inserts.sh | 2 +- .../0_stateless/01509_parallel_quorum_and_merge.sh | 4 ++-- ...01593_concurrent_alter_mutations_kill.reference | 2 +- .../01593_concurrent_alter_mutations_kill.sh | 2 +- ...nt_alter_mutations_kill_many_replicas.reference | 10 +++++----- ...oncurrent_alter_mutations_kill_many_replicas.sh | 6 +++--- .../queries/0_stateless/01671_ddl_hang_timeout.sh | 2 +- .../01753_system_zookeeper_query_param_path.sh | 4 ++-- 49 files changed, 92 insertions(+), 95 deletions(-) diff --git a/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh b/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh index 86f1d1f161c..3360b8da83d 100755 --- a/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh +++ b/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh @@ -4,12 +4,11 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh - ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_optimize_exception" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_optimize_exception_replicated" ${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_optimize_exception (date Date) ENGINE=MergeTree() PARTITION BY toYYYYMM(date) ORDER BY date" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_optimize_exception_replicated (date Date) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_00029/optimize', 'r1') PARTITION BY toYYYYMM(date) ORDER BY date" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_optimize_exception_replicated (date Date) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/optimize', 'r1') PARTITION BY toYYYYMM(date) ORDER BY date" ${CLICKHOUSE_CLIENT} --query="INSERT INTO test_optimize_exception VALUES (toDate('2017-09-09')), (toDate('2017-09-10'))" ${CLICKHOUSE_CLIENT} --query="INSERT INTO test_optimize_exception VALUES (toDate('2017-09-09')), (toDate('2017-09-10'))" diff --git a/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh b/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh index 60de1822318..5c5ecd4564b 100755 --- a/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh +++ b/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh @@ -8,8 +8,8 @@ ch="$CLICKHOUSE_CLIENT --stacktrace -q" $ch "DROP TABLE IF EXISTS clear_column1" $ch "DROP TABLE IF EXISTS clear_column2" -$ch "CREATE TABLE clear_column1 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column_concurrent', '1', d, d, 8192)" -$ch "CREATE TABLE clear_column2 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column_concurrent', '2', d, d, 8192)" +$ch "CREATE TABLE clear_column1 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/tables/clear_column_concurrent', '1', d, d, 8192)" +$ch "CREATE TABLE clear_column2 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/tables/clear_column_concurrent', '2', d, d, 8192)" $ch "ALTER TABLE clear_column1 CLEAR COLUMN VasyaUnexistingColumn IN PARTITION '200001'" --replication_alter_partitions_sync=2 1>/dev/null 2>/dev/null rc=$? diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index f5f667d084b..443f2856c88 100755 --- a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -35,8 +35,8 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst_r1;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst_r2;" $CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst_r1 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00626/dst_1', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst_r2 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00626/dst_1', '2') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst_r1 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst_1', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst_r2 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst_1', '2') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);" diff --git a/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh b/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh index 817da08bfa0..def8e8f4cfe 100755 --- a/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh +++ b/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh @@ -10,10 +10,10 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS a" ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS b" ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS c" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE root (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00633/root', '1') ORDER BY d" -${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW a (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00633/a', '1') ORDER BY d AS SELECT * FROM root" -${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW b (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00633/b', '1') ORDER BY d SETTINGS parts_to_delay_insert=1, parts_to_throw_insert=1 AS SELECT * FROM root" -${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW c (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00633/c', '1') ORDER BY d AS SELECT * FROM root" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE root (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/root', '1') ORDER BY d" +${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW a (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/a', '1') ORDER BY d AS SELECT * FROM root" +${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW b (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/b', '1') ORDER BY d SETTINGS parts_to_delay_insert=1, parts_to_throw_insert=1 AS SELECT * FROM root" +${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW c (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/c', '1') ORDER BY d AS SELECT * FROM root" ${CLICKHOUSE_CLIENT} --query "INSERT INTO root VALUES (1)"; ${CLICKHOUSE_CLIENT} --query "SELECT _table, d FROM merge('${CLICKHOUSE_DATABASE}', '^[abc]\$') ORDER BY _table" @@ -33,7 +33,7 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE c" # Deduplication check for non-replicated root table echo ${CLICKHOUSE_CLIENT} --query "CREATE TABLE root (d UInt64) ENGINE = Null" -${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW d (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00633/d', '1') ORDER BY d AS SELECT * FROM root" +${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW d (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/d', '1') ORDER BY d AS SELECT * FROM root" ${CLICKHOUSE_CLIENT} --query "INSERT INTO root VALUES (1)"; ${CLICKHOUSE_CLIENT} --query "INSERT INTO root VALUES (1)"; ${CLICKHOUSE_CLIENT} --query "SELECT * FROM d"; diff --git a/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh b/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh index 02f552c250d..58295e17790 100755 --- a/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh +++ b/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh @@ -11,7 +11,7 @@ ${CLICKHOUSE_CLIENT} --multiquery << EOF DROP TABLE IF EXISTS mutations_r1; DROP TABLE IF EXISTS for_subquery; -CREATE TABLE mutations_r1(x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}/mutations', 'r1') ORDER BY x; +CREATE TABLE mutations_r1(x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations', 'r1') ORDER BY x; INSERT INTO mutations_r1 VALUES (123, 1), (234, 2), (345, 3); CREATE TABLE for_subquery(x UInt32) ENGINE TinyLog; diff --git a/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh b/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh index 08a39c58c3e..3ec6e4e3e90 100755 --- a/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh +++ b/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh @@ -10,8 +10,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations_r1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations_r2" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_r1(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00652/mutations', 'r1', d, intDiv(x, 10), 8192)" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_r2(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00652/mutations', 'r2', d, intDiv(x, 10), 8192)" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_r1(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations', 'r1', d, intDiv(x, 10), 8192)" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_r2(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations', 'r2', d, intDiv(x, 10), 8192)" # Test a mutation on empty table ${CLICKHOUSE_CLIENT} --query="ALTER TABLE mutations_r1 DELETE WHERE x = 1 SETTINGS mutations_sync = 2" @@ -51,11 +51,11 @@ ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations_cleaner_r1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations_cleaner_r2" # Create 2 replicas with finished_mutations_to_keep = 2 -${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_cleaner_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00652/mutations_cleaner', 'r1') ORDER BY x SETTINGS \ +${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_cleaner_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations_cleaner', 'r1') ORDER BY x SETTINGS \ finished_mutations_to_keep = 2, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_cleaner_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00652/mutations_cleaner', 'r2') ORDER BY x SETTINGS \ +${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_cleaner_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations_cleaner', 'r2') ORDER BY x SETTINGS \ finished_mutations_to_keep = 2, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0" diff --git a/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh b/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh index 54b6c80f2ac..48833d2643c 100755 --- a/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh +++ b/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh @@ -11,8 +11,8 @@ ${CLICKHOUSE_CLIENT} -n --query=" DROP TABLE IF EXISTS fetches_r1; DROP TABLE IF EXISTS fetches_r2" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00715/fetches', 'r1') ORDER BY x" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00715/fetches', 'r2') ORDER BY x \ +${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/fetches', 'r1') ORDER BY x" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/fetches', 'r2') ORDER BY x \ SETTINGS prefer_fetch_merged_part_time_threshold=0, \ prefer_fetch_merged_part_size_threshold=0" diff --git a/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh b/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh index d1f938f73fe..92ab6814235 100755 --- a/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh @@ -10,8 +10,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS kill_mutation_r1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS kill_mutation_r2" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE kill_mutation_r1(d Date, x UInt32, s String) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00834/kill_mutation', '1') ORDER BY x PARTITION BY d" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE kill_mutation_r2(d Date, x UInt32, s String) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00834/kill_mutation', '2') ORDER BY x PARTITION BY d" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE kill_mutation_r1(d Date, x UInt32, s String) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/kill_mutation', '1') ORDER BY x PARTITION BY d" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE kill_mutation_r2(d Date, x UInt32, s String) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/kill_mutation', '2') ORDER BY x PARTITION BY d" ${CLICKHOUSE_CLIENT} --query="INSERT INTO kill_mutation_r1 VALUES ('2000-01-01', 1, 'a')" ${CLICKHOUSE_CLIENT} --query="INSERT INTO kill_mutation_r1 VALUES ('2001-01-01', 2, 'b')" diff --git a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh index bbc2d957937..d7a27693e98 100755 --- a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh +++ b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh @@ -15,7 +15,7 @@ CREATE TABLE elog ( engine_id UInt32, referrer String ) -ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00953/elog', 'test') +ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog', 'test') PARTITION BY date ORDER BY (engine_id) SETTINGS replicated_deduplication_window = 2, cleanup_delay_period=4, cleanup_delay_period_random_add=0;" @@ -28,35 +28,35 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 3, 'h $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 3 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 1, 'hello')" $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 4 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')" $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 5 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')" diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh index 81c0c563db1..a3ac5692caa 100755 --- a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh @@ -17,7 +17,7 @@ CREATE TABLE indices_mutaions1 i64 Int64, i32 Int32, INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00975/indices_mutaions', 'r1') +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/indices_mutaions', 'r1') PARTITION BY i32 ORDER BY u64 SETTINGS index_granularity = 2; @@ -27,7 +27,7 @@ CREATE TABLE indices_mutaions2 i64 Int64, i32 Int32, INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00975/indices_mutaions', 'r2') +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/indices_mutaions', 'r2') PARTITION BY i32 ORDER BY u64 SETTINGS index_granularity = 2;" diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh index 613e032f42a..1e61c8d64f3 100755 --- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh @@ -10,8 +10,8 @@ $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS alter_table; DROP TABLE IF EXISTS alter_table2; - CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_DATABASE.alter_table', 'r1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; - CREATE TABLE alter_table2 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_DATABASE.alter_table', 'r2') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0 + CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; + CREATE TABLE alter_table2 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r2') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0 " function thread1() diff --git a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh index 1731148f71f..d960d8ff91d 100755 --- a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh +++ b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh @@ -52,7 +52,7 @@ function thread6() while true; do REPLICA=$(($RANDOM % 10)) $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS alter_table_$REPLICA; - CREATE TABLE alter_table_$REPLICA (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00993/alter_table', 'r_$REPLICA') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; + CREATE TABLE alter_table_$REPLICA (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$REPLICA') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; sleep 0.$RANDOM; done } diff --git a/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh b/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh index 724caa7f414..89b178a38ea 100755 --- a/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh +++ b/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh @@ -4,7 +4,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh - R1=table_1013_1 R2=table_1013_2 @@ -12,8 +11,8 @@ ${CLICKHOUSE_CLIENT} -n -q " DROP TABLE IF EXISTS $R1; DROP TABLE IF EXISTS $R2; - CREATE TABLE $R1 (x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1013', 'r1') ORDER BY x; - CREATE TABLE $R2 (x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1013', 'r2') ORDER BY x; + CREATE TABLE $R1 (x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_1013', 'r1') ORDER BY x; + CREATE TABLE $R2 (x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_1013', 'r2') ORDER BY x; SYSTEM STOP FETCHES $R2; INSERT INTO $R1 VALUES (1) diff --git a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh index d7d0dab71b9..a10e5fb2788 100755 --- a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh +++ b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh @@ -4,7 +4,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh - R1=table_1017_1 R2=table_1017_2 T1=table_1017_merge @@ -29,8 +28,8 @@ ${CLICKHOUSE_CLIENT} -n -q " CREATE TABLE lookup_table (y UInt32, y_new UInt32) ENGINE = Join(ANY, LEFT, y); INSERT INTO lookup_table VALUES(1,1001),(2,1002); - CREATE TABLE $R1 (x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1017', 'r1') ORDER BY x; - CREATE TABLE $R2 (x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1017', 'r2') ORDER BY x; + CREATE TABLE $R1 (x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_1017', 'r1') ORDER BY x; + CREATE TABLE $R2 (x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_1017', 'r2') ORDER BY x; CREATE TABLE $T1 (x UInt32, y UInt32) ENGINE MergeTree() ORDER BY x; INSERT INTO $R1 VALUES (0, 1)(1, 2)(2, 3)(3, 4); diff --git a/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh index 11c7296932c..ae3dd7851c8 100755 --- a/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh @@ -26,8 +26,8 @@ function query_with_retry $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS src;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src1', '1') PARTITION BY p ORDER BY k;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst1', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/src1', '1') PARTITION BY p ORDER BY k;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst1', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);" @@ -56,8 +56,8 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE dst;" $CLICKHOUSE_CLIENT --query="SELECT 'MOVE incompatible schema missing column';" -$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src2', '1') PARTITION BY p ORDER BY (d, p);" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst2', '1') PARTITION BY p ORDER BY (d, p) SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/src2', '1') PARTITION BY p ORDER BY (d, p);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst2', '1') PARTITION BY p ORDER BY (d, p) SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);" @@ -75,8 +75,8 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE dst;" $CLICKHOUSE_CLIENT --query="SELECT 'MOVE incompatible schema different order by';" -$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src3', '1') PARTITION BY p ORDER BY (p, k, d);" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst3', '1') PARTITION BY p ORDER BY (d, k, p);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CURR_DATABASE/src3', '1') PARTITION BY p ORDER BY (p, k, d);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CURR_DATABASE/dst3', '1') PARTITION BY p ORDER BY (d, k, p);" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" diff --git a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh index 4eb3cb9a7bd..7c15b795c36 100755 --- a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh @@ -9,8 +9,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS src;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY k;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/src', '1') PARTITION BY p ORDER BY k;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" function thread1() { diff --git a/tests/queries/0_stateless/01045_zookeeper_system_mutations_with_parts_names.sh b/tests/queries/0_stateless/01045_zookeeper_system_mutations_with_parts_names.sh index c035a692d12..6510fcf408d 100755 --- a/tests/queries/0_stateless/01045_zookeeper_system_mutations_with_parts_names.sh +++ b/tests/queries/0_stateless/01045_zookeeper_system_mutations_with_parts_names.sh @@ -47,7 +47,7 @@ ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS table_for_mutations" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS replicated_table_for_mutations" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE replicated_table_for_mutations(k UInt32, v1 UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_01045/replicated_table_for_mutations', '1') ORDER BY k PARTITION BY modulo(k, 2)" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE replicated_table_for_mutations(k UInt32, v1 UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/replicated_table_for_mutations', '1') ORDER BY k PARTITION BY modulo(k, 2)" ${CLICKHOUSE_CLIENT} --query="SYSTEM STOP MERGES replicated_table_for_mutations" diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index ca453ee8f0d..efe518046a1 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -20,7 +20,7 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_mutate_mt_$i (key UInt64, value1 UInt64, value2 String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01076/concurrent_mutate_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_mutate_mt_$i (key UInt64, value1 UInt64, value2 String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_mutate_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_mt_1 SELECT number, number + 10, toString(number) from numbers(10)" diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference b/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference index ebefe4b2a29..67510b28a34 100644 --- a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference +++ b/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference @@ -1,6 +1,6 @@ Wrong column name. -CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01079/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01079/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 syntax error at begin of string. 7 Hello diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh b/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh index 1c0206453b7..6452b830f38 100755 --- a/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh +++ b/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh @@ -10,7 +10,7 @@ $CLICKHOUSE_CLIENT -n --query "CREATE TABLE table_for_bad_alters ( key UInt64, value1 UInt8, value2 String -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01079/table_for_bad_alters', '1') +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_for_bad_alters', '1') ORDER BY key;" $CLICKHOUSE_CLIENT --query "INSERT INTO table_for_bad_alters VALUES(1, 1, 'Hello');" diff --git a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh index b3a5de8f9bc..7a3e3cf155f 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh @@ -12,7 +12,7 @@ done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_add_drop_$i (key UInt64, value0 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01079/concurrent_alter_add_drop_column', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_add_drop_$i (key UInt64, value0 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_add_drop_column', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_1 SELECT number, number + 10 from numbers(100000)" diff --git a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh index d5f0c987e5d..83f3196253a 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh @@ -11,7 +11,7 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_detach_$i (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01079/concurrent_alter_detach', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_detach_$i (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_detach', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10)" diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh index 5b14c5a8543..9cca73b5eef 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh @@ -11,7 +11,7 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01079/concurrent_alter_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10)" diff --git a/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh b/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh index 287a63f858b..72459e21b69 100755 --- a/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh +++ b/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh @@ -26,7 +26,7 @@ function thread3() { while true; do $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS concurrent_optimize_table; - CREATE TABLE concurrent_optimize_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01103/concurrent_optimize_table', '1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; + CREATE TABLE concurrent_optimize_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_optimize_table', '1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; sleep 0.$RANDOM; sleep 0.$RANDOM; sleep 0.$RANDOM; diff --git a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh index cddf1ebcda6..d564594291a 100755 --- a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh +++ b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) for i in $(seq 4); do $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS replica_01108_$i" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS replica_01108_${i}_tmp" - $CLICKHOUSE_CLIENT -q "CREATE TABLE replica_01108_$i (n int) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_01108/replica_01108_$i', 'replica') ORDER BY tuple()" + $CLICKHOUSE_CLIENT -q "CREATE TABLE replica_01108_$i (n int) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/replica_01108_$i', 'replica') ORDER BY tuple()" $CLICKHOUSE_CLIENT -q "INSERT INTO replica_01108_$i SELECT * FROM system.numbers LIMIT $i * 10, 10" done diff --git a/tests/queries/0_stateless/01192_rename_database_zookeeper.sh b/tests/queries/0_stateless/01192_rename_database_zookeeper.sh index 90b9baf4ebf..58bdfbf71ad 100755 --- a/tests/queries/0_stateless/01192_rename_database_zookeeper.sh +++ b/tests/queries/0_stateless/01192_rename_database_zookeeper.sh @@ -36,7 +36,7 @@ $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01192_renamed.mt" # 5. check moving tables from Ordinary to Atomic (can be used to "alter" database engine) $CLICKHOUSE_CLIENT --default_database_engine=Ordinary -q "CREATE DATABASE test_01192" $CLICKHOUSE_CLIENT -q "CREATE TABLE test_01192.mt AS test_01192_renamed.mt ENGINE=MergeTree ORDER BY n" -$CLICKHOUSE_CLIENT -q "CREATE TABLE test_01192.rmt AS test_01192_renamed.mt ENGINE=ReplicatedMergeTree('/test/01192/', '1') ORDER BY n" +$CLICKHOUSE_CLIENT -q "CREATE TABLE test_01192.rmt AS test_01192_renamed.mt ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/', '1') ORDER BY n" $CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW test_01192.mv TO test_01192.rmt AS SELECT * FROM test_01192.mt" $CLICKHOUSE_CLIENT -q "INSERT INTO test_01192.mt SELECT number FROM numbers(10)" && echo "inserted" diff --git a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference index fc2a74d1a93..35385731ad3 100644 --- a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference +++ b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference @@ -1,7 +1,7 @@ 1 -CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date,\n `key` UInt64,\n `value1` String,\n `value2` String,\n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01213/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date,\n `key` UInt64,\n `value1` String,\n `value2` String,\n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01213_alter_rename_column_zookeeper_default/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 renamed_value1 -CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date,\n `key` UInt64,\n `renamed_value1` String,\n `value2` String,\n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01213/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date,\n `key` UInt64,\n `renamed_value1` String,\n `value2` String,\n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01213_alter_rename_column_zookeeper_default/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 1 date key renamed_value1 value2 value3 2019-10-02 1 1 1 1 diff --git a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh index 5ab0e800d39..5da8de70c46 100755 --- a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh +++ b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh @@ -15,7 +15,7 @@ CREATE TABLE table_for_rename_replicated value2 String, value3 String ) -ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01213/table_for_rename_replicated', '1') +ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_for_rename_replicated', '1') PARTITION BY date ORDER BY key; " diff --git a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh index 5dd3d2b38d6..01bb9af461c 100755 --- a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh +++ b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh @@ -13,13 +13,13 @@ function thread() # Ignore "Replica already exists" exception while true; do $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1 NO DELAY; - CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01305/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | + CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now|is already finished removing by another replica right now|Removing leftovers from table|Another replica was suddenly created|was successfully removed from ZooKeeper|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time|already exists' done else while true; do $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1; - CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01305/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | + CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now|is already finished removing by another replica right now|Removing leftovers from table|Another replica was suddenly created|was successfully removed from ZooKeeper|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time' done fi diff --git a/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh b/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh index 24c6199a94a..21fc88d7c2d 100755 --- a/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh +++ b/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh @@ -12,7 +12,7 @@ DATA_SIZE=200 SEQ=$(seq 0 $(($NUM_REPLICAS - 1))) for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "DROP TABLE IF EXISTS r$REPLICA"; done -for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE r$REPLICA (x UInt64) ENGINE = ReplicatedMergeTree('/test_01307/table', 'r$REPLICA') ORDER BY x SETTINGS min_bytes_for_wide_part = '10M';"; done +for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE r$REPLICA (x UInt64) ENGINE = ReplicatedMergeTree('/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r$REPLICA') ORDER BY x SETTINGS min_bytes_for_wide_part = '10M';"; done function thread() { diff --git a/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh b/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh index ced668e9849..a05304c670c 100755 --- a/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh +++ b/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh @@ -11,7 +11,7 @@ $CLICKHOUSE_CLIENT --query " key UInt64, value String ) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01318/mutation_table', '1') + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutation_table', '1') ORDER BY key PARTITION BY key % 10 " diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh index a15d8c8d2cd..97c200c651f 100755 --- a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh @@ -11,7 +11,7 @@ $CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01320 ENGINE=Ordinary" # Diff function thread1() { - while true; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/test_01320/table', 'r') ORDER BY x; DROP TABLE test_01320.r;"; done + while true; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r') ORDER BY x; DROP TABLE test_01320.r;"; done } function thread2() diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference index e7db2788824..b4ed8efab63 100644 --- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference +++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference @@ -1,3 +1,3 @@ 10 5 -CREATE TABLE default.alter_mt\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01338/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_mt\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01338_long_select_and_alter_zookeeper_default/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh index d990a8a1c08..4aeecc7343d 100755 --- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh +++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" -$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_01338/alter_mt', '1') ORDER BY key" +$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_mt', '1') ORDER BY key" $CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number, toString(number) FROM numbers(5)" diff --git a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh index 30b2b665658..b604ace85cc 100755 --- a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh +++ b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh @@ -12,8 +12,8 @@ SCALE=5000 $CLICKHOUSE_CLIENT -n --query " DROP TABLE IF EXISTS r1; DROP TABLE IF EXISTS r2; - CREATE TABLE r1 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01396/r', '1') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10; - CREATE TABLE r2 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01396/r', '2') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10; + CREATE TABLE r1 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', '1') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10; + CREATE TABLE r2 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', '2') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10; DETACH TABLE r2; " @@ -29,16 +29,16 @@ for _ in {1..60}; do done -$CLICKHOUSE_CLIENT --query "SELECT numChildren < $((SCALE / 4)) FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01396/r' AND name = 'log'"; +$CLICKHOUSE_CLIENT --query "SELECT numChildren < $((SCALE / 4)) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r' AND name = 'log'"; echo -e '\n---\n'; -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01396/r/replicas/1' AND name = 'is_lost'"; -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01396/r/replicas/2' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r/replicas/1' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r/replicas/2' AND name = 'is_lost'"; echo -e '\n---\n'; $CLICKHOUSE_CLIENT --query "ATTACH TABLE r2" $CLICKHOUSE_CLIENT --receive_timeout 600 --query "SYSTEM SYNC REPLICA r2" # Need to increase timeout, otherwise it timed out in debug build -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01396/r/replicas/2' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r/replicas/2' AND name = 'is_lost'"; $CLICKHOUSE_CLIENT -n --query " DROP TABLE IF EXISTS r1; diff --git a/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh b/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh index ceeeed41049..6e1a6e01757 100755 --- a/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh +++ b/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh @@ -12,7 +12,7 @@ $CLICKHOUSE_CLIENT --query " key UInt64, value String ) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01414/mutation_table', '1') + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutation_table', '1') ORDER BY tuple() PARTITION BY date " diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh index 480daeefa46..bb935a950ff 100755 --- a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh +++ b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh @@ -11,7 +11,7 @@ FREEZE_OUT_STRUCTURE='backup_name String, backup_path String , part_backup_path # setup ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze_replicated;" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE table_for_freeze_replicated (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test_01417/table_for_freeze_replicated', '1') ORDER BY key PARTITION BY key % 10;" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE table_for_freeze_replicated (key UInt64, value String) ENGINE = ReplicatedMergeTree('/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_for_freeze_replicated', '1') ORDER BY key PARTITION BY key % 10;" ${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze_replicated SELECT number, toString(number) from numbers(10);" ${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \ diff --git a/tests/queries/0_stateless/01459_manual_write_to_replicas.sh b/tests/queries/0_stateless/01459_manual_write_to_replicas.sh index 467c29d3d33..cf239fd7032 100755 --- a/tests/queries/0_stateless/01459_manual_write_to_replicas.sh +++ b/tests/queries/0_stateless/01459_manual_write_to_replicas.sh @@ -11,7 +11,7 @@ NUM_REPLICAS=10 for i in $(seq 1 $NUM_REPLICAS); do $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS r$i; - CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/01459_manual_write_ro_replicas/r', 'r$i') ORDER BY x; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x; " done diff --git a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh index 376ee58859e..8c322798173 100755 --- a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh +++ b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh @@ -11,7 +11,7 @@ NUM_REPLICAS=10 for i in $(seq 1 $NUM_REPLICAS); do $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS r$i; - CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/01459_manual_write_ro_replicas_quorum/r', 'r$i') ORDER BY x; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x; " done diff --git a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh index 4cb4734b448..156deb60ff9 100755 --- a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh +++ b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh @@ -8,7 +8,7 @@ $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_renames0" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_renames50" -$CLICKHOUSE_CLIENT --query "CREATE TABLE table_for_renames0 (value UInt64, data String) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_01508/concurrent_rename', '1') ORDER BY tuple() SETTINGS cleanup_delay_period = 1, cleanup_delay_period_random_add = 0, min_rows_for_compact_part = 100000, min_rows_for_compact_part = 10000000, write_ahead_log_max_bytes = 1" +$CLICKHOUSE_CLIENT --query "CREATE TABLE table_for_renames0 (value UInt64, data String) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_rename', '1') ORDER BY tuple() SETTINGS cleanup_delay_period = 1, cleanup_delay_period_random_add = 0, min_rows_for_compact_part = 100000, min_rows_for_compact_part = 10000000, write_ahead_log_max_bytes = 1" $CLICKHOUSE_CLIENT --query "INSERT INTO table_for_renames0 SELECT number, toString(number) FROM numbers(1000)" diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh index c5ffad1c4ca..b71654e7e6c 100755 --- a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh +++ b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh @@ -11,7 +11,7 @@ NUM_REPLICAS=10 for i in $(seq 1 $NUM_REPLICAS); do $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS r$i; - CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_quorum_many', 'r$i') ORDER BY x; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/parallel_quorum_many', 'r$i') ORDER BY x; " done diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh index 898a68d9c77..78336ea073b 100755 --- a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh +++ b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh @@ -12,7 +12,7 @@ NUM_INSERTS=5 for i in $(seq 1 $NUM_REPLICAS); do $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS r$i; - CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_quorum', 'r$i') ORDER BY x; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/parallel_quorum', 'r$i') ORDER BY x; " done diff --git a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh b/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh index ca5f58512a3..fbeb65419ce 100755 --- a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh +++ b/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh @@ -10,9 +10,9 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parallel_q1" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parallel_q2" -$CLICKHOUSE_CLIENT -q "CREATE TABLE parallel_q1 (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_q', 'r1') ORDER BY tuple() SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE parallel_q1 (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/parallel_q', 'r1') ORDER BY tuple() SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0" -$CLICKHOUSE_CLIENT -q "CREATE TABLE parallel_q2 (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_q', 'r2') ORDER BY tuple() SETTINGS always_fetch_merged_part = 1" +$CLICKHOUSE_CLIENT -q "CREATE TABLE parallel_q2 (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/parallel_q', 'r2') ORDER BY tuple() SETTINGS always_fetch_merged_part = 1" $CLICKHOUSE_CLIENT -q "SYSTEM STOP REPLICATION QUEUES parallel_q2" diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.reference b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.reference index 94e15c09768..4b07f533f5a 100644 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.reference +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.reference @@ -1,2 +1,2 @@ -CREATE TABLE default.concurrent_mutate_kill\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593/concurrent_mutate_kill\', \'1\')\nPARTITION BY key % 100\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_mutate_kill\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_default/concurrent_mutate_kill\', \'1\')\nPARTITION BY key % 100\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 499999500000 diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh index 6ae103bdf6e..d40406222c2 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_mutate_kill" -$CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_mutate_kill (key UInt64, value String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01593/concurrent_mutate_kill', '1') ORDER BY key PARTITION BY key % 100 SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" +$CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_mutate_kill (key UInt64, value String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_mutate_kill', '1') ORDER BY key PARTITION BY key % 100 SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_kill SELECT number, toString(number) FROM numbers(1000000)" diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.reference b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.reference index cb1eace24a2..80b9977aac7 100644 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.reference +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.reference @@ -4,13 +4,13 @@ 499999500000 499999500000 Metadata version on replica 1 equal with first replica, OK -CREATE TABLE default.concurrent_kill_1\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593_concurrent_kill\', \'1\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_kill_1\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'1\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 Metadata version on replica 2 equal with first replica, OK -CREATE TABLE default.concurrent_kill_2\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593_concurrent_kill\', \'2\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_kill_2\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'2\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 Metadata version on replica 3 equal with first replica, OK -CREATE TABLE default.concurrent_kill_3\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593_concurrent_kill\', \'3\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_kill_3\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'3\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 Metadata version on replica 4 equal with first replica, OK -CREATE TABLE default.concurrent_kill_4\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593_concurrent_kill\', \'4\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_kill_4\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'4\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 Metadata version on replica 5 equal with first replica, OK -CREATE TABLE default.concurrent_kill_5\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593_concurrent_kill\', \'5\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_kill_5\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'5\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 499999500000 diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh index bfa68328c06..4cb3fd35294 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh @@ -11,7 +11,7 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_kill_$i (key UInt64, value String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01593_concurrent_kill', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_kill_$i (key UInt64, value String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_kill_1 SELECT number, toString(number) FROM numbers(1000000)" @@ -77,9 +77,9 @@ while true; do done -metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01593_concurrent_kill/replicas/$i/' and name = 'metadata_version'") +metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/replicas/$i/' and name = 'metadata_version'") for i in $(seq $REPLICAS); do - replica_metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01593_concurrent_kill/replicas/$i/' and name = 'metadata_version'") + replica_metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/replicas/$i/' and name = 'metadata_version'") if [ "$metadata_version" != "$replica_metadata_version" ]; then echo "Metadata version on replica $i differs from the first replica, FAIL" else diff --git a/tests/queries/0_stateless/01671_ddl_hang_timeout.sh b/tests/queries/0_stateless/01671_ddl_hang_timeout.sh index 2ca97e3978b..641eba2d8fa 100755 --- a/tests/queries/0_stateless/01671_ddl_hang_timeout.sh +++ b/tests/queries/0_stateless/01671_ddl_hang_timeout.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function thread_create_drop_table { while true; do REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS t1 (x UInt64, s Array(Nullable(String))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01671/test_01671', 'r_$REPLICA') order by x" 2>/dev/null + $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS t1 (x UInt64, s Array(Nullable(String))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_01671', 'r_$REPLICA') order by x" 2>/dev/null sleep 0.0$RANDOM $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS t1" done diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh index 1f4ba412a19..d3046e73b93 100755 --- a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh +++ b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh @@ -6,9 +6,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_01753"; -${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_01753 (n Int8) ENGINE=ReplicatedMergeTree('/$CLICKHOUSE_DATABASE/test_01753/test', '1') ORDER BY n" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_01753 (n Int8) ENGINE=ReplicatedMergeTree('/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_01753/test', '1') ORDER BY n" -${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.zookeeper WHERE path = {path:String}" --param_path "$CLICKHOUSE_DATABASE/test_01753" +${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.zookeeper WHERE path = {path:String}" --param_path "$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_01753" ${CLICKHOUSE_CLIENT} --query="DROP TABLE test_01753 SYNC"; From a03a9051e1882adbf43b8ea23c6ef7a38c92cd98 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 15 Mar 2021 21:03:24 +0300 Subject: [PATCH 144/233] Mark some tests as long From [1] and [2]: [1]: https://clickhouse-test-reports.s3.yandex.net/21724/dacb6066237b78511ad5f07cc65045b8cf4e5e7b/functional_stateless_tests_flaky_check_(address).html#fail1 [2]: https://clickhouse-test-reports.s3.yandex.net/21724/ee8ece157987f3cd8b68ca4a5b7df709f4688208/functional_stateless_tests_flaky_check_(address)/test_run.txt.out.log [3]: https://clickhouse-test-reports.s3.yandex.net/21724/e4485ce0b1d1b21c4360bbdecacd2485d1634a91/functional_stateless_tests_flaky_check_(address)/test_run.txt.out.log --- ...75_indices_mutation_replicated_zookeeper_long.reference} | 0 ... => 00975_indices_mutation_replicated_zookeeper_long.sh} | 0 ...92_system_parts_race_condition_zookeeper_long.reference} | 0 ... => 00992_system_parts_race_condition_zookeeper_long.sh} | 0 ....reference => 01079_bad_alters_zookeeper_long.reference} | 4 ++-- ...ters_zookeeper.sh => 01079_bad_alters_zookeeper_long.sh} | 0 ...=> 01079_parallel_alter_modify_zookeeper_long.reference} | 0 ...per.sh => 01079_parallel_alter_modify_zookeeper_long.sh} | 0 ...08_race_condition_rename_clear_zookeeper_long.reference} | 0 ... => 01508_race_condition_rename_clear_zookeeper_long.sh} | 0 ...01509_check_many_parallel_quorum_inserts_long.reference} | 0 ....sh => 01509_check_many_parallel_quorum_inserts_long.sh} | 0 ...e => 01509_check_parallel_quorum_inserts_long.reference} | 0 ...serts.sh => 01509_check_parallel_quorum_inserts_long.sh} | 0 ...rence => 01509_parallel_quorum_and_merge_long.reference} | 0 ...and_merge.sh => 01509_parallel_quorum_and_merge_long.sh} | 0 ...eout.reference => 01671_ddl_hang_timeout_long.reference} | 0 ...1_ddl_hang_timeout.sh => 01671_ddl_hang_timeout_long.sh} | 0 ... 01753_system_zookeeper_query_param_path_long.reference} | 0 ...h.sh => 01753_system_zookeeper_query_param_path_long.sh} | 0 tests/queries/skip_list.json | 6 +++--- 21 files changed, 5 insertions(+), 5 deletions(-) rename tests/queries/0_stateless/{00975_indices_mutation_replicated_zookeeper.reference => 00975_indices_mutation_replicated_zookeeper_long.reference} (100%) rename tests/queries/0_stateless/{00975_indices_mutation_replicated_zookeeper.sh => 00975_indices_mutation_replicated_zookeeper_long.sh} (100%) rename tests/queries/0_stateless/{00992_system_parts_race_condition_zookeeper.reference => 00992_system_parts_race_condition_zookeeper_long.reference} (100%) rename tests/queries/0_stateless/{00992_system_parts_race_condition_zookeeper.sh => 00992_system_parts_race_condition_zookeeper_long.sh} (100%) rename tests/queries/0_stateless/{01079_bad_alters_zookeeper.reference => 01079_bad_alters_zookeeper_long.reference} (52%) rename tests/queries/0_stateless/{01079_bad_alters_zookeeper.sh => 01079_bad_alters_zookeeper_long.sh} (100%) rename tests/queries/0_stateless/{01079_parallel_alter_modify_zookeeper.reference => 01079_parallel_alter_modify_zookeeper_long.reference} (100%) rename tests/queries/0_stateless/{01079_parallel_alter_modify_zookeeper.sh => 01079_parallel_alter_modify_zookeeper_long.sh} (100%) rename tests/queries/0_stateless/{01508_race_condition_rename_clear_zookeeper.reference => 01508_race_condition_rename_clear_zookeeper_long.reference} (100%) rename tests/queries/0_stateless/{01508_race_condition_rename_clear_zookeeper.sh => 01508_race_condition_rename_clear_zookeeper_long.sh} (100%) rename tests/queries/0_stateless/{01509_check_many_parallel_quorum_inserts.reference => 01509_check_many_parallel_quorum_inserts_long.reference} (100%) rename tests/queries/0_stateless/{01509_check_many_parallel_quorum_inserts.sh => 01509_check_many_parallel_quorum_inserts_long.sh} (100%) rename tests/queries/0_stateless/{01509_check_parallel_quorum_inserts.reference => 01509_check_parallel_quorum_inserts_long.reference} (100%) rename tests/queries/0_stateless/{01509_check_parallel_quorum_inserts.sh => 01509_check_parallel_quorum_inserts_long.sh} (100%) rename tests/queries/0_stateless/{01509_parallel_quorum_and_merge.reference => 01509_parallel_quorum_and_merge_long.reference} (100%) rename tests/queries/0_stateless/{01509_parallel_quorum_and_merge.sh => 01509_parallel_quorum_and_merge_long.sh} (100%) rename tests/queries/0_stateless/{01671_ddl_hang_timeout.reference => 01671_ddl_hang_timeout_long.reference} (100%) rename tests/queries/0_stateless/{01671_ddl_hang_timeout.sh => 01671_ddl_hang_timeout_long.sh} (100%) rename tests/queries/0_stateless/{01753_system_zookeeper_query_param_path.reference => 01753_system_zookeeper_query_param_path_long.reference} (100%) rename tests/queries/0_stateless/{01753_system_zookeeper_query_param_path.sh => 01753_system_zookeeper_query_param_path_long.sh} (100%) diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.reference b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.reference similarity index 100% rename from tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.reference rename to tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.reference diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh similarity index 100% rename from tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh rename to tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.reference b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.reference similarity index 100% rename from tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.reference rename to tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.reference diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh similarity index 100% rename from tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh rename to tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference b/tests/queries/0_stateless/01079_bad_alters_zookeeper_long.reference similarity index 52% rename from tests/queries/0_stateless/01079_bad_alters_zookeeper.reference rename to tests/queries/0_stateless/01079_bad_alters_zookeeper_long.reference index 67510b28a34..731cd871b3b 100644 --- a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference +++ b/tests/queries/0_stateless/01079_bad_alters_zookeeper_long.reference @@ -1,6 +1,6 @@ Wrong column name. -CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_long_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_long_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 syntax error at begin of string. 7 Hello diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh b/tests/queries/0_stateless/01079_bad_alters_zookeeper_long.sh similarity index 100% rename from tests/queries/0_stateless/01079_bad_alters_zookeeper.sh rename to tests/queries/0_stateless/01079_bad_alters_zookeeper_long.sh diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.reference b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.reference similarity index 100% rename from tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.reference rename to tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.reference diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh similarity index 100% rename from tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh rename to tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh diff --git a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.reference b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.reference similarity index 100% rename from tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.reference rename to tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.reference diff --git a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh similarity index 100% rename from tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh rename to tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.reference b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.reference similarity index 100% rename from tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.reference rename to tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.reference diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh similarity index 100% rename from tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh rename to tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.reference b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.reference similarity index 100% rename from tests/queries/0_stateless/01509_check_parallel_quorum_inserts.reference rename to tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.reference diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.sh similarity index 100% rename from tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh rename to tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.sh diff --git a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.reference b/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.reference similarity index 100% rename from tests/queries/0_stateless/01509_parallel_quorum_and_merge.reference rename to tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.reference diff --git a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh b/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh similarity index 100% rename from tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh rename to tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh diff --git a/tests/queries/0_stateless/01671_ddl_hang_timeout.reference b/tests/queries/0_stateless/01671_ddl_hang_timeout_long.reference similarity index 100% rename from tests/queries/0_stateless/01671_ddl_hang_timeout.reference rename to tests/queries/0_stateless/01671_ddl_hang_timeout_long.reference diff --git a/tests/queries/0_stateless/01671_ddl_hang_timeout.sh b/tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh similarity index 100% rename from tests/queries/0_stateless/01671_ddl_hang_timeout.sh rename to tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path_long.reference similarity index 100% rename from tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference rename to tests/queries/0_stateless/01753_system_zookeeper_query_param_path_long.reference diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path_long.sh similarity index 100% rename from tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh rename to tests/queries/0_stateless/01753_system_zookeeper_query_param_path_long.sh diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 974ef48ef3c..9501343754c 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -307,7 +307,7 @@ "00954_client_prepared_statements", "00956_sensitive_data_masking", "00969_columns_clause", - "00975_indices_mutation_replicated_zookeeper", + "00975_indices_mutation_replicated_zookeeper_long", "00975_values_list", "00976_system_stop_ttl_merges", "00977_int_div", @@ -442,8 +442,8 @@ "01504_compression_multiple_streams", "01508_explain_header", "01508_partition_pruning_long", - "01509_check_parallel_quorum_inserts", - "01509_parallel_quorum_and_merge", + "01509_check_parallel_quorum_inserts_long", + "01509_parallel_quorum_and_merge_long", "01515_mv_and_array_join_optimisation_bag", "01516_create_table_primary_key", "01517_drop_mv_with_inner_table", From 0bb665428d166d1e589886b62a3f645d6a5dc5d6 Mon Sep 17 00:00:00 2001 From: Vitaliy Fedorchenko Date: Tue, 16 Mar 2021 09:05:08 +0200 Subject: [PATCH 145/233] Update gui.md: add SeekTable --- docs/en/interfaces/third-party/gui.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index fa123d8b23d..5d14b3aa3cc 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -167,4 +167,21 @@ Features: [How to configure ClickHouse in Looker.](https://docs.looker.com/setup-and-management/database-config/clickhouse) +### SeekTable {#seektable} + +[SeekTable](https://www.seektable.com) is a self-service BI tool for data exploration and operational reporting. SeekTable is available both as a cloud service and a self-hosted version. SeekTable reports may be embedded into any web-app. + +Features: + +- Business users-friendly reports builder. +- Powerful report parameters for SQL filtering and report-specific query customizations. +- Can connect to ClickHouse both with a native TCP/IP endpoint and a HTTP(S) interface (2 different drivers). +- It is possible to use all power of CH SQL dialect in dimensions/measures definitions +- [Web API](https://www.seektable.com/help/web-api-integration) for automated reports generation. +- Supports reports development flow with account data [backup/restore](https://www.seektable.com/help/self-hosted-backup-restore), data models (cubes) / reports configuration is a human-readable XML and can be stored under version control. + +SeekTable is [free](https://www.seektable.com/help/cloud-pricing) for personal/individual usage. + +[How to configure ClickHouse connection in SeekTable.](https://www.seektable.com/help/clickhouse-pivot-table) + [Original article](https://clickhouse.tech/docs/en/interfaces/third-party/gui/) From 27b8d10bd8edcd0778ae06c2f480bf5762db4cd6 Mon Sep 17 00:00:00 2001 From: Ali Demirci Date: Tue, 16 Mar 2021 13:30:05 +0300 Subject: [PATCH 146/233] docs(fix): typo --- docs/en/sql-reference/functions/machine-learning-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/machine-learning-functions.md b/docs/en/sql-reference/functions/machine-learning-functions.md index c9abd130ef3..60dabd73781 100644 --- a/docs/en/sql-reference/functions/machine-learning-functions.md +++ b/docs/en/sql-reference/functions/machine-learning-functions.md @@ -9,7 +9,7 @@ toc_title: Machine Learning Prediction using fitted regression models uses `evalMLMethod` function. See link in `linearRegression`. -## stochasticLinearRegressionn {#stochastic-linear-regression} +## stochasticLinearRegression {#stochastic-linear-regression} The [stochasticLinearRegression](../../sql-reference/aggregate-functions/reference/stochasticlinearregression.md#agg_functions-stochasticlinearregression) aggregate function implements stochastic gradient descent method using linear model and MSE loss function. Uses `evalMLMethod` to predict on new data. From eadf0248d0e9cf57be70f5707277e2cda3c5855e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 14:07:53 +0300 Subject: [PATCH 147/233] Build fixes --- base/common/LocalDateTime.h | 24 +++++- base/mysqlxx/tests/CMakeLists.txt | 3 - base/mysqlxx/tests/failover.xml | 21 ----- base/mysqlxx/tests/mysqlxx_test.cpp | 77 ------------------- programs/odbc-bridge/ODBCBlockInputStream.cpp | 4 +- .../odbc-bridge/ODBCBlockOutputStream.cpp | 2 +- src/Storages/tests/CMakeLists.txt | 3 - src/Storages/tests/part_name.cpp | 21 ----- utils/wikistat-loader/main.cpp | 2 +- 9 files changed, 27 insertions(+), 130 deletions(-) delete mode 100644 base/mysqlxx/tests/failover.xml delete mode 100644 base/mysqlxx/tests/mysqlxx_test.cpp delete mode 100644 src/Storages/tests/part_name.cpp diff --git a/base/common/LocalDateTime.h b/base/common/LocalDateTime.h index 4c2cf0e637d..dde283e5ebb 100644 --- a/base/common/LocalDateTime.h +++ b/base/common/LocalDateTime.h @@ -106,8 +106,30 @@ public: void second(unsigned char x) { m_second = x; } LocalDate toDate() const { return LocalDate(m_year, m_month, m_day); } + LocalDateTime toStartOfDate() const { return LocalDateTime(m_year, m_month, m_day, 0, 0, 0); } - LocalDateTime toStartOfDate() { return LocalDateTime(m_year, m_month, m_day, 0, 0, 0); } + std::string toString() const + { + std::string s{"0000-00-00 00:00:00"}; + + s[0] += m_year / 1000; + s[1] += (m_year / 100) % 10; + s[2] += (m_year / 10) % 10; + s[3] += m_year % 10; + s[5] += m_month / 10; + s[6] += m_month % 10; + s[8] += m_day / 10; + s[9] += m_day % 10; + + s[11] += m_hour / 10; + s[12] += m_hour % 10; + s[14] += m_minute / 10; + s[15] += m_minute % 10; + s[17] += m_second / 10; + s[18] += m_second % 10; + + return s; + } bool operator< (const LocalDateTime & other) const { diff --git a/base/mysqlxx/tests/CMakeLists.txt b/base/mysqlxx/tests/CMakeLists.txt index 2cf19d78418..6473a927308 100644 --- a/base/mysqlxx/tests/CMakeLists.txt +++ b/base/mysqlxx/tests/CMakeLists.txt @@ -1,5 +1,2 @@ -add_executable (mysqlxx_test mysqlxx_test.cpp) -target_link_libraries (mysqlxx_test PRIVATE mysqlxx) - add_executable (mysqlxx_pool_test mysqlxx_pool_test.cpp) target_link_libraries (mysqlxx_pool_test PRIVATE mysqlxx) diff --git a/base/mysqlxx/tests/failover.xml b/base/mysqlxx/tests/failover.xml deleted file mode 100644 index 73702eabb29..00000000000 --- a/base/mysqlxx/tests/failover.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - 3306 - root - Metrica - qwerty - - example02t - 0 - - - example02t - 3306 - root - qwerty - Metrica - 1 - - - diff --git a/base/mysqlxx/tests/mysqlxx_test.cpp b/base/mysqlxx/tests/mysqlxx_test.cpp deleted file mode 100644 index c505d34a58d..00000000000 --- a/base/mysqlxx/tests/mysqlxx_test.cpp +++ /dev/null @@ -1,77 +0,0 @@ -#include -#include - - -int main(int, char **) -{ - try - { - mysqlxx::Connection connection("test", "127.0.0.1", "root", "qwerty", 3306); - std::cerr << "Connected." << std::endl; - - { - mysqlxx::Query query = connection.query(); - query << "SELECT 1 x, '2010-01-01 01:01:01' d"; - mysqlxx::UseQueryResult result = query.use(); - std::cerr << "use() called." << std::endl; - - while (mysqlxx::Row row = result.fetch()) - { - std::cerr << "Fetched row." << std::endl; - std::cerr << row[0] << ", " << row["x"] << std::endl; - std::cerr << row[1] << ", " << row["d"] - << ", " << row[1].getDate() - << ", " << row[1].getDateTime() - << ", " << row[1].getDate() - << ", " << row[1].getDateTime() - << std::endl - << row[1].getDate() << ", " << row[1].getDateTime() << std::endl - << row[1].getDate() << ", " << row[1].getDateTime() << std::endl - << row[1].getDate() << ", " << row[1].getDateTime() << std::endl - << row[1].getDate() << ", " << row[1].getDateTime() << std::endl - ; - - time_t t1 = row[0]; - time_t t2 = row[1]; - std::cerr << t1 << ", " << LocalDateTime(t1) << std::endl; - std::cerr << t2 << ", " << LocalDateTime(t2) << std::endl; - } - } - - { - mysqlxx::UseQueryResult result = connection.query("SELECT 'abc\\\\def' x").use(); - mysqlxx::Row row = result.fetch(); - std::cerr << row << std::endl; - std::cerr << row << std::endl; - } - - { - /// Копирование Query - mysqlxx::Query query1 = connection.query("SELECT"); - mysqlxx::Query query2 = query1; - query2 << " 1"; - - std::cerr << query1.str() << ", " << query2.str() << std::endl; - } - - { - /// NULL - mysqlxx::Null x = mysqlxx::null; - std::cerr << (x == mysqlxx::null ? "Ok" : "Fail") << std::endl; - std::cerr << (x == 0 ? "Fail" : "Ok") << std::endl; - std::cerr << (x.isNull() ? "Ok" : "Fail") << std::endl; - x = 1; - std::cerr << (x == mysqlxx::null ? "Fail" : "Ok") << std::endl; - std::cerr << (x == 0 ? "Fail" : "Ok") << std::endl; - std::cerr << (x == 1 ? "Ok" : "Fail") << std::endl; - std::cerr << (x.isNull() ? "Fail" : "Ok") << std::endl; - } - } - catch (const mysqlxx::Exception & e) - { - std::cerr << e.code() << ", " << e.message() << std::endl; - throw; - } - - return 0; -} diff --git a/programs/odbc-bridge/ODBCBlockInputStream.cpp b/programs/odbc-bridge/ODBCBlockInputStream.cpp index 3e2a2d0c7d4..b8a4209ac94 100644 --- a/programs/odbc-bridge/ODBCBlockInputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockInputStream.cpp @@ -87,8 +87,8 @@ namespace case ValueType::vtDateTime: { Poco::DateTime datetime = value.convert(); - assert_cast(column).insertValue(time_t{LocalDateTime( - datetime.year(), datetime.month(), datetime.day(), datetime.hour(), datetime.minute(), datetime.second())}); + assert_cast(column).insertValue(DateLUT::instance().makeDateTime( + datetime.year(), datetime.month(), datetime.day(), datetime.hour(), datetime.minute(), datetime.second())); break; } case ValueType::vtUUID: diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.cpp b/programs/odbc-bridge/ODBCBlockOutputStream.cpp index 4d8b9fa6bdf..db3c9441419 100644 --- a/programs/odbc-bridge/ODBCBlockOutputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockOutputStream.cpp @@ -81,7 +81,7 @@ namespace case ValueType::vtDate: return Poco::Dynamic::Var(LocalDate(DayNum(field.get())).toString()).convert(); case ValueType::vtDateTime: - return Poco::Dynamic::Var(std::to_string(LocalDateTime(time_t(field.get())))).convert(); + return Poco::Dynamic::Var(DateLUT::instance().timeToString(time_t(field.get()))).convert(); case ValueType::vtUUID: return Poco::Dynamic::Var(UUID(field.get()).toUnderType().toHexString()).convert(); default: diff --git a/src/Storages/tests/CMakeLists.txt b/src/Storages/tests/CMakeLists.txt index b58fed9edf5..59d44829363 100644 --- a/src/Storages/tests/CMakeLists.txt +++ b/src/Storages/tests/CMakeLists.txt @@ -1,6 +1,3 @@ -add_executable (part_name part_name.cpp) -target_link_libraries (part_name PRIVATE dbms) - add_executable (remove_symlink_directory remove_symlink_directory.cpp) target_link_libraries (remove_symlink_directory PRIVATE dbms) diff --git a/src/Storages/tests/part_name.cpp b/src/Storages/tests/part_name.cpp deleted file mode 100644 index 227e19cf17c..00000000000 --- a/src/Storages/tests/part_name.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include -#include -#include - - -int main(int, char **) -{ - const DayNum today(DateLUT::instance().toDayNum(time(nullptr)).toUnderType()); - - for (DayNum date = today; DayNum(date + 10) > today; --date) - { - DB::MergeTreePartInfo part_info("partition", 0, 0, 0); - std::string name = part_info.getPartNameV0(date, date); - std::cerr << name << '\n'; - - time_t time = DateLUT::instance().YYYYMMDDToDate(DB::parse(name)); - std::cerr << LocalDateTime(time) << '\n'; - } - - return 0; -} diff --git a/utils/wikistat-loader/main.cpp b/utils/wikistat-loader/main.cpp index f2adcc43a3a..31ade014c74 100644 --- a/utils/wikistat-loader/main.cpp +++ b/utils/wikistat-loader/main.cpp @@ -151,7 +151,7 @@ try std::string time_str = options.at("time").as(); LocalDateTime time(time_str); - LocalDate date(time); + LocalDate date(time_str); DB::ReadBufferFromFileDescriptor in(STDIN_FILENO); DB::WriteBufferFromFileDescriptor out(STDOUT_FILENO); From f8a99804c178487ab2e7ca3e391cc7b823877054 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 16 Mar 2021 10:20:31 +0000 Subject: [PATCH 148/233] Add double quotes --- src/Databases/PostgreSQL/DatabasePostgreSQL.cpp | 3 ++- src/TableFunctions/TableFunctionPostgreSQL.cpp | 4 +++- tests/integration/test_storage_postgresql/test.py | 11 +++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 722b9c64edb..d2eb6797c84 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace DB @@ -162,7 +163,7 @@ StoragePtr DatabasePostgreSQL::fetchTable(const String & table_name, const Conte return StoragePtr{}; auto use_nulls = context.getSettingsRef().external_table_functions_use_nulls; - auto columns = fetchPostgreSQLTableStructure(connection->conn(), table_name, use_nulls); + auto columns = fetchPostgreSQLTableStructure(connection->conn(), doubleQuoteString(table_name), use_nulls); if (!columns) return StoragePtr{}; diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index 0e3f1c5da24..ae34ce41e47 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -11,6 +11,7 @@ #include "registerTableFunctions.h" #include #include +#include namespace DB @@ -41,7 +42,8 @@ ColumnsDescription TableFunctionPostgreSQL::getActualTableStructure(const Contex const bool use_nulls = context.getSettingsRef().external_table_functions_use_nulls; auto columns = fetchPostgreSQLTableStructure( connection->conn(), - remote_table_schema.empty() ? remote_table_name : remote_table_schema + '.' + remote_table_name, + remote_table_schema.empty() ? doubleQuoteString(remote_table_name) + : doubleQuoteString(remote_table_schema) + '.' + doubleQuoteString(remote_table_name), use_nulls); return ColumnsDescription{*columns}; diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 58f3233bacc..8be72f5a7c7 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -152,6 +152,17 @@ def test_non_default_scema(started_cluster): result = node1.query('SELECT * FROM {}'.format(table_function)) assert(result == expected) + cursor.execute('''CREATE SCHEMA "test.nice.schema"''') + cursor.execute('''CREATE TABLE "test.nice.schema"."test.nice.table" (a integer)''') + cursor.execute('INSERT INTO "test.nice.schema"."test.nice.table" SELECT i FROM generate_series(0, 99) as t(i)') + + node1.query(''' + CREATE TABLE test_pg_table_schema_with_dots (a UInt32) + ENGINE PostgreSQL('postgres1:5432', 'clickhouse', 'test.nice.table', 'postgres', 'mysecretpassword', 'test.nice.schema'); + ''') + result = node1.query('SELECT * FROM test_pg_table_schema_with_dots') + assert(result == expected) + if __name__ == '__main__': cluster.start() From de091114f3cee6ac5a775a9811855295ff29a34e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 15:22:52 +0300 Subject: [PATCH 149/233] Fix UBSan report --- base/common/DateLUTImpl.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 98b767ccbcc..43fc1b8befd 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -63,7 +63,7 @@ private: template friend inline LUTIndex operator+(const LUTIndex & index, const T v) { - return LUTIndex{(index.toUnderType() + v) & date_lut_mask}; + return LUTIndex{(index.toUnderType() + UInt32(v)) & date_lut_mask}; } template @@ -80,7 +80,7 @@ private: template friend inline LUTIndex operator-(const LUTIndex & index, const T v) { - return LUTIndex{(index.toUnderType() - v) & date_lut_mask}; + return LUTIndex{(index.toUnderType() - UInt32(v)) & date_lut_mask}; } template @@ -97,7 +97,7 @@ private: template friend inline LUTIndex operator*(const LUTIndex & index, const T v) { - return LUTIndex{(index.toUnderType() * v) & date_lut_mask}; + return LUTIndex{(index.toUnderType() * UInt32(v)) & date_lut_mask}; } template @@ -109,13 +109,13 @@ private: template friend inline LUTIndex operator/(const LUTIndex & index, const T v) { - return LUTIndex{(index.toUnderType() / v) & date_lut_mask}; + return LUTIndex{(index.toUnderType() / UInt32(v)) & date_lut_mask}; } template friend inline LUTIndex operator/(const T v, const LUTIndex & index) { - return LUTIndex{(v / index.toUnderType()) & date_lut_mask}; + return LUTIndex{(UInt32(v) / index.toUnderType()) & date_lut_mask}; } public: From 172f668fa3fe78e49fec00f0eaa2faf0a124eff6 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 16 Mar 2021 15:45:08 +0300 Subject: [PATCH 150/233] fix doc --- docs/en/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 70cf7f2212e..0b551040ba1 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -529,7 +529,7 @@ CREATE TABLE table_for_aggregation y Int ) ENGINE = MergeTree -ORDER BY k1, k2 +ORDER BY (k1, k2) TTL d + INTERVAL 1 MONTH GROUP BY k1, k2 SET x = max(x), y = min(y); ``` From 1dd730d2d6a48fb03d8f9f692888bee9f5969743 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 15:59:14 +0300 Subject: [PATCH 151/233] Fix perf test --- tests/performance/date_time_long.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/performance/date_time_long.xml b/tests/performance/date_time_long.xml index 3a61a5992e5..ae41602e825 100644 --- a/tests/performance/date_time_long.xml +++ b/tests/performance/date_time_long.xml @@ -77,7 +77,6 @@ toYYYYMMDDhhmmss toRelativeQuarterNum - toUnixTimestamp From 637b683ae9afedbad922ba6a7216ac8095891d12 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 18:35:14 +0300 Subject: [PATCH 152/233] Fix clang-tidy --- src/DataTypes/DataTypeDateTime.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/DataTypes/DataTypeDateTime.cpp b/src/DataTypes/DataTypeDateTime.cpp index 09dcb5f3e2e..510747f6ef9 100644 --- a/src/DataTypes/DataTypeDateTime.cpp +++ b/src/DataTypes/DataTypeDateTime.cpp @@ -91,7 +91,7 @@ void DataTypeDateTime::deserializeWholeText(IColumn & column, ReadBuffer & istr, void DataTypeDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - time_t x; + time_t x = 0; readTextHelper(x, istr, settings, time_zone, utc_time_zone); if (x < 0) x = 0; @@ -107,7 +107,8 @@ void DataTypeDateTime::serializeTextQuoted(const IColumn & column, size_t row_nu void DataTypeDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - time_t x; + time_t x = 0; + if (checkChar('\'', istr)) /// Cases: '2017-08-31 18:36:48' or '1504193808' { readTextHelper(x, istr, settings, time_zone, utc_time_zone); @@ -131,7 +132,8 @@ void DataTypeDateTime::serializeTextJSON(const IColumn & column, size_t row_num, void DataTypeDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - time_t x; + time_t x = 0; + if (checkChar('"', istr)) { readTextHelper(x, istr, settings, time_zone, utc_time_zone); @@ -157,7 +159,7 @@ void DataTypeDateTime::serializeTextCSV(const IColumn & column, size_t row_num, void DataTypeDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - time_t x; + time_t x = 0; if (istr.eof()) throwReadAfterEOF(); From 02a06eb448d63668c1973c361feb7276cc14388a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 16 Mar 2021 18:39:39 +0300 Subject: [PATCH 153/233] Update mergetree.md --- .../table-engines/mergetree-family/mergetree.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 0b551040ba1..a24b7229d17 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -353,7 +353,7 @@ The `set` index can be used with all functions. Function subsets for other index | Function (operator) / Index | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter | |------------------------------------------------------------------------------------------------------------|-------------|--------|-------------|-------------|---------------| | [equals (=, ==)](../../../sql-reference/functions/comparison-functions.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notEquals(!=, \<\>)](../../../sql-reference/functions/comparison-functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notEquals(!=, <>)](../../../sql-reference/functions/comparison-functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | | [like](../../../sql-reference/functions/string-search-functions.md#function-like) | ✔ | ✔ | ✔ | ✔ | ✗ | | [notLike](../../../sql-reference/functions/string-search-functions.md#function-notlike) | ✔ | ✔ | ✔ | ✔ | ✗ | | [startsWith](../../../sql-reference/functions/string-functions.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | @@ -361,10 +361,10 @@ The `set` index can be used with all functions. Function subsets for other index | [multiSearchAny](../../../sql-reference/functions/string-search-functions.md#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | | [in](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | | [notIn](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [less (\<)](../../../sql-reference/functions/comparison-functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [greater (\>)](../../../sql-reference/functions/comparison-functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [lessOrEquals (\<=)](../../../sql-reference/functions/comparison-functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [greaterOrEquals (\>=)](../../../sql-reference/functions/comparison-functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [less (<)](../../../sql-reference/functions/comparison-functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greater (>)](../../../sql-reference/functions/comparison-functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [lessOrEquals (<=)](../../../sql-reference/functions/comparison-functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greaterOrEquals (>=)](../../../sql-reference/functions/comparison-functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | | [empty](../../../sql-reference/functions/array-functions.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | | [notEmpty](../../../sql-reference/functions/array-functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | | hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | From e5bef75728ed16f3c23146a9c1986461695b6f5f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 16 Mar 2021 19:15:31 +0300 Subject: [PATCH 154/233] fix --- .../Transforms/PartialSortingTransform.cpp | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/Processors/Transforms/PartialSortingTransform.cpp b/src/Processors/Transforms/PartialSortingTransform.cpp index 33ff639f10d..7c29f506617 100644 --- a/src/Processors/Transforms/PartialSortingTransform.cpp +++ b/src/Processors/Transforms/PartialSortingTransform.cpp @@ -91,10 +91,19 @@ size_t getFilterMask(const ColumnRawPtrs & lhs, const ColumnRawPtrs & rhs, size_ void PartialSortingTransform::transform(Chunk & chunk) { - auto rows_num = chunk.getNumRows(); - + if (chunk.getColumns().empty()) + { + // Sometimes we can have Chunks w/o columns, e.g. in case of + // `select count() over () from numbers(4) where number < 2`. + // We don't have to modify this Chunk, but we have to preserve the input + // number of rows. The following code uses Block for sorting, and Block + // is incapable of recording the number of rows when there is no columns. + // The simplest solution is to specifically check for Chunk with no + // columns and not modify it, which is what we do here. + return; + } if (read_rows) - read_rows->add(rows_num); + read_rows->add(chunk.getNumRows()); auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()); @@ -103,6 +112,7 @@ void PartialSortingTransform::transform(Chunk & chunk) */ if (!threshold_block_columns.empty()) { + UInt64 rows_num = block.rows(); auto block_columns = extractColumns(block, description); size_t result_size_hint = getFilterMask( @@ -117,15 +127,13 @@ void PartialSortingTransform::transform(Chunk & chunk) { for (auto & column : block) column.column = column.column->filter(filter, result_size_hint); - - rows_num = block.rows(); } } sortBlock(block, description, limit); /// Check if we can use this block for optimization. - if (min_limit_for_partial_sort_optimization <= limit && limit <= rows_num) + if (min_limit_for_partial_sort_optimization <= limit && limit <= block.rows()) { auto block_columns = extractColumns(block, description); @@ -137,9 +145,7 @@ void PartialSortingTransform::transform(Chunk & chunk) } } - assert(block.columns() == 0 || block.rows() == rows_num); - - chunk.setColumns(block.getColumns(), rows_num); + chunk.setColumns(block.getColumns(), block.rows()); } } From 9f05fc22d062391dec8b704987a3d269f4f13741 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 19:31:25 +0300 Subject: [PATCH 155/233] Modern JavaScript --- programs/server/play.html | 64 +++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/programs/server/play.html b/programs/server/play.html index 81fc13f1f86..7d0ceeeaeb1 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -306,10 +306,10 @@ /// Incremental request number. When response is received, /// if it's request number does not equal to the current request number, response will be ignored. /// This is to avoid race conditions. - var request_num = 0; + let request_num = 0; /// Save query in history only if it is different. - var previous_query = ''; + let previous_query = ''; /// Substitute the address of the server where the page is served. if (location.protocol != 'file:') { @@ -317,7 +317,7 @@ } /// Substitute user name if it's specified in the query string - var user_from_url = (new URL(window.location)).searchParams.get('user'); + let user_from_url = (new URL(window.location)).searchParams.get('user'); if (user_from_url) { document.getElementById('user').value = user_from_url; } @@ -326,10 +326,10 @@ { /// TODO: Check if URL already contains query string (append parameters). - var user = document.getElementById('user').value; - var password = document.getElementById('password').value; + let user = document.getElementById('user').value; + let password = document.getElementById('password').value; - var url = document.getElementById('url').value + + let url = document.getElementById('url').value + /// Ask server to allow cross-domain requests. '?add_http_cors_header=1' + '&user=' + encodeURIComponent(user) + @@ -338,7 +338,7 @@ /// Safety settings to prevent results that browser cannot display. '&max_result_rows=1000&max_result_bytes=10000000&result_overflow_mode=break'; - var xhr = new XMLHttpRequest; + let xhr = new XMLHttpRequest; xhr.open('POST', url, true); @@ -352,13 +352,13 @@ /// The query is saved in browser history (in state JSON object) /// as well as in URL fragment identifier. if (query != previous_query) { - var state = { + let state = { query: query, status: this.status, response: this.response.length > 100000 ? null : this.response /// Lower than the browser's limit. }; - var title = "ClickHouse Query: " + query; - var url = window.location.pathname + '?user=' + encodeURIComponent(user) + '#' + window.btoa(query); + let title = "ClickHouse Query: " + query; + let url = window.location.pathname + '?user=' + encodeURIComponent(user) + '#' + window.btoa(query); if (previous_query == '') { history.replaceState(state, title, url); } else { @@ -382,7 +382,7 @@ document.getElementById('hourglass').style.display = 'none'; if (status === 200) { - var json; + let json; try { json = JSON.parse(response); } catch (e) {} if (json !== undefined && json.statistics !== undefined) { renderResult(json); @@ -415,7 +415,7 @@ function post() { ++request_num; - var query = document.getElementById('query').value; + let query = document.getElementById('query').value; postImpl(request_num, query); } @@ -434,7 +434,7 @@ function clear() { - var table = document.getElementById('data-table'); + let table = document.getElementById('data-table'); while (table.firstChild) { table.removeChild(table.lastChild); } @@ -456,30 +456,30 @@ //console.log(response); clear(); - var stats = document.getElementById('stats'); + let stats = document.getElementById('stats'); stats.innerText = 'Elapsed: ' + response.statistics.elapsed.toFixed(3) + " sec, read " + response.statistics.rows_read + " rows."; - var thead = document.createElement('thead'); - for (var idx in response.meta) { - var th = document.createElement('th'); - var name = document.createTextNode(response.meta[idx].name); + let thead = document.createElement('thead'); + for (let idx in response.meta) { + let th = document.createElement('th'); + let name = document.createTextNode(response.meta[idx].name); th.appendChild(name); thead.appendChild(th); } /// To prevent hanging the browser, limit the number of cells in a table. /// It's important to have the limit on number of cells, not just rows, because tables may be wide or narrow. - var max_rows = 10000 / response.meta.length; - var row_num = 0; + let max_rows = 10000 / response.meta.length; + let row_num = 0; - var tbody = document.createElement('tbody'); - for (var row_idx in response.data) { - var tr = document.createElement('tr'); - for (var col_idx in response.data[row_idx]) { - var td = document.createElement('td'); - var cell = response.data[row_idx][col_idx]; - var is_null = (cell === null); - var content = document.createTextNode(is_null ? 'ᴺᵁᴸᴸ' : cell); + let tbody = document.createElement('tbody'); + for (let row_idx in response.data) { + let tr = document.createElement('tr'); + for (let col_idx in response.data[row_idx]) { + let td = document.createElement('td'); + let cell = response.data[row_idx][col_idx]; + let is_null = (cell === null); + let content = document.createTextNode(is_null ? 'ᴺᵁᴸᴸ' : cell); td.appendChild(content); /// TODO: Execute regexp only once for each column. td.className = response.meta[col_idx].type.match(/^(U?Int|Decimal|Float)/) ? 'right' : 'left'; @@ -496,7 +496,7 @@ } } - var table = document.getElementById('data-table'); + let table = document.getElementById('data-table'); table.appendChild(thead); table.appendChild(tbody); } @@ -505,7 +505,7 @@ function renderUnparsedResult(response) { clear(); - var data = document.getElementById('data-unparsed') + let data = document.getElementById('data-unparsed') if (response === '') { /// TODO: Fade or remove previous result when new request will be performed. @@ -531,12 +531,12 @@ } /// The choice of color theme is saved in browser. - var theme = window.localStorage.getItem('theme'); + let theme = window.localStorage.getItem('theme'); if (theme) { setColorTheme(theme); } else { /// Obtain system-level user preference - var media_query_list = window.matchMedia('prefers-color-scheme: dark') + let media_query_list = window.matchMedia('prefers-color-scheme: dark') if (media_query_list.matches) { /// Set without saving to localstorage From bc25624b885bf74ad0898c16f18229fa32d09c48 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 19:54:16 +0300 Subject: [PATCH 156/233] Better formatting for Array and Map in Web UI --- programs/server/play.html | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/programs/server/play.html b/programs/server/play.html index 7d0ceeeaeb1..e9404d13acb 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -472,17 +472,30 @@ let max_rows = 10000 / response.meta.length; let row_num = 0; + let column_classes = response.meta.map(elem => elem.type.match(/^(U?Int|Decimal|Float)/) ? 'right' : 'left'); + let tbody = document.createElement('tbody'); for (let row_idx in response.data) { let tr = document.createElement('tr'); for (let col_idx in response.data[row_idx]) { let td = document.createElement('td'); let cell = response.data[row_idx][col_idx]; + let is_null = (cell === null); - let content = document.createTextNode(is_null ? 'ᴺᵁᴸᴸ' : cell); - td.appendChild(content); + + /// Test: SELECT number, toString(number) AS str, number % 2 ? number : NULL AS nullable, range(number) AS arr, CAST((['hello', 'world'], [number, number % 2]) AS Map(String, UInt64)) AS map FROM numbers(10) + let text; + if (is_null) { + text = 'ᴺᵁᴸᴸ'; + } else if (typeof(cell) === 'object') { + text = JSON.stringify(cell); + } else { + text = cell; + } + + td.appendChild(document.createTextNode(text)); /// TODO: Execute regexp only once for each column. - td.className = response.meta[col_idx].type.match(/^(U?Int|Decimal|Float)/) ? 'right' : 'left'; + td.className = column_classes[col_idx]; if (is_null) { td.className += ' null'; } From 174bce9ca4966b56ee60ee876a5f2337fddbfa1e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 19:55:56 +0300 Subject: [PATCH 157/233] TODO is resolved --- programs/server/play.html | 1 - 1 file changed, 1 deletion(-) diff --git a/programs/server/play.html b/programs/server/play.html index e9404d13acb..0c039097ce1 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -494,7 +494,6 @@ } td.appendChild(document.createTextNode(text)); - /// TODO: Execute regexp only once for each column. td.className = column_classes[col_idx]; if (is_null) { td.className += ' null'; From 94da7f422c3ca954d422741edb76ed4f9a735b4a Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Mar 2021 20:57:17 +0300 Subject: [PATCH 158/233] updated translation --- docs/ru/sql-reference/statements/detach.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/statements/detach.md b/docs/ru/sql-reference/statements/detach.md index bec8f4c5ff7..90e9663def9 100644 --- a/docs/ru/sql-reference/statements/detach.md +++ b/docs/ru/sql-reference/statements/detach.md @@ -15,13 +15,13 @@ DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] Но ни данные, ни метаданные таблицы или материализованного представления не удаляются. При следующем запуске сервера, если не было использовано `PERMANENTLY`, сервер прочитает метаданные и снова узнает о таблице/представлении. Если таблица или представление были откреплено перманентно, сервер не прикрепит их обратно автоматически. -Независимо от того, каким способом таблица была откреплена, ее можно прикрепить обратно с помощью запроса [ATTACH](../../sql-reference/statements/attach.md). Системные log таблицы также могут быть прикреплены обратно (к примеру `query_log`, `text_log` и др.) Другие системные таблицы не могут быть прикреплены обратно, но на следующем запуске сервер снова вспомнит об этих таблицах. +Независимо от того, каким способом таблица была откреплена, ее можно прикрепить обратно с помощью запроса [ATTACH](../../sql-reference/statements/attach.md). Системные log таблицы также могут быть прикреплены обратно (к примеру `query_log`, `text_log` и др.) Другие системные таблицы не могут быть прикреплены обратно, но на следующем запуске сервер снова "вспомнит" об этих таблицах. `ATTACH MATERIALIZED VIEW` не может быть использован с кратким синтаксисом (без `SELECT`), но можно прикрепить представление с помощью запроса `ATTACH TABLE`. Обратите внимание, что нельзя перманентно открепить таблицу, которая уже временно откреплена. Для этого ее сначала надо прикрепить обратно, а затем снова открепить перманентно. -Также нельзя использовать [DROP](../../sql-reference/statements/drop.md#drop-table) с открепленной таблицей или создавать таблицу с помощью [CREATE TABLE](../../sql-reference/statements/create/table.md) с таким же именем, как уже открепленная таблица. Еще нельзя заменить открепленную таблицу другой с помощью запроса [RENAME TABLE](../../sql-reference/statements/rename.md). +Также нельзя использовать [DROP](../../sql-reference/statements/drop.md#drop-table) с открепленной таблицей или создавать таблицу с помощью [CREATE TABLE](../../sql-reference/statements/create/table.md) с таким же именем, как у уже открепленной таблицы. Еще нельзя заменить открепленную таблицу другой с помощью запроса [RENAME TABLE](../../sql-reference/statements/rename.md). **Пример** From 3c61e7d3f97c34325dfb09de03d4b1559ace193e Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 20:59:28 +0300 Subject: [PATCH 159/233] Update docs/ru/operations/external-authenticators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/index.md b/docs/ru/operations/external-authenticators/index.md index 6b75e864fb8..c8ac7459cfa 100644 --- a/docs/ru/operations/external-authenticators/index.md +++ b/docs/ru/operations/external-authenticators/index.md @@ -6,7 +6,7 @@ toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" # Внешние аутентификаторы пользователей и каталоги {#external-authenticators} -ClickHouse поддерживает аунтетификацию и управление пользователями внешними сервисами. +ClickHouse поддерживает аутентификацию и управление пользователями при помощи внешних сервисов. Поддерживаются следующие внешние аутентификаторы и каталоги: From dc18ad6359940d13e429569d03e22315916f0265 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:00:31 +0300 Subject: [PATCH 160/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 7f901898a99..e3c9f17d2af 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -1,6 +1,6 @@ # LDAP {#external-authenticators-ldap} -Для аутентификации пользователей ClickHouse можно использовать сервер LDAP. Существует два подхода: +Для аутентификации пользователей ClickHouse можно использовать сервер LDAP. Существуют два подхода: - Использовать LDAP как внешний аутентификатор для существующих пользователей, которые определены в `users.xml` или в локальных путях управления контролем. - Использовать LDAP как внешний пользовательский каталог и разрешить аутентификацию локально неопределенных пользователей, если они есть на LDAP сервере. From 19d989c6871e12596d29f5c8669a6a6421021d81 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:00:47 +0300 Subject: [PATCH 161/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index e3c9f17d2af..86b096c7d32 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -2,7 +2,7 @@ Для аутентификации пользователей ClickHouse можно использовать сервер LDAP. Существуют два подхода: -- Использовать LDAP как внешний аутентификатор для существующих пользователей, которые определены в `users.xml` или в локальных путях управления контролем. +- Использовать LDAP как внешний аутентификатор для существующих пользователей, которые определены в `users.xml` или в локальных параметрах управления доступом. - Использовать LDAP как внешний пользовательский каталог и разрешить аутентификацию локально неопределенных пользователей, если они есть на LDAP сервере. Для обоих подходов необходимо определить в конфиге ClickHouse LDAP сервер с внутренним именем, чтобы другие части конфига могли ссылаться на него. From 225aa5c3fbb666a43634031e45c3271a3832e2bf Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:01:04 +0300 Subject: [PATCH 162/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 86b096c7d32..6a436c775ec 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -5,7 +5,7 @@ - Использовать LDAP как внешний аутентификатор для существующих пользователей, которые определены в `users.xml` или в локальных параметрах управления доступом. - Использовать LDAP как внешний пользовательский каталог и разрешить аутентификацию локально неопределенных пользователей, если они есть на LDAP сервере. -Для обоих подходов необходимо определить в конфиге ClickHouse LDAP сервер с внутренним именем, чтобы другие части конфига могли ссылаться на него. +Для обоих подходов необходимо определить внутреннее имя LDAP сервера в конфигурации ClickHouse, чтобы другие параметры конфигурации могли ссылаться на это имя. ## Определение LDAP сервера {#ldap-server-definition} From 916c50017aaab06ce04cc5a4c4955bf142a8e37b Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:01:10 +0300 Subject: [PATCH 163/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 6a436c775ec..d07c9dae3cd 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -39,7 +39,7 @@ **Параметры** -- `host` — имя хоста сервера LDAP или его IP. Этот параметр обязательный и не может быть оставлен пустым. +- `host` — имя хоста сервера LDAP или его IP. Этот параметр обязательный и не может быть пустым. - `port` — порт сервера LDAP. По-умолчанию: `636` при значении `true` настройки `enable_tls`, иначе `389`. - `bind_dn` — шаблон для создания DN для привязки. - Конечный DN будет создан заменой всех подстрок `{user_name}` шаблона на фактическое имя пользователя при каждой попытке аутентификации. From ad92537251f7c8ebc50e33c3af53b01d7d8d571a Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:01:15 +0300 Subject: [PATCH 164/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index d07c9dae3cd..bc3a647bda7 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -40,7 +40,7 @@ **Параметры** - `host` — имя хоста сервера LDAP или его IP. Этот параметр обязательный и не может быть пустым. -- `port` — порт сервера LDAP. По-умолчанию: `636` при значении `true` настройки `enable_tls`, иначе `389`. +- `port` — порт сервера LDAP. Если настройка `enable_tls` равна `true`, то по умолчанию используется порт `636`, иначе — порт `389`. - `bind_dn` — шаблон для создания DN для привязки. - Конечный DN будет создан заменой всех подстрок `{user_name}` шаблона на фактическое имя пользователя при каждой попытке аутентификации. - `verification_cooldown` — промежуток времени (в секундах) после успешной попытки привязки, в течение которого пользователь будет считаться успешно аутентифицированным, и сможет совершать запросы без контакта с серверов LDAP. From 06b01bed1720fb0eb43242c1d46aedbe0ff7ed74 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:01:34 +0300 Subject: [PATCH 165/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index bc3a647bda7..d1c13f7534a 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -42,7 +42,7 @@ - `host` — имя хоста сервера LDAP или его IP. Этот параметр обязательный и не может быть пустым. - `port` — порт сервера LDAP. Если настройка `enable_tls` равна `true`, то по умолчанию используется порт `636`, иначе — порт `389`. - `bind_dn` — шаблон для создания DN для привязки. - - Конечный DN будет создан заменой всех подстрок `{user_name}` шаблона на фактическое имя пользователя при каждой попытке аутентификации. + - При формировании DN все подстроки `{user_name}` в шаблоне будут заменяться на фактическое имя пользователя при каждой попытке аутентификации. - `verification_cooldown` — промежуток времени (в секундах) после успешной попытки привязки, в течение которого пользователь будет считаться успешно аутентифицированным, и сможет совершать запросы без контакта с серверов LDAP. - Укажите `0` (по-умолчанию), чтобы отключить кеширование и заставить связываться с сервером LDAP для каждого запроса аутентификации. - `enable_tls` — флаг, включающий использование защищенного соединения с сервером LDAP. From d5a0f58c0fbf6452d95d7a53e58f26d07605a3b9 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:01:48 +0300 Subject: [PATCH 166/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index d1c13f7534a..ae1d3bb935b 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -43,7 +43,7 @@ - `port` — порт сервера LDAP. Если настройка `enable_tls` равна `true`, то по умолчанию используется порт `636`, иначе — порт `389`. - `bind_dn` — шаблон для создания DN для привязки. - При формировании DN все подстроки `{user_name}` в шаблоне будут заменяться на фактическое имя пользователя при каждой попытке аутентификации. -- `verification_cooldown` — промежуток времени (в секундах) после успешной попытки привязки, в течение которого пользователь будет считаться успешно аутентифицированным, и сможет совершать запросы без контакта с серверов LDAP. +- `verification_cooldown` — промежуток времени (в секундах) после успешной попытки привязки, в течение которого пользователь будет считаться аутентифицированным и сможет выполнять запросы без повторного обращения к серверам LDAP. - Укажите `0` (по-умолчанию), чтобы отключить кеширование и заставить связываться с сервером LDAP для каждого запроса аутентификации. - `enable_tls` — флаг, включающий использование защищенного соединения с сервером LDAP. - Укажите `no` для текстового `ldap://` протокола (не рекомендовано). From 09ee74f1972c3af8277c97b22eb4f1584ef237bb Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:01:55 +0300 Subject: [PATCH 167/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index ae1d3bb935b..9e8727cfa76 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -44,7 +44,7 @@ - `bind_dn` — шаблон для создания DN для привязки. - При формировании DN все подстроки `{user_name}` в шаблоне будут заменяться на фактическое имя пользователя при каждой попытке аутентификации. - `verification_cooldown` — промежуток времени (в секундах) после успешной попытки привязки, в течение которого пользователь будет считаться аутентифицированным и сможет выполнять запросы без повторного обращения к серверам LDAP. - - Укажите `0` (по-умолчанию), чтобы отключить кеширование и заставить связываться с сервером LDAP для каждого запроса аутентификации. + - Чтобы отключить кеширование и заставить обращаться к серверу LDAP для каждого запроса аутентификации, укажите `0` (значение по умолчанию). - `enable_tls` — флаг, включающий использование защищенного соединения с сервером LDAP. - Укажите `no` для текстового `ldap://` протокола (не рекомендовано). - Укажите `yes` для LDAP через SSL/TLS `ldaps://` протокола (рекомендовано, используется по-умолчанию). From 9eee949e4a735cf534ff6dae5fdfaeb402d0a4a1 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:02:19 +0300 Subject: [PATCH 168/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 9e8727cfa76..44474502f46 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -46,9 +46,9 @@ - `verification_cooldown` — промежуток времени (в секундах) после успешной попытки привязки, в течение которого пользователь будет считаться аутентифицированным и сможет выполнять запросы без повторного обращения к серверам LDAP. - Чтобы отключить кеширование и заставить обращаться к серверу LDAP для каждого запроса аутентификации, укажите `0` (значение по умолчанию). - `enable_tls` — флаг, включающий использование защищенного соединения с сервером LDAP. - - Укажите `no` для текстового `ldap://` протокола (не рекомендовано). - - Укажите `yes` для LDAP через SSL/TLS `ldaps://` протокола (рекомендовано, используется по-умолчанию). - - Укажите `starttls` для устаревшего StartTLS протокола (текстовый `ldap://` протокол, модернизированный до TLS). + - Укажите `no` для использования текстового протокола `ldap://` (не рекомендовано). + - Укажите `yes` для обращения к LDAP по протоколу SSL/TLS `ldaps://` (рекомендовано, используется по умолчанию). + - Укажите `starttls` для использования устаревшего протокола StartTLS (текстовый `ldap://` протокол, модернизированный до TLS). - `tls_minimum_protocol_version` — минимальная версия протокола SSL/TLS. - Принимаемые значения: `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, `tls1.2` (по-умолчанию). - `tls_require_cert` — поведение при проверке сертификата SSL/TLS. From a38264a78d191567adca9be5fef080aa1c9d3d9c Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:02:27 +0300 Subject: [PATCH 169/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 44474502f46..86fa6322b76 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -50,9 +50,9 @@ - Укажите `yes` для обращения к LDAP по протоколу SSL/TLS `ldaps://` (рекомендовано, используется по умолчанию). - Укажите `starttls` для использования устаревшего протокола StartTLS (текстовый `ldap://` протокол, модернизированный до TLS). - `tls_minimum_protocol_version` — минимальная версия протокола SSL/TLS. - - Принимаемые значения: `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, `tls1.2` (по-умолчанию). + - Возможные значения: `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, `tls1.2` (по-умолчанию). - `tls_require_cert` — поведение при проверке сертификата SSL/TLS. - - Принимаемые значения: `never`, `allow`, `try`, `demand` (по-умолчанию). + - Возможные значения: `never`, `allow`, `try`, `demand` (по-умолчанию). - `tls_cert_file` — путь к файлу сертификата. - `tls_key_file` — путь к файлу ключа сертификата. - `tls_ca_cert_file` — путь к файлу ЦС сертификата. From 38df6717c827f9b0ea5dd81ad16cf6510694018b Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:02:34 +0300 Subject: [PATCH 170/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 86fa6322b76..3ca3369f4df 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -56,7 +56,7 @@ - `tls_cert_file` — путь к файлу сертификата. - `tls_key_file` — путь к файлу ключа сертификата. - `tls_ca_cert_file` — путь к файлу ЦС сертификата. -- `tls_ca_cert_dir` — путь к каталогу, содержащая сертификаты ЦС. +- `tls_ca_cert_dir` — путь к каталогу, содержащему сертификаты ЦС. - `tls_cipher_suite` — разрешенный набор шифров (в нотации OpenSSL). ## Внешний аутентификатор LDAP {#ldap-external-authenticator} From eab348f139c0ba2f58e905253cab3083750d4640 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:02:46 +0300 Subject: [PATCH 171/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 3ca3369f4df..83c92af0130 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -61,7 +61,7 @@ ## Внешний аутентификатор LDAP {#ldap-external-authenticator} -Удаленный сервер LDAP можно использовать как метод верификации паролей локально определенных пользователей (пользователей, которые определены в `users.xml` или в локальных путях управления контролем). Для этого укажите имя определенного до этого сервера LDAP вместо `password` или другой похожей секции в определении пользователя. +Удаленный сервер LDAP можно использовать для верификации паролей локально определенных пользователей (пользователей, которые определены в `users.xml` или в локальных параметрах управления доступом). Для этого укажите имя определенного ранее сервера LDAP вместо `password` или другой аналогичной секции в настройках пользователя. При каждой попытке авторизации, ClickHouse пытается "привязаться" к DN, указанному в [определении LDAP сервера](#ldap-server-definition) параметром `bind_dn`, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается аутентифицированным. Обычно это называют методом "простой привязки". From e0d4487528759574c1f6f167688990941eafc2ea Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:03:03 +0300 Subject: [PATCH 172/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 83c92af0130..441cb519917 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -63,7 +63,7 @@ Удаленный сервер LDAP можно использовать для верификации паролей локально определенных пользователей (пользователей, которые определены в `users.xml` или в локальных параметрах управления доступом). Для этого укажите имя определенного ранее сервера LDAP вместо `password` или другой аналогичной секции в настройках пользователя. -При каждой попытке авторизации, ClickHouse пытается "привязаться" к DN, указанному в [определении LDAP сервера](#ldap-server-definition) параметром `bind_dn`, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается аутентифицированным. Обычно это называют методом "простой привязки". +При каждой попытке авторизации ClickHouse пытается "привязаться" к DN, указанному в [определении LDAP сервера](#ldap-server-definition), используя параметр `bind_dn` и предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается аутентифицированным. Обычно это называют методом "простой привязки". **Пример** From d64cef7990c90502cc1db650f0eff65fad9ae941 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:03:21 +0300 Subject: [PATCH 173/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 441cb519917..1cdc3e4e6ea 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -94,7 +94,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; ## Внешний пользовательский каталог LDAP {#ldap-external-user-directory} -В добавок к локально определенным пользователям, удаленный LDAP сервер может быть использован как источник определения пользователей. Для этого укажите имя определенного до этого сервера LDAP (см. [Определение LDAP сервера](#ldap-server-definition)) в секции `ldap` внутри секции `users_directories` файла `config.xml`. +В дополнение к локально определенным пользователям, удаленный LDAP сервер может служить источником определения пользователей. Для этого укажите имя определенного ранее сервера LDAP (см. [Определение LDAP сервера](#ldap-server-definition)) в секции `ldap` внутри секции `users_directories` файла `config.xml`. При каждой попытке авторизации ClicHouse пытается локально найти определение пользователя и авторизовать его как обычно. Если определение не будет найдено, ClickHouse предполагает, что оно находится во внешнем LDAP каталоге, и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю будут присвоены роли из списка, указанного в секции `roles`. Кроме того, может быть выполнен LDAP поиск, а его результаты могут быть преобразованы в имена ролей и присвоены пользователям, если была настроена секция `role_mapping`. Все это работает при условии, что SQL-ориентированное [Управлением доступом](../access-rights.md#access-control) включено, а роли созданы выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). From 53b005ade334394859cf2d82b17055b3b7ba452f Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:03:52 +0300 Subject: [PATCH 174/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 1cdc3e4e6ea..691de8004ff 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -96,7 +96,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; В дополнение к локально определенным пользователям, удаленный LDAP сервер может служить источником определения пользователей. Для этого укажите имя определенного ранее сервера LDAP (см. [Определение LDAP сервера](#ldap-server-definition)) в секции `ldap` внутри секции `users_directories` файла `config.xml`. -При каждой попытке авторизации ClicHouse пытается локально найти определение пользователя и авторизовать его как обычно. Если определение не будет найдено, ClickHouse предполагает, что оно находится во внешнем LDAP каталоге, и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю будут присвоены роли из списка, указанного в секции `roles`. Кроме того, может быть выполнен LDAP поиск, а его результаты могут быть преобразованы в имена ролей и присвоены пользователям, если была настроена секция `role_mapping`. Все это работает при условии, что SQL-ориентированное [Управлением доступом](../access-rights.md#access-control) включено, а роли созданы выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). +При каждой попытке авторизации ClickHouse пытается локально найти определение пользователя и аутентифицировать его как обычно. Если пользователь не находится локально, ClickHouse предполагает, что он определяется во внешнем LDAP каталоге и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю присваиваются роли из списка, указанного в секции `roles`. Кроме того, если настроена секция `role_mapping`, то выполняется LDAP поиск, а его результаты преобразуются в имена ролей и присваиваются пользователям. Все это работает при условии, что SQL-ориентированное [управлением доступом](../access-rights.md#access-control) включено, а роли созданы выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). **Пример** From 44a3b9dd6530957e41fd2f1bf3adff6a0f080e7b Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:04:09 +0300 Subject: [PATCH 175/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 691de8004ff..d10cd0cfe3d 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -131,7 +131,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; - `server` — одно из имен сервера LDAP, определенного в секции конфига `ldap_servers` выше. Этот параметр обязательный и не может быть оставлен пустым. - `roles` — секция со списком локально определенных ролей, которые будут присвоены каждому пользователю, полученному от сервера LDAP. - - Если роли не указаны здесь или в секции `role_mapping` (ниже), пользователь не сможет выполнять никаких операций после аутентификации. + - Если роли не указаны ни здесь, ни в секции `role_mapping` (см. ниже), пользователь после аутентификации не сможет выполнять никаких действий. - `role_mapping` — секция c параметрами LDAP поиска и правилами отображения. - При аутентификации пользователя, пока еще связанного с LDAP, производится LDAP поиск с помощью `search_filter` и имени этого пользователя. Для каждой записи, найденной в ходе поиска, выделяется значение указанного атрибута. У каждого атрибута, имеющего указанный префикс, удаляется этот префикс, а остальная часть значения становится именем локальной роли, определенной в ClickHouse, причем предполагается, что эта роль была создана выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) до этого. - Внутри одной секции `ldap` может быть несколько секций `role_mapping`. Все они будут применены. From 82dff38fe673c7415445a064faa26a8b6bd1a516 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:04:18 +0300 Subject: [PATCH 176/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index d10cd0cfe3d..95cbb2952f5 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -133,7 +133,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; - `roles` — секция со списком локально определенных ролей, которые будут присвоены каждому пользователю, полученному от сервера LDAP. - Если роли не указаны ни здесь, ни в секции `role_mapping` (см. ниже), пользователь после аутентификации не сможет выполнять никаких действий. - `role_mapping` — секция c параметрами LDAP поиска и правилами отображения. - - При аутентификации пользователя, пока еще связанного с LDAP, производится LDAP поиск с помощью `search_filter` и имени этого пользователя. Для каждой записи, найденной в ходе поиска, выделяется значение указанного атрибута. У каждого атрибута, имеющего указанный префикс, удаляется этот префикс, а остальная часть значения становится именем локальной роли, определенной в ClickHouse, причем предполагается, что эта роль была создана выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) до этого. + - При аутентификации пользователя, пока еще связанного с LDAP, производится LDAP поиск с помощью `search_filter` и имени этого пользователя. Для каждой записи, найденной в ходе поиска, выделяется значение указанного атрибута. У каждого атрибута, имеющего указанный префикс, этот префикс удаляется, а остальная часть значения становится именем локальной роли, определенной в ClickHouse, причем предполагается, что эта роль была ранее создана выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) до этого. - Внутри одной секции `ldap` может быть несколько секций `role_mapping`. Все они будут применены. - `base_dn` — шаблон, который используется для создания базового DN для LDAP поиска. - конечный DN будет создан заменой всех подстрок `{user_name}` и `{bind_dn}` шаблона на фактическое имя пользователя и DN привязки соответственно при каждом LDAP поиске. From 6e17c26fe3620bdc22fbdbb68d374b8cfba72a4c Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:04:25 +0300 Subject: [PATCH 177/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 95cbb2952f5..b2e088aa721 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -136,7 +136,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; - При аутентификации пользователя, пока еще связанного с LDAP, производится LDAP поиск с помощью `search_filter` и имени этого пользователя. Для каждой записи, найденной в ходе поиска, выделяется значение указанного атрибута. У каждого атрибута, имеющего указанный префикс, этот префикс удаляется, а остальная часть значения становится именем локальной роли, определенной в ClickHouse, причем предполагается, что эта роль была ранее создана выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) до этого. - Внутри одной секции `ldap` может быть несколько секций `role_mapping`. Все они будут применены. - `base_dn` — шаблон, который используется для создания базового DN для LDAP поиска. - - конечный DN будет создан заменой всех подстрок `{user_name}` и `{bind_dn}` шаблона на фактическое имя пользователя и DN привязки соответственно при каждом LDAP поиске. + - При формировании DN все подстроки `{user_name}` и `{bind_dn}` в шаблоне будут заменяться на фактическое имя пользователя и DN привязки соответственно при каждом LDAP поиске. - `scope` — Область LDAP поиска. - Принимаемые значения: `base`, `one_level`, `children`, `subtree` (по-умолчанию). - `search_filter` — шаблон, который используется для создания фильтра для каждого LDAP поиска. From ce5f88a14e45f711d9fed731f27aae422434d66c Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:04:33 +0300 Subject: [PATCH 178/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index b2e088aa721..88b804c10a2 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -138,7 +138,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; - `base_dn` — шаблон, который используется для создания базового DN для LDAP поиска. - При формировании DN все подстроки `{user_name}` и `{bind_dn}` в шаблоне будут заменяться на фактическое имя пользователя и DN привязки соответственно при каждом LDAP поиске. - `scope` — Область LDAP поиска. - - Принимаемые значения: `base`, `one_level`, `children`, `subtree` (по-умолчанию). + - Возможные значения: `base`, `one_level`, `children`, `subtree` (по умолчанию). - `search_filter` — шаблон, который используется для создания фильтра для каждого LDAP поиска. - Конечный фильтр будет создан заменой всех подстрок `{user_name}`, `{bind_dn}` и `{base_dn}` шаблона на фактическое имя пользователя, DN привязи и базовый DN при соответственно каждом LDAP поиске. - Обратите внимание, что специальные символы должны быть правильно экранированы в XML. From 8563b3b822c93ad4f42eedbf69f1357139e757f9 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:05:01 +0300 Subject: [PATCH 179/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 88b804c10a2..96e598a5eb2 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -143,6 +143,6 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; - Конечный фильтр будет создан заменой всех подстрок `{user_name}`, `{bind_dn}` и `{base_dn}` шаблона на фактическое имя пользователя, DN привязи и базовый DN при соответственно каждом LDAP поиске. - Обратите внимание, что специальные символы должны быть правильно экранированы в XML. - `attribute` — имя атрибута, значение которого будет возвращаться LDAP поиском. - - `prefix` — префикс, который, как предполагается, будет находиться перед началом каждой строки в исходном списке строк, возвращаемых LDAP поиском. Префикс будет удален из исходных строк, а сами они будут рассматриваться как имена локальных ролей. По-умолчанию пусто. + - `prefix` — префикс, который, как предполагается, будет находиться перед началом каждой строки в исходном списке строк, возвращаемых LDAP поиском. Префикс будет удален из исходных строк, а сами они будут рассматриваться как имена локальных ролей. По умолчанию: пустая строка. [Оригинальная статья](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap.md) From 37e5578a03aadb98888605994e882da986a20c5c Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Mar 2021 21:12:53 +0300 Subject: [PATCH 180/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 96e598a5eb2..86ccb14896d 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -129,7 +129,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; **Параметры** -- `server` — одно из имен сервера LDAP, определенного в секции конфига `ldap_servers` выше. Этот параметр обязательный и не может быть оставлен пустым. +- `server` — имя одного из серверов LDAP, определенных в секции `ldap_servers` в файле конфигурации (см.выше). Этот параметр обязательный и не может быть пустым. - `roles` — секция со списком локально определенных ролей, которые будут присвоены каждому пользователю, полученному от сервера LDAP. - Если роли не указаны ни здесь, ни в секции `role_mapping` (см. ниже), пользователь после аутентификации не сможет выполнять никаких действий. - `role_mapping` — секция c параметрами LDAP поиска и правилами отображения. From 5f9a854617caea599e35fc1e7b12bf00d384572c Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Mar 2021 21:13:22 +0300 Subject: [PATCH 181/233] Some updates --- docs/en/operations/external-authenticators/ldap.md | 2 +- docs/ru/operations/external-authenticators/ldap.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index e528e2a7c07..fc3dd466ea9 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -84,7 +84,7 @@ At each login attempt, ClickHouse tries to "bind" to the specified DN defined by Note, that user `my_user` refers to `my_ldap_server`. This LDAP server must be configured in the main `config.xml` file as described previously. -When SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled, users that are authenticated by LDAP servers can also be created using the [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement) statement. +When SQL-driven [Access Control and Account Management](../access-rights.md) is enabled, users that are authenticated by LDAP servers can also be created using the [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement) statement. Query: diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 7f901898a99..6fcb20b8b3b 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -84,7 +84,7 @@ Обратите внимание, что пользователь `my_user` ссылается на `my_ldap_server`. Этот LDAP сервер должен быть настроен в основном файле `config.xml`, как это было описано ранее. -При включенном SQL-ориентированным [Управлением доступом](../access-rights.md#access-control) пользователи, аутентифицированные LDAP серверами, могут также быть созданы выражением [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement). +При включенном SQL-ориентированным [Управлении доступом](../access-rights.md) пользователи, аутентифицированные LDAP серверами, могут также быть созданы выражением [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement). Запрос: From 1130fd0654d8ff71d6d76fdbf0f3a77d7658a9f3 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Mar 2021 21:14:45 +0300 Subject: [PATCH 182/233] minor fix --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 60640030fd1..49b4d13ccb2 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -84,7 +84,7 @@ Обратите внимание, что пользователь `my_user` ссылается на `my_ldap_server`. Этот LDAP сервер должен быть настроен в основном файле `config.xml`, как это было описано ранее. -При включенном SQL-ориентированным [Управлении доступом](../access-rights.md) пользователи, аутентифицированные LDAP серверами, могут также быть созданы выражением [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement). +При включенном SQL-ориентированном [управлении доступом](../access-rights.md) пользователи, аутентифицированные LDAP серверами, могут также быть созданы выражением [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement). Запрос: From 2c48ea6f59fe2e3c8f0230b498996e81dab49e96 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Mar 2021 21:18:08 +0300 Subject: [PATCH 183/233] fixed description --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 49b4d13ccb2..63e0c73c63e 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -140,7 +140,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; - `scope` — Область LDAP поиска. - Возможные значения: `base`, `one_level`, `children`, `subtree` (по умолчанию). - `search_filter` — шаблон, который используется для создания фильтра для каждого LDAP поиска. - - Конечный фильтр будет создан заменой всех подстрок `{user_name}`, `{bind_dn}` и `{base_dn}` шаблона на фактическое имя пользователя, DN привязи и базовый DN при соответственно каждом LDAP поиске. + - при формировании фильтра все подстроки `{user_name}`, `{bind_dn}` и `{base_dn}` в шаблоне будут заменяться на фактическое имя пользователя, DN привязки и базовый DN соответственно при каждом LDAP поиске. - Обратите внимание, что специальные символы должны быть правильно экранированы в XML. - `attribute` — имя атрибута, значение которого будет возвращаться LDAP поиском. - `prefix` — префикс, который, как предполагается, будет находиться перед началом каждой строки в исходном списке строк, возвращаемых LDAP поиском. Префикс будет удален из исходных строк, а сами они будут рассматриваться как имена локальных ролей. По умолчанию: пустая строка. From 9801241760aace052c36090d908726e25f42ede8 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Mar 2021 21:31:59 +0300 Subject: [PATCH 184/233] fixed links --- docs/en/operations/external-authenticators/ldap.md | 2 +- docs/ru/operations/external-authenticators/ldap.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index fc3dd466ea9..158a44a7492 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -97,7 +97,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; In addition to the locally defined users, a remote LDAP server can be used as a source of user definitions. To achieve this, specify previously defined LDAP server name (see [LDAP Server Definition](#ldap-server-definition)) in the `ldap` section inside the `users_directories` section of the `config.xml` file. -At each login attempt, ClickHouse tries to find the user definition locally and authenticate it as usual. If the user is not defined, ClickHouse will assume the definition exists in the external LDAP directory and will try to "bind" to the specified DN at the LDAP server using the provided credentials. If successful, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. Additionally, LDAP "search" can be performed and results can be transformed and treated as role names and then be assigned to the user if the `role_mapping` section is also configured. All this implies that the SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled and roles are created using the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. +At each login attempt, ClickHouse tries to find the user definition locally and authenticate it as usual. If the user is not defined, ClickHouse will assume the definition exists in the external LDAP directory and will try to "bind" to the specified DN at the LDAP server using the provided credentials. If successful, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. Additionally, LDAP "search" can be performed and results can be transformed and treated as role names and then be assigned to the user if the `role_mapping` section is also configured. All this implies that the SQL-driven [Access Control and Account Management](../access-rights.md) is enabled and roles are created using the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. **Example** diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 63e0c73c63e..f93581123fd 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -96,7 +96,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; В дополнение к локально определенным пользователям, удаленный LDAP сервер может служить источником определения пользователей. Для этого укажите имя определенного ранее сервера LDAP (см. [Определение LDAP сервера](#ldap-server-definition)) в секции `ldap` внутри секции `users_directories` файла `config.xml`. -При каждой попытке авторизации ClickHouse пытается локально найти определение пользователя и аутентифицировать его как обычно. Если пользователь не находится локально, ClickHouse предполагает, что он определяется во внешнем LDAP каталоге и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю присваиваются роли из списка, указанного в секции `roles`. Кроме того, если настроена секция `role_mapping`, то выполняется LDAP поиск, а его результаты преобразуются в имена ролей и присваиваются пользователям. Все это работает при условии, что SQL-ориентированное [управлением доступом](../access-rights.md#access-control) включено, а роли созданы выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). +При каждой попытке авторизации ClickHouse пытается локально найти определение пользователя и аутентифицировать его как обычно. Если пользователь не находится локально, ClickHouse предполагает, что он определяется во внешнем LDAP каталоге и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю присваиваются роли из списка, указанного в секции `roles`. Кроме того, если настроена секция `role_mapping`, то выполняется LDAP поиск, а его результаты преобразуются в имена ролей и присваиваются пользователям. Все это работает при условии, что SQL-ориентированное [управлением доступом](../access-rights.md) включено, а роли созданы выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). **Пример** From 4d24cef69c6ed2bf3a32a1dcae41411b0baced5c Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Mar 2021 21:46:05 +0300 Subject: [PATCH 185/233] fixed links --- docs/en/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index 158a44a7492..1cd38ba8430 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -146,4 +146,4 @@ Note that `my_ldap_server` referred in the `ldap` section inside the `user_direc - `attribute` — Attribute name whose values will be returned by the LDAP search. - `prefix` — Prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. The prefix will be removed from the original strings and the resulting strings will be treated as local role names. Empty by default. -[Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap.md) +[Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap/) From b39f8cc6ac2e470b01826458f3f0f286367da64e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 22:05:51 +0300 Subject: [PATCH 186/233] Move ErrorCodes::increment() into module part --- src/Common/ErrorCodes.cpp | 11 +++++++++++ src/Common/ErrorCodes.h | 11 +---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 40ce23fffb2..f6c15848553 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -584,6 +584,17 @@ namespace ErrorCodes } ErrorCode end() { return END + 1; } + + void increment(ErrorCode error_code) + { + if (error_code >= end()) + { + /// For everything outside the range, use END. + /// (end() is the pointer pass the end, while END is the last value that has an element in values array). + error_code = end() - 1; + } + values[error_code].fetch_add(1, std::memory_order_relaxed); + } } } diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index cc610c5d927..919a4afdabf 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -31,16 +31,7 @@ namespace ErrorCodes ErrorCode end(); /// Add value for specified error_code. - inline void increment(ErrorCode error_code) - { - if (error_code >= end()) - { - /// For everything outside the range, use END. - /// (end() is the pointer pass the end, while END is the last value that has an element in values array). - error_code = end() - 1; - } - values[error_code].fetch_add(1, std::memory_order_relaxed); - } + void increment(ErrorCode error_code); } } From 259e5ba88e35546279bb46511a5e6ad18b4457d6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 22:05:51 +0300 Subject: [PATCH 187/233] Separate accounting of remote exceptions in system.errors --- docs/en/operations/system-tables/errors.md | 3 ++- src/Common/ErrorCodes.cpp | 9 ++++--- src/Common/ErrorCodes.h | 9 +++++-- src/Common/Exception.cpp | 12 +++++----- src/Storages/System/StorageSystemErrors.cpp | 24 ++++++++++++------- .../0_stateless/01545_system_errors.reference | 3 ++- .../0_stateless/01545_system_errors.sh | 13 +++++++--- 7 files changed, 49 insertions(+), 24 deletions(-) diff --git a/docs/en/operations/system-tables/errors.md b/docs/en/operations/system-tables/errors.md index ec874efd711..bf3e33f5275 100644 --- a/docs/en/operations/system-tables/errors.md +++ b/docs/en/operations/system-tables/errors.md @@ -7,11 +7,12 @@ Columns: - `name` ([String](../../sql-reference/data-types/string.md)) — name of the error (`errorCodeToName`). - `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — code number of the error. - `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — the number of times this error has been happened. +- `remote` ([UInt8](../../sql-reference/data-types/int-uint.md)) — remote exception (i.e. received during one of the distributed query). **Example** ``` sql -SELECT * +SELECT name, code, value FROM system.errors WHERE value > 0 ORDER BY code ASC diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index f6c15848553..ec3bf9c8917 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -563,7 +563,7 @@ namespace ErrorCodes #undef M constexpr Value END = 3000; - std::atomic values[END + 1]{}; + ValuePair values[END + 1]{}; struct ErrorCodesNames { @@ -585,7 +585,7 @@ namespace ErrorCodes ErrorCode end() { return END + 1; } - void increment(ErrorCode error_code) + void increment(ErrorCode error_code, bool remote) { if (error_code >= end()) { @@ -593,7 +593,10 @@ namespace ErrorCodes /// (end() is the pointer pass the end, while END is the last value that has an element in values array). error_code = end() - 1; } - values[error_code].fetch_add(1, std::memory_order_relaxed); + if (remote) + values[error_code].remote.fetch_add(1, std::memory_order_relaxed); + else + values[error_code].local.fetch_add(1, std::memory_order_relaxed); } } diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 919a4afdabf..c4a9ae2907b 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -25,13 +25,18 @@ namespace ErrorCodes std::string_view getName(ErrorCode error_code); /// ErrorCode identifier -> current value of error_code. - extern std::atomic values[]; + struct ValuePair + { + std::atomic local; + std::atomic remote; + }; + extern ValuePair values[]; /// Get index just after last error_code identifier. ErrorCode end(); /// Add value for specified error_code. - void increment(ErrorCode error_code); + void increment(ErrorCode error_code, bool remote); } } diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 16f15d4e6f2..1963c1513b9 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -34,9 +34,9 @@ namespace ErrorCodes extern const int CANNOT_MREMAP; } -/// Aborts the process if error code is LOGICAL_ERROR. -/// Increments error codes statistics. -void handle_error_code([[maybe_unused]] const std::string & msg, int code) +/// - Aborts the process if error code is LOGICAL_ERROR. +/// - Increments error codes statistics. +void handle_error_code([[maybe_unused]] const std::string & msg, int code, bool remote) { // In debug builds and builds with sanitizers, treat LOGICAL_ERROR as an assertion failure. // Log the message before we fail. @@ -47,20 +47,20 @@ void handle_error_code([[maybe_unused]] const std::string & msg, int code) abort(); } #endif - ErrorCodes::increment(code); + ErrorCodes::increment(code, remote); } Exception::Exception(const std::string & msg, int code, bool remote_) : Poco::Exception(msg, code) , remote(remote_) { - handle_error_code(msg, code); + handle_error_code(msg, code, remote); } Exception::Exception(const std::string & msg, const Exception & nested, int code) : Poco::Exception(msg, nested, code) { - handle_error_code(msg, code); + handle_error_code(msg, code, remote); } Exception::Exception(CreateFromPocoTag, const Poco::Exception & exc) diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index 89df058900b..1a29484e169 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -13,27 +13,35 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() { "name", std::make_shared() }, { "code", std::make_shared() }, { "value", std::make_shared() }, + { "remote", std::make_shared() }, }; } void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { + auto add_row = [&](std::string_view name, size_t code, size_t value, bool remote) + { + if (value || context.getSettingsRef().system_events_show_zero_values) + { + size_t col_num = 0; + res_columns[col_num++]->insert(name); + res_columns[col_num++]->insert(code); + res_columns[col_num++]->insert(value); + res_columns[col_num++]->insert(remote); + } + }; + for (size_t i = 0, end = ErrorCodes::end(); i < end; ++i) { - UInt64 value = ErrorCodes::values[i]; + const auto & error = ErrorCodes::values[i]; std::string_view name = ErrorCodes::getName(i); if (name.empty()) continue; - if (value || context.getSettingsRef().system_events_show_zero_values) - { - size_t col_num = 0; - res_columns[col_num++]->insert(name); - res_columns[col_num++]->insert(i); - res_columns[col_num++]->insert(value); - } + add_row(name, i, error.local, 0 /* remote=0 */); + add_row(name, i, error.remote, 1 /* remote=1 */); } } diff --git a/tests/queries/0_stateless/01545_system_errors.reference b/tests/queries/0_stateless/01545_system_errors.reference index d00491fd7e5..0e7f2447090 100644 --- a/tests/queries/0_stateless/01545_system_errors.reference +++ b/tests/queries/0_stateless/01545_system_errors.reference @@ -1 +1,2 @@ -1 +local=1 +remote=1 diff --git a/tests/queries/0_stateless/01545_system_errors.sh b/tests/queries/0_stateless/01545_system_errors.sh index 63af6bb8d43..970fd403866 100755 --- a/tests/queries/0_stateless/01545_system_errors.sh +++ b/tests/queries/0_stateless/01545_system_errors.sh @@ -4,7 +4,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -prev="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO'")" +# local +prev="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO' AND NOT remote")" $CLICKHOUSE_CLIENT -q 'SELECT throwIf(1)' >& /dev/null -cur="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO'")" -echo $((cur - prev)) +cur="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO' AND NOT remote")" +echo local=$((cur - prev)) + +# remote +prev="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO' AND remote")" +${CLICKHOUSE_CLIENT} -q "SELECT * FROM remote('127.2', system.one) where throwIf(not dummy)" >& /dev/null +cur="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO' AND remote")" +echo remote=$((cur - prev)) From 7f73ac2b7a94ed809f4a2eaa59c13d7585756a6c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 22:05:51 +0300 Subject: [PATCH 188/233] Fix ErrorCodes::Value/ErrorCode types (sigh) Note, that system.errors already uses correct types --- src/Common/ErrorCodes.cpp | 2 +- src/Common/ErrorCodes.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index ec3bf9c8917..6f1ff8e4f3a 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -562,7 +562,7 @@ namespace ErrorCodes APPLY_FOR_ERROR_CODES(M) #undef M - constexpr Value END = 3000; + constexpr ErrorCode END = 3000; ValuePair values[END + 1]{}; struct ErrorCodesNames diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index c4a9ae2907b..6373ad6d0f9 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -17,8 +17,8 @@ namespace DB namespace ErrorCodes { /// ErrorCode identifier (index in array). - using ErrorCode = size_t; - using Value = int; + using ErrorCode = int; + using Value = size_t; /// Get name of error_code by identifier. /// Returns statically allocated string. From 0d01eaf94fd122213b3d31593248fe3f3e9c6a40 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 23:08:04 +0300 Subject: [PATCH 189/233] Guard ErrorCodes with mutex over atomic --- src/Common/ErrorCodes.cpp | 30 +++++++++++++++++---- src/Common/ErrorCodes.h | 25 +++++++++++++---- src/Storages/System/StorageSystemErrors.cpp | 2 +- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 6f1ff8e4f3a..32c9c4a452b 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -563,7 +563,7 @@ namespace ErrorCodes #undef M constexpr ErrorCode END = 3000; - ValuePair values[END + 1]{}; + ValuePairHolder values[END + 1]{}; struct ErrorCodesNames { @@ -593,10 +593,30 @@ namespace ErrorCodes /// (end() is the pointer pass the end, while END is the last value that has an element in values array). error_code = end() - 1; } - if (remote) - values[error_code].remote.fetch_add(1, std::memory_order_relaxed); - else - values[error_code].local.fetch_add(1, std::memory_order_relaxed); + + ValuePair inc_value{ + !remote, /* local */ + remote, /* remote */ + }; + values[error_code].increment(inc_value); + } + + ValuePair & ValuePair::operator+=(const ValuePair & value) + { + local += value.local; + remote += value.remote; + return *this; + } + + void ValuePairHolder::increment(const ValuePair & value_) + { + std::lock_guard lock(mutex); + value += value_; + } + ValuePair ValuePairHolder::get() + { + std::lock_guard lock(mutex); + return value; } } diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 6373ad6d0f9..0db877db205 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include @@ -24,13 +24,28 @@ namespace ErrorCodes /// Returns statically allocated string. std::string_view getName(ErrorCode error_code); - /// ErrorCode identifier -> current value of error_code. struct ValuePair { - std::atomic local; - std::atomic remote; + Value local = 0; + Value remote = 0; + + ValuePair & operator+=(const ValuePair & value); }; - extern ValuePair values[]; + + /// Thread-safe + struct ValuePairHolder + { + public: + void increment(const ValuePair & value_); + ValuePair get(); + + private: + ValuePair value; + std::mutex mutex; + }; + + /// ErrorCode identifier -> current value of error_code. + extern ValuePairHolder values[]; /// Get index just after last error_code identifier. ErrorCode end(); diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index 1a29484e169..d57e8a0a670 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -34,7 +34,7 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & for (size_t i = 0, end = ErrorCodes::end(); i < end; ++i) { - const auto & error = ErrorCodes::values[i]; + const auto & error = ErrorCodes::values[i].get(); std::string_view name = ErrorCodes::getName(i); if (name.empty()) From c8852331a2f3356069155d6ab45ba79f1ecada57 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 23:16:46 +0300 Subject: [PATCH 190/233] Add system.errors.last_error_time column --- docs/en/operations/system-tables/errors.md | 1 + src/Common/ErrorCodes.cpp | 5 +++++ src/Common/ErrorCodes.h | 1 + src/Storages/System/StorageSystemErrors.cpp | 9 ++++++--- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/system-tables/errors.md b/docs/en/operations/system-tables/errors.md index bf3e33f5275..b440ae4d787 100644 --- a/docs/en/operations/system-tables/errors.md +++ b/docs/en/operations/system-tables/errors.md @@ -7,6 +7,7 @@ Columns: - `name` ([String](../../sql-reference/data-types/string.md)) — name of the error (`errorCodeToName`). - `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — code number of the error. - `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — the number of times this error has been happened. +- `last_error_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — time when the last error happened. - `remote` ([UInt8](../../sql-reference/data-types/int-uint.md)) — remote exception (i.e. received during one of the distributed query). **Example** diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 32c9c4a452b..3532c063651 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -1,4 +1,5 @@ #include +#include /** Previously, these constants were located in one enum. * But in this case there is a problem: when you add a new constant, you need to recompile @@ -605,6 +606,10 @@ namespace ErrorCodes { local += value.local; remote += value.remote; + + const auto now = std::chrono::system_clock::now(); + last_error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + return *this; } diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 0db877db205..c8c454b51a7 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -28,6 +28,7 @@ namespace ErrorCodes { Value local = 0; Value remote = 0; + UInt64 last_error_time_ms = 0; ValuePair & operator+=(const ValuePair & value); }; diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index d57e8a0a670..a3d68ff5d86 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -13,6 +14,7 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() { "name", std::make_shared() }, { "code", std::make_shared() }, { "value", std::make_shared() }, + { "last_error_time", std::make_shared() }, { "remote", std::make_shared() }, }; } @@ -20,7 +22,7 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - auto add_row = [&](std::string_view name, size_t code, size_t value, bool remote) + auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, bool remote) { if (value || context.getSettingsRef().system_events_show_zero_values) { @@ -28,6 +30,7 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & res_columns[col_num++]->insert(name); res_columns[col_num++]->insert(code); res_columns[col_num++]->insert(value); + res_columns[col_num++]->insert(last_error_time_ms / 1000); res_columns[col_num++]->insert(remote); } }; @@ -40,8 +43,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, 0 /* remote=0 */); - add_row(name, i, error.remote, 1 /* remote=1 */); + add_row(name, i, error.local, error.last_error_time_ms, 0 /* remote=0 */); + add_row(name, i, error.remote, error.last_error_time_ms, 1 /* remote=1 */); } } From 775f8f76827378b98d0b679eaa564ac7697ee81f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 23:31:51 +0300 Subject: [PATCH 191/233] Add system.errors.last_error_message column --- docs/en/operations/system-tables/errors.md | 1 + src/Common/ErrorCodes.cpp | 5 ++++- src/Common/ErrorCodes.h | 3 ++- src/Common/Exception.cpp | 2 +- src/Storages/System/StorageSystemErrors.cpp | 8 +++++--- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/en/operations/system-tables/errors.md b/docs/en/operations/system-tables/errors.md index b440ae4d787..f8ac1de29a8 100644 --- a/docs/en/operations/system-tables/errors.md +++ b/docs/en/operations/system-tables/errors.md @@ -8,6 +8,7 @@ Columns: - `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — code number of the error. - `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — the number of times this error has been happened. - `last_error_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — time when the last error happened. +- `last_error_message` ([String](../../sql-reference/data-types/string.md)) — message for the last error. - `remote` ([UInt8](../../sql-reference/data-types/int-uint.md)) — remote exception (i.e. received during one of the distributed query). **Example** diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 3532c063651..6c9de122a26 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -586,7 +586,7 @@ namespace ErrorCodes ErrorCode end() { return END + 1; } - void increment(ErrorCode error_code, bool remote) + void increment(ErrorCode error_code, bool remote, const std::string & message) { if (error_code >= end()) { @@ -598,6 +598,8 @@ namespace ErrorCodes ValuePair inc_value{ !remote, /* local */ remote, /* remote */ + 0, /* last_error_time_ms */ + message, /* message */ }; values[error_code].increment(inc_value); } @@ -606,6 +608,7 @@ namespace ErrorCodes { local += value.local; remote += value.remote; + message = value.message; const auto now = std::chrono::system_clock::now(); last_error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index c8c454b51a7..962b1f8a20a 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -29,6 +29,7 @@ namespace ErrorCodes Value local = 0; Value remote = 0; UInt64 last_error_time_ms = 0; + std::string message; ValuePair & operator+=(const ValuePair & value); }; @@ -52,7 +53,7 @@ namespace ErrorCodes ErrorCode end(); /// Add value for specified error_code. - void increment(ErrorCode error_code, bool remote); + void increment(ErrorCode error_code, bool remote, const std::string & message); } } diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 1963c1513b9..1fe224edc6e 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -47,7 +47,7 @@ void handle_error_code([[maybe_unused]] const std::string & msg, int code, bool abort(); } #endif - ErrorCodes::increment(code, remote); + ErrorCodes::increment(code, remote, msg); } Exception::Exception(const std::string & msg, int code, bool remote_) diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index a3d68ff5d86..c06bb13beb6 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -15,6 +15,7 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() { "code", std::make_shared() }, { "value", std::make_shared() }, { "last_error_time", std::make_shared() }, + { "last_error_message",std::make_shared() }, { "remote", std::make_shared() }, }; } @@ -22,7 +23,7 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, bool remote) + auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, const std::string & message, bool remote) { if (value || context.getSettingsRef().system_events_show_zero_values) { @@ -31,6 +32,7 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & res_columns[col_num++]->insert(code); res_columns[col_num++]->insert(value); res_columns[col_num++]->insert(last_error_time_ms / 1000); + res_columns[col_num++]->insert(message); res_columns[col_num++]->insert(remote); } }; @@ -43,8 +45,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, error.last_error_time_ms, 0 /* remote=0 */); - add_row(name, i, error.remote, error.last_error_time_ms, 1 /* remote=1 */); + add_row(name, i, error.local, error.last_error_time_ms, error.message, 0 /* remote=0 */); + add_row(name, i, error.remote, error.last_error_time_ms, error.message, 1 /* remote=1 */); } } From 44c9dc753da4add80dd0a92ce14152790138e85a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 23:39:19 +0300 Subject: [PATCH 192/233] Add system.errors.last_error_stacktrace column --- docs/en/operations/system-tables/errors.md | 1 + src/Common/ErrorCodes.cpp | 12 +++++++----- src/Common/ErrorCodes.h | 3 ++- src/Common/Exception.cpp | 8 ++++---- src/Storages/System/StorageSystemErrors.cpp | 20 +++++++++++--------- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/docs/en/operations/system-tables/errors.md b/docs/en/operations/system-tables/errors.md index f8ac1de29a8..72a537f15b9 100644 --- a/docs/en/operations/system-tables/errors.md +++ b/docs/en/operations/system-tables/errors.md @@ -9,6 +9,7 @@ Columns: - `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — the number of times this error has been happened. - `last_error_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — time when the last error happened. - `last_error_message` ([String](../../sql-reference/data-types/string.md)) — message for the last error. +- `last_error_stacktrace` ([String](../../sql-reference/data-types/string.md)) — stacktrace for the last error. - `remote` ([UInt8](../../sql-reference/data-types/int-uint.md)) — remote exception (i.e. received during one of the distributed query). **Example** diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 6c9de122a26..14182467351 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -586,7 +586,7 @@ namespace ErrorCodes ErrorCode end() { return END + 1; } - void increment(ErrorCode error_code, bool remote, const std::string & message) + void increment(ErrorCode error_code, bool remote, const std::string & message, const std::string & stacktrace) { if (error_code >= end()) { @@ -596,10 +596,11 @@ namespace ErrorCodes } ValuePair inc_value{ - !remote, /* local */ - remote, /* remote */ - 0, /* last_error_time_ms */ - message, /* message */ + !remote, /* local */ + remote, /* remote */ + 0, /* last_error_time_ms */ + message, /* message */ + stacktrace, /* stacktrace */ }; values[error_code].increment(inc_value); } @@ -609,6 +610,7 @@ namespace ErrorCodes local += value.local; remote += value.remote; message = value.message; + stacktrace = value.stacktrace; const auto now = std::chrono::system_clock::now(); last_error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 962b1f8a20a..4c79614d55d 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -30,6 +30,7 @@ namespace ErrorCodes Value remote = 0; UInt64 last_error_time_ms = 0; std::string message; + std::string stacktrace; ValuePair & operator+=(const ValuePair & value); }; @@ -53,7 +54,7 @@ namespace ErrorCodes ErrorCode end(); /// Add value for specified error_code. - void increment(ErrorCode error_code, bool remote, const std::string & message); + void increment(ErrorCode error_code, bool remote, const std::string & message, const std::string & stacktrace); } } diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 1fe224edc6e..08afd0397f5 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -36,7 +36,7 @@ namespace ErrorCodes /// - Aborts the process if error code is LOGICAL_ERROR. /// - Increments error codes statistics. -void handle_error_code([[maybe_unused]] const std::string & msg, int code, bool remote) +void handle_error_code([[maybe_unused]] const std::string & msg, const std::string & stacktrace, int code, bool remote) { // In debug builds and builds with sanitizers, treat LOGICAL_ERROR as an assertion failure. // Log the message before we fail. @@ -47,20 +47,20 @@ void handle_error_code([[maybe_unused]] const std::string & msg, int code, bool abort(); } #endif - ErrorCodes::increment(code, remote, msg); + ErrorCodes::increment(code, remote, msg, stacktrace); } Exception::Exception(const std::string & msg, int code, bool remote_) : Poco::Exception(msg, code) , remote(remote_) { - handle_error_code(msg, code, remote); + handle_error_code(msg, getStackTraceString(), code, remote); } Exception::Exception(const std::string & msg, const Exception & nested, int code) : Poco::Exception(msg, nested, code) { - handle_error_code(msg, code, remote); + handle_error_code(msg, getStackTraceString(), code, remote); } Exception::Exception(CreateFromPocoTag, const Poco::Exception & exc) diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index c06bb13beb6..87cf3f2f603 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -11,19 +11,20 @@ namespace DB NamesAndTypesList StorageSystemErrors::getNamesAndTypes() { return { - { "name", std::make_shared() }, - { "code", std::make_shared() }, - { "value", std::make_shared() }, - { "last_error_time", std::make_shared() }, - { "last_error_message",std::make_shared() }, - { "remote", std::make_shared() }, + { "name", std::make_shared() }, + { "code", std::make_shared() }, + { "value", std::make_shared() }, + { "last_error_time", std::make_shared() }, + { "last_error_message", std::make_shared() }, + { "last_error_stacktrace", std::make_shared() }, + { "remote", std::make_shared() }, }; } void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, const std::string & message, bool remote) + auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, const std::string & message, const std::string & stacktrace, bool remote) { if (value || context.getSettingsRef().system_events_show_zero_values) { @@ -33,6 +34,7 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & res_columns[col_num++]->insert(value); res_columns[col_num++]->insert(last_error_time_ms / 1000); res_columns[col_num++]->insert(message); + res_columns[col_num++]->insert(stacktrace); res_columns[col_num++]->insert(remote); } }; @@ -45,8 +47,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, error.last_error_time_ms, error.message, 0 /* remote=0 */); - add_row(name, i, error.remote, error.last_error_time_ms, error.message, 1 /* remote=1 */); + add_row(name, i, error.local, error.last_error_time_ms, error.message, error.stacktrace, 0 /* remote=0 */); + add_row(name, i, error.remote, error.last_error_time_ms, error.message, error.stacktrace, 1 /* remote=1 */); } } From efdd04c958960233fad73f480366d3cfcbffba0d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 23:43:58 +0300 Subject: [PATCH 193/233] Drop last_ prefix for ErrorCodes::ValuePair::error_time_ms --- src/Common/ErrorCodes.cpp | 4 ++-- src/Common/ErrorCodes.h | 2 +- src/Storages/System/StorageSystemErrors.cpp | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 14182467351..d7e0d5fb16a 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -598,7 +598,7 @@ namespace ErrorCodes ValuePair inc_value{ !remote, /* local */ remote, /* remote */ - 0, /* last_error_time_ms */ + 0, /* error_time_ms */ message, /* message */ stacktrace, /* stacktrace */ }; @@ -613,7 +613,7 @@ namespace ErrorCodes stacktrace = value.stacktrace; const auto now = std::chrono::system_clock::now(); - last_error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); return *this; } diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 4c79614d55d..1c8f0a58884 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -28,7 +28,7 @@ namespace ErrorCodes { Value local = 0; Value remote = 0; - UInt64 last_error_time_ms = 0; + UInt64 error_time_ms = 0; std::string message; std::string stacktrace; diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index 87cf3f2f603..c16eba6754b 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -24,7 +24,7 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, const std::string & message, const std::string & stacktrace, bool remote) + auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 error_time_ms, const std::string & message, const std::string & stacktrace, bool remote) { if (value || context.getSettingsRef().system_events_show_zero_values) { @@ -32,7 +32,7 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & res_columns[col_num++]->insert(name); res_columns[col_num++]->insert(code); res_columns[col_num++]->insert(value); - res_columns[col_num++]->insert(last_error_time_ms / 1000); + res_columns[col_num++]->insert(error_time_ms / 1000); res_columns[col_num++]->insert(message); res_columns[col_num++]->insert(stacktrace); res_columns[col_num++]->insert(remote); @@ -47,8 +47,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, error.last_error_time_ms, error.message, error.stacktrace, 0 /* remote=0 */); - add_row(name, i, error.remote, error.last_error_time_ms, error.message, error.stacktrace, 1 /* remote=1 */); + add_row(name, i, error.local, error.error_time_ms, error.message, error.stacktrace, 0 /* remote=0 */); + add_row(name, i, error.remote, error.error_time_ms, error.message, error.stacktrace, 1 /* remote=1 */); } } From cc87bcfb63fbca0a9024364156d60dc703521c63 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 9 Mar 2021 09:09:28 +0300 Subject: [PATCH 194/233] Fix errorCodeToName() for signed integers - https://clickhouse-test-reports.s3.yandex.net/21529/2ce2772d35eb3d81628f4d294d5799e9f05333fd/functional_stateless_tests_(address).html#fail1 - https://clickhouse-test-reports.s3.yandex.net/21529/2ce2772d35eb3d81628f4d294d5799e9f05333fd/functional_stateless_tests_(ubsan).html#fail1 - https://clickhouse-test-reports.s3.yandex.net/21529/2ce2772d35eb3d81628f4d294d5799e9f05333fd/stress_test_(address).html#fail1 --- src/Common/ErrorCodes.cpp | 2 +- tests/queries/0_stateless/01544_errorCodeToName.reference | 1 + tests/queries/0_stateless/01544_errorCodeToName.sql | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index d7e0d5fb16a..879784bb43a 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -579,7 +579,7 @@ namespace ErrorCodes std::string_view getName(ErrorCode error_code) { - if (error_code >= END) + if (error_code < 0 || error_code >= END) return std::string_view(); return error_codes_names.names[error_code]; } diff --git a/tests/queries/0_stateless/01544_errorCodeToName.reference b/tests/queries/0_stateless/01544_errorCodeToName.reference index ace588644e1..fefccf984be 100644 --- a/tests/queries/0_stateless/01544_errorCodeToName.reference +++ b/tests/queries/0_stateless/01544_errorCodeToName.reference @@ -1,4 +1,5 @@ + OK UNSUPPORTED_METHOD diff --git a/tests/queries/0_stateless/01544_errorCodeToName.sql b/tests/queries/0_stateless/01544_errorCodeToName.sql index 9e28ed1116c..aa32270f00b 100644 --- a/tests/queries/0_stateless/01544_errorCodeToName.sql +++ b/tests/queries/0_stateless/01544_errorCodeToName.sql @@ -1,4 +1,5 @@ SELECT errorCodeToName(toUInt32(-1)); +SELECT errorCodeToName(-1); SELECT errorCodeToName(600); /* gap in error codes */ SELECT errorCodeToName(0); SELECT errorCodeToName(1); From 9921e7ca284b7d274af540d13812881dd6a0e578 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 9 Mar 2021 09:12:08 +0300 Subject: [PATCH 195/233] Add 01545_system_errors into skip_list.parallel https://clickhouse-test-reports.s3.yandex.net/21529/2ce2772d35eb3d81628f4d294d5799e9f05333fd/functional_stateless_tests_flaky_check_(address).html#fail1 --- tests/queries/skip_list.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index bded0807db9..caab92636b3 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -739,6 +739,7 @@ "01541_max_memory_usage_for_user_long", "01542_dictionary_load_exception_race", "01560_optimize_on_insert_zookeeper", + "01545_system_errors", // looks at the difference of values in system.errors "01575_disable_detach_table_of_dictionary", "01593_concurrent_alter_mutations_kill", "01593_concurrent_alter_mutations_kill_many_replicas", From a337691b060e9ad2c2cf53e9762691352f732837 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 9 Mar 2021 10:05:56 +0300 Subject: [PATCH 196/233] Fix modernize-use-bool-literals clang-tidy warning in StorageSystemErrors --- src/Storages/System/StorageSystemErrors.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index c16eba6754b..c9aac9ce007 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -47,8 +47,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, error.error_time_ms, error.message, error.stacktrace, 0 /* remote=0 */); - add_row(name, i, error.remote, error.error_time_ms, error.message, error.stacktrace, 1 /* remote=1 */); + add_row(name, i, error.local, error.error_time_ms, error.message, error.stacktrace, false /* remote=0 */); + add_row(name, i, error.remote, error.error_time_ms, error.message, error.stacktrace, true /* remote=1 */); } } From 9dee842b6082a3e90c65a8a45e5a357de30106a6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 16 Mar 2021 21:31:14 +0300 Subject: [PATCH 197/233] Distinguish remote and local error info --- src/Common/ErrorCodes.cpp | 34 +++++++-------------- src/Common/ErrorCodes.h | 26 ++++++++++------ src/Storages/System/StorageSystemErrors.cpp | 16 +++++----- 3 files changed, 35 insertions(+), 41 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 879784bb43a..a4e2f8742ca 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -564,7 +564,7 @@ namespace ErrorCodes #undef M constexpr ErrorCode END = 3000; - ValuePairHolder values[END + 1]{}; + ErrorPairHolder values[END + 1]{}; struct ErrorCodesNames { @@ -595,35 +595,23 @@ namespace ErrorCodes error_code = end() - 1; } - ValuePair inc_value{ - !remote, /* local */ - remote, /* remote */ - 0, /* error_time_ms */ - message, /* message */ - stacktrace, /* stacktrace */ - }; - values[error_code].increment(inc_value); + values[error_code].increment(remote, message, stacktrace); } - ValuePair & ValuePair::operator+=(const ValuePair & value) + void ErrorPairHolder::increment(bool remote, const std::string & message, const std::string & stacktrace) { - local += value.local; - remote += value.remote; - message = value.message; - stacktrace = value.stacktrace; - const auto now = std::chrono::system_clock::now(); - error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); - return *this; - } - - void ValuePairHolder::increment(const ValuePair & value_) - { std::lock_guard lock(mutex); - value += value_; + + auto & error = remote ? value.remote : value.local; + + ++error.count; + error.message = message; + error.stacktrace = stacktrace; + error.error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); } - ValuePair ValuePairHolder::get() + ErrorPair ErrorPairHolder::get() { std::lock_guard lock(mutex); return value; diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 1c8f0a58884..edb9be9e0c0 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -24,31 +24,37 @@ namespace ErrorCodes /// Returns statically allocated string. std::string_view getName(ErrorCode error_code); - struct ValuePair + struct Error { - Value local = 0; - Value remote = 0; + /// Number of times Exception with this ErrorCode had been throw. + Value count; + /// Time of the last error. UInt64 error_time_ms = 0; + /// Message for the last error. std::string message; + /// Stacktrace for the last error. std::string stacktrace; - - ValuePair & operator+=(const ValuePair & value); + }; + struct ErrorPair + { + Error local; + Error remote; }; /// Thread-safe - struct ValuePairHolder + struct ErrorPairHolder { public: - void increment(const ValuePair & value_); - ValuePair get(); + ErrorPair get(); + void increment(bool remote, const std::string & message, const std::string & stacktrace); private: - ValuePair value; + ErrorPair value; std::mutex mutex; }; /// ErrorCode identifier -> current value of error_code. - extern ValuePairHolder values[]; + extern ErrorPairHolder values[]; /// Get index just after last error_code identifier. ErrorCode end(); diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index c9aac9ce007..5243cb11aa3 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -24,17 +24,17 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 error_time_ms, const std::string & message, const std::string & stacktrace, bool remote) + auto add_row = [&](std::string_view name, size_t code, const auto & error, bool remote) { - if (value || context.getSettingsRef().system_events_show_zero_values) + if (error.count || context.getSettingsRef().system_events_show_zero_values) { size_t col_num = 0; res_columns[col_num++]->insert(name); res_columns[col_num++]->insert(code); - res_columns[col_num++]->insert(value); - res_columns[col_num++]->insert(error_time_ms / 1000); - res_columns[col_num++]->insert(message); - res_columns[col_num++]->insert(stacktrace); + res_columns[col_num++]->insert(error.count); + res_columns[col_num++]->insert(error.error_time_ms / 1000); + res_columns[col_num++]->insert(error.message); + res_columns[col_num++]->insert(error.stacktrace); res_columns[col_num++]->insert(remote); } }; @@ -47,8 +47,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, error.error_time_ms, error.message, error.stacktrace, false /* remote=0 */); - add_row(name, i, error.remote, error.error_time_ms, error.message, error.stacktrace, true /* remote=1 */); + add_row(name, i, error.local, /* remote= */ false); + add_row(name, i, error.remote, /* remote= */ true); } } From 37a17749ea57b9d1958b3f0dba70eaff0e61883a Mon Sep 17 00:00:00 2001 From: 3ldar-nasyrov <80788015+3ldar-nasyrov@users.noreply.github.com> Date: Tue, 16 Mar 2021 23:17:07 +0300 Subject: [PATCH 198/233] fixed exceeded amount of tries typo 'retires' -> 'retries' --- src/Interpreters/DDLWorker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 0ecb27ee3aa..eceb48ae773 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -872,7 +872,7 @@ bool DDLWorker::tryExecuteQueryOnLeaderReplica( else /// If we exceeded amount of tries { LOG_WARNING(log, "Task {} was not executed by anyone, maximum number of retries exceeded", task.entry_name); - task.execution_status = ExecutionStatus(ErrorCodes::UNFINISHED, "Cannot execute replicated DDL query, maximum retires exceeded"); + task.execution_status = ExecutionStatus(ErrorCodes::UNFINISHED, "Cannot execute replicated DDL query, maximum retries exceeded"); } return false; } From 0855f5fb183d9b8ef8caaef395e8fa792205096e Mon Sep 17 00:00:00 2001 From: George Date: Wed, 17 Mar 2021 01:34:40 +0300 Subject: [PATCH 199/233] Troubleshooting --- docs/en/operations/external-authenticators/ldap.md | 4 ++-- docs/ru/operations/external-authenticators/ldap.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index 1cd38ba8430..1b65ecc968b 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -84,7 +84,7 @@ At each login attempt, ClickHouse tries to "bind" to the specified DN defined by Note, that user `my_user` refers to `my_ldap_server`. This LDAP server must be configured in the main `config.xml` file as described previously. -When SQL-driven [Access Control and Account Management](../access-rights.md) is enabled, users that are authenticated by LDAP servers can also be created using the [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement) statement. +When SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled, users that are authenticated by LDAP servers can also be created using the [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement) statement. Query: @@ -97,7 +97,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; In addition to the locally defined users, a remote LDAP server can be used as a source of user definitions. To achieve this, specify previously defined LDAP server name (see [LDAP Server Definition](#ldap-server-definition)) in the `ldap` section inside the `users_directories` section of the `config.xml` file. -At each login attempt, ClickHouse tries to find the user definition locally and authenticate it as usual. If the user is not defined, ClickHouse will assume the definition exists in the external LDAP directory and will try to "bind" to the specified DN at the LDAP server using the provided credentials. If successful, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. Additionally, LDAP "search" can be performed and results can be transformed and treated as role names and then be assigned to the user if the `role_mapping` section is also configured. All this implies that the SQL-driven [Access Control and Account Management](../access-rights.md) is enabled and roles are created using the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. +At each login attempt, ClickHouse tries to find the user definition locally and authenticate it as usual. If the user is not defined, ClickHouse will assume the definition exists in the external LDAP directory and will try to "bind" to the specified DN at the LDAP server using the provided credentials. If successful, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. Additionally, LDAP "search" can be performed and results can be transformed and treated as role names and then be assigned to the user if the `role_mapping` section is also configured. All this implies that the SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled and roles are created using the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. **Example** diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index f93581123fd..102cc36eaa2 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -84,7 +84,7 @@ Обратите внимание, что пользователь `my_user` ссылается на `my_ldap_server`. Этот LDAP сервер должен быть настроен в основном файле `config.xml`, как это было описано ранее. -При включенном SQL-ориентированном [управлении доступом](../access-rights.md) пользователи, аутентифицированные LDAP серверами, могут также быть созданы выражением [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement). +При включенном SQL-ориентированном [управлении доступом](../access-rights.md#access-control) пользователи, аутентифицированные LDAP серверами, могут также быть созданы выражением [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement). Запрос: @@ -96,7 +96,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; В дополнение к локально определенным пользователям, удаленный LDAP сервер может служить источником определения пользователей. Для этого укажите имя определенного ранее сервера LDAP (см. [Определение LDAP сервера](#ldap-server-definition)) в секции `ldap` внутри секции `users_directories` файла `config.xml`. -При каждой попытке авторизации ClickHouse пытается локально найти определение пользователя и аутентифицировать его как обычно. Если пользователь не находится локально, ClickHouse предполагает, что он определяется во внешнем LDAP каталоге и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю присваиваются роли из списка, указанного в секции `roles`. Кроме того, если настроена секция `role_mapping`, то выполняется LDAP поиск, а его результаты преобразуются в имена ролей и присваиваются пользователям. Все это работает при условии, что SQL-ориентированное [управлением доступом](../access-rights.md) включено, а роли созданы выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). +При каждой попытке авторизации ClickHouse пытается локально найти определение пользователя и аутентифицировать его как обычно. Если пользователь не находится локально, ClickHouse предполагает, что он определяется во внешнем LDAP каталоге и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю присваиваются роли из списка, указанного в секции `roles`. Кроме того, если настроена секция `role_mapping`, то выполняется LDAP поиск, а его результаты преобразуются в имена ролей и присваиваются пользователям. Все это работает при условии, что SQL-ориентированное [управлением доступом](../access-rights.md#access-control) включено, а роли созданы выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). **Пример** @@ -145,4 +145,4 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; - `attribute` — имя атрибута, значение которого будет возвращаться LDAP поиском. - `prefix` — префикс, который, как предполагается, будет находиться перед началом каждой строки в исходном списке строк, возвращаемых LDAP поиском. Префикс будет удален из исходных строк, а сами они будут рассматриваться как имена локальных ролей. По умолчанию: пустая строка. -[Оригинальная статья](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap.md) +[Оригинальная статья](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap) From 3060d9bb3be814ba8429b1e39d3eee8051078caa Mon Sep 17 00:00:00 2001 From: George Date: Wed, 17 Mar 2021 01:49:02 +0300 Subject: [PATCH 200/233] fixed links --- docs/en/operations/external-authenticators/index.md | 2 +- docs/ru/operations/external-authenticators/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/external-authenticators/index.md b/docs/en/operations/external-authenticators/index.md index fe4e6a42974..f4b64865910 100644 --- a/docs/en/operations/external-authenticators/index.md +++ b/docs/en/operations/external-authenticators/index.md @@ -12,4 +12,4 @@ The following external authenticators and directories are supported: - [LDAP](./ldap.md#external-authenticators-ldap) [Authenticator](./ldap.md#ldap-external-authenticator) and [Directory](./ldap.md#ldap-external-user-directory) -[Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/index.md) +[Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/index/) diff --git a/docs/ru/operations/external-authenticators/index.md b/docs/ru/operations/external-authenticators/index.md index c8ac7459cfa..beb9bb5742c 100644 --- a/docs/ru/operations/external-authenticators/index.md +++ b/docs/ru/operations/external-authenticators/index.md @@ -12,4 +12,4 @@ ClickHouse поддерживает аутентификацию и управл - [LDAP](./ldap.md#external-authenticators-ldap) [аутентификатор](./ldap.md#ldap-external-authenticator) и [каталог](./ldap.md#ldap-external-user-directory) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/external-authenticators/index.md) +[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/external-authenticators/index/) From 9ca0566132016abeb25282f2a14127da1ee463f6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 17 Mar 2021 02:08:23 +0300 Subject: [PATCH 201/233] Minor modification --- src/Compression/LZ4_decompress_faster.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/LZ4_decompress_faster.h b/src/Compression/LZ4_decompress_faster.h index dd923279ebf..30a0d7acb22 100644 --- a/src/Compression/LZ4_decompress_faster.h +++ b/src/Compression/LZ4_decompress_faster.h @@ -95,7 +95,7 @@ struct PerformanceStatistics /// How to select method to run. /// -1 - automatically, based on statistics (default); - /// 0..3 - always choose specified method (for performance testing); + /// >= 0 - always choose specified method (for performance testing); /// -2 - choose methods in round robin fashion (for performance testing). ssize_t choose_method = -1; From 9969124cc43555369f7ba36a3b3e911f3d441f8a Mon Sep 17 00:00:00 2001 From: George Date: Wed, 17 Mar 2021 03:09:42 +0300 Subject: [PATCH 202/233] Change wording --- docs/en/sql-reference/statements/detach.md | 2 +- docs/ru/sql-reference/statements/attach.md | 4 ++-- docs/ru/sql-reference/statements/detach.md | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index cb0d7cf7b66..e9c9ed3693c 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -5,7 +5,7 @@ toc_title: DETACH # DETACH Statement {#detach} -Deletes information about the table or materialized view from the server. The server stops knowing about their existence. +Makes the server "forget" about the existence of the table or materialized view. Syntax: diff --git a/docs/ru/sql-reference/statements/attach.md b/docs/ru/sql-reference/statements/attach.md index be5b0b6d44a..55d4db80099 100644 --- a/docs/ru/sql-reference/statements/attach.md +++ b/docs/ru/sql-reference/statements/attach.md @@ -10,7 +10,7 @@ toc_title: ATTACH - вместо слова `CREATE` используется слово `ATTACH`; - запрос не создаёт данные на диске, а предполагает, что данные уже лежат в соответствующих местах, и всего лишь добавляет информацию о таблице на сервер. После выполнения запроса `ATTACH` сервер будет знать о существовании таблицы. -Если таблица перед этим была откреплена ([DETACH](../../sql-reference/statements/detach.md)), т.е. её структура известна, можно использовать сокращенную форму записи без определения структуры. +Если таблица перед этим была отключена ([DETACH](../../sql-reference/statements/detach.md)), т.е. её структура известна, можно использовать сокращенную форму записи без определения структуры. ``` sql ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] @@ -18,7 +18,7 @@ ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] Этот запрос используется при старте сервера. Сервер хранит метаданные таблиц в виде файлов с запросами `ATTACH`, которые он просто исполняет при запуске (за исключением некоторых системных таблиц, которые явно создаются на сервере). -Если таблица была откреплена перманентно, она не будет прикреплена обратно во время старта сервера, так что нужно явно использовать запрос `ATTACH`, чтобы прикрепить ее. +Если таблица была отключена перманентно, она не будет подключена обратно во время старта сервера, так что нужно явно использовать запрос `ATTACH`, чтобы подключить ее. [Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/attach/) diff --git a/docs/ru/sql-reference/statements/detach.md b/docs/ru/sql-reference/statements/detach.md index 90e9663def9..1809d85b958 100644 --- a/docs/ru/sql-reference/statements/detach.md +++ b/docs/ru/sql-reference/statements/detach.md @@ -5,7 +5,7 @@ toc_title: DETACH # DETACH {#detach-statement} -Удаляет из сервера информацию о таблице или материализованном представлении. Сервер перестаёт знать о существовании таблицы. +Заставляет сервер "забыть" о существовании таблицы или материализованного представления. Синтаксис: @@ -13,15 +13,15 @@ toc_title: DETACH DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] ``` -Но ни данные, ни метаданные таблицы или материализованного представления не удаляются. При следующем запуске сервера, если не было использовано `PERMANENTLY`, сервер прочитает метаданные и снова узнает о таблице/представлении. Если таблица или представление были откреплено перманентно, сервер не прикрепит их обратно автоматически. +Но ни данные, ни метаданные таблицы или материализованного представления не удаляются. При следующем запуске сервера, если не было использовано `PERMANENTLY`, сервер прочитает метаданные и снова узнает о таблице/представлении. Если таблица или представление были отключены перманентно, сервер не подключит их обратно автоматически. -Независимо от того, каким способом таблица была откреплена, ее можно прикрепить обратно с помощью запроса [ATTACH](../../sql-reference/statements/attach.md). Системные log таблицы также могут быть прикреплены обратно (к примеру `query_log`, `text_log` и др.) Другие системные таблицы не могут быть прикреплены обратно, но на следующем запуске сервер снова "вспомнит" об этих таблицах. +Независимо от того, каким способом таблица была отключена, ее можно подключить обратно с помощью запроса [ATTACH](../../sql-reference/statements/attach.md). Системные log таблицы также могут быть подключены обратно (к примеру `query_log`, `text_log` и др.) Другие системные таблицы не могут быть подключены обратно, но на следующем запуске сервер снова "вспомнит" об этих таблицах. -`ATTACH MATERIALIZED VIEW` не может быть использован с кратким синтаксисом (без `SELECT`), но можно прикрепить представление с помощью запроса `ATTACH TABLE`. +`ATTACH MATERIALIZED VIEW` не может быть использован с кратким синтаксисом (без `SELECT`), но можно подключить представление с помощью запроса `ATTACH TABLE`. -Обратите внимание, что нельзя перманентно открепить таблицу, которая уже временно откреплена. Для этого ее сначала надо прикрепить обратно, а затем снова открепить перманентно. +Обратите внимание, что нельзя перманентно отключить таблицу, которая уже временно отключена. Для этого ее сначала надо подключить обратно, а затем снова отключить перманентно. -Также нельзя использовать [DROP](../../sql-reference/statements/drop.md#drop-table) с открепленной таблицей или создавать таблицу с помощью [CREATE TABLE](../../sql-reference/statements/create/table.md) с таким же именем, как у уже открепленной таблицы. Еще нельзя заменить открепленную таблицу другой с помощью запроса [RENAME TABLE](../../sql-reference/statements/rename.md). +Также нельзя использовать [DROP](../../sql-reference/statements/drop.md#drop-table) с отключенной таблицей или создавать таблицу с помощью [CREATE TABLE](../../sql-reference/statements/create/table.md) с таким же именем, как у отключенной таблицы. Еще нельзя заменить отключенную таблицу другой с помощью запроса [RENAME TABLE](../../sql-reference/statements/rename.md). **Пример** @@ -51,7 +51,7 @@ SELECT * FROM test; └────────┘ ``` -Открепление таблицы: +Отключение таблицы: Запрос: From 0640bb4cbaadafed46c74fc1f528200a99851145 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 17 Mar 2021 14:33:57 +0800 Subject: [PATCH 203/233] Fix arena data race in two level merge --- src/Interpreters/Aggregator.cpp | 39 +++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index ea81155e26a..ee8132cd40c 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1389,35 +1389,46 @@ BlocksList Aggregator::prepareBlocksAndFillTwoLevelImpl( for (size_t i = data_variants.aggregates_pools.size(); i < max_threads; ++i) data_variants.aggregates_pools.push_back(std::make_shared()); - auto converter = [&](size_t bucket, ThreadGroupStatusPtr thread_group) + std::atomic next_bucket_to_merge = 0; + + auto converter = [&](size_t thread_id, ThreadGroupStatusPtr thread_group) { if (thread_group) CurrentThread::attachToIfDetached(thread_group); - /// Select Arena to avoid race conditions - size_t thread_number = static_cast(bucket) % max_threads; - Arena * arena = data_variants.aggregates_pools.at(thread_number).get(); + BlocksList blocks; + while (true) + { + UInt32 bucket = next_bucket_to_merge.fetch_add(1); - return convertOneBucketToBlock(data_variants, method, arena, final, bucket); + if (bucket >= Method::Data::NUM_BUCKETS) + break; + + if (method.data.impls[bucket].empty()) + continue; + + /// Select Arena to avoid race conditions + Arena * arena = data_variants.aggregates_pools.at(thread_id).get(); + blocks.emplace_back(convertOneBucketToBlock(data_variants, method, arena, final, bucket)); + } + return blocks; }; /// packaged_task is used to ensure that exceptions are automatically thrown into the main stream. - std::vector> tasks(Method::Data::NUM_BUCKETS); + std::vector> tasks(max_threads); try { - for (size_t bucket = 0; bucket < Method::Data::NUM_BUCKETS; ++bucket) + for (size_t thread_id = 0; thread_id < max_threads; ++thread_id) { - if (method.data.impls[bucket].empty()) - continue; - - tasks[bucket] = std::packaged_task([group = CurrentThread::getGroup(), bucket, &converter]{ return converter(bucket, group); }); + tasks[thread_id] = std::packaged_task( + [group = CurrentThread::getGroup(), thread_id, &converter] { return converter(thread_id, group); }); if (thread_pool) - thread_pool->scheduleOrThrowOnError([bucket, &tasks] { tasks[bucket](); }); + thread_pool->scheduleOrThrowOnError([thread_id, &tasks] { tasks[thread_id](); }); else - tasks[bucket](); + tasks[thread_id](); } } catch (...) @@ -1439,7 +1450,7 @@ BlocksList Aggregator::prepareBlocksAndFillTwoLevelImpl( if (!task.valid()) continue; - blocks.emplace_back(task.get_future().get()); + blocks.splice(blocks.end(), task.get_future().get()); } return blocks; From d8fe02ad910c3c6f9a63f78ba97ee5543207dacd Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Wed, 17 Mar 2021 09:58:51 +0300 Subject: [PATCH 204/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 102cc36eaa2..b5892b2aa97 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -133,7 +133,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; - `roles` — секция со списком локально определенных ролей, которые будут присвоены каждому пользователю, полученному от сервера LDAP. - Если роли не указаны ни здесь, ни в секции `role_mapping` (см. ниже), пользователь после аутентификации не сможет выполнять никаких действий. - `role_mapping` — секция c параметрами LDAP поиска и правилами отображения. - - При аутентификации пользователя, пока еще связанного с LDAP, производится LDAP поиск с помощью `search_filter` и имени этого пользователя. Для каждой записи, найденной в ходе поиска, выделяется значение указанного атрибута. У каждого атрибута, имеющего указанный префикс, этот префикс удаляется, а остальная часть значения становится именем локальной роли, определенной в ClickHouse, причем предполагается, что эта роль была ранее создана выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) до этого. + - При аутентификации пользователя, пока еще связанного с LDAP, производится LDAP поиск с помощью `search_filter` и имени этого пользователя. Для каждой записи, найденной в ходе поиска, выделяется значение указанного атрибута. У каждого атрибута, имеющего указанный префикс, этот префикс удаляется, а остальная часть значения становится именем локальной роли, определенной в ClickHouse, причем предполагается, что эта роль была ранее создана запросом [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) до этого. - Внутри одной секции `ldap` может быть несколько секций `role_mapping`. Все они будут применены. - `base_dn` — шаблон, который используется для создания базового DN для LDAP поиска. - При формировании DN все подстроки `{user_name}` и `{bind_dn}` в шаблоне будут заменяться на фактическое имя пользователя и DN привязки соответственно при каждом LDAP поиске. From 349c7bf0d6445d0113fe317e78cf7d4528d4661a Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Wed, 17 Mar 2021 09:58:55 +0300 Subject: [PATCH 205/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index b5892b2aa97..4afaa210cfe 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -96,7 +96,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; В дополнение к локально определенным пользователям, удаленный LDAP сервер может служить источником определения пользователей. Для этого укажите имя определенного ранее сервера LDAP (см. [Определение LDAP сервера](#ldap-server-definition)) в секции `ldap` внутри секции `users_directories` файла `config.xml`. -При каждой попытке авторизации ClickHouse пытается локально найти определение пользователя и аутентифицировать его как обычно. Если пользователь не находится локально, ClickHouse предполагает, что он определяется во внешнем LDAP каталоге и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю присваиваются роли из списка, указанного в секции `roles`. Кроме того, если настроена секция `role_mapping`, то выполняется LDAP поиск, а его результаты преобразуются в имена ролей и присваиваются пользователям. Все это работает при условии, что SQL-ориентированное [управлением доступом](../access-rights.md#access-control) включено, а роли созданы выражением [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). +При каждой попытке авторизации ClickHouse пытается локально найти определение пользователя и аутентифицировать его как обычно. Если пользователь не находится локально, ClickHouse предполагает, что он определяется во внешнем LDAP каталоге и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю присваиваются роли из списка, указанного в секции `roles`. Кроме того, если настроена секция `role_mapping`, то выполняется LDAP поиск, а его результаты преобразуются в имена ролей и присваиваются пользователям. Все это работает при условии, что SQL-ориентированное [управлением доступом](../access-rights.md#access-control) включено, а роли созданы запросом [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). **Пример** From 8272d69f7495ddb1657bf24437742069781776db Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Wed, 17 Mar 2021 09:59:01 +0300 Subject: [PATCH 206/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 4afaa210cfe..0c941cee0d5 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -84,7 +84,7 @@ Обратите внимание, что пользователь `my_user` ссылается на `my_ldap_server`. Этот LDAP сервер должен быть настроен в основном файле `config.xml`, как это было описано ранее. -При включенном SQL-ориентированном [управлении доступом](../access-rights.md#access-control) пользователи, аутентифицированные LDAP серверами, могут также быть созданы выражением [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement). +При включенном SQL-ориентированном [управлении доступом](../access-rights.md#access-control) пользователи, аутентифицированные LDAP серверами, могут также быть созданы запросом [CREATE USER](../../sql-reference/statements/create/user.md#create-user-statement). Запрос: From 4878ad5b6cc094a42b0a3f98194f30ab1dce6e03 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Wed, 17 Mar 2021 09:59:09 +0300 Subject: [PATCH 207/233] Update docs/ru/operations/external-authenticators/ldap.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index 0c941cee0d5..b53c4cba121 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -2,7 +2,7 @@ Для аутентификации пользователей ClickHouse можно использовать сервер LDAP. Существуют два подхода: -- Использовать LDAP как внешний аутентификатор для существующих пользователей, которые определены в `users.xml` или в локальных параметрах управления доступом. +- Использовать LDAP как внешний аутентификатор для существующих пользователей, которые определены в `users.xml`, или в локальных параметрах управления доступом. - Использовать LDAP как внешний пользовательский каталог и разрешить аутентификацию локально неопределенных пользователей, если они есть на LDAP сервере. Для обоих подходов необходимо определить внутреннее имя LDAP сервера в конфигурации ClickHouse, чтобы другие параметры конфигурации могли ссылаться на это имя. From f69b6ecf0e030716574fae31b69b06db6ff9eae1 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 17 Mar 2021 13:16:41 +0200 Subject: [PATCH 208/233] Fixed cross-links to other pages --- docs/en/sql-reference/statements/optimize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index ea1fac90466..49a7404d76e 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -23,7 +23,7 @@ When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engin ### BY expression {#by-expression} -If you want to perform deduplication on custom set of columns rather than on all, you can specify list of columns explicitly or use any combination of [`*`](../../../sql-reference/statements/select/index.md#asterisk), [`COLUMNS`](../../../sql-reference/statements/select/index.md#columns-expression) or [`EXCEPT`](../../../sql-reference/statements/select/index.md#except-modifier) expressions. The explictly written or implicitly expanded list of columns must include all columns specified in row ordering expression (both primary and sorting keys) and partitioning expression (partitioning key). +If you want to perform deduplication on custom set of columns rather than on all, you can specify list of columns explicitly or use any combination of [`*`](../../sql-reference/statements/select/index.md#asterisk), [`COLUMNS`](../../sql-reference/statements/select/index.md#columns-expression) or [`EXCEPT`](../../sql-reference/statements/select/index.md#except-modifier) expressions. The explictly written or implicitly expanded list of columns must include all columns specified in row ordering expression (both primary and sorting keys) and partitioning expression (partitioning key). Note that `*` behaves just like in `SELECT`: `MATERIALIZED`, and `ALIAS` columns are not used for expansion. Also, it is an error to specify empty list of columns, or write an expression that results in an empty list of columns, or deduplicate by an ALIAS column. From e6158be4dac1f19dfad279035dc721618c75bb16 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 17 Mar 2021 14:54:14 +0300 Subject: [PATCH 209/233] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 928991dc937..43531b60267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## ClickHouse release 21.3 +## ClickHouse release 21.3 (LTS) ### ClickHouse release v21.3, 2021-03-12 From f8fc4281f2c540128cdb689dd6c909b63b037ef3 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 17 Mar 2021 15:56:47 +0300 Subject: [PATCH 210/233] Update PostgreSQLReplicaConnection.h --- src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h index 9465d4a119b..e58d4bc8100 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h +++ b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h @@ -25,6 +25,7 @@ public: private: + /// Highest priority is 0, the bigger the number in map, the less the priority using ReplicasByPriority = std::map; Poco::Logger * log; From 1cd9f28bd4f8d1cda7347da8ad81a93975a4c840 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 17 Mar 2021 18:31:30 +0300 Subject: [PATCH 211/233] another way --- src/Interpreters/InterpreterSelectQuery.cpp | 8 +++++++- .../Transforms/PartialSortingTransform.cpp | 17 ++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index d0c8966cf07..6be43408997 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2038,7 +2038,13 @@ void InterpreterSelectQuery::executeWindow(QueryPlan & query_plan) for (size_t i = 0; i < windows_sorted.size(); ++i) { const auto & w = *windows_sorted[i]; - if (i == 0 || !sortIsPrefix(w, *windows_sorted[i - 1])) + + // We don't need to sort again if the input from previous window already + // has suitable sorting. Also don't create sort steps when there are no + // columns to sort by, because the sort nodes are confused by this. It + // happens in case of `over ()`. + if (!w.full_sort_description.empty() + && (i == 0 || !sortIsPrefix(w, *windows_sorted[i - 1]))) { auto partial_sorting = std::make_unique( query_plan.getCurrentDataStream(), diff --git a/src/Processors/Transforms/PartialSortingTransform.cpp b/src/Processors/Transforms/PartialSortingTransform.cpp index 7c29f506617..3a75571872f 100644 --- a/src/Processors/Transforms/PartialSortingTransform.cpp +++ b/src/Processors/Transforms/PartialSortingTransform.cpp @@ -10,6 +10,8 @@ PartialSortingTransform::PartialSortingTransform( : ISimpleTransform(header_, header_, false) , description(description_), limit(limit_) { + // Sorting by no columns doesn't make sense. + assert(!description.empty()); } static ColumnRawPtrs extractColumns(const Block & block, const SortDescription & description) @@ -91,17 +93,14 @@ size_t getFilterMask(const ColumnRawPtrs & lhs, const ColumnRawPtrs & rhs, size_ void PartialSortingTransform::transform(Chunk & chunk) { - if (chunk.getColumns().empty()) + if (chunk.getNumRows()) { - // Sometimes we can have Chunks w/o columns, e.g. in case of - // `select count() over () from numbers(4) where number < 2`. - // We don't have to modify this Chunk, but we have to preserve the input - // number of rows. The following code uses Block for sorting, and Block - // is incapable of recording the number of rows when there is no columns. - // The simplest solution is to specifically check for Chunk with no - // columns and not modify it, which is what we do here. - return; + // The following code works with Blocks and will lose the number of + // rows when there are no columns. We shouldn't get such block, because + // we have to sort by at least one column. + assert(chunk.getNumColumns()); } + if (read_rows) read_rows->add(chunk.getNumRows()); From 738cb1af62a9f65846bba21757c9032583c9d00f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 11 Mar 2021 13:34:15 +0300 Subject: [PATCH 212/233] Fix filter push down columns order. --- src/Interpreters/ActionsDAG.cpp | 30 ++++++++++++++----- src/Interpreters/ActionsDAG.h | 4 +-- .../Optimizations/filterPushDown.cpp | 3 +- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 83844176f3b..31abbc89634 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1360,7 +1360,8 @@ ColumnsWithTypeAndName prepareFunctionArguments(const std::vector conjunction) + +ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunction, const ColumnsWithTypeAndName & all_inputs) { if (conjunction.empty()) return nullptr; @@ -1374,6 +1375,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc std::make_shared())); std::unordered_map nodes_mapping; + std::unordered_map> added_inputs; struct Frame { @@ -1416,16 +1418,30 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc child = nodes_mapping[child]; if (node.type == ActionType::INPUT) - { - actions->inputs.emplace_back(&node); - actions->index.insert(&node); - } + added_inputs[node.result_name].push_back(&node); stack.pop(); } } } + + for (const auto & col : all_inputs) + { + Node * input; + auto & list = added_inputs[col.name]; + if (list.empty()) + input = &const_cast(actions->addInput(col)); + else + { + input = list.front(); + list.pop_front(); + actions->inputs.push_back(input); + } + + actions->index.insert(input); + } + Node * result_predicate = nodes_mapping[*conjunction.begin()]; if (conjunction.size() > 1) @@ -1442,7 +1458,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc return actions; } -ActionsDAGPtr ActionsDAG::splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs) +ActionsDAGPtr ActionsDAG::splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs, const ColumnsWithTypeAndName & all_inputs) { Node * predicate; @@ -1480,7 +1496,7 @@ ActionsDAGPtr ActionsDAG::splitActionsForFilter(const std::string & filter_name, } auto conjunction = getConjunctionNodes(predicate, allowed_nodes); - auto actions = cloneActionsForConjunction(conjunction.allowed); + auto actions = cloneActionsForConjunction(conjunction.allowed, all_inputs); if (!actions) return nullptr; diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 165f712a627..a3cfd9cb776 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -292,7 +292,7 @@ public: /// Otherwise, return actions which inputs are from available_inputs. /// Returned actions add single column which may be used for filter. /// Also, replace some nodes of current inputs to constant 1 in case they are filtered. - ActionsDAGPtr splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs); + ActionsDAGPtr splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs, const ColumnsWithTypeAndName & all_inputs); private: Node & addNode(Node node, bool can_replace = false, bool add_to_index = true); @@ -323,7 +323,7 @@ private: void compileFunctions(); - ActionsDAGPtr cloneActionsForConjunction(std::vector conjunction); + ActionsDAGPtr cloneActionsForConjunction(std::vector conjunction, const ColumnsWithTypeAndName & all_inputs); }; diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index d64f082b7ee..f6a4eecbad1 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -43,7 +43,8 @@ static size_t tryAddNewFilterStep( // std::cerr << "Filter: \n" << expression->dumpDAG() << std::endl; - auto split_filter = expression->splitActionsForFilter(filter_column_name, removes_filter, allowed_inputs); + const auto & all_inputs = child->getInputStreams().front().header.getColumnsWithTypeAndName(); + auto split_filter = expression->splitActionsForFilter(filter_column_name, removes_filter, allowed_inputs, all_inputs); if (!split_filter) return 0; From bb17f14d837f0aabf47aafbcd6161f9d09c06e49 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 17 Mar 2021 18:55:53 +0300 Subject: [PATCH 213/233] fix --- programs/client/Client.cpp | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 4a61662c238..4ba96280939 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -65,6 +65,7 @@ #include #include #include +#include #include #include #include @@ -116,6 +117,31 @@ namespace ErrorCodes } +static bool queryHasWithClause(const IAST * ast) +{ + if (const auto * select = dynamic_cast(ast); + select && select->with()) + { + return true; + } + + // This is a bit too much, because most of the children are not queries, + // but on the other hand it will let us to avoid breakage when the AST + // structure changes and some new variant of query nesting is added. This + // function is used in fuzzer, so it's better to be defensive and avoid + // weird unexpected errors. + for (const auto & child : ast->children) + { + if (queryHasWithClause(child.get())) + { + return true; + } + } + + return false; +} + + class Client : public Poco::Util::Application { public: @@ -1429,7 +1455,11 @@ private: // when `lambda()` function gets substituted into a wrong place. // To avoid dealing with these cases, run the check only for the // queries we were able to successfully execute. - if (!have_error) + // The final caveat is that sometimes WITH queries are not executed, + // if they are not referenced by the main SELECT, so they can still + // have the abovementioned problems. Disable this check for such + // queries, for lack of a better solution. + if (!have_error && queryHasWithClause(parsed_query.get())) { ASTPtr parsed_formatted_query; try From cb92d578e1913107f9e25536fea91025aa2d75df Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 17 Mar 2021 18:57:13 +0300 Subject: [PATCH 214/233] Update programs/client/Client.cpp --- programs/client/Client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 4ba96280939..b879fb0a0ee 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1457,7 +1457,7 @@ private: // queries we were able to successfully execute. // The final caveat is that sometimes WITH queries are not executed, // if they are not referenced by the main SELECT, so they can still - // have the abovementioned problems. Disable this check for such + // have the aforementioned problems. Disable this check for such // queries, for lack of a better solution. if (!have_error && queryHasWithClause(parsed_query.get())) { From 3eba817a688f95744679779775358dc88310a56d Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 17 Mar 2021 19:08:46 +0300 Subject: [PATCH 215/233] Add comments. --- src/Interpreters/ActionsDAG.cpp | 13 ++++++++----- src/Interpreters/ActionsDAG.h | 18 +++++++++++++++++- .../QueryPlan/Optimizations/filterPushDown.cpp | 2 +- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 31abbc89634..e67ab82a5ca 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1360,7 +1360,6 @@ ColumnsWithTypeAndName prepareFunctionArguments(const std::vector conjunction, const ColumnsWithTypeAndName & all_inputs) { if (conjunction.empty()) @@ -1375,7 +1374,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc std::make_shared())); std::unordered_map nodes_mapping; - std::unordered_map> added_inputs; + std::unordered_map> required_inputs; struct Frame { @@ -1418,7 +1417,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc child = nodes_mapping[child]; if (node.type == ActionType::INPUT) - added_inputs[node.result_name].push_back(&node); + required_inputs[node.result_name].push_back(&node); stack.pop(); } @@ -1429,7 +1428,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc for (const auto & col : all_inputs) { Node * input; - auto & list = added_inputs[col.name]; + auto & list = required_inputs[col.name]; if (list.empty()) input = &const_cast(actions->addInput(col)); else @@ -1458,7 +1457,11 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc return actions; } -ActionsDAGPtr ActionsDAG::splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs, const ColumnsWithTypeAndName & all_inputs) +ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown( + const std::string & filter_name, + bool can_remove_filter, + const Names & available_inputs, + const ColumnsWithTypeAndName & all_inputs) { Node * predicate; diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index a3cfd9cb776..e0e0e9c8957 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -292,7 +292,23 @@ public: /// Otherwise, return actions which inputs are from available_inputs. /// Returned actions add single column which may be used for filter. /// Also, replace some nodes of current inputs to constant 1 in case they are filtered. - ActionsDAGPtr splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs, const ColumnsWithTypeAndName & all_inputs); + /// + /// @param all_inputs should containt inputs from previous step, which will be used for result actions. + /// It is expected that all_inputs contain columns from available_inputs. + /// This parameter is needed to enforce result actions save columns order in block. + /// Otherwise for some queries, e.g. with GROUP BY, columns colum be swapped. + /// Example: SELECT sum(x), y, z FROM tab WHERE z > 0 and sum(x) > 0 + /// Pushed condition: z > 0 + /// GROUP BY step will transform columns `x, y, z` -> `sum(x), y, z` + /// If we just add filter step with actions `z -> z > 0` before GROUP BY, + /// columns will be transformed like `x, y, z` -> `z, z > 0, x, y` -(remove filter)-> `z, z, y`. + /// To avoid it, add inputs from `all_inputs` list, + /// so actions `x, y, z -> x, y, z, z > 0` -(remove filter)-> `x, y, z` will not change columns order. + ActionsDAGPtr cloneActionsForFilterPushDown( + const std::string & filter_name, + bool can_remove_filter, + const Names & available_inputs, + const ColumnsWithTypeAndName & all_inputs); private: Node & addNode(Node node, bool can_replace = false, bool add_to_index = true); diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index f6a4eecbad1..0b988f9803f 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -44,7 +44,7 @@ static size_t tryAddNewFilterStep( // std::cerr << "Filter: \n" << expression->dumpDAG() << std::endl; const auto & all_inputs = child->getInputStreams().front().header.getColumnsWithTypeAndName(); - auto split_filter = expression->splitActionsForFilter(filter_column_name, removes_filter, allowed_inputs, all_inputs); + auto split_filter = expression->cloneActionsForFilterPushDown(filter_column_name, removes_filter, allowed_inputs, all_inputs); if (!split_filter) return 0; From 5dcddbd4681a2182788e62c433a7c37357796e52 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 17 Mar 2021 19:52:50 +0300 Subject: [PATCH 216/233] boop --- docker/test/fuzzer/run-fuzzer.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 6858e838850..611fb411d6c 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -38,7 +38,6 @@ function download ln -s ./clickhouse ./clickhouse-server ln -s ./clickhouse ./clickhouse-client - # clickhouse-server is in the current dir export PATH="$PWD:$PATH" } From 2c1539f641aaac5267700a26e410c30fd54c2dd1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 17 Mar 2021 20:08:30 +0300 Subject: [PATCH 217/233] Fix added input. --- src/Interpreters/ActionsDAG.cpp | 6 +++--- src/Interpreters/ActionsDAG.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index e67ab82a5ca..e5ae2dcfcf9 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -91,7 +91,7 @@ const ActionsDAG::Node & ActionsDAG::addInput(std::string name, DataTypePtr type return addNode(std::move(node), can_replace, add_to_index); } -const ActionsDAG::Node & ActionsDAG::addInput(ColumnWithTypeAndName column, bool can_replace) +const ActionsDAG::Node & ActionsDAG::addInput(ColumnWithTypeAndName column, bool can_replace, bool add_to_index) { Node node; node.type = ActionType::INPUT; @@ -99,7 +99,7 @@ const ActionsDAG::Node & ActionsDAG::addInput(ColumnWithTypeAndName column, bool node.result_name = std::move(column.name); node.column = std::move(column.column); - return addNode(std::move(node), can_replace); + return addNode(std::move(node), can_replace, add_to_index); } const ActionsDAG::Node & ActionsDAG::addColumn(ColumnWithTypeAndName column, bool can_replace, bool materialize) @@ -1430,7 +1430,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc Node * input; auto & list = required_inputs[col.name]; if (list.empty()) - input = &const_cast(actions->addInput(col)); + input = &const_cast(actions->addInput(col, true, false)); else { input = list.front(); diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index e0e0e9c8957..fc6a0545ebd 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -200,7 +200,7 @@ public: std::string dumpDAG() const; const Node & addInput(std::string name, DataTypePtr type, bool can_replace = false, bool add_to_index = true); - const Node & addInput(ColumnWithTypeAndName column, bool can_replace = false); + const Node & addInput(ColumnWithTypeAndName column, bool can_replace = false, bool add_to_index = true); const Node & addColumn(ColumnWithTypeAndName column, bool can_replace = false, bool materialize = false); const Node & addAlias(const std::string & name, std::string alias, bool can_replace = false); const Node & addArrayJoin(const std::string & source_name, std::string result_name); @@ -301,7 +301,7 @@ public: /// Pushed condition: z > 0 /// GROUP BY step will transform columns `x, y, z` -> `sum(x), y, z` /// If we just add filter step with actions `z -> z > 0` before GROUP BY, - /// columns will be transformed like `x, y, z` -> `z, z > 0, x, y` -(remove filter)-> `z, z, y`. + /// columns will be transformed like `x, y, z` -> `z, z > 0, x, y` -(remove filter)-> `z, x, y`. /// To avoid it, add inputs from `all_inputs` list, /// so actions `x, y, z -> x, y, z, z > 0` -(remove filter)-> `x, y, z` will not change columns order. ActionsDAGPtr cloneActionsForFilterPushDown( From 6397099d3cf51b6721f1d3df372fe77e6401b329 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 17 Mar 2021 20:08:49 +0300 Subject: [PATCH 218/233] Added test. --- .../01763_filter_push_down_bugs.reference | 6 +++ .../01763_filter_push_down_bugs.sql | 37 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 tests/queries/0_stateless/01763_filter_push_down_bugs.reference create mode 100644 tests/queries/0_stateless/01763_filter_push_down_bugs.sql diff --git a/tests/queries/0_stateless/01763_filter_push_down_bugs.reference b/tests/queries/0_stateless/01763_filter_push_down_bugs.reference new file mode 100644 index 00000000000..66ea84a07c1 --- /dev/null +++ b/tests/queries/0_stateless/01763_filter_push_down_bugs.reference @@ -0,0 +1,6 @@ +1 2 +1 2 +[1] 2 +[[1]] 2 +String1_0 String2_0 String3_0 String4_0 1 +String1_0 String2_0 String3_0 String4_0 1 diff --git a/tests/queries/0_stateless/01763_filter_push_down_bugs.sql b/tests/queries/0_stateless/01763_filter_push_down_bugs.sql new file mode 100644 index 00000000000..5000eb38878 --- /dev/null +++ b/tests/queries/0_stateless/01763_filter_push_down_bugs.sql @@ -0,0 +1,37 @@ +SELECT * FROM (SELECT col1, col2 FROM (select '1' as col1, '2' as col2) GROUP by col1, col2) AS expr_qry WHERE col2 != ''; +SELECT * FROM (SELECT materialize('1') AS s1, materialize('2') AS s2 GROUP BY s1, s2) WHERE s2 = '2'; +SELECT * FROM (SELECT materialize([1]) AS s1, materialize('2') AS s2 GROUP BY s1, s2) WHERE s2 = '2'; +SELECT * FROM (SELECT materialize([[1]]) AS s1, materialize('2') AS s2 GROUP BY s1, s2) WHERE s2 = '2'; + +DROP TABLE IF EXISTS Test; + +CREATE TABLE Test +ENGINE = MergeTree() +PRIMARY KEY (String1,String2) +ORDER BY (String1,String2) +AS +SELECT + 'String1_' || toString(number) as String1, + 'String2_' || toString(number) as String2, + 'String3_' || toString(number) as String3, + 'String4_' || toString(number%4) as String4 +FROM numbers(1); + +SELECT * +FROM + ( + SELECT String1,String2,String3,String4,COUNT(*) + FROM Test + GROUP by String1,String2,String3,String4 + ) AS expr_qry; + +SELECT * +FROM + ( + SELECT String1,String2,String3,String4,COUNT(*) + FROM Test + GROUP by String1,String2,String3,String4 + ) AS expr_qry +WHERE String4 ='String4_0'; + +DROP TABLE IF EXISTS Test; From 89e79185a0f764e1009a061ba01ea4cb93704c55 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 17 Mar 2021 20:09:29 +0300 Subject: [PATCH 219/233] Update tests.md --- docs/en/development/tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index fb453e55417..7547497b9af 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -233,7 +233,7 @@ Google OSS-Fuzz can be found at `docker/fuzz`. We also use simple fuzz test to generate random SQL queries and to check that the server doesn’t die executing them. You can find it in `00746_sql_fuzzy.pl`. This test should be run continuously (overnight and longer). -We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. It does random permutations and substitutions in queries AST. It remembers AST nodes from previous tests to use them for fuzzing of subsequent tests while processing them in random order. +We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. It does random permutations and substitutions in queries AST. It remembers AST nodes from previous tests to use them for fuzzing of subsequent tests while processing them in random order. You can learn more about this fuzzer in [this blog article](https://clickhouse.tech/blog/en/2021/fuzzing-clickhouse/). ## Stress test From 8cb19d63aef4777ee8ebfa3fe94f229dbbd6dc4b Mon Sep 17 00:00:00 2001 From: robert Date: Wed, 17 Mar 2021 20:38:49 +0300 Subject: [PATCH 220/233] zookeeper-dump-tree: added ctime option to dump node ctime --- utils/zookeeper-dump-tree/main.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/zookeeper-dump-tree/main.cpp b/utils/zookeeper-dump-tree/main.cpp index 5ab7bc2d536..5dcc260762f 100644 --- a/utils/zookeeper-dump-tree/main.cpp +++ b/utils/zookeeper-dump-tree/main.cpp @@ -17,6 +17,7 @@ int main(int argc, char ** argv) "addresses of ZooKeeper instances, comma separated. Example: example01e.yandex.ru:2181") ("path,p", boost::program_options::value()->default_value("/"), "where to start") + ("ctime,c", "print node ctime") ; boost::program_options::variables_map options; @@ -79,7 +80,11 @@ int main(int argc, char ** argv) throw; } - std::cout << it->first << '\t' << response.stat.numChildren << '\t' << response.stat.dataLength << '\n'; + std::cout << it->first << '\t' << response.stat.numChildren << '\t' << response.stat.dataLength; + if (options.count("ctime")) { + std::cout << '\t' << response.stat.ctime; + } + std::cout << '\n'; for (const auto & name : response.names) { From e1b11c786bf23844b4132d5560e4fac0248f741c Mon Sep 17 00:00:00 2001 From: robert Date: Wed, 17 Mar 2021 21:10:51 +0300 Subject: [PATCH 221/233] zookeeper-dump-tree: move ctime option to bool --- utils/zookeeper-dump-tree/main.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/zookeeper-dump-tree/main.cpp b/utils/zookeeper-dump-tree/main.cpp index 5dcc260762f..47e23c5af71 100644 --- a/utils/zookeeper-dump-tree/main.cpp +++ b/utils/zookeeper-dump-tree/main.cpp @@ -31,6 +31,8 @@ int main(int argc, char ** argv) return 1; } + bool dump_ctime = (options.count("ctime")) ? true : false; + zkutil::ZooKeeperPtr zookeeper = std::make_shared(options.at("address").as()); std::string initial_path = options.at("path").as(); @@ -81,7 +83,7 @@ int main(int argc, char ** argv) } std::cout << it->first << '\t' << response.stat.numChildren << '\t' << response.stat.dataLength; - if (options.count("ctime")) { + if (dump_ctime) { std::cout << '\t' << response.stat.ctime; } std::cout << '\n'; From 9fdb0e667fbeef9e13750afc4be90a29618c292e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 17 Mar 2021 21:40:26 +0300 Subject: [PATCH 222/233] Add comment. --- src/Interpreters/ActionsDAG.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index e5ae2dcfcf9..94d9b72b8e9 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1424,7 +1424,8 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc } } - + /// Actions must have the same inputs as in all_inputs list. + /// See comment to cloneActionsForFilterPushDown. for (const auto & col : all_inputs) { Node * input; From 45fead90621dfd2cf19bbd350aca703f4604b77f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 17 Mar 2021 21:42:42 +0300 Subject: [PATCH 223/233] Fix typos --- src/Interpreters/ActionsDAG.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index fc6a0545ebd..d5a15324280 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -293,10 +293,10 @@ public: /// Returned actions add single column which may be used for filter. /// Also, replace some nodes of current inputs to constant 1 in case they are filtered. /// - /// @param all_inputs should containt inputs from previous step, which will be used for result actions. + /// @param all_inputs should contain inputs from previous step, which will be used for result actions. /// It is expected that all_inputs contain columns from available_inputs. /// This parameter is needed to enforce result actions save columns order in block. - /// Otherwise for some queries, e.g. with GROUP BY, columns colum be swapped. + /// Otherwise for some queries, e.g. with GROUP BY, columns will be mixed. /// Example: SELECT sum(x), y, z FROM tab WHERE z > 0 and sum(x) > 0 /// Pushed condition: z > 0 /// GROUP BY step will transform columns `x, y, z` -> `sum(x), y, z` From 9b1d256f5460bdda9387c5656e1b5d4f43fbf3f4 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 17 Mar 2021 21:44:43 +0300 Subject: [PATCH 224/233] clang-tidy...... --- programs/client/Client.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 4ba96280939..2528357565c 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -130,11 +130,17 @@ static bool queryHasWithClause(const IAST * ast) // structure changes and some new variant of query nesting is added. This // function is used in fuzzer, so it's better to be defensive and avoid // weird unexpected errors. - for (const auto & child : ast->children) + // clang-tidy is confused by this function: it thinks that if `select` is + // nullptr, `ast` is also nullptr, and complains about nullptr dereference. + // NOLINTNEXTLINE + if (ast->children) { - if (queryHasWithClause(child.get())) + for (const auto & child : ast->children) /* NOLINT */ { - return true; + if (queryHasWithClause(child.get())) + { + return true; + } } } From bb460dd7f4e6da81a06920234b5e8cdcb15c9f91 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 17 Mar 2021 21:55:06 +0300 Subject: [PATCH 225/233] fix --- programs/client/Client.cpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index d77b196dff1..c2450c9e48f 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -125,22 +125,19 @@ static bool queryHasWithClause(const IAST * ast) return true; } - // This is a bit too much, because most of the children are not queries, - // but on the other hand it will let us to avoid breakage when the AST - // structure changes and some new variant of query nesting is added. This - // function is used in fuzzer, so it's better to be defensive and avoid - // weird unexpected errors. + // This full recursive walk is somewhat excessive, because most of the + // children are not queries, but on the other hand it will let us to avoid + // breakage when the AST structure changes and some new variant of query + // nesting is added. This function is used in fuzzer, so it's better to be + // defensive and avoid weird unexpected errors. // clang-tidy is confused by this function: it thinks that if `select` is // nullptr, `ast` is also nullptr, and complains about nullptr dereference. // NOLINTNEXTLINE - if (ast->children) + for (const auto & child : ast->children) { - for (const auto & child : ast->children) /* NOLINT */ + if (queryHasWithClause(child.get())) { - if (queryHasWithClause(child.get())) - { - return true; - } + return true; } } From a3c1096fdbb59cdc52666e321dea93cbf17a0085 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 17 Mar 2021 21:59:03 +0300 Subject: [PATCH 226/233] Update run-fuzzer.sh --- docker/test/fuzzer/run-fuzzer.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 611fb411d6c..2a6cf7cb0d2 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -20,11 +20,8 @@ function clone # the repo from the CI as well. For local runs, start directly from the "fuzz" # stage. rm -rf ch ||: - mkdir ch - cd ch wget -nv -nd -c "https://clickhouse-test-reports.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/repo/clickhouse_no_subs.tar.gz" - tar -xvf clickhouse_no_subs.tar.gz - tree ||: + tar -xf -C ch --strip-components=1 clickhouse_no_subs.tar.gz ls -lath ||: } From 3a8d7e9a48d254eb9adb6f5c9623834b65658df6 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 17 Mar 2021 22:12:10 +0300 Subject: [PATCH 227/233] Update main.cpp --- utils/zookeeper-dump-tree/main.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/zookeeper-dump-tree/main.cpp b/utils/zookeeper-dump-tree/main.cpp index 47e23c5af71..893056564bb 100644 --- a/utils/zookeeper-dump-tree/main.cpp +++ b/utils/zookeeper-dump-tree/main.cpp @@ -31,7 +31,7 @@ int main(int argc, char ** argv) return 1; } - bool dump_ctime = (options.count("ctime")) ? true : false; + bool dump_ctime = options.count("ctime"); zkutil::ZooKeeperPtr zookeeper = std::make_shared(options.at("address").as()); @@ -83,9 +83,8 @@ int main(int argc, char ** argv) } std::cout << it->first << '\t' << response.stat.numChildren << '\t' << response.stat.dataLength; - if (dump_ctime) { + if (dump_ctime) std::cout << '\t' << response.stat.ctime; - } std::cout << '\n'; for (const auto & name : response.names) From 51cd10c8eb440421ce7b7d7f94c4a4a471ccc1d9 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 18 Mar 2021 00:22:36 +0300 Subject: [PATCH 228/233] Update run-fuzzer.sh --- docker/test/fuzzer/run-fuzzer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 2a6cf7cb0d2..2f6aac5c183 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -21,7 +21,7 @@ function clone # stage. rm -rf ch ||: wget -nv -nd -c "https://clickhouse-test-reports.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/repo/clickhouse_no_subs.tar.gz" - tar -xf -C ch --strip-components=1 clickhouse_no_subs.tar.gz + tar -C ch --strip-components=1 -xf clickhouse_no_subs.tar.gz ls -lath ||: } From ebd20e6052888474ba1ef9309a07ab1ea9ca8649 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 18 Mar 2021 05:09:38 +0300 Subject: [PATCH 229/233] Update run-fuzzer.sh --- docker/test/fuzzer/run-fuzzer.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 2f6aac5c183..7707d7e0459 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -20,6 +20,7 @@ function clone # the repo from the CI as well. For local runs, start directly from the "fuzz" # stage. rm -rf ch ||: + mkdir ch ||: wget -nv -nd -c "https://clickhouse-test-reports.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/repo/clickhouse_no_subs.tar.gz" tar -C ch --strip-components=1 -xf clickhouse_no_subs.tar.gz ls -lath ||: From 1f5904fea9f332a9986634c5614b08b4b2362c42 Mon Sep 17 00:00:00 2001 From: TCeason Date: Wed, 17 Mar 2021 15:54:55 +0800 Subject: [PATCH 230/233] fix integration MaterializeMySQL test https://github.com/ClickHouse/ClickHouse/pull/21759 --- tests/integration/README.md | 5 +++-- .../materialize_with_ddl.py | 22 +++++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index cdfb6b1a70a..e4073f96449 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -12,7 +12,7 @@ You must install latest Docker from https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#set-up-the-repository Don't use Docker from your system repository. -* [pip](https://pypi.python.org/pypi/pip) and `libpq-dev`. To install: `sudo apt-get install python3-pip libpq-dev zlib1g-dev libcrypto++-dev libssl-dev` +* [pip](https://pypi.python.org/pypi/pip) and `libpq-dev`. To install: `sudo apt-get install python3-pip libpq-dev zlib1g-dev libcrypto++-dev libssl-dev libkrb5-dev` * [py.test](https://docs.pytest.org/) testing framework. To install: `sudo -H pip install pytest` * [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install: @@ -39,7 +39,8 @@ sudo -H pip install \ redis \ tzlocal \ urllib3 \ - requests-kerberos + requests-kerberos \ + dict2xml ``` (highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of "pip": `sudo apt install -y docker docker-compose python3-pytest python3-dicttoxml python3-docker python3-pymysql python3-pymongo python3-tzlocal python3-kazoo python3-psycopg2 kafka-python python3-pytest-timeout python3-minio` diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index f906c309443..1675b72e0c4 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -653,10 +653,17 @@ def mysql_kill_sync_thread_restore_test(clickhouse_node, mysql_node, service_nam check_query(clickhouse_node, "SELECT * FROM test_database.test_table FORMAT TSV", '1\n') check_query(clickhouse_node, "SELECT * FROM test_database_auto.test_table FORMAT TSV", '11\n') + + # When ClickHouse dump all history data we can query it on ClickHouse + # but it don't mean that the sync thread is already to connect to MySQL. + # So After ClickHouse can query data, insert some rows to MySQL. Use this to re-check sync successed. + mysql_node.query("INSERT INTO test_database_auto.test_table VALUES (22)") + mysql_node.query("INSERT INTO test_database.test_table VALUES (2)") + check_query(clickhouse_node, "SELECT * FROM test_database.test_table ORDER BY id FORMAT TSV", '1\n2\n') + check_query(clickhouse_node, "SELECT * FROM test_database_auto.test_table ORDER BY id FORMAT TSV", '11\n22\n') + get_sync_id_query = "select id from information_schema.processlist where STATE='Master has sent all binlog to slave; waiting for more updates'" result = mysql_node.query_and_get_data(get_sync_id_query) - assert len(result) == 2 - for row in result: row_result = {} query = "kill " + str(row[0]) + ";" @@ -671,13 +678,13 @@ def mysql_kill_sync_thread_restore_test(clickhouse_node, mysql_node, service_nam clickhouse_node.query("DETACH DATABASE test_database") clickhouse_node.query("ATTACH DATABASE test_database") - check_query(clickhouse_node, "SELECT * FROM test_database.test_table FORMAT TSV", '1\n') - - mysql_node.query("INSERT INTO test_database.test_table VALUES (2)") check_query(clickhouse_node, "SELECT * FROM test_database.test_table ORDER BY id FORMAT TSV", '1\n2\n') - mysql_node.query("INSERT INTO test_database_auto.test_table VALUES (12)") - check_query(clickhouse_node, "SELECT * FROM test_database_auto.test_table ORDER BY id FORMAT TSV", '11\n12\n') + mysql_node.query("INSERT INTO test_database.test_table VALUES (3)") + check_query(clickhouse_node, "SELECT * FROM test_database.test_table ORDER BY id FORMAT TSV", '1\n2\n3\n') + + mysql_node.query("INSERT INTO test_database_auto.test_table VALUES (33)") + check_query(clickhouse_node, "SELECT * FROM test_database_auto.test_table ORDER BY id FORMAT TSV", '11\n22\n33\n') clickhouse_node.query("DROP DATABASE test_database") clickhouse_node.query("DROP DATABASE test_database_auto") @@ -756,6 +763,7 @@ def utf8mb4_test(clickhouse_node, mysql_node, service_name): mysql_node.query("CREATE TABLE utf8mb4_test.test (id INT(11) NOT NULL PRIMARY KEY, name VARCHAR(255)) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4") mysql_node.query("INSERT INTO utf8mb4_test.test VALUES(1, '🦄'),(2, '\u2601')") clickhouse_node.query("CREATE DATABASE utf8mb4_test ENGINE = MaterializeMySQL('{}:3306', 'utf8mb4_test', 'root', 'clickhouse')".format(service_name)) + check_query(clickhouse_node, "SHOW TABLES FROM utf8mb4_test FORMAT TSV", "test\n") check_query(clickhouse_node, "SELECT id, name FROM utf8mb4_test.test ORDER BY id", "1\t\U0001F984\n2\t\u2601\n") def system_parts_test(clickhouse_node, mysql_node, service_name): From c4dfd97709972c271b3cc9e306f7be80cbd3af9f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 18 Mar 2021 10:59:48 +0300 Subject: [PATCH 231/233] Fix ActionsDAG::Index::insert --- src/Interpreters/ActionsDAG.h | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index d5a15324280..4e334bd1be8 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -120,8 +120,31 @@ public: /// Insert method doesn't check if map already have node with the same name. /// If node with the same name exists, it is removed from map, but not list. /// It is expected and used for project(), when result may have several columns with the same name. - void insert(Node * node) { map[node->result_name] = list.emplace(list.end(), node); } - void prepend(Node * node) { map[node->result_name] = list.emplace(list.begin(), node); } + void insert(Node * node) + { + auto it = list.emplace(list.end(), node); + if (auto handle = map.extract(node->result_name)) + { + handle.key() = node->result_name; /// Change string_view + handle.mapped() = it; + map.insert(std::move(handle)); + } + else + map[node->result_name] = it; + } + + void prepend(Node * node) + { + auto it = list.emplace(list.begin(), node); + if (auto handle = map.extract(node->result_name)) + { + handle.key() = node->result_name; /// Change string_view + handle.mapped() = it; + map.insert(std::move(handle)); + } + else + map[node->result_name] = it; + } /// If node with same name exists in index, replace it. Otherwise insert new node to index. void replace(Node * node) From c068538a8e2388f8096c8fedd6ff38eed7aae9aa Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 18 Mar 2021 17:04:52 +0300 Subject: [PATCH 232/233] Update ldap.md --- docs/ru/operations/external-authenticators/ldap.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/operations/external-authenticators/ldap.md b/docs/ru/operations/external-authenticators/ldap.md index b53c4cba121..312020000ea 100644 --- a/docs/ru/operations/external-authenticators/ldap.md +++ b/docs/ru/operations/external-authenticators/ldap.md @@ -55,7 +55,7 @@ - Возможные значения: `never`, `allow`, `try`, `demand` (по-умолчанию). - `tls_cert_file` — путь к файлу сертификата. - `tls_key_file` — путь к файлу ключа сертификата. -- `tls_ca_cert_file` — путь к файлу ЦС сертификата. +- `tls_ca_cert_file` — путь к файлу ЦС (certification authority) сертификата. - `tls_ca_cert_dir` — путь к каталогу, содержащему сертификаты ЦС. - `tls_cipher_suite` — разрешенный набор шифров (в нотации OpenSSL). @@ -96,7 +96,7 @@ CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; В дополнение к локально определенным пользователям, удаленный LDAP сервер может служить источником определения пользователей. Для этого укажите имя определенного ранее сервера LDAP (см. [Определение LDAP сервера](#ldap-server-definition)) в секции `ldap` внутри секции `users_directories` файла `config.xml`. -При каждой попытке авторизации ClickHouse пытается локально найти определение пользователя и аутентифицировать его как обычно. Если пользователь не находится локально, ClickHouse предполагает, что он определяется во внешнем LDAP каталоге и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю присваиваются роли из списка, указанного в секции `roles`. Кроме того, если настроена секция `role_mapping`, то выполняется LDAP поиск, а его результаты преобразуются в имена ролей и присваиваются пользователям. Все это работает при условии, что SQL-ориентированное [управлением доступом](../access-rights.md#access-control) включено, а роли созданы запросом [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). +При каждой попытке аутентификации ClickHouse пытается локально найти определение пользователя и аутентифицировать его как обычно. Если пользователь не находится локально, ClickHouse предполагает, что он определяется во внешнем LDAP каталоге и пытается "привязаться" к DN, указанному на LDAP сервере, используя предоставленные реквизиты для входа. Если попытка оказалась успешной, пользователь считается существующим и аутентифицированным. Пользователю присваиваются роли из списка, указанного в секции `roles`. Кроме того, если настроена секция `role_mapping`, то выполняется LDAP поиск, а его результаты преобразуются в имена ролей и присваиваются пользователям. Все это работает при условии, что SQL-ориентированное [управлением доступом](../access-rights.md#access-control) включено, а роли созданы запросом [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement). **Пример** From b5e344bf49155402f9de14f1eec9eb6e919a09da Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 19 Mar 2021 02:19:58 +0300 Subject: [PATCH 233/233] Update date_time_long.xml --- tests/performance/date_time_long.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/performance/date_time_long.xml b/tests/performance/date_time_long.xml index ae41602e825..0c3d85f9659 100644 --- a/tests/performance/date_time_long.xml +++ b/tests/performance/date_time_long.xml @@ -118,9 +118,9 @@ - SELECT count() FROM numbers(100000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {datetime_transform}(t, '{time_zone}')) - SELECT count() FROM numbers(100000000) WHERE NOT ignore(toDate('2017-01-01') + number % 1000 + rand() % 10 AS t, {date_transform}(t)) - SELECT count() FROM numbers(100000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {binary_function}(t, 1)) - SELECT count() FROM numbers(100000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, toStartOfInterval(t, INTERVAL 1 month)) - SELECT count() FROM numbers(100000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, date_trunc('month', t)) + SELECT count() FROM numbers(50000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {datetime_transform}(t, '{time_zone}')) + SELECT count() FROM numbers(50000000) WHERE NOT ignore(toDate('2017-01-01') + number % 1000 + rand() % 10 AS t, {date_transform}(t)) + SELECT count() FROM numbers(50000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {binary_function}(t, 1)) + SELECT count() FROM numbers(50000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, toStartOfInterval(t, INTERVAL 1 month)) + SELECT count() FROM numbers(50000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, date_trunc('month', t))