From c1c4cabe9ffd5de032bc26fbb2ebd19b316e0336 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 27 Aug 2024 12:41:02 +0200 Subject: [PATCH 01/10] Update toReference*Num function documentation --- .../functions/date-time-functions.md | 306 +++++++++++++++++- 1 file changed, 298 insertions(+), 8 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 4f5e5a5d716..cb32d1a8d63 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1625,37 +1625,327 @@ If unit `WEEK` was specified, `toStartOfInterval` assumes that weeks start on Mo Converts a date with time to a certain fixed date, while preserving the time. +**Syntax** + +```sql +toTime(date[,timezone]) +``` + +**Arguments** + +- `date` — Date to convert to a time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). +- `timezone` (optional) — Timezone for the returned value. [String](../data-types/string.md). + +**Returned value** + +- DateTime with date equated to `1970-01-02` while preserving the time. [DateTime](../data-types/datetime.md). + +:::note +If the `date` input argument contained sub-second components, +they will be dropped in the returned `DateTime` value with second-accuracy. +::: + +**Example** + +Query: + +```sql +SELECT toTime(toDateTime64('1970-12-10 01:20:30.3000',3)) AS result, toTypeName(result); +``` + +Result: + +```response + +``` + ## toRelativeYearNum -Converts a date, or date with time, to the number of the year, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of years elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeYearNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of years from a fixed reference point in the past. [UInt16](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeYearNum(toDate('2002-12-08')) AS y1, + toRelativeYearNum(toDate('2010-10-26')) AS y2 +``` + +Result: + +```response +┌───y1─┬───y2─┐ +│ 2002 │ 2010 │ +└──────┴──────┘ +``` ## toRelativeQuarterNum -Converts a date, or date with time, to the number of the quarter, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of quarters elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeQuarterNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of quarters from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeQuarterNum(toDate('1993-11-25')) AS q1, + toRelativeQuarterNum(toDate('2005-01-05')) AS q2 +``` + +Result: + +```response +┌───q1─┬───q2─┐ +│ 7975 │ 8020 │ +└──────┴──────┘ +``` ## toRelativeMonthNum -Converts a date, or date with time, to the number of the month, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of months elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeMonthNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of months from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeMonthNum(toDate('2001-04-25')) AS m1, + toRelativeMonthNum(toDate('2009-07-08')) AS m2 +``` + +Result: + +```response +┌────m1─┬────m2─┐ +│ 24016 │ 24115 │ +└───────┴───────┘ +``` ## toRelativeWeekNum -Converts a date, or date with time, to the number of the week, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of weeks elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeWeekNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of weeks from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeWeekNum(toDate('2000-02-29')) AS w1, + toRelativeWeekNum(toDate('2001-01-12')) AS w2 +``` + +Result: + +```response +┌───w1─┬───w2─┐ +│ 1574 │ 1619 │ +└──────┴──────┘ +``` ## toRelativeDayNum -Converts a date, or date with time, to the number of the day, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of days elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeDayNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of days from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeDayNum(toDate('1993-10-05')) AS d1, + toRelativeDayNum(toDate('2000-09-20')) AS d2 +``` + +Result: + +```response +┌───d1─┬────d2─┐ +│ 8678 │ 11220 │ +└──────┴───────┘ +``` ## toRelativeHourNum -Converts a date, or date with time, to the number of the hour, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of hours elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeHourNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of hours from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeHourNum(toDateTime('1993-10-05 05:20:36')) AS h1, + toRelativeHourNum(toDateTime('2000-09-20 14:11:29')) AS h2 +``` + +Result: + +```response +┌─────h1─┬─────h2─┐ +│ 208276 │ 269292 │ +└────────┴────────┘ +``` ## toRelativeMinuteNum -Converts a date, or date with time, to the number of the minute, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of minutes elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeMinuteNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of minutes from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeMinuteNum(toDateTime('1993-10-05 05:20:36')) AS m1, + toRelativeMinuteNum(toDateTime('2000-09-20 14:11:29')) AS m2 +``` + +Result: + +```response +┌───────m1─┬───────m2─┐ +│ 12496580 │ 16157531 │ +└──────────┴──────────┘ +``` ## toRelativeSecondNum -Converts a date, or date with time, to the number of the second, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of the seconds elapsed since a certain fixed point in the past. + +**Syntax** + +```sql +toRelativeSecondNum(date) +``` + +**Arguments** + +- `date` — Date or date with time. [Date](../data-types/date.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Returned value** + +- The number of seconds from a fixed reference point in the past. [UInt32](../data-types/int-uint.md). + +**Example** + +Query: + +```sql +SELECT + toRelativeSecondNum(toDateTime('1993-10-05 05:20:36')) AS s1, + toRelativeSecondNum(toDateTime('2000-09-20 14:11:29')) AS s2 +``` + +Result: + +```response +┌────────s1─┬────────s2─┐ +│ 749794836 │ 969451889 │ +└───────────┴───────────┘ +``` ## toISOYear From 29664d12fc7eedb977147872bdebced05ecfd058 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 27 Aug 2024 12:44:28 +0200 Subject: [PATCH 02/10] update example of toTime --- docs/en/sql-reference/functions/date-time-functions.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index cb32d1a8d63..46d36dbe92f 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1656,7 +1656,9 @@ SELECT toTime(toDateTime64('1970-12-10 01:20:30.3000',3)) AS result, toTypeName( Result: ```response - +┌──────────────result─┬─toTypeName(result)─┐ +│ 1970-01-02 01:20:30 │ DateTime │ +└─────────────────────┴────────────────────┘ ``` ## toRelativeYearNum From 6febfc78b3418c27d2c5c12787da554ee3a13fb3 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 27 Aug 2024 14:52:46 +0200 Subject: [PATCH 03/10] updates docs for DateTime64OrZero/Null/Default --- .../functions/type-conversion-functions.md | 131 +++++++++++++++++- 1 file changed, 129 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index edd04580f27..625ed4462d7 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -3857,7 +3857,7 @@ Result: ## toDateTime64 -Converts the argument to the [DateTime64](../data-types/datetime64.md) data type. +Converts an input value to a value of type [DateTime64](../data-types/datetime64.md). **Syntax** @@ -3869,7 +3869,7 @@ toDateTime64(expr, scale, [timezone]) - `expr` — The value. [String](../data-types/string.md), [UInt32](../data-types/int-uint.md), [Float](../data-types/float.md) or [DateTime](../data-types/datetime.md). - `scale` - Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. -- `timezone` - Time zone of the specified datetime64 object. +- `timezone` (optional) - Time zone of the specified datetime64 object. **Returned value** @@ -3928,10 +3928,137 @@ SELECT toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul') AS value, toTypeN ## toDateTime64OrZero +Like [toDateTime64](#todatetime64), this function converts an input value to a value of type [DateTime64](../data-types/datetime64.md) but returns the min value of [DateTime64](../data-types/datetime64.md) if an invalid argument is received. + +**Syntax** + +``` sql +toDateTime64OrZero(expr, scale, [timezone]) +``` + +**Arguments** + +- `expr` — The value. [String](../data-types/string.md), [UInt32](../data-types/int-uint.md), [Float](../data-types/float.md) or [DateTime](../data-types/datetime.md). +- `scale` - Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. +- `timezone` (optional) - Time zone of the specified DateTime64 object. + +**Returned value** + +- A calendar date and time of day, with sub-second precision, otherwise the minimum value of `DateTime64`: `1970-01-01 01:00:00.000`. [DateTime64](../data-types/datetime64.md). + +**Example** + +Query: + +```sql +SELECT toDateTime64OrZero('2008-10-12 00:00:00 00:30:30', 3) AS invalid_arg +``` + +Result: + +```response +┌─────────────invalid_arg─┐ +│ 1970-01-01 01:00:00.000 │ +└─────────────────────────┘ +``` + +**See also** + +- [toDateTime64](#todatetime64). +- [toDateTime64OrNull](#todatetime64ornull). +- [toDateTime64OrDefault](#todatetime64ordefault). + ## toDateTime64OrNull +Like [toDateTime64](#todatetime64), this function converts an input value to a value of type [DateTime64](../data-types/datetime64.md) but returns `NULL` if an invalid argument is received. + +**Syntax** + +``` sql +toDateTime64OrNull(expr, scale, [timezone]) +``` + +**Arguments** + +- `expr` — The value. [String](../data-types/string.md), [UInt32](../data-types/int-uint.md), [Float](../data-types/float.md) or [DateTime](../data-types/datetime.md). +- `scale` - Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. +- `timezone` (optional) - Time zone of the specified DateTime64 object. + +**Returned value** + +- A calendar date and time of day, with sub-second precision, otherwise `NULL`. [DateTime64](../data-types/datetime64.md)/[NULL](../data-types/nullable.md). + +**Example** + +Query: + +```sql +SELECT + toDateTime64OrNull('1976-10-18 00:00:00.30', 3) AS valid_arg, + toDateTime64OrNull('1976-10-18 00:00:00 30', 3) AS invalid_arg +``` + +Result: + +```response +┌───────────────valid_arg─┬─invalid_arg─┐ +│ 1976-10-18 00:00:00.300 │ ᴺᵁᴸᴸ │ +└─────────────────────────┴─────────────┘ +``` + +**See also** + +- [toDateTime64](#todatetime64). +- [toDateTime64OrZero](#todatetime64orzero). +- [toDateTime64OrDefault](#todatetime64ordefault). + ## toDateTime64OrDefault +Like [toDateTime64](#todatetime64), this function converts an input value to a value of type [DateTime64](../data-types/datetime64.md), +but returns either the default value of [DateTime64](../data-types/datetime64.md) +or the provided default if an invalid argument is received. + +**Syntax** + +``` sql +toDateTime64OrNull(expr, scale, [timezone, default]) +``` + +**Arguments** + +- `expr` — The value. [String](../data-types/string.md), [UInt32](../data-types/int-uint.md), [Float](../data-types/float.md) or [DateTime](../data-types/datetime.md). +- `scale` - Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. +- `timezone` (optional) - Time zone of the specified DateTime64 object. +- `default` (optional) - Default value to return if an invalid argument is received. [DateTime64](../data-types/datetime64.md). + +**Returned value** + +- A calendar date and time of day, with sub-second precision, otherwise the minimum value of `DateTime64` or the `default` value if provided. [DateTime64](../data-types/datetime64.md). + +**Example** + +Query: + +```sql +SELECT + toDateTime64OrDefault('1976-10-18 00:00:00 30', 3) AS invalid_arg, + toDateTime64OrDefault('1976-10-18 00:00:00 30', 3, 'UTC', toDateTime64('2001-01-01 00:00:00.00',3)) AS invalid_arg_with_default +``` + +Result: + +```response +┌─────────────invalid_arg─┬─invalid_arg_with_default─┐ +│ 1970-01-01 01:00:00.000 │ 2000-12-31 23:00:00.000 │ +└─────────────────────────┴──────────────────────────┘ +``` + +**See also** + +- [toDateTime64](#todatetime64). +- [toDateTime64OrZero](#todatetime64orzero). +- [toDateTime64OrNull](#todatetime64ornull). + ## toDecimal32 Converts an input value to a value of type [`Decimal(9, S)`](../data-types/decimal.md) with scale of `S`. Throws an exception in case of an error. From ad08db39e5309602736eefd1dbb535a223993c1f Mon Sep 17 00:00:00 2001 From: imddba Date: Wed, 28 Aug 2024 20:05:06 +0800 Subject: [PATCH 04/10] Add CKibana as third-party GUI --- docs/en/interfaces/third-party/gui.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index 8d9dce983bc..16d9b66e17f 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -233,6 +233,16 @@ Features: - Useful tools: Zookeeper data exploration, query EXPLAIN, kill queries, etc. - Visualization metric charts: queries and resource usage, number of merges/mutation, merge performance, query performance, etc. +### CKibana {#ckibana} + +[CKibana](https://github.com/TongchengOpenSource/ckibana) is a lightweight service that allows you to effortlessly search, explore, and visualize ClickHouse data using the native Kibana UI. + +Features: + +- Translates chart requests from the native Kibana UI into ClickHouse query syntax. +- Supports advanced features such as sampling and caching to enhance query performance. +- Minimizes the learning cost for users after migrating from ElasticSearch to ClickHouse. + ## Commercial {#commercial} ### DataGrip {#datagrip} From 9d07d002f109a45b1348297342dce7f575f31bfe Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 6 Sep 2024 16:20:30 +0000 Subject: [PATCH 05/10] update keeper client docs --- docs/en/operations/utilities/clickhouse-keeper-client.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/operations/utilities/clickhouse-keeper-client.md b/docs/en/operations/utilities/clickhouse-keeper-client.md index a66ecbc1372..fbfdd66d1a3 100644 --- a/docs/en/operations/utilities/clickhouse-keeper-client.md +++ b/docs/en/operations/utilities/clickhouse-keeper-client.md @@ -47,6 +47,8 @@ keeper foo bar - `ls '[path]'` -- Lists the nodes for the given path (default: cwd) - `cd '[path]'` -- Changes the working path (default `.`) +- `cp '' ''` -- Copies 'src' node to 'dest' path +- `mv '' ''` -- Moves 'src' node to the 'dest' path - `exists ''` -- Returns `1` if node exists, `0` otherwise - `set '' [version]` -- Updates the node's value. Only updates if version matches (default: -1) - `create '' [mode]` -- Creates new node with the set value From 3431e2ec4ab7239d04b98212ea52171d8bb8e5ae Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 6 Sep 2024 16:52:31 +0000 Subject: [PATCH 06/10] add dest to aspell ignore --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index d10db5f0d3d..d4166e221a7 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -2583,6 +2583,7 @@ sqlinsert sqlite sqrt src +dest srcReplicas sshkey stackoverflow From 942d0fab4a7f5583992dedc63a31db6b0f1f39ee Mon Sep 17 00:00:00 2001 From: imddba Date: Mon, 9 Sep 2024 20:24:16 +0800 Subject: [PATCH 07/10] spelling fixes --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 14b1d90c319..89f63f73928 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -151,6 +151,7 @@ ChannelID Cidr Ciphertext CityHash +CKibana Clangd ClickBench ClickCat @@ -247,6 +248,7 @@ DoubleDelta Doxygen Durre ECMA +ElasticSearch ETag Ecto EdgeAngle @@ -471,6 +473,7 @@ KeeperMap KeeperOutstandingRequests Kerberos Khanna +Kibana KittenHouse Klickhouse Kolmogorov @@ -1366,6 +1369,7 @@ cipherList ciphertext cityHash cityhash +ckibana ckman clangd cli From e6e79c3c4ada9f2eb82e6e9c63c736663cfb8e02 Mon Sep 17 00:00:00 2001 From: Pablo Marcos Date: Mon, 9 Sep 2024 11:36:18 +0000 Subject: [PATCH 08/10] Fix minor typos --- src/Client/ConnectionPoolWithFailover.h | 2 +- src/Client/HedgedConnectionsFactory.cpp | 2 +- src/Core/Defines.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index 308644ebbdb..19232e0ee7c 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -98,7 +98,7 @@ public: std::vector getShuffledPools(const Settings & settings, GetPriorityFunc priority_func = {}, bool use_slowdown_count = false); - size_t getMaxErrorCup() const { return Base::max_error_cap; } + size_t getMaxErrorCap() const { return Base::max_error_cap; } void updateSharedError(std::vector & shuffled_pools) { diff --git a/src/Client/HedgedConnectionsFactory.cpp b/src/Client/HedgedConnectionsFactory.cpp index be7397b0fad..df63a124539 100644 --- a/src/Client/HedgedConnectionsFactory.cpp +++ b/src/Client/HedgedConnectionsFactory.cpp @@ -327,7 +327,7 @@ HedgedConnectionsFactory::State HedgedConnectionsFactory::processFinishedConnect ShuffledPool & shuffled_pool = shuffled_pools[index]; LOG_INFO(log, "Connection failed at try №{}, reason: {}", (shuffled_pool.error_count + 1), fail_message); - shuffled_pool.error_count = std::min(pool->getMaxErrorCup(), shuffled_pool.error_count + 1); + shuffled_pool.error_count = std::min(pool->getMaxErrorCap(), shuffled_pool.error_count + 1); shuffled_pool.slowdown_count = 0; if (shuffled_pool.error_count >= max_tries) diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 6df335a9c8f..3341669fed2 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -40,7 +40,7 @@ static constexpr auto SHOW_CHARS_ON_SYNTAX_ERROR = ptrdiff_t(160); /// each period reduces the error counter by 2 times /// too short a period can cause errors to disappear immediately after creation. static constexpr auto DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD = 60; -/// replica error max cap, this is to prevent replica from accumulating too many errors and taking to long to recover. +/// replica error max cap, this is to prevent replica from accumulating too many errors and taking too long to recover. static constexpr auto DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT = 1000; /// The boundary on which the blocks for asynchronous file operations should be aligned. From 67b57eb89fb8724c542cc197a7371c3e60009566 Mon Sep 17 00:00:00 2001 From: Pablo Marcos Date: Mon, 9 Sep 2024 13:55:14 +0000 Subject: [PATCH 09/10] Fix undefined behavior if all connection tries fail For some reason, getManyCheckedForInsert was calling getManyImpl with skip_unavailable_endpoints=nullptr, which resulted in getManyImpl using the `skip_unavailable_shards` settings. Since this is true by default, min_entries was set to 0. Having min_entries set to 0 while using PoolMode::GET_ONE was strange, to say the least. There was an edge case where if all connection attempts failed and min_entries was 0, it was returning an empty vector. That was not considered to be possible by the caller and it was getting the front entry of an empty vector, causing undefined behavior. Conclusion: set `skip_unavailable_endpoints=false` explicitly in getManyCheckedForInsert so that min_entries=1. In case all connections fail, an exception will be thrown. Also, add some defensive programming to ensure we never ever try to get the front element of an empty vector. --- src/Client/ConnectionPoolWithFailover.cpp | 2 +- src/Client/ConnectionPoolWithFailover.h | 2 +- src/Common/PoolWithFailoverBase.h | 8 +++++++- src/Storages/Distributed/DistributedAsyncInsertBatch.cpp | 6 ++---- .../Distributed/DistributedAsyncInsertDirectoryQueue.cpp | 3 +-- src/Storages/Distributed/DistributedSink.cpp | 3 +-- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index fb895d17763..a5c14dc9957 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -168,7 +168,7 @@ std::vector ConnectionPoolWithFailover::g { return tryGetEntry(pool, timeouts, fail_message, settings, &table_to_check, /*async_callback=*/ {}); }; return getManyImpl(settings, pool_mode, try_get_entry, - /*skip_unavailable_endpoints=*/ std::nullopt, + /*skip_unavailable_endpoints=*/ false, /// skip_unavailable_endpoints is used to get the min number of entries, and we need at least one /*priority_func=*/ {}, settings.distributed_insert_skip_read_only_replicas); } diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index 19232e0ee7c..6db52140854 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -42,7 +42,7 @@ public: size_t max_error_cap = DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT); using Entry = IConnectionPool::Entry; - using PoolWithFailoverBase::checkTryResultIsValid; + using PoolWithFailoverBase::getValidTryResult; /** Allocates connection to work. */ Entry get(const ConnectionTimeouts & timeouts) override; diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index 53a746c316e..a0da72071f9 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -122,12 +122,18 @@ public: return result.entry.isNull() || !result.is_usable || (skip_read_only_replicas && result.is_readonly); } - void checkTryResultIsValid(const TryResult & result, bool skip_read_only_replicas) const + TryResult getValidTryResult(const std::vector & results, bool skip_read_only_replicas) const { + if (results.empty()) + throw DB::Exception(DB::ErrorCodes::ALL_CONNECTION_TRIES_FAILED, "Cannot get any valid connection. Probably all connection tries failed"); + + auto result = results.front(); if (isTryResultInvalid(result, skip_read_only_replicas)) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Got an invalid connection result: entry.isNull {}, is_usable {}, is_up_to_date {}, delay {}, is_readonly {}, skip_read_only_replicas {}", result.entry.isNull(), result.is_usable, result.is_up_to_date, result.delay, result.is_readonly, skip_read_only_replicas); + + return result; } size_t getPoolSize() const { return nested_pools.size(); } diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp index 2db2bdf3981..625c64128e7 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp @@ -242,8 +242,7 @@ void DistributedAsyncInsertBatch::sendBatch(const SettingsChanges & settings_cha auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName()); - auto result = results.front(); - parent.pool->checkTryResultIsValid(result, insert_settings.distributed_insert_skip_read_only_replicas); + auto result = parent.pool->getValidTryResult(results, insert_settings.distributed_insert_skip_read_only_replicas); connection = std::move(result.entry); compression_expected = connection->getCompression() == Protocol::Compression::Enable; @@ -302,8 +301,7 @@ void DistributedAsyncInsertBatch::sendSeparateFiles(const SettingsChanges & sett auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName()); - auto result = results.front(); - parent.pool->checkTryResultIsValid(result, insert_settings.distributed_insert_skip_read_only_replicas); + auto result = parent.pool->getValidTryResult(results, insert_settings.distributed_insert_skip_read_only_replicas); auto connection = std::move(result.entry); bool compression_expected = connection->getCompression() == Protocol::Compression::Enable; diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp index 2400de4c07c..7f368102dfd 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -415,8 +415,7 @@ void DistributedAsyncInsertDirectoryQueue::processFile(std::string & file_path, auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName()); - auto result = results.front(); - pool->checkTryResultIsValid(result, insert_settings.distributed_insert_skip_read_only_replicas); + auto result = pool->getValidTryResult(results, insert_settings.distributed_insert_skip_read_only_replicas); auto connection = std::move(result.entry); LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)", diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 39f75fd7d56..f01ea10065c 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -377,8 +377,7 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si /// NOTE: INSERT will also take into account max_replica_delay_for_distributed_queries /// (anyway fallback_to_stale_replicas_for_distributed_queries=true by default) auto results = shard_info.pool->getManyCheckedForInsert(timeouts, settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName()); - auto result = results.front(); - shard_info.pool->checkTryResultIsValid(result, settings.distributed_insert_skip_read_only_replicas); + auto result = shard_info.pool->getValidTryResult(results, settings.distributed_insert_skip_read_only_replicas); job.connection_entry = std::move(result.entry); } else From f7dee100309a0c47d6daaad19164b4e968fd3b8d Mon Sep 17 00:00:00 2001 From: Pablo Marcos Date: Mon, 9 Sep 2024 14:38:06 +0000 Subject: [PATCH 10/10] Make a more assertive exception text --- src/Common/PoolWithFailoverBase.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index a0da72071f9..989ffd888f8 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -125,7 +125,7 @@ public: TryResult getValidTryResult(const std::vector & results, bool skip_read_only_replicas) const { if (results.empty()) - throw DB::Exception(DB::ErrorCodes::ALL_CONNECTION_TRIES_FAILED, "Cannot get any valid connection. Probably all connection tries failed"); + throw DB::Exception(DB::ErrorCodes::ALL_CONNECTION_TRIES_FAILED, "Cannot get any valid connection because all connection tries failed"); auto result = results.front(); if (isTryResultInvalid(result, skip_read_only_replicas))