diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index cbd3bd7bec4..c52a58eac8a 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -683,3 +683,4 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 finish_check.py + python3 merge_pr.py diff --git a/.github/workflows/docs_check.yml b/.github/workflows/docs_check.yml index a513eb9216d..d69020d810e 100644 --- a/.github/workflows/docs_check.yml +++ b/.github/workflows/docs_check.yml @@ -169,3 +169,4 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 finish_check.py + python3 merge_pr.py --check-approved diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index aecf3799a5d..c677ec4bf5c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -4388,3 +4388,4 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 finish_check.py + python3 merge_pr.py --check-approved diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 2582b599d58..3458cf905da 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -5,6 +5,7 @@ set -x # core.COMM.PID-TID sysctl kernel.core_pattern='core.%e.%p-%P' +dmesg --clear ||: set -e set -u @@ -368,6 +369,7 @@ if [ -f core.zst ]; then fi rg --text -F '' server.log > fatal.log ||: +dmesg -T > dmesg.log ||: zstd --threads=0 server.log @@ -396,6 +398,7 @@ p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-s fuzzer.log server.log.zst main.log + dmesg.log ${CORE_LINK}

diff --git a/docs/en/engines/database-engines/postgresql.md b/docs/en/engines/database-engines/postgresql.md index ac19794c167..939995a61c5 100644 --- a/docs/en/engines/database-engines/postgresql.md +++ b/docs/en/engines/database-engines/postgresql.md @@ -136,3 +136,7 @@ DESCRIBE TABLE test_database.test_table; │ data │ Nullable(String) │ └────────┴───────────────────┘ ``` + +## Related content + +- Blog: [ClickHouse and PostgreSQL - a match made in data heaven - part 1](https://clickhouse.com/blog/migrating-data-between-clickhouse-postgres) diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md index 7f9659400b8..b73d28c8508 100644 --- a/docs/en/engines/table-engines/integrations/postgresql.md +++ b/docs/en/engines/table-engines/integrations/postgresql.md @@ -175,3 +175,6 @@ CREATE TABLE pg_table_schema_with_dots (a UInt32) - [The `postgresql` table function](../../../sql-reference/table-functions/postgresql.md) - [Using PostgreSQL as a dictionary source](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-postgresql) + +## Related content +- Blog: [ClickHouse and PostgreSQL - a match made in data heaven - part 1](https://clickhouse.com/blog/migrating-data-between-clickhouse-postgres) diff --git a/docs/en/operations/utilities/clickhouse-local.md b/docs/en/operations/utilities/clickhouse-local.md index 963ddfe7a02..a4fa5579638 100644 --- a/docs/en/operations/utilities/clickhouse-local.md +++ b/docs/en/operations/utilities/clickhouse-local.md @@ -120,5 +120,6 @@ Read 186 rows, 4.15 KiB in 0.035 sec., 5302 rows/sec., 118.34 KiB/sec. ## Related Content +- [Extracting, converting, and querying data in local files using clickhouse-local](https://clickhouse.com/blog/extracting-converting-querying-local-files-with-sql-clickhouse-local) - [Getting Data Into ClickHouse - Part 1](https://clickhouse.com/blog/getting-data-into-clickhouse-part-1) - [Exploring massive, real-world data sets: 100+ Years of Weather Records in ClickHouse](https://clickhouse.com/blog/real-world-data-noaa-climate-data) diff --git a/docs/en/sql-reference/aggregate-functions/reference/index.md b/docs/en/sql-reference/aggregate-functions/reference/index.md index 5d9423e0a55..bd8e72e0fec 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/index.md +++ b/docs/en/sql-reference/aggregate-functions/reference/index.md @@ -57,6 +57,7 @@ ClickHouse-specific aggregate functions: - [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md) - [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md) - [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md) +- [uniqTheta](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md) - [quantile](../../../sql-reference/aggregate-functions/reference/quantile.md) - [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md) - [quantileExact](../../../sql-reference/aggregate-functions/reference/quantileexact.md) @@ -79,4 +80,4 @@ ClickHouse-specific aggregate functions: - [cramersVBiasCorrected](./cramersvbiascorrected.md) - [theilsU](./theilsu.md) - [maxIntersections](./maxintersections.md) -- [maxIntersectionsPosition](./maxintersectionsposition.md) \ No newline at end of file +- [maxIntersectionsPosition](./maxintersectionsposition.md) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantileinterpolatedweighted.md b/docs/en/sql-reference/aggregate-functions/reference/quantileinterpolatedweighted.md new file mode 100644 index 00000000000..07fcd187217 --- /dev/null +++ b/docs/en/sql-reference/aggregate-functions/reference/quantileinterpolatedweighted.md @@ -0,0 +1,68 @@ +--- +slug: /en/sql-reference/aggregate-functions/reference/quantileInterpolatedWeighted +sidebar_position: 203 +--- + +# quantileInterpolatedWeighted + +Computes [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence using linear interpolation, taking into account the weight of each element. + +To get the interpolated value, all the passed values are combined into an array, which are then sorted by their corresponding weights. Quantile interpolation is then performed using the [weighted percentile method](https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method) by building a cumulative distribution based on weights and then a linear interpolation is performed using the weights and the values to compute the quantiles. + +When using multiple `quantile*` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) function. + +**Syntax** + +``` sql +quantileInterpolatedWeighted(level)(expr, weight) +``` + +Alias: `medianInterpolatedWeighted`. + +**Arguments** + +- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` value in the range of `[0.01, 0.99]`. Default value: 0.5. At `level=0.5` the function calculates [median](https://en.wikipedia.org/wiki/Median). +- `expr` — Expression over the column values resulting in numeric [data types](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) or [DateTime](../../../sql-reference/data-types/datetime.md). +- `weight` — Column with weights of sequence members. Weight is a number of value occurrences. + +**Returned value** + +- Quantile of the specified level. + +Type: + +- [Float64](../../../sql-reference/data-types/float.md) for numeric data type input. +- [Date](../../../sql-reference/data-types/date.md) if input values have the `Date` type. +- [DateTime](../../../sql-reference/data-types/datetime.md) if input values have the `DateTime` type. + +**Example** + +Input table: + +``` text +┌─n─┬─val─┐ +│ 0 │ 3 │ +│ 1 │ 2 │ +│ 2 │ 1 │ +│ 5 │ 4 │ +└───┴─────┘ +``` + +Query: + +``` sql +SELECT quantileInterpolatedWeighted(n, val) FROM t +``` + +Result: + +``` text +┌─quantileInterpolatedWeighted(n, val)─┐ +│ 1 │ +└──────────────────────────────────────┘ +``` + +**See Also** + +- [median](../../../sql-reference/aggregate-functions/reference/median.md#median) +- [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md index 5c9120fb8f4..57151915336 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md @@ -9,7 +9,7 @@ sidebar_position: 201 Syntax: `quantiles(level1, level2, …)(x)` -All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`, `quantilesBFloat16`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values. +All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantileInterpolatedWeighted`, `quantilesTDigest`, `quantilesBFloat16`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values. ## quantilesExactExclusive diff --git a/docs/en/sql-reference/data-types/json.md b/docs/en/sql-reference/data-types/json.md index ab1596b1760..d9099ba5ad3 100644 --- a/docs/en/sql-reference/data-types/json.md +++ b/docs/en/sql-reference/data-types/json.md @@ -6,6 +6,10 @@ sidebar_label: JSON # JSON +:::warning +This feature is experimental and is not production ready. If you need to work with JSON documents, consider using [this guide](/docs/en/guides/developer/working-with-json/json-load-data.md) instead. +::: + Stores JavaScript Object Notation (JSON) documents in a single column. `JSON` is an alias for `Object('json')`. diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index c044b972754..9d2f89c1837 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -121,7 +121,7 @@ Accepts an empty array and returns a one-element array that is equal to the defa ## range(end), range(\[start, \] end \[, step\]) -Returns an array of `UInt` numbers from `start` to `end - 1` by `step`. +Returns an array of numbers from `start` to `end - 1` by `step`. The supported types are [UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64](../data-types/int-uint.md). **Syntax** ``` sql @@ -130,31 +130,30 @@ range([start, ] end [, step]) **Arguments** -- `start` — The first element of the array. Optional, required if `step` is used. Default value: 0. [UInt](../data-types/int-uint.md) -- `end` — The number before which the array is constructed. Required. [UInt](../data-types/int-uint.md) -- `step` — Determines the incremental step between each element in the array. Optional. Default value: 1. [UInt](../data-types/int-uint.md) +- `start` — The first element of the array. Optional, required if `step` is used. Default value: 0. +- `end` — The number before which the array is constructed. Required. +- `step` — Determines the incremental step between each element in the array. Optional. Default value: 1. **Returned value** -- Array of `UInt` numbers from `start` to `end - 1` by `step`. +- Array of numbers from `start` to `end - 1` by `step`. **Implementation details** -- All arguments must be positive values: `start`, `end`, `step` are `UInt` data types, as well as elements of the returned array. +- All arguments `start`, `end`, `step` must be below data types: `UInt8`, `UInt16`, `UInt32`, `UInt64`,`Int8`, `Int16`, `Int32`, `Int64`, as well as elements of the returned array, which's type is a super type of all arguments's. - An exception is thrown if query results in arrays with a total length of more than number of elements specified by the [function_range_max_elements_in_block](../../operations/settings/settings.md#settings-function_range_max_elements_in_block) setting. - **Examples** Query: ``` sql -SELECT range(5), range(1, 5), range(1, 5, 2); +SELECT range(5), range(1, 5), range(1, 5, 2), range(-1, 5, 2); ``` Result: ```txt -┌─range(5)────┬─range(1, 5)─┬─range(1, 5, 2)─┐ -│ [0,1,2,3,4] │ [1,2,3,4] │ [1,3] │ -└─────────────┴─────────────┴────────────────┘ +┌─range(5)────┬─range(1, 5)─┬─range(1, 5, 2)─┬─range(-1, 5, 2)─┐ +│ [0,1,2,3,4] │ [1,2,3,4] │ [1,3] │ [-1,1,3] │ +└─────────────┴─────────────┴────────────────┴─────────────────┘ ``` ## array(x1, …), operator \[x1, …\] diff --git a/docs/en/sql-reference/table-functions/generate.md b/docs/en/sql-reference/table-functions/generate.md index dd56b47cd3a..380c8364090 100644 --- a/docs/en/sql-reference/table-functions/generate.md +++ b/docs/en/sql-reference/table-functions/generate.md @@ -39,3 +39,16 @@ SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64( │ [68] │ -67417.0770 │ ('2080-03-12 14:17:31.269','110425e5-413f-10a6-05ba-fa6b3e929f15') │ └──────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ ``` + +```sql +CREATE TABLE random (a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) engine=Memory; +INSERT INTO random SELECT * FROM generateRandom() LIMIT 2; +SELECT * FROM random; +``` + +```text +┌─a────────────────────────────┬────────────d─┬─c──────────────────────────────────────────────────────────────────┐ +│ [] │ 68091.8197 │ ('2037-10-02 12:44:23.368','039ecab7-81c2-45ee-208c-844e5c6c5652') │ +│ [8,-83,0,-22,65,9,-30,28,64] │ -186233.4909 │ ('2062-01-11 00:06:04.124','69563ea1-5ad1-f870-16d8-67061da0df25') │ +└──────────────────────────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ +``` \ No newline at end of file diff --git a/docs/zh/sql-reference/functions/array-functions.md b/docs/zh/sql-reference/functions/array-functions.md index 565304710cc..d150b94b8af 100644 --- a/docs/zh/sql-reference/functions/array-functions.md +++ b/docs/zh/sql-reference/functions/array-functions.md @@ -117,7 +117,7 @@ SELECT notEmpty([1,2]); ## range(end), range(\[start, \] end \[, step\]) {#range} -返回一个以`step`作为增量步长的从`start`到`end - 1`的`UInt`类型数字数组。 +返回一个以`step`作为增量步长的从`start`到`end - 1`的整形数字数组， 支持类型包括[`UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64`](../data-types/int-uint.md)。 **语法** ``` sql @@ -126,31 +126,30 @@ range([start, ] end [, step]) **参数** -- `start` — 数组的第一个元素。可选项，如果设置了`step`时同样需要`start`，默认值为：0，类型为[UInt](../data-types/int-uint.md)。 -- `end` — 计数到`end`结束，但不包括`end`，必填项，类型为[UInt](../data-types/int-uint.md)。 -- `step` — 确定数组中每个元素之间的增量步长。可选项，默认值为：1，类型为[UInt](../data-types/int-uint.md)。 +- `start` — 数组的第一个元素。可选项，如果设置了`step`时同样需要`start`，默认值为：0。 +- `end` — 计数到`end`结束，但不包括`end`，必填项。 +- `step` — 确定数组中每个元素之间的增量步长。可选项，默认值为：1。 **返回值** -- 以`step`作为增量步长的从`start`到`end - 1`的`UInt`类型数字数组。 +- 以`step`作为增量步长的从`start`到`end - 1`的数字数组。 **注意事项** -- 所有参数必须是正值：`start`、`end`、`step`，类型均为`UInt`，结果数组的元素与此相同。 +- 所有参数`start`、`end`、`step`必须属于以下几种类型之一：[`UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64`](../data-types/int-uint.md)。结果数组的元素数据类型为所有入参类型的最小超类，也必须属于以上几种类型之一。 - 如果查询结果的数组总长度超过[function_range_max_elements_in_block](../../operations/settings/settings.md#settings-function_range_max_elements_in_block)指定的元素数，将会抛出异常。 - **示例** 查询语句: ``` sql -SELECT range(5), range(1, 5), range(1, 5, 2); +SELECT range(5), range(1, 5), range(1, 5, 2), range(-1, 5, 2); ``` 结果: ```txt -┌─range(5)────┬─range(1, 5)─┬─range(1, 5, 2)─┐ -│ [0,1,2,3,4] │ [1,2,3,4] │ [1,3] │ -└─────────────┴─────────────┴────────────────┘ +┌─range(5)────┬─range(1, 5)─┬─range(1, 5, 2)─┬─range(-1, 5, 2)─┐ +│ [0,1,2,3,4] │ [1,2,3,4] │ [1,3] │ [-1,1,3] │ +└─────────────┴─────────────┴────────────────┴─────────────────┘ ``` ## array(x1, …), operator \[x1, …\] {#arrayx1-operator-x1} diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 742838d6433..419b80ccff2 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -140,6 +140,7 @@ namespace CurrentMetrics namespace ProfileEvents { extern const Event MainConfigLoads; + extern const Event ServerStartupMilliseconds; } namespace fs = std::filesystem; @@ -652,6 +653,8 @@ static void sanityChecks(Server & server) int Server::main(const std::vector & /*args*/) try { + Stopwatch startup_watch; + Poco::Logger * log = &logger(); UseSSL use_ssl; @@ -1822,6 +1825,9 @@ try LOG_INFO(log, "Ready for connections."); } + startup_watch.stop(); + ProfileEvents::increment(ProfileEvents::ServerStartupMilliseconds, startup_watch.elapsedMilliseconds()); + try { global_context->startClusterDiscovery(); diff --git a/src/AggregateFunctions/AggregateFunctionHistogram.h b/src/AggregateFunctions/AggregateFunctionHistogram.h index c559b3f115f..ac81f7466fa 100644 --- a/src/AggregateFunctions/AggregateFunctionHistogram.h +++ b/src/AggregateFunctions/AggregateFunctionHistogram.h @@ -207,7 +207,7 @@ private: { // Fuse points if their text representations differ only in last digit auto min_diff = 10 * (points[left].mean + points[right].mean) * std::numeric_limits::epsilon(); - if (points[left].mean + min_diff >= points[right].mean) + if (points[left].mean + std::fabs(min_diff) >= points[right].mean) { points[left] = points[left] + points[right]; } diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h index 6427d03f089..49157acf690 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -232,6 +232,9 @@ struct NameQuantilesExactInclusive { static constexpr auto name = "quantilesExac struct NameQuantileExactWeighted { static constexpr auto name = "quantileExactWeighted"; }; struct NameQuantilesExactWeighted { static constexpr auto name = "quantilesExactWeighted"; }; +struct NameQuantileInterpolatedWeighted { static constexpr auto name = "quantileInterpolatedWeighted"; }; +struct NameQuantilesInterpolatedWeighted { static constexpr auto name = "quantilesInterpolatedWeighted"; }; + struct NameQuantileTiming { static constexpr auto name = "quantileTiming"; }; struct NameQuantileTimingWeighted { static constexpr auto name = "quantileTimingWeighted"; }; struct NameQuantilesTiming { static constexpr auto name = "quantilesTiming"; }; diff --git a/src/AggregateFunctions/AggregateFunctionQuantileInterpolatedWeighted.cpp b/src/AggregateFunctions/AggregateFunctionQuantileInterpolatedWeighted.cpp new file mode 100644 index 00000000000..68b42376df7 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionQuantileInterpolatedWeighted.cpp @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +struct Settings; + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +namespace +{ + + template using FuncQuantileInterpolatedWeighted = AggregateFunctionQuantile, NameQuantileInterpolatedWeighted, true, void, false>; + template using FuncQuantilesInterpolatedWeighted = AggregateFunctionQuantile, NameQuantilesInterpolatedWeighted, true, void, true>; + + template