diff --git a/base/base/IPv4andIPv6.h b/base/base/IPv4andIPv6.h
new file mode 100644
index 00000000000..0e97d83b07e
--- /dev/null
+++ b/base/base/IPv4andIPv6.h
@@ -0,0 +1,53 @@
+#pragma once
+
+#include
+#include
+#include
+
+namespace DB
+{
+
+ using IPv4 = StrongTypedef;
+
+ struct IPv6 : StrongTypedef
+ {
+ constexpr IPv6() = default;
+ constexpr explicit IPv6(const UInt128 & x) : StrongTypedef(x) {}
+ constexpr explicit IPv6(UInt128 && x) : StrongTypedef(std::move(x)) {}
+
+ IPv6 & operator=(const UInt128 & rhs) { StrongTypedef::operator=(rhs); return *this; }
+ IPv6 & operator=(UInt128 && rhs) { StrongTypedef::operator=(std::move(rhs)); return *this; }
+
+ bool operator<(const IPv6 & rhs) const
+ {
+ return
+ memcmp16(
+ reinterpret_cast(toUnderType().items),
+ reinterpret_cast(rhs.toUnderType().items)
+ ) < 0;
+ }
+
+ bool operator>(const IPv6 & rhs) const
+ {
+ return
+ memcmp16(
+ reinterpret_cast(toUnderType().items),
+ reinterpret_cast(rhs.toUnderType().items)
+ ) > 0;
+ }
+
+ bool operator==(const IPv6 & rhs) const
+ {
+ return
+ memcmp16(
+ reinterpret_cast(toUnderType().items),
+ reinterpret_cast(rhs.toUnderType().items)
+ ) == 0;
+ }
+
+ bool operator<=(const IPv6 & rhs) const { return !operator>(rhs); }
+ bool operator>=(const IPv6 & rhs) const { return !operator<(rhs); }
+ bool operator!=(const IPv6 & rhs) const { return !operator==(rhs); }
+ };
+
+}
diff --git a/base/base/TypeName.h b/base/base/TypeName.h
index b3ea47fad5d..9005b5a2bf4 100644
--- a/base/base/TypeName.h
+++ b/base/base/TypeName.h
@@ -2,6 +2,7 @@
#include "Decimal.h"
#include "UUID.h"
+#include "IPv4andIPv6.h"
namespace DB
{
@@ -35,6 +36,8 @@ TN_MAP(Float32)
TN_MAP(Float64)
TN_MAP(String)
TN_MAP(UUID)
+TN_MAP(IPv4)
+TN_MAP(IPv6)
TN_MAP(Decimal32)
TN_MAP(Decimal64)
TN_MAP(Decimal128)
diff --git a/docker/images.json b/docker/images.json
index bc8e9924955..323f53c0ae6 100644
--- a/docker/images.json
+++ b/docker/images.json
@@ -63,10 +63,6 @@
"name": "clickhouse/integration-tests-runner",
"dependent": []
},
- "docker/test/testflows/runner": {
- "name": "clickhouse/testflows-runner",
- "dependent": []
- },
"docker/test/fasttest": {
"name": "clickhouse/fasttest",
"dependent": []
diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine
index 22d6282d71c..8b85b886fec 100644
--- a/docker/server/Dockerfile.alpine
+++ b/docker/server/Dockerfile.alpine
@@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \
# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="22.12.1.1752"
+ARG VERSION="22.12.2.25"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# user/group precreated explicitly with fixed uid/gid on purpose.
diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu
index 3135ec508de..9348bfdcf7a 100644
--- a/docker/server/Dockerfile.ubuntu
+++ b/docker/server/Dockerfile.ubuntu
@@ -21,7 +21,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="22.12.1.1752"
+ARG VERSION="22.12.2.25"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# set non-empty deb_location_url url to create a docker image
diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index 164f2e28d76..54ea9e0b00f 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -262,14 +262,17 @@ quit
if [ "$server_died" == 1 ]
then
# The server has died.
- if ! grep --text -ao "Received signal.*\|Logical error.*\|Assertion.*failed\|Failed assertion.*\|.*runtime error: .*\|.*is located.*\|SUMMARY: AddressSanitizer:.*\|SUMMARY: MemorySanitizer:.*\|SUMMARY: ThreadSanitizer:.*\|.*_LIBCPP_ASSERT.*" server.log > description.txt
+ if ! grep -E --text -o 'Received signal.*|Logical error.*|Assertion.*failed|Failed assertion.*|.*runtime error: .*|.*is located.*|(SUMMARY|ERROR): [a-zA-Z]+Sanitizer:.*|.*_LIBCPP_ASSERT.*' server.log > description.txt
then
echo "Lost connection to server. See the logs." > description.txt
fi
- if grep -E --text 'Sanitizer: (out-of-memory|failed to allocate)' description.txt
+ IS_SANITIZED=$(clickhouse-local --query "SELECT value LIKE '%-fsanitize=%' FROM system.build_options WHERE name = 'CXX_FLAGS'")
+
+ if [ "${IS_SANITIZED}" -eq "1" ] && grep -E --text 'Sanitizer: (out-of-memory|out of memory|failed to allocate|Child process was terminated by signal 9)' description.txt
then
# OOM of sanitizer is not a problem we can handle - treat it as success, but preserve the description.
+ # Why? Because sanitizers have the memory overhead, that is not controllable from inside clickhouse-server.
task_exit_code=0
echo "success" > status.txt
else
diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile
index ccfd63c8ed0..ce5bae2a031 100644
--- a/docker/test/integration/runner/Dockerfile
+++ b/docker/test/integration/runner/Dockerfile
@@ -8,6 +8,7 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN apt-get update \
&& env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
+ adduser \
ca-certificates \
bash \
btrfs-progs \
diff --git a/docs/changelogs/v22.12.2.25-stable.md b/docs/changelogs/v22.12.2.25-stable.md
new file mode 100644
index 00000000000..e38fd2045ac
--- /dev/null
+++ b/docs/changelogs/v22.12.2.25-stable.md
@@ -0,0 +1,29 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v22.12.2.25-stable (c790cfd4465) FIXME as compared to v22.12.1.1752-stable (688e488e930)
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#44381](https://github.com/ClickHouse/ClickHouse/issues/44381): In rare cases, we don't rebuild binaries, because another task with a similar prefix succeeded. E.g. `binary_darwin` didn't restart because `binary_darwin_aarch64`. [#44311](https://github.com/ClickHouse/ClickHouse/pull/44311) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#44561](https://github.com/ClickHouse/ClickHouse/issues/44561): Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in official stable or prestable release)
+
+* Backported in [#44739](https://github.com/ClickHouse/ClickHouse/issues/44739): [#40651](https://github.com/ClickHouse/ClickHouse/issues/40651) [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)).
+* Backported in [#44764](https://github.com/ClickHouse/ClickHouse/issues/44764): Fix parsing of bad version from compatibility setting. [#44224](https://github.com/ClickHouse/ClickHouse/pull/44224) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#44435](https://github.com/ClickHouse/ClickHouse/issues/44435): Fix possible crash in case function `IN` with constant arguments was used as a constant argument together with `LowCardinality`. Fixes [#44221](https://github.com/ClickHouse/ClickHouse/issues/44221). [#44346](https://github.com/ClickHouse/ClickHouse/pull/44346) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Bump libdivide (to gain some new optimizations) [#44132](https://github.com/ClickHouse/ClickHouse/pull/44132) ([Azat Khuzhin](https://github.com/azat)).
+* Add some settings under `compatibility` [#44209](https://github.com/ClickHouse/ClickHouse/pull/44209) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix deadlock in StorageSystemDatabases [#44272](https://github.com/ClickHouse/ClickHouse/pull/44272) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Get rid of global Git object [#44273](https://github.com/ClickHouse/ClickHouse/pull/44273) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Add check for submodules sanity [#44386](https://github.com/ClickHouse/ClickHouse/pull/44386) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix crash on delete from materialized view [#44705](https://github.com/ClickHouse/ClickHouse/pull/44705) ([Alexander Gololobov](https://github.com/davenger)).
+
diff --git a/docs/en/development/continuous-integration.md b/docs/en/development/continuous-integration.md
index ef3efa75d66..cb86ef06952 100644
--- a/docs/en/development/continuous-integration.md
+++ b/docs/en/development/continuous-integration.md
@@ -154,10 +154,6 @@ Runs [stateful functional tests](tests.md#functional-tests). Treat them in the s
Runs [integration tests](tests.md#integration-tests).
-## Testflows Check
-Runs some tests using Testflows test system. See [here](https://github.com/ClickHouse/ClickHouse/tree/master/tests/testflows#running-tests-locally) how to run them locally.
-
-
## Stress Test
Runs stateless functional tests concurrently from several clients to detect
concurrency-related errors. If it fails:
diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md
index f8195a06b6e..f917286029a 100644
--- a/docs/en/development/tests.md
+++ b/docs/en/development/tests.md
@@ -281,10 +281,6 @@ We also track test coverage but only for functional tests and only for clickhous
There is automated check for flaky tests. It runs all new tests 100 times (for functional tests) or 10 times (for integration tests). If at least single time the test failed, it is considered flaky.
-## Testflows
-
-[Testflows](https://testflows.com/) is an enterprise-grade open-source testing framework, which is used to test a subset of ClickHouse.
-
## Test Automation {#test-automation}
We run tests with [GitHub Actions](https://github.com/features/actions).
diff --git a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md
index b07f4a29396..104ec049ec4 100644
--- a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md
@@ -19,7 +19,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
(
Path String,
Time DateTime,
- Value ,
+ Value Float64,
Version
...
) ENGINE = GraphiteMergeTree(config_section)
@@ -37,7 +37,7 @@ A table for the Graphite data should have the following columns for the followin
- Time of measuring the metric. Data type: `DateTime`.
-- Value of the metric. Data type: any numeric.
+- Value of the metric. Data type: `Float64`.
- Version of the metric. Data type: any numeric (ClickHouse saves the rows with the highest version or the last written if versions are the same. Other rows are deleted during the merge of data parts).
@@ -65,7 +65,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
EventDate Date,
Path String,
Time DateTime,
- Value ,
+ Value Float64,
Version
...
) ENGINE [=] GraphiteMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, config_section)
diff --git a/docs/en/engines/table-engines/special/join.md b/docs/en/engines/table-engines/special/join.md
index a49214bd00a..3f408db2933 100644
--- a/docs/en/engines/table-engines/special/join.md
+++ b/docs/en/engines/table-engines/special/join.md
@@ -86,7 +86,18 @@ When creating a table, the following settings are applied:
[join_any_take_last_row](/docs/en/operations/settings/settings.md/#settings-join_any_take_last_row)
#### join_use_nulls
-[persistent](/docs/en/operations/settings/settings.md/#persistent)
+#### persistent
+
+Disables persistency for the Join and [Set](/docs/en/engines/table-engines/special/set.md) table engines.
+
+Reduces the I/O overhead. Suitable for scenarios that pursue performance and do not require persistence.
+
+Possible values:
+
+- 1 — Enabled.
+- 0 — Disabled.
+
+Default value: `1`.
The `Join`-engine tables can’t be used in `GLOBAL JOIN` operations.
diff --git a/docs/en/engines/table-engines/special/set.md b/docs/en/engines/table-engines/special/set.md
index 3a3e7c4d5de..451835f2b95 100644
--- a/docs/en/engines/table-engines/special/set.md
+++ b/docs/en/engines/table-engines/special/set.md
@@ -19,5 +19,16 @@ For a rough server restart, the block of data on the disk might be lost or damag
When creating a table, the following settings are applied:
-- [persistent](../../../operations/settings/settings.md#persistent)
+#### persistent
+
+Disables persistency for the Set and [Join](/docs/en/engines/table-engines/special/join.md/#join) table engines.
+
+Reduces the I/O overhead. Suitable for scenarios that pursue performance and do not require persistence.
+
+Possible values:
+
+- 1 — Enabled.
+- 0 — Disabled.
+
+Default value: `1`.
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index c2afd740ff2..bde0a0e8290 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -2441,19 +2441,6 @@ Result
└──────────────────────────┴───────┴───────────────────────────────────────────────────────┘
```
-## persistent {#persistent}
-
-Disables persistency for the [Set](../../engines/table-engines/special/set.md/#set) and [Join](../../engines/table-engines/special/join.md/#join) table engines.
-
-Reduces the I/O overhead. Suitable for scenarios that pursue performance and do not require persistence.
-
-Possible values:
-
-- 1 — Enabled.
-- 0 — Disabled.
-
-Default value: `1`.
-
## allow_nullable_key {#allow-nullable-key}
Allows using of the [Nullable](../../sql-reference/data-types/nullable.md/#data_type-nullable)-typed values in a sorting and a primary key for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md/#table_engines-mergetree) tables.
diff --git a/docs/en/operations/system-tables/data_skipping_indices.md b/docs/en/operations/system-tables/data_skipping_indices.md
index 338c6d02206..f1e233b33f7 100644
--- a/docs/en/operations/system-tables/data_skipping_indices.md
+++ b/docs/en/operations/system-tables/data_skipping_indices.md
@@ -11,6 +11,7 @@ Columns:
- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
- `name` ([String](../../sql-reference/data-types/string.md)) — Index name.
- `type` ([String](../../sql-reference/data-types/string.md)) — Index type.
+- `type_full` ([String](../../sql-reference/data-types/string.md)) — Index type expression from create statement.
- `expr` ([String](../../sql-reference/data-types/string.md)) — Expression for the index calculation.
- `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of granules in the block.
- `data_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The size of compressed data, in bytes.
@@ -30,6 +31,7 @@ database: default
table: user_actions
name: clicks_idx
type: minmax
+type_full: minmax
expr: clicks
granularity: 1
data_compressed_bytes: 58
@@ -42,6 +44,7 @@ database: default
table: users
name: contacts_null_idx
type: minmax
+type_full: minmax
expr: assumeNotNull(contacts_null)
granularity: 1
data_compressed_bytes: 58
diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md
index 5f6fad9ab9a..225f2b162ab 100644
--- a/docs/en/sql-reference/functions/date-time-functions.md
+++ b/docs/en/sql-reference/functions/date-time-functions.md
@@ -806,7 +806,8 @@ Aliases: `dateSub`, `DATE_SUB`.
**Arguments**
-- `unit` — The type of interval to subtract. [String](../../sql-reference/data-types/string.md).
+- `unit` — The type of interval to subtract. Note: The unit should be unquoted.
+
Possible values:
- `second`
diff --git a/docs/en/sql-reference/functions/ext-dict-functions.md b/docs/en/sql-reference/functions/ext-dict-functions.md
index d9e811a5703..b4b7ec5ab21 100644
--- a/docs/en/sql-reference/functions/ext-dict-functions.md
+++ b/docs/en/sql-reference/functions/ext-dict-functions.md
@@ -416,6 +416,7 @@ Functions:
- `dictGetDateTime`
- `dictGetUUID`
- `dictGetString`
+- `dictGetIPv4`, `dictGetIPv6`
All these functions have the `OrDefault` modification. For example, `dictGetDateOrDefault`.
diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md
index e0c8b41147a..fea834b1b8a 100644
--- a/docs/zh/development/continuous-integration.md
+++ b/docs/zh/development/continuous-integration.md
@@ -110,9 +110,6 @@ git push
## 集成测试 {#integration-tests}
运行[集成测试](./tests.md#integration-tests).
-## Testflows 检查{#testflows-check}
-使用Testflows测试系统去运行一些测试, 在[此处](https://github.com/ClickHouse/ClickHouse/tree/master/tests/testflows#running-tests-locally)查看如何在本地运行它们.
-
## 压力测试 {#stress-test}
从多个客户端并发运行无状态功能测试, 用以检测与并发相关的错误.如果失败:
```
diff --git a/docs/zh/development/tests.md b/docs/zh/development/tests.md
index 6f1118e5e63..c16bc26935a 100644
--- a/docs/zh/development/tests.md
+++ b/docs/zh/development/tests.md
@@ -281,10 +281,6 @@ We also track test coverage but only for functional tests and only for clickhous
There is automated check for flaky tests. It runs all new tests 100 times (for functional tests) or 10 times (for integration tests). If at least single time the test failed, it is considered flaky.
-## Testflows
-
-[Testflows](https://testflows.com/) is an enterprise-grade open-source testing framework, which is used to test a subset of ClickHouse.
-
## Test Automation {#test-automation}
We run tests with [GitHub Actions](https://github.com/features/actions).
diff --git a/packages/clickhouse-keeper.yaml b/packages/clickhouse-keeper.yaml
index f2095dda02a..7586fa580e6 100644
--- a/packages/clickhouse-keeper.yaml
+++ b/packages/clickhouse-keeper.yaml
@@ -12,8 +12,6 @@ priority: "optional"
conflicts:
- clickhouse-server
-depends:
-- adduser
suggests:
- clickhouse-keeper-dbg
diff --git a/packages/clickhouse-server.yaml b/packages/clickhouse-server.yaml
index fe59828ca43..018e88ef828 100644
--- a/packages/clickhouse-server.yaml
+++ b/packages/clickhouse-server.yaml
@@ -12,8 +12,6 @@ priority: "optional"
conflicts:
- clickhouse-keeper
-depends:
-- adduser
replaces:
- clickhouse-server-common
- clickhouse-server-base
diff --git a/src/AggregateFunctions/AggregateFunctionMap.h b/src/AggregateFunctions/AggregateFunctionMap.h
index dc19bf3f71c..91530698bf4 100644
--- a/src/AggregateFunctions/AggregateFunctionMap.h
+++ b/src/AggregateFunctions/AggregateFunctionMap.h
@@ -116,13 +116,9 @@ public:
static DataTypePtr getKeyType(const DataTypes & types, const AggregateFunctionPtr & nested)
{
- if (types.empty())
+ if (types.size() != 1)
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
- "Aggregate function {}Map requires at least one argument", nested->getName());
-
- if (types.size() > 1)
- throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
- "Aggregate function {}Map requires only one map argument", nested->getName());
+ "Aggregate function {}Map requires one map argument, but {} found", nested->getName(), types.size());
const auto * map_type = checkAndGetDataType(types[0].get());
if (!map_type)
diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.h b/src/AggregateFunctions/AggregateFunctionOrFill.h
index eeec630be9a..4679e086506 100644
--- a/src/AggregateFunctions/AggregateFunctionOrFill.h
+++ b/src/AggregateFunctions/AggregateFunctionOrFill.h
@@ -36,8 +36,8 @@ public:
AggregateFunctionOrFill(AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params)
: IAggregateFunctionHelper{arguments, params, createResultType(nested_function_->getResultType())}
, nested_function{nested_function_}
- , size_of_data {nested_function->sizeOfData()}
- , inner_nullable {nested_function->getResultType()->isNullable()}
+ , size_of_data{nested_function->sizeOfData()}
+ , inner_nullable{nested_function->getResultType()->isNullable()}
{
// nothing
}
diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h
index 4a1088a87bd..6c62adb2e65 100644
--- a/src/AggregateFunctions/AggregateFunctionSumMap.h
+++ b/src/AggregateFunctions/AggregateFunctionSumMap.h
@@ -428,10 +428,7 @@ public:
}
bool keepKey(const T & key) const { return static_cast(*this).keepKey(key); }
- String getName() const override { return getNameImpl(); }
-
-private:
- static String getNameImpl() { return Derived::getNameImpl(); }
+ String getName() const override { return Derived::getNameImpl(); }
};
template
diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h
index a5d1887f85e..7a57a433cba 100644
--- a/src/AggregateFunctions/IAggregateFunction.h
+++ b/src/AggregateFunctions/IAggregateFunction.h
@@ -65,9 +65,9 @@ class IAggregateFunction : public std::enable_shared_from_this();
}
-ColumnsWithTypeAndName FunctionNode::getArgumentTypes() const
+ColumnsWithTypeAndName FunctionNode::getArgumentColumns() const
{
- ColumnsWithTypeAndName argument_types;
- for (const auto & arg : getArguments().getNodes())
+ const auto & arguments = getArguments().getNodes();
+ ColumnsWithTypeAndName argument_columns;
+ argument_columns.reserve(arguments.size());
+
+ for (const auto & arg : arguments)
{
ColumnWithTypeAndName argument;
argument.type = arg->getResultType();
if (auto * constant = arg->as())
argument.column = argument.type->createColumnConst(1, constant->getValue());
- argument_types.push_back(argument);
+ argument_columns.push_back(std::move(argument));
}
- return argument_types;
+ return argument_columns;
}
void FunctionNode::resolveAsFunction(FunctionBasePtr function_value)
diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h
index 501d439e55e..49e66ba32c1 100644
--- a/src/Analyzer/FunctionNode.h
+++ b/src/Analyzer/FunctionNode.h
@@ -1,12 +1,14 @@
#pragma once
#include
-#include
+#include
+#include
#include
#include
-#include
#include
#include
+#include
+#include
namespace DB
{
@@ -19,12 +21,6 @@ namespace ErrorCodes
class IFunctionOverloadResolver;
using FunctionOverloadResolverPtr = std::shared_ptr;
-class IFunctionBase;
-using FunctionBasePtr = std::shared_ptr;
-
-class IAggregateFunction;
-using AggregateFunctionPtr = std::shared_ptr;
-
/** Function node represents function in query tree.
* Function syntax: function_name(parameter_1, ...)(argument_1, ...).
* If function does not have parameters its syntax is function_name(argument_1, ...).
@@ -63,66 +59,36 @@ public:
explicit FunctionNode(String function_name_);
/// Get function name
- const String & getFunctionName() const
- {
- return function_name;
- }
+ const String & getFunctionName() const { return function_name; }
/// Get parameters
- const ListNode & getParameters() const
- {
- return children[parameters_child_index]->as();
- }
+ const ListNode & getParameters() const { return children[parameters_child_index]->as(); }
/// Get parameters
- ListNode & getParameters()
- {
- return children[parameters_child_index]->as();
- }
+ ListNode & getParameters() { return children[parameters_child_index]->as(); }
/// Get parameters node
- const QueryTreeNodePtr & getParametersNode() const
- {
- return children[parameters_child_index];
- }
+ const QueryTreeNodePtr & getParametersNode() const { return children[parameters_child_index]; }
/// Get parameters node
- QueryTreeNodePtr & getParametersNode()
- {
- return children[parameters_child_index];
- }
+ QueryTreeNodePtr & getParametersNode() { return children[parameters_child_index]; }
/// Get arguments
- const ListNode & getArguments() const
- {
- return children[arguments_child_index]->as();
- }
+ const ListNode & getArguments() const { return children[arguments_child_index]->as(); }
/// Get arguments
- ListNode & getArguments()
- {
- return children[arguments_child_index]->as();
- }
+ ListNode & getArguments() { return children[arguments_child_index]->as(); }
/// Get arguments node
- const QueryTreeNodePtr & getArgumentsNode() const
- {
- return children[arguments_child_index];
- }
+ const QueryTreeNodePtr & getArgumentsNode() const { return children[arguments_child_index]; }
/// Get arguments node
- QueryTreeNodePtr & getArgumentsNode()
- {
- return children[arguments_child_index];
- }
+ QueryTreeNodePtr & getArgumentsNode() { return children[arguments_child_index]; }
- ColumnsWithTypeAndName getArgumentTypes() const;
+ ColumnsWithTypeAndName getArgumentColumns() const;
/// Returns true if function node has window, false otherwise
- bool hasWindow() const
- {
- return children[window_child_index] != nullptr;
- }
+ bool hasWindow() const { return children[window_child_index] != nullptr; }
/** Get window node.
* Valid only for window function node.
@@ -130,18 +96,12 @@ public:
* 1. It can be identifier node if window function is defined as expr OVER window_name.
* 2. It can be window node if window function is defined as expr OVER (window_name ...).
*/
- const QueryTreeNodePtr & getWindowNode() const
- {
- return children[window_child_index];
- }
+ const QueryTreeNodePtr & getWindowNode() const { return children[window_child_index]; }
/** Get window node.
* Valid only for window function node.
*/
- QueryTreeNodePtr & getWindowNode()
- {
- return children[window_child_index];
- }
+ QueryTreeNodePtr & getWindowNode() { return children[window_child_index]; }
/** Get non aggregate function.
* If function is not resolved nullptr returned.
@@ -150,7 +110,7 @@ public:
{
if (kind != FunctionKind::ORDINARY)
return {};
- return std::reinterpret_pointer_cast(function);
+ return std::static_pointer_cast(function);
}
/** Get aggregate function.
@@ -161,32 +121,20 @@ public:
{
if (kind == FunctionKind::UNKNOWN || kind == FunctionKind::ORDINARY)
return {};
- return std::reinterpret_pointer_cast(function);
+ return std::static_pointer_cast(function);
}
/// Is function node resolved
- bool isResolved() const
- {
- return function != nullptr;
- }
+ bool isResolved() const { return function != nullptr; }
/// Is function node window function
- bool isWindowFunction() const
- {
- return hasWindow();
- }
+ bool isWindowFunction() const { return hasWindow(); }
/// Is function node aggregate function
- bool isAggregateFunction() const
- {
- return kind == FunctionKind::AGGREGATE;
- }
+ bool isAggregateFunction() const { return kind == FunctionKind::AGGREGATE; }
/// Is function node ordinary function
- bool isOrdinaryFunction() const
- {
- return kind == FunctionKind::ORDINARY;
- }
+ bool isOrdinaryFunction() const { return kind == FunctionKind::ORDINARY; }
/** Resolve function node as non aggregate function.
* It is important that function name is updated with resolved function name.
@@ -208,10 +156,7 @@ public:
*/
void resolveAsWindowFunction(AggregateFunctionPtr window_function_value);
- QueryTreeNodeType getNodeType() const override
- {
- return QueryTreeNodeType::FUNCTION;
- }
+ QueryTreeNodeType getNodeType() const override { return QueryTreeNodeType::FUNCTION; }
DataTypePtr getResultType() const override
{
diff --git a/src/Analyzer/Passes/CustomizeFunctionsPass.cpp b/src/Analyzer/Passes/CustomizeFunctionsPass.cpp
index 7eb4a040970..407614dbe50 100644
--- a/src/Analyzer/Passes/CustomizeFunctionsPass.cpp
+++ b/src/Analyzer/Passes/CustomizeFunctionsPass.cpp
@@ -155,7 +155,7 @@ public:
inline void resolveOrdinaryFunctionNode(FunctionNode & function_node, const String & function_name) const
{
auto function = FunctionFactory::instance().get(function_name, context);
- function_node.resolveAsFunction(function->build(function_node.getArgumentTypes()));
+ function_node.resolveAsFunction(function->build(function_node.getArgumentColumns()));
}
private:
diff --git a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp
index 0c5a450135f..aff932e39db 100644
--- a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp
+++ b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp
@@ -193,7 +193,7 @@ private:
inline void resolveOrdinaryFunctionNode(FunctionNode & function_node, const String & function_name) const
{
auto function = FunctionFactory::instance().get(function_name, context);
- function_node.resolveAsFunction(function->build(function_node.getArgumentTypes()));
+ function_node.resolveAsFunction(function->build(function_node.getArgumentColumns()));
}
ContextPtr & context;
diff --git a/src/Analyzer/Passes/FuseFunctionsPass.cpp b/src/Analyzer/Passes/FuseFunctionsPass.cpp
index f354a7b1ec3..c73e1048dc2 100644
--- a/src/Analyzer/Passes/FuseFunctionsPass.cpp
+++ b/src/Analyzer/Passes/FuseFunctionsPass.cpp
@@ -65,7 +65,7 @@ QueryTreeNodePtr createResolvedFunction(const ContextPtr & context, const String
auto function = FunctionFactory::instance().get(name, context);
function_node->getArguments().getNodes() = std::move(arguments);
- function_node->resolveAsFunction(function->build(function_node->getArgumentTypes()));
+ function_node->resolveAsFunction(function->build(function_node->getArgumentColumns()));
return function_node;
}
@@ -88,7 +88,7 @@ FunctionNodePtr createResolvedAggregateFunction(const String & name, const Query
{ argument->getResultType() },
parameters,
properties);
- function_node->resolveAsAggregateFunction(aggregate_function);
+ function_node->resolveAsAggregateFunction(std::move(aggregate_function));
return function_node;
}
diff --git a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp
index 020edfe4820..077ba331ead 100644
--- a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp
+++ b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp
@@ -56,7 +56,7 @@ public:
auto multi_if_function = std::make_shared("multiIf");
multi_if_function->getArguments().getNodes() = std::move(multi_if_arguments);
- multi_if_function->resolveAsFunction(multi_if_function_ptr->build(multi_if_function->getArgumentTypes()));
+ multi_if_function->resolveAsFunction(multi_if_function_ptr->build(multi_if_function->getArgumentColumns()));
node = std::move(multi_if_function);
}
diff --git a/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp b/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp
index 776fe63c803..dd1c211a053 100644
--- a/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp
+++ b/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp
@@ -52,7 +52,7 @@ QueryTreeNodePtr createCastFunction(QueryTreeNodePtr from, DataTypePtr result_ty
auto function_node = std::make_shared("_CAST");
function_node->getArguments().getNodes() = std::move(arguments);
- function_node->resolveAsFunction(cast_function->build(function_node->getArgumentTypes()));
+ function_node->resolveAsFunction(cast_function->build(function_node->getArgumentColumns()));
return function_node;
}
@@ -71,7 +71,7 @@ void changeIfArguments(
auto if_resolver = FunctionFactory::instance().get("if", context);
- if_node.resolveAsFunction(if_resolver->build(if_node.getArgumentTypes()));
+ if_node.resolveAsFunction(if_resolver->build(if_node.getArgumentColumns()));
}
/// transform(value, array_from, array_to, default_value) will be transformed to transform(value, array_from, _CAST(array_to, Array(Enum...)), _CAST(default_value, Enum...))
@@ -93,7 +93,7 @@ void changeTransformArguments(
auto transform_resolver = FunctionFactory::instance().get("transform", context);
- transform_node.resolveAsFunction(transform_resolver->build(transform_node.getArgumentTypes()));
+ transform_node.resolveAsFunction(transform_resolver->build(transform_node.getArgumentColumns()));
}
void wrapIntoToString(FunctionNode & function_node, QueryTreeNodePtr arg, ContextPtr context)
@@ -102,7 +102,7 @@ void wrapIntoToString(FunctionNode & function_node, QueryTreeNodePtr arg, Contex
QueryTreeNodes arguments{ std::move(arg) };
function_node.getArguments().getNodes() = std::move(arguments);
- function_node.resolveAsFunction(to_string_function->build(function_node.getArgumentTypes()));
+ function_node.resolveAsFunction(to_string_function->build(function_node.getArgumentColumns()));
assert(isString(function_node.getResultType()));
}
diff --git a/src/Analyzer/Passes/MultiIfToIfPass.cpp b/src/Analyzer/Passes/MultiIfToIfPass.cpp
index 7e13675bf98..9eb2a4da817 100644
--- a/src/Analyzer/Passes/MultiIfToIfPass.cpp
+++ b/src/Analyzer/Passes/MultiIfToIfPass.cpp
@@ -27,7 +27,7 @@ public:
return;
auto result_type = function_node->getResultType();
- function_node->resolveAsFunction(if_function_ptr->build(function_node->getArgumentTypes()));
+ function_node->resolveAsFunction(if_function_ptr->build(function_node->getArgumentColumns()));
}
private:
diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp
index 8d923d2a69d..e93548d34ed 100644
--- a/src/Analyzer/Passes/QueryAnalysisPass.cpp
+++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp
@@ -4333,7 +4333,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
? AggregateFunctionFactory::instance().get(function_name + "OrNull", argument_types, parameters, properties)
: AggregateFunctionFactory::instance().get(function_name, argument_types, parameters, properties);
- function_node.resolveAsWindowFunction(aggregate_function);
+ function_node.resolveAsWindowFunction(std::move(aggregate_function));
bool window_node_is_identifier = function_node.getWindowNode()->getNodeType() == QueryTreeNodeType::IDENTIFIER;
ProjectionName window_projection_name = resolveWindow(function_node.getWindowNode(), scope);
@@ -4396,7 +4396,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
auto aggregate_function = need_add_or_null
? AggregateFunctionFactory::instance().get(function_name + "OrNull", argument_types, parameters, properties)
: AggregateFunctionFactory::instance().get(function_name, argument_types, parameters, properties);
- function_node.resolveAsAggregateFunction(aggregate_function);
+ function_node.resolveAsAggregateFunction(std::move(aggregate_function));
return result_projection_names;
}
diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.cpp b/src/Analyzer/Passes/SumIfToCountIfPass.cpp
index 7e120b6828d..879eb4d4a8d 100644
--- a/src/Analyzer/Passes/SumIfToCountIfPass.cpp
+++ b/src/Analyzer/Passes/SumIfToCountIfPass.cpp
@@ -122,7 +122,7 @@ public:
auto & not_function_arguments = not_function->getArguments().getNodes();
not_function_arguments.push_back(std::move(nested_if_function_arguments_nodes[0]));
- not_function->resolveAsFunction(FunctionFactory::instance().get("not", context)->build(not_function->getArgumentTypes()));
+ not_function->resolveAsFunction(FunctionFactory::instance().get("not", context)->build(not_function->getArgumentColumns()));
function_node_arguments_nodes[0] = std::move(not_function);
function_node_arguments_nodes.resize(1);
diff --git a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp
index 37bad70da57..3ce5ec4a24c 100644
--- a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp
+++ b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp
@@ -75,7 +75,6 @@ public:
function_node->getAggregateFunction()->getParameters(),
properties);
- auto function_result_type = function_node->getResultType();
function_node->resolveAsAggregateFunction(std::move(aggregate_function));
}
};
diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp
index 06a1fec4698..4148d42ee23 100644
--- a/src/Analyzer/QueryTreePassManager.cpp
+++ b/src/Analyzer/QueryTreePassManager.cpp
@@ -59,7 +59,7 @@ class ValidationChecker : public InDepthQueryTreeVisitor
if (!function->isResolved())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Function {} is not resolved after running {} pass",
- function->dumpTree(), pass_name);
+ function->toAST()->formatForErrorMessage(), pass_name);
}
public:
diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h
index 27faf4bd2ad..a295eeb5c84 100644
--- a/src/Columns/ColumnUnique.h
+++ b/src/Columns/ColumnUnique.h
@@ -355,6 +355,8 @@ size_t ColumnUnique::uniqueInsert(const Field & x)
void operator() (const Int128 & x) { res = {reinterpret_cast(&x), sizeof(x)}; }
void operator() (const Int256 & x) { res = {reinterpret_cast(&x), sizeof(x)}; }
void operator() (const UUID & x) { res = {reinterpret_cast(&x), sizeof(x)}; }
+ void operator() (const IPv4 & x) { res = {reinterpret_cast(&x), sizeof(x)}; }
+ void operator() (const IPv6 & x) { res = {reinterpret_cast(&x), sizeof(x)}; }
void operator() (const Float64 & x) { res = {reinterpret_cast(&x), sizeof(x)}; }
void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; }
void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; }
diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp
index 30e7423fde0..a97da8fe9c6 100644
--- a/src/Columns/ColumnVector.cpp
+++ b/src/Columns/ColumnVector.cpp
@@ -942,5 +942,7 @@ template class ColumnVector;
template class ColumnVector;
template class ColumnVector;
template class ColumnVector;
+template class ColumnVector;
+template class ColumnVector;
}
diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h
index a601dd8b405..ded66430160 100644
--- a/src/Columns/ColumnVector.h
+++ b/src/Columns/ColumnVector.h
@@ -557,5 +557,7 @@ extern template class ColumnVector;
extern template class ColumnVector;
extern template class ColumnVector;
extern template class ColumnVector;
+extern template class ColumnVector;
+extern template class ColumnVector;
}
diff --git a/src/Columns/ColumnsNumber.h b/src/Columns/ColumnsNumber.h
index 654d0c73254..ae7eddb0b22 100644
--- a/src/Columns/ColumnsNumber.h
+++ b/src/Columns/ColumnsNumber.h
@@ -27,5 +27,7 @@ using ColumnFloat32 = ColumnVector;
using ColumnFloat64 = ColumnVector;
using ColumnUUID = ColumnVector;
+using ColumnIPv4 = ColumnVector;
+using ColumnIPv6 = ColumnVector;
}
diff --git a/src/Columns/MaskOperations.cpp b/src/Columns/MaskOperations.cpp
index e320e1d57a3..24cb81a6528 100644
--- a/src/Columns/MaskOperations.cpp
+++ b/src/Columns/MaskOperations.cpp
@@ -72,6 +72,8 @@ INSTANTIATE(Decimal256)
INSTANTIATE(DateTime64)
INSTANTIATE(char *)
INSTANTIATE(UUID)
+INSTANTIATE(IPv4)
+INSTANTIATE(IPv6)
#undef INSTANTIATE
diff --git a/src/Columns/tests/gtest_column_vector.cpp b/src/Columns/tests/gtest_column_vector.cpp
index 5017d687791..14bf36434b6 100644
--- a/src/Columns/tests/gtest_column_vector.cpp
+++ b/src/Columns/tests/gtest_column_vector.cpp
@@ -1,4 +1,5 @@
#include
+#include
#include
#include
#include
@@ -14,6 +15,12 @@ static constexpr size_t MAX_ROWS = 10000;
static const std::vector filter_ratios = {1, 2, 5, 11, 32, 64, 100, 1000};
static const size_t K = filter_ratios.size();
+template
+struct HasUnderlyingType : std::false_type {};
+
+template
+struct HasUnderlyingType> : std::true_type {};
+
template
static MutableColumnPtr createColumn(size_t n)
{
@@ -21,7 +28,10 @@ static MutableColumnPtr createColumn(size_t n)
auto & values = column->getData();
for (size_t i = 0; i < n; ++i)
- values.push_back(static_cast(i));
+ if constexpr (HasUnderlyingType::value)
+ values.push_back(static_cast(i));
+ else
+ values.push_back(static_cast(i));
return column;
}
@@ -85,6 +95,8 @@ TEST(ColumnVector, Filter)
testFilter();
testFilter();
testFilter();
+ testFilter();
+ testFilter();
}
template
diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index 95333eccbcd..531d7292ae2 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -643,6 +643,8 @@
M(672, INVALID_SCHEDULER_NODE) \
M(673, RESOURCE_ACCESS_DENIED) \
M(674, RESOURCE_NOT_FOUND) \
+ M(675, CANNOT_PARSE_IPV4) \
+ M(676, CANNOT_PARSE_IPV6) \
\
M(999, KEEPER_EXCEPTION) \
M(1000, POCO_EXCEPTION) \
diff --git a/src/Common/FieldVisitorConvertToNumber.h b/src/Common/FieldVisitorConvertToNumber.h
index 466d312406e..94071e4f3fa 100644
--- a/src/Common/FieldVisitorConvertToNumber.h
+++ b/src/Common/FieldVisitorConvertToNumber.h
@@ -55,6 +55,8 @@ public:
T operator() (const Int64 & x) const { return T(x); }
T operator() (const Int128 & x) const { return T(x); }
T operator() (const UUID & x) const { return T(x.toUnderType()); }
+ T operator() (const IPv4 & x) const { return T(x.toUnderType()); }
+ T operator() (const IPv6 & x) const { return T(x.toUnderType()); }
T operator() (const Float64 & x) const
{
diff --git a/src/Common/FieldVisitorDump.cpp b/src/Common/FieldVisitorDump.cpp
index fc3d56c3503..be4331ca478 100644
--- a/src/Common/FieldVisitorDump.cpp
+++ b/src/Common/FieldVisitorDump.cpp
@@ -37,6 +37,8 @@ String FieldVisitorDump::operator() (const UInt256 & x) const { return formatQuo
String FieldVisitorDump::operator() (const Int128 & x) const { return formatQuotedWithPrefix(x, "Int128_"); }
String FieldVisitorDump::operator() (const Int256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); }
String FieldVisitorDump::operator() (const UUID & x) const { return formatQuotedWithPrefix(x, "UUID_"); }
+String FieldVisitorDump::operator() (const IPv4 & x) const { return formatQuotedWithPrefix(x, "IPv4_"); }
+String FieldVisitorDump::operator() (const IPv6 & x) const { return formatQuotedWithPrefix(x, "IPv6_"); }
String FieldVisitorDump::operator() (const bool & x) const { return formatQuotedWithPrefix(x, "Bool_"); }
diff --git a/src/Common/FieldVisitorDump.h b/src/Common/FieldVisitorDump.h
index dc67ccf7da3..6ffd91bb400 100644
--- a/src/Common/FieldVisitorDump.h
+++ b/src/Common/FieldVisitorDump.h
@@ -17,6 +17,8 @@ public:
String operator() (const Int128 & x) const;
String operator() (const Int256 & x) const;
String operator() (const UUID & x) const;
+ String operator() (const IPv4 & x) const;
+ String operator() (const IPv6 & x) const;
String operator() (const Float64 & x) const;
String operator() (const String & x) const;
String operator() (const Array & x) const;
diff --git a/src/Common/FieldVisitorHash.cpp b/src/Common/FieldVisitorHash.cpp
index b6750fdcd03..d759635c65b 100644
--- a/src/Common/FieldVisitorHash.cpp
+++ b/src/Common/FieldVisitorHash.cpp
@@ -49,6 +49,20 @@ void FieldVisitorHash::operator() (const UUID & x) const
hash.update(x);
}
+void FieldVisitorHash::operator() (const IPv4 & x) const
+{
+ UInt8 type = Field::Types::IPv4;
+ hash.update(type);
+ hash.update(x);
+}
+
+void FieldVisitorHash::operator() (const IPv6 & x) const
+{
+ UInt8 type = Field::Types::IPv6;
+ hash.update(type);
+ hash.update(x);
+}
+
void FieldVisitorHash::operator() (const Float64 & x) const
{
UInt8 type = Field::Types::Float64;
diff --git a/src/Common/FieldVisitorHash.h b/src/Common/FieldVisitorHash.h
index e574b0456eb..1350956146b 100644
--- a/src/Common/FieldVisitorHash.h
+++ b/src/Common/FieldVisitorHash.h
@@ -23,6 +23,8 @@ public:
void operator() (const Int128 & x) const;
void operator() (const Int256 & x) const;
void operator() (const UUID & x) const;
+ void operator() (const IPv4 & x) const;
+ void operator() (const IPv6 & x) const;
void operator() (const Float64 & x) const;
void operator() (const String & x) const;
void operator() (const Array & x) const;
diff --git a/src/Common/FieldVisitorSum.cpp b/src/Common/FieldVisitorSum.cpp
index db7b4850204..ed4dd373049 100644
--- a/src/Common/FieldVisitorSum.cpp
+++ b/src/Common/FieldVisitorSum.cpp
@@ -33,6 +33,8 @@ bool FieldVisitorSum::operator() (Tuple &) const { throw Exception("Cannot sum T
bool FieldVisitorSum::operator() (Map &) const { throw Exception("Cannot sum Maps", ErrorCodes::LOGICAL_ERROR); }
bool FieldVisitorSum::operator() (Object &) const { throw Exception("Cannot sum Objects", ErrorCodes::LOGICAL_ERROR); }
bool FieldVisitorSum::operator() (UUID &) const { throw Exception("Cannot sum UUIDs", ErrorCodes::LOGICAL_ERROR); }
+bool FieldVisitorSum::operator() (IPv4 &) const { throw Exception("Cannot sum IPv4s", ErrorCodes::LOGICAL_ERROR); }
+bool FieldVisitorSum::operator() (IPv6 &) const { throw Exception("Cannot sum IPv6s", ErrorCodes::LOGICAL_ERROR); }
bool FieldVisitorSum::operator() (AggregateFunctionStateData &) const
{
diff --git a/src/Common/FieldVisitorSum.h b/src/Common/FieldVisitorSum.h
index c28e2058b05..5b9c83d1dd1 100644
--- a/src/Common/FieldVisitorSum.h
+++ b/src/Common/FieldVisitorSum.h
@@ -28,6 +28,8 @@ public:
bool operator() (Map &) const;
bool operator() (Object &) const;
bool operator() (UUID &) const;
+ bool operator() (IPv4 &) const;
+ bool operator() (IPv6 &) const;
bool operator() (AggregateFunctionStateData &) const;
bool operator() (bool &) const;
diff --git a/src/Common/FieldVisitorToString.cpp b/src/Common/FieldVisitorToString.cpp
index e0e138d744c..d7113b8c724 100644
--- a/src/Common/FieldVisitorToString.cpp
+++ b/src/Common/FieldVisitorToString.cpp
@@ -65,6 +65,8 @@ String FieldVisitorToString::operator() (const UInt128 & x) const { return forma
String FieldVisitorToString::operator() (const UInt256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Int256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const UUID & x) const { return formatQuoted(x); }
+String FieldVisitorToString::operator() (const IPv4 & x) const { return formatQuoted(x); }
+String FieldVisitorToString::operator() (const IPv6 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const AggregateFunctionStateData & x) const { return formatQuoted(x.data); }
String FieldVisitorToString::operator() (const bool & x) const { return x ? "true" : "false"; }
diff --git a/src/Common/FieldVisitorToString.h b/src/Common/FieldVisitorToString.h
index cca29a8f7e0..6a10de12ff9 100644
--- a/src/Common/FieldVisitorToString.h
+++ b/src/Common/FieldVisitorToString.h
@@ -17,6 +17,8 @@ public:
String operator() (const Int128 & x) const;
String operator() (const Int256 & x) const;
String operator() (const UUID & x) const;
+ String operator() (const IPv4 & x) const;
+ String operator() (const IPv6 & x) const;
String operator() (const Float64 & x) const;
String operator() (const String & x) const;
String operator() (const Array & x) const;
diff --git a/src/Common/FieldVisitorWriteBinary.cpp b/src/Common/FieldVisitorWriteBinary.cpp
index 85c32cee3c0..c9b0cf16414 100644
--- a/src/Common/FieldVisitorWriteBinary.cpp
+++ b/src/Common/FieldVisitorWriteBinary.cpp
@@ -16,6 +16,8 @@ void FieldVisitorWriteBinary::operator() (const Int128 & x, WriteBuffer & buf) c
void FieldVisitorWriteBinary::operator() (const UInt256 & x, WriteBuffer & buf) const { writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const Int256 & x, WriteBuffer & buf) const { writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const UUID & x, WriteBuffer & buf) const { writeBinary(x, buf); }
+void FieldVisitorWriteBinary::operator() (const IPv4 & x, WriteBuffer & buf) const { writeBinary(x, buf); }
+void FieldVisitorWriteBinary::operator() (const IPv6 & x, WriteBuffer & buf) const { writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField & x, WriteBuffer & buf) const { writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField & x, WriteBuffer & buf) const { writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField & x, WriteBuffer & buf) const { writeBinary(x.getValue(), buf); }
diff --git a/src/Common/FieldVisitorWriteBinary.h b/src/Common/FieldVisitorWriteBinary.h
index ff2740383f7..bc75150bed2 100644
--- a/src/Common/FieldVisitorWriteBinary.h
+++ b/src/Common/FieldVisitorWriteBinary.h
@@ -16,6 +16,8 @@ public:
void operator() (const Int128 & x, WriteBuffer & buf) const;
void operator() (const Int256 & x, WriteBuffer & buf) const;
void operator() (const UUID & x, WriteBuffer & buf) const;
+ void operator() (const IPv4 & x, WriteBuffer & buf) const;
+ void operator() (const IPv6 & x, WriteBuffer & buf) const;
void operator() (const Float64 & x, WriteBuffer & buf) const;
void operator() (const String & x, WriteBuffer & buf) const;
void operator() (const Array & x, WriteBuffer & buf) const;
diff --git a/src/Common/HashTable/Hash.h b/src/Common/HashTable/Hash.h
index 2e0ef400da8..01758c1b9fb 100644
--- a/src/Common/HashTable/Hash.h
+++ b/src/Common/HashTable/Hash.h
@@ -259,7 +259,7 @@ inline size_t DefaultHash64(T key)
static_cast(key) ^
static_cast(key >> 64));
}
- else if constexpr (std::is_same_v)
+ else if constexpr (std::is_same_v || std::is_same_v)
{
return intHash64(
static_cast(key.toUnderType()) ^
diff --git a/src/Common/formatIPv6.cpp b/src/Common/formatIPv6.cpp
index 9c71debaa1e..cfa93bf4920 100644
--- a/src/Common/formatIPv6.cpp
+++ b/src/Common/formatIPv6.cpp
@@ -9,36 +9,55 @@
namespace DB
{
-// To be used in formatIPv4, maps a byte to it's string form prefixed with length (so save strlen call).
-extern const char one_byte_to_string_lookup_table[256][4] =
+/** Further we want to generate constexpr array of strings with sizes from sequence of unsigned ints [0..N)
+ * in order to use this arrey for fast conversion of unsigned integers to strings
+ */
+namespace detail
{
- {1, '0'}, {1, '1'}, {1, '2'}, {1, '3'}, {1, '4'}, {1, '5'}, {1, '6'}, {1, '7'}, {1, '8'}, {1, '9'},
- {2, '1', '0'}, {2, '1', '1'}, {2, '1', '2'}, {2, '1', '3'}, {2, '1', '4'}, {2, '1', '5'}, {2, '1', '6'}, {2, '1', '7'}, {2, '1', '8'}, {2, '1', '9'},
- {2, '2', '0'}, {2, '2', '1'}, {2, '2', '2'}, {2, '2', '3'}, {2, '2', '4'}, {2, '2', '5'}, {2, '2', '6'}, {2, '2', '7'}, {2, '2', '8'}, {2, '2', '9'},
- {2, '3', '0'}, {2, '3', '1'}, {2, '3', '2'}, {2, '3', '3'}, {2, '3', '4'}, {2, '3', '5'}, {2, '3', '6'}, {2, '3', '7'}, {2, '3', '8'}, {2, '3', '9'},
- {2, '4', '0'}, {2, '4', '1'}, {2, '4', '2'}, {2, '4', '3'}, {2, '4', '4'}, {2, '4', '5'}, {2, '4', '6'}, {2, '4', '7'}, {2, '4', '8'}, {2, '4', '9'},
- {2, '5', '0'}, {2, '5', '1'}, {2, '5', '2'}, {2, '5', '3'}, {2, '5', '4'}, {2, '5', '5'}, {2, '5', '6'}, {2, '5', '7'}, {2, '5', '8'}, {2, '5', '9'},
- {2, '6', '0'}, {2, '6', '1'}, {2, '6', '2'}, {2, '6', '3'}, {2, '6', '4'}, {2, '6', '5'}, {2, '6', '6'}, {2, '6', '7'}, {2, '6', '8'}, {2, '6', '9'},
- {2, '7', '0'}, {2, '7', '1'}, {2, '7', '2'}, {2, '7', '3'}, {2, '7', '4'}, {2, '7', '5'}, {2, '7', '6'}, {2, '7', '7'}, {2, '7', '8'}, {2, '7', '9'},
- {2, '8', '0'}, {2, '8', '1'}, {2, '8', '2'}, {2, '8', '3'}, {2, '8', '4'}, {2, '8', '5'}, {2, '8', '6'}, {2, '8', '7'}, {2, '8', '8'}, {2, '8', '9'},
- {2, '9', '0'}, {2, '9', '1'}, {2, '9', '2'}, {2, '9', '3'}, {2, '9', '4'}, {2, '9', '5'}, {2, '9', '6'}, {2, '9', '7'}, {2, '9', '8'}, {2, '9', '9'},
- {3, '1', '0', '0'}, {3, '1', '0', '1'}, {3, '1', '0', '2'}, {3, '1', '0', '3'}, {3, '1', '0', '4'}, {3, '1', '0', '5'}, {3, '1', '0', '6'}, {3, '1', '0', '7'}, {3, '1', '0', '8'}, {3, '1', '0', '9'},
- {3, '1', '1', '0'}, {3, '1', '1', '1'}, {3, '1', '1', '2'}, {3, '1', '1', '3'}, {3, '1', '1', '4'}, {3, '1', '1', '5'}, {3, '1', '1', '6'}, {3, '1', '1', '7'}, {3, '1', '1', '8'}, {3, '1', '1', '9'},
- {3, '1', '2', '0'}, {3, '1', '2', '1'}, {3, '1', '2', '2'}, {3, '1', '2', '3'}, {3, '1', '2', '4'}, {3, '1', '2', '5'}, {3, '1', '2', '6'}, {3, '1', '2', '7'}, {3, '1', '2', '8'}, {3, '1', '2', '9'},
- {3, '1', '3', '0'}, {3, '1', '3', '1'}, {3, '1', '3', '2'}, {3, '1', '3', '3'}, {3, '1', '3', '4'}, {3, '1', '3', '5'}, {3, '1', '3', '6'}, {3, '1', '3', '7'}, {3, '1', '3', '8'}, {3, '1', '3', '9'},
- {3, '1', '4', '0'}, {3, '1', '4', '1'}, {3, '1', '4', '2'}, {3, '1', '4', '3'}, {3, '1', '4', '4'}, {3, '1', '4', '5'}, {3, '1', '4', '6'}, {3, '1', '4', '7'}, {3, '1', '4', '8'}, {3, '1', '4', '9'},
- {3, '1', '5', '0'}, {3, '1', '5', '1'}, {3, '1', '5', '2'}, {3, '1', '5', '3'}, {3, '1', '5', '4'}, {3, '1', '5', '5'}, {3, '1', '5', '6'}, {3, '1', '5', '7'}, {3, '1', '5', '8'}, {3, '1', '5', '9'},
- {3, '1', '6', '0'}, {3, '1', '6', '1'}, {3, '1', '6', '2'}, {3, '1', '6', '3'}, {3, '1', '6', '4'}, {3, '1', '6', '5'}, {3, '1', '6', '6'}, {3, '1', '6', '7'}, {3, '1', '6', '8'}, {3, '1', '6', '9'},
- {3, '1', '7', '0'}, {3, '1', '7', '1'}, {3, '1', '7', '2'}, {3, '1', '7', '3'}, {3, '1', '7', '4'}, {3, '1', '7', '5'}, {3, '1', '7', '6'}, {3, '1', '7', '7'}, {3, '1', '7', '8'}, {3, '1', '7', '9'},
- {3, '1', '8', '0'}, {3, '1', '8', '1'}, {3, '1', '8', '2'}, {3, '1', '8', '3'}, {3, '1', '8', '4'}, {3, '1', '8', '5'}, {3, '1', '8', '6'}, {3, '1', '8', '7'}, {3, '1', '8', '8'}, {3, '1', '8', '9'},
- {3, '1', '9', '0'}, {3, '1', '9', '1'}, {3, '1', '9', '2'}, {3, '1', '9', '3'}, {3, '1', '9', '4'}, {3, '1', '9', '5'}, {3, '1', '9', '6'}, {3, '1', '9', '7'}, {3, '1', '9', '8'}, {3, '1', '9', '9'},
- {3, '2', '0', '0'}, {3, '2', '0', '1'}, {3, '2', '0', '2'}, {3, '2', '0', '3'}, {3, '2', '0', '4'}, {3, '2', '0', '5'}, {3, '2', '0', '6'}, {3, '2', '0', '7'}, {3, '2', '0', '8'}, {3, '2', '0', '9'},
- {3, '2', '1', '0'}, {3, '2', '1', '1'}, {3, '2', '1', '2'}, {3, '2', '1', '3'}, {3, '2', '1', '4'}, {3, '2', '1', '5'}, {3, '2', '1', '6'}, {3, '2', '1', '7'}, {3, '2', '1', '8'}, {3, '2', '1', '9'},
- {3, '2', '2', '0'}, {3, '2', '2', '1'}, {3, '2', '2', '2'}, {3, '2', '2', '3'}, {3, '2', '2', '4'}, {3, '2', '2', '5'}, {3, '2', '2', '6'}, {3, '2', '2', '7'}, {3, '2', '2', '8'}, {3, '2', '2', '9'},
- {3, '2', '3', '0'}, {3, '2', '3', '1'}, {3, '2', '3', '2'}, {3, '2', '3', '3'}, {3, '2', '3', '4'}, {3, '2', '3', '5'}, {3, '2', '3', '6'}, {3, '2', '3', '7'}, {3, '2', '3', '8'}, {3, '2', '3', '9'},
- {3, '2', '4', '0'}, {3, '2', '4', '1'}, {3, '2', '4', '2'}, {3, '2', '4', '3'}, {3, '2', '4', '4'}, {3, '2', '4', '5'}, {3, '2', '4', '6'}, {3, '2', '4', '7'}, {3, '2', '4', '8'}, {3, '2', '4', '9'},
- {3, '2', '5', '0'}, {3, '2', '5', '1'}, {3, '2', '5', '2'}, {3, '2', '5', '3'}, {3, '2', '5', '4'}, {3, '2', '5', '5'},
-};
+ template
+ struct ToChars
+ {
+ static const char value[];
+ static const size_t size;
+ };
+
+ template
+ constexpr char ToChars::value[] = {('0' + digits)..., 0};
+
+ template
+ constexpr size_t ToChars::size = sizeof...(digits);
+
+ template
+ struct Decompose : Decompose {};
+
+ template
+ struct Decompose<0, digits...> : ToChars {};
+
+ template <>
+ struct Decompose<0> : ToChars<0> {};
+
+ template
+ struct NumToString : Decompose {};
+
+ template
+ consteval std::array, sizeof...(ints)> str_make_array_impl(std::integer_sequence)
+ {
+ return std::array, sizeof...(ints)> { std::pair {NumToString::value, NumToString::size}... };
+ }
+}
+
+/** str_make_array() - generates static array of std::pair for numbers [0..N), where:
+ * first - null-terminated string representing number
+ * second - size of the string as would returned by strlen()
+ */
+template
+consteval std::array, N> str_make_array()
+{
+ return detail::str_make_array_impl(std::make_integer_sequence{});
+}
+
+/// This will generate static array of pair for [0..255] at compile time
+extern constexpr auto one_byte_to_string_lookup_table = str_make_array<256>();
/// integer logarithm, return ceil(log(value, base)) (the smallest integer greater or equal than log(value, base)
static constexpr UInt32 intLog(const UInt32 value, const UInt32 base, const bool carry)
diff --git a/src/Common/formatIPv6.h b/src/Common/formatIPv6.h
index 14093594cff..69963336cef 100644
--- a/src/Common/formatIPv6.h
+++ b/src/Common/formatIPv6.h
@@ -3,6 +3,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -17,6 +18,7 @@ constexpr size_t IPV6_MAX_TEXT_LENGTH = 45; /// Does not count tail zero byt
namespace DB
{
+extern const std::array, 256> one_byte_to_string_lookup_table;
/** Rewritten inet_ntop6 from http://svn.apache.org/repos/asf/apr/apr/trunk/network_io/unix/inet_pton.c
* performs significantly faster than the reference implementation due to the absence of sprintf calls,
@@ -30,17 +32,38 @@ void formatIPv6(const unsigned char * src, char *& dst, uint8_t zeroed_tail_byte
* which should be long enough.
* That is "127.0.0.1" becomes 0x7f000001.
*
- * In case of failure returns false and doesn't modify buffer pointed by `dst`.
+ * In case of failure doesn't modify buffer pointed by `dst`.
*
- * @param src - input string, expected to be non-null and null-terminated right after the IPv4 string value.
- * @param dst - where to put output bytes, expected to be non-null and at IPV4_BINARY_LENGTH-long.
- * @return false if parsing failed, true otherwise.
+ * WARNING - this function is adapted to work with ReadBuffer, where src is the position reference (ReadBuffer::position())
+ * and eof is the ReadBuffer::eof() - therefore algorithm below does not rely on buffer's continuity.
+ * To parse strings use overloads below.
+ *
+ * @param src - iterator (reference to pointer) over input string - warning - continuity is not guaranteed.
+ * @param eof - function returning true if iterator riched the end - warning - can break iterator's continuity.
+ * @param dst - where to put output bytes, expected to be non-null and at IPV4_BINARY_LENGTH-long.
+ * @param first_octet - preparsed first octet
+ * @return - true if parsed successfully, false otherwise.
*/
-inline bool parseIPv4(const char * src, unsigned char * dst)
+template
+requires (std::is_same::type, char>::value)
+inline bool parseIPv4(T * &src, EOFfunction eof, unsigned char * dst, int32_t first_octet = -1)
{
+ if (src == nullptr || first_octet > 255)
+ return false;
+
UInt32 result = 0;
- for (int offset = 24; offset >= 0; offset -= 8)
+ int offset = 24;
+ if (first_octet >= 0)
{
+ result |= first_octet << offset;
+ offset -= 8;
+ }
+
+ for (; true; offset -= 8, ++src)
+ {
+ if (eof())
+ return false;
+
UInt32 value = 0;
size_t len = 0;
while (isNumericASCII(*src) && len <= 3)
@@ -48,135 +71,331 @@ inline bool parseIPv4(const char * src, unsigned char * dst)
value = value * 10 + (*src - '0');
++len;
++src;
+ if (eof())
+ break;
}
- if (len == 0 || value > 255 || (offset > 0 && *src != '.'))
+ if (len == 0 || value > 255 || (offset > 0 && (eof() || *src != '.')))
return false;
result |= value << offset;
- ++src;
- }
- if (*(src - 1) != '\0')
- return false;
- memcpy(dst, &result, sizeof(result));
+ if (offset == 0)
+ break;
+ }
+
+ if constexpr (std::endian::native == std::endian::little)
+ memcpy(dst, &result, sizeof(result));
+ else
+ reverseMemcpy(dst, &result, sizeof(result));
+
return true;
}
+/// returns pointer to the right after parsed sequence or null on failed parsing
+inline const char * parseIPv4(const char * src, const char * end, unsigned char * dst)
+{
+ if (parseIPv4(src, [&src, end](){ return src == end; }, dst))
+ return src;
+ return nullptr;
+}
+
+/// returns true if whole buffer was parsed successfully
+inline bool parseIPv4whole(const char * src, const char * end, unsigned char * dst)
+{
+ return parseIPv4(src, end, dst) == end;
+}
+
+/// returns pointer to the right after parsed sequence or null on failed parsing
+inline const char * parseIPv4(const char * src, unsigned char * dst)
+{
+ if (parseIPv4(src, [](){ return false; }, dst))
+ return src;
+ return nullptr;
+}
+
+/// returns true if whole null-terminated string was parsed successfully
+inline bool parseIPv4whole(const char * src, unsigned char * dst)
+{
+ const char * end = parseIPv4(src, dst);
+ return end != nullptr && *end == '\0';
+}
+
/** Unsafe (no bounds-checking for src nor dst), optimized version of parsing IPv6 string.
*
-* Slightly altered implementation from http://svn.apache.org/repos/asf/apr/apr/trunk/network_io/unix/inet_pton.c
* Parses the input string `src` and stores binary big-endian value into buffer pointed by `dst`,
-* which should be long enough. In case of failure zeroes
-* IPV6_BINARY_LENGTH bytes of buffer pointed by `dst`.
+* which should be long enough. In case of failure zeroes IPV6_BINARY_LENGTH bytes of buffer pointed by `dst`.
*
-* @param src - input string, expected to be non-null and null-terminated right after the IPv6 string value.
-* @param dst - where to put output bytes, expected to be non-null and at IPV6_BINARY_LENGTH-long.
-* @return false if parsing failed, true otherwise.
+* WARNING - this function is adapted to work with ReadBuffer, where src is the position reference (ReadBuffer::position())
+* and eof is the ReadBuffer::eof() - therefore algorithm below does not rely on buffer's continuity.
+* To parse strings use overloads below.
+*
+* @param src - iterator (reference to pointer) over input string - warning - continuity is not guaranteed.
+* @param eof - function returning true if iterator riched the end - warning - can break iterator's continuity.
+* @param dst - where to put output bytes, expected to be non-null and at IPV6_BINARY_LENGTH-long.
+* @param first_block - preparsed first block
+* @return - true if parsed successfully, false otherwise.
*/
-inline bool parseIPv6(const char * src, unsigned char * dst)
+template
+requires (std::is_same::type, char>::value)
+inline bool parseIPv6(T * &src, EOFfunction eof, unsigned char * dst, int32_t first_block = -1)
{
const auto clear_dst = [dst]()
{
- memset(dst, '\0', IPV6_BINARY_LENGTH);
+ std::memset(dst, '\0', IPV6_BINARY_LENGTH);
return false;
};
- /// Leading :: requires some special handling.
- if (*src == ':')
- if (*++src != ':')
- return clear_dst();
+ if (src == nullptr || eof())
+ return clear_dst();
- unsigned char tmp[IPV6_BINARY_LENGTH]{};
- unsigned char * tp = tmp;
- unsigned char * endp = tp + IPV6_BINARY_LENGTH;
- const char * curtok = src;
- bool saw_xdigit = false;
- UInt32 val{};
- unsigned char * colonp = nullptr;
+ int groups = 0; /// number of parsed groups
+ unsigned char * iter = dst; /// iterator over dst buffer
+ unsigned char * zptr = nullptr; /// pointer into dst buffer array where all-zeroes block ("::") is started
- /// Assuming zero-terminated string.
- while (char ch = *src++)
+ std::memset(dst, '\0', IPV6_BINARY_LENGTH);
+
+ if (first_block >= 0)
{
- UInt8 num = unhex(ch);
-
- if (num != 0xFF)
+ *iter++ = static_cast((first_block >> 8) & 0xffu);
+ *iter++ = static_cast(first_block & 0xffu);
+ if (*src == ':')
{
- val <<= 4;
- val |= num;
- if (val > 0xffffu)
+ zptr = iter;
+ ++src;
+ }
+ ++groups;
+ }
+
+ bool group_start = true;
+
+ while (!eof() && groups < 8)
+ {
+ if (*src == ':')
+ {
+ ++src;
+ if (eof()) /// trailing colon is not allowed
return clear_dst();
- saw_xdigit = true;
- continue;
- }
+ group_start = true;
- if (ch == ':')
- {
- curtok = src;
- if (!saw_xdigit)
+ if (*src == ':')
{
- if (colonp)
+ if (zptr != nullptr) /// multiple all-zeroes blocks are not allowed
return clear_dst();
-
- colonp = tp;
+ zptr = iter;
+ ++src;
continue;
}
-
- if (tp + sizeof(UInt16) > endp)
+ if (groups == 0) /// leading colon is not allowed
return clear_dst();
-
- *tp++ = static_cast((val >> 8) & 0xffu);
- *tp++ = static_cast(val & 0xffu);
- saw_xdigit = false;
- val = 0;
- continue;
}
- if (ch == '.' && (tp + IPV4_BINARY_LENGTH) <= endp)
+ if (*src == '.') /// mixed IPv4 parsing
{
- if (!parseIPv4(curtok, tp))
+ if (groups <= 1 && zptr == nullptr) /// IPv4 block can't be the first
+ return clear_dst();
+
+ ++src;
+ if (eof())
+ return clear_dst();
+
+ /// last parsed group should be reinterpreted as a decimal value - it's the first octet of IPv4
+ --groups;
+ iter -= 2;
+
+ UInt16 num = 0;
+ for (int i = 0; i < 2; ++i)
+ {
+ unsigned char first = (iter[i] >> 4) & 0x0fu;
+ unsigned char second = iter[i] & 0x0fu;
+ if (first > 9 || second > 9)
+ return clear_dst();
+ (num *= 100) += first * 10 + second;
+ }
+ if (num > 255)
+ return clear_dst();
+
+ /// parse IPv4 with known first octet
+ if (!parseIPv4(src, eof, iter, num))
return clear_dst();
if constexpr (std::endian::native == std::endian::little)
- std::reverse(tp, tp + IPV4_BINARY_LENGTH);
+ std::reverse(iter, iter + IPV4_BINARY_LENGTH);
- tp += IPV4_BINARY_LENGTH;
- saw_xdigit = false;
- break; /* '\0' was seen by ipv4_scan(). */
+ iter += 4;
+ groups += 2;
+ break; /// IPv4 block is the last - end of parsing
}
- return clear_dst();
+ if (!group_start) /// end of parsing
+ break;
+ group_start = false;
+
+ UInt16 val = 0; /// current decoded group
+ int xdigits = 0; /// number of decoded hex digits in current group
+
+ for (; !eof() && xdigits < 4; ++src, ++xdigits)
+ {
+ UInt8 num = unhex(*src);
+ if (num == 0xFF)
+ break;
+ (val <<= 4) |= num;
+ }
+
+ if (xdigits == 0) /// end of parsing
+ break;
+
+ *iter++ = static_cast((val >> 8) & 0xffu);
+ *iter++ = static_cast(val & 0xffu);
+ ++groups;
}
- if (saw_xdigit)
+ /// either all 8 groups or all-zeroes block should be present
+ if (groups < 8 && zptr == nullptr)
+ return clear_dst();
+
+ if (zptr != nullptr) /// process all-zeroes block
{
- if (tp + sizeof(UInt16) > endp)
+ size_t msize = iter - zptr;
+ std::memmove(dst + IPV6_BINARY_LENGTH - msize, zptr, msize);
+ std::memset(zptr, '\0', IPV6_BINARY_LENGTH - (iter - dst));
+ }
+
+ return true;
+}
+
+/// returns pointer to the right after parsed sequence or null on failed parsing
+inline const char * parseIPv6(const char * src, const char * end, unsigned char * dst)
+{
+ if (parseIPv6(src, [&src, end](){ return src == end; }, dst))
+ return src;
+ return nullptr;
+}
+
+/// returns true if whole buffer was parsed successfully
+inline bool parseIPv6whole(const char * src, const char * end, unsigned char * dst)
+{
+ return parseIPv6(src, end, dst) == end;
+}
+
+/// returns pointer to the right after parsed sequence or null on failed parsing
+inline const char * parseIPv6(const char * src, unsigned char * dst)
+{
+ if (parseIPv6(src, [](){ return false; }, dst))
+ return src;
+ return nullptr;
+}
+
+/// returns true if whole null-terminated string was parsed successfully
+inline bool parseIPv6whole(const char * src, unsigned char * dst)
+{
+ const char * end = parseIPv6(src, dst);
+ return end != nullptr && *end == '\0';
+}
+
+/** Unsafe (no bounds-checking for src nor dst), optimized version of parsing IPv6 string.
+*
+* Parses the input string `src` IPv6 or possible IPv4 into IPv6 and stores binary big-endian value into buffer pointed by `dst`,
+* which should be long enough. In case of failure zeroes IPV6_BINARY_LENGTH bytes of buffer pointed by `dst`.
+*
+* WARNING - this function is adapted to work with ReadBuffer, where src is the position reference (ReadBuffer::position())
+* and eof is the ReadBuffer::eof() - therefore algorithm below does not rely on buffer's continuity.
+*
+* @param src - iterator (reference to pointer) over input string - warning - continuity is not guaranteed.
+* @param eof - function returning true if iterator riched the end - warning - can break iterator's continuity.
+* @param dst - where to put output bytes, expected to be non-null and at IPV6_BINARY_LENGTH-long.
+* @return - true if parsed successfully, false otherwise.
+*/
+template
+requires (std::is_same::type, char>::value)
+inline bool parseIPv6orIPv4(T * &src, EOFfunction eof, unsigned char * dst)
+{
+ const auto clear_dst = [dst]()
+ {
+ std::memset(dst, '\0', IPV6_BINARY_LENGTH);
+ return false;
+ };
+
+ if (src == nullptr)
+ return clear_dst();
+
+ bool leading_zero = false;
+ uint16_t val = 0;
+ int digits = 0;
+ /// parse up to 4 first digits as hexadecimal
+ for (; !eof() && digits < 4; ++src, ++digits)
+ {
+ if (*src == ':' || *src == '.')
+ break;
+
+ if (digits == 0 && *src == '0')
+ leading_zero = true;
+
+ UInt8 num = unhex(*src);
+ if (num == 0xFF)
+ return clear_dst();
+ (val <<= 4) |= num;
+ }
+
+ if (eof())
+ return clear_dst();
+
+ if (*src == ':') /// IPv6
+ {
+ if (digits == 0) /// leading colon - no preparsed group
+ return parseIPv6(src, eof, dst);
+ ++src;
+ return parseIPv6(src, eof, dst, val); /// parse with first preparsed group
+ }
+
+ if (*src == '.') /// IPv4
+ {
+ /// should has some digits
+ if (digits == 0)
+ return clear_dst();
+ /// should not has leading zeroes, should has no more than 3 digits
+ if ((leading_zero && digits > 1) || digits > 3)
return clear_dst();
- *tp++ = static_cast((val >> 8) & 0xffu);
- *tp++ = static_cast(val & 0xffu);
- }
-
- if (colonp)
- {
- /*
- * Since some memmove()'s erroneously fail to handle
- * overlapping regions, we'll do the shift by hand.
- */
- const auto n = tp - colonp;
-
- for (int i = 1; i <= n; ++i)
+ /// recode first group as decimal
+ UInt16 num = 0;
+ for (int exp = 1; exp < 1000; exp *= 10)
{
- endp[- i] = colonp[n - i];
- colonp[n - i] = 0;
+ int n = val & 0x0fu;
+ if (n > 9)
+ return clear_dst();
+ num += n * exp;
+ val >>= 4;
}
- tp = endp;
+ if (num > 255)
+ return clear_dst();
+
+ ++src;
+ if (!parseIPv4(src, eof, dst, num)) /// try to parse as IPv4 with preparsed first octet
+ return clear_dst();
+
+ /// convert into IPv6
+ if constexpr (std::endian::native == std::endian::little)
+ {
+ dst[15] = dst[0]; dst[0] = 0;
+ dst[14] = dst[1]; dst[1] = 0;
+ dst[13] = dst[2]; dst[2] = 0;
+ dst[12] = dst[3]; dst[3] = 0;
+ }
+ else
+ {
+ dst[15] = dst[3]; dst[3] = 0;
+ dst[14] = dst[2]; dst[2] = 0;
+ dst[13] = dst[1]; dst[1] = 0;
+ dst[12] = dst[0]; dst[0] = 0;
+ }
+
+ dst[11] = 0xff;
+ dst[10] = 0xff;
+
+ return true;
}
- if (tp != endp)
- return clear_dst();
-
- memcpy(dst, tmp, sizeof(tmp));
- return true;
+ return clear_dst();
}
/** Format 4-byte binary sequesnce as IPv4 text: 'aaa.bbb.ccc.ddd',
@@ -198,22 +417,27 @@ inline bool parseIPv6(const char * src, unsigned char * dst)
* formatIPv4(&0x7f000001, dst, mask_tail_octets = 1, "0");
* > dst == "127.0.0.0"
*/
-inline void formatIPv4(const unsigned char * src, char *& dst, uint8_t mask_tail_octets = 0, const char * mask_string = "xxx")
+inline void formatIPv4(const unsigned char * src, size_t src_size, char *& dst, uint8_t mask_tail_octets = 0, const char * mask_string = "xxx")
{
- extern const char one_byte_to_string_lookup_table[256][4];
-
const size_t mask_length = mask_string ? strlen(mask_string) : 0;
const size_t limit = std::min(IPV4_BINARY_LENGTH, IPV4_BINARY_LENGTH - mask_tail_octets);
- for (size_t octet = 0; octet < limit; ++octet)
+ const size_t padding = std::min(4 - src_size, limit);
+
+ for (size_t octet = 0; octet < padding; ++octet)
+ {
+ *dst++ = '0';
+ *dst++ = '.';
+ }
+
+ for (size_t octet = 4 - src_size; octet < limit; ++octet)
{
uint8_t value = 0;
if constexpr (std::endian::native == std::endian::little)
value = static_cast(src[IPV4_BINARY_LENGTH - octet - 1]);
else
value = static_cast(src[octet]);
- const auto * rep = one_byte_to_string_lookup_table[value];
- const uint8_t len = rep[0];
- const char* str = rep + 1;
+ const uint8_t len = one_byte_to_string_lookup_table[value].second;
+ const char* str = one_byte_to_string_lookup_table[value].first;
memcpy(dst, str, len);
dst += len;
@@ -231,4 +455,9 @@ inline void formatIPv4(const unsigned char * src, char *& dst, uint8_t mask_tail
dst[-1] = '\0';
}
+inline void formatIPv4(const unsigned char * src, char *& dst, uint8_t mask_tail_octets = 0, const char * mask_string = "xxx")
+{
+ formatIPv4(src, 4, dst, mask_tail_octets, mask_string);
+}
+
}
diff --git a/src/Common/typeid_cast.h b/src/Common/typeid_cast.h
index 3c3f236f740..75bc9eb22b1 100644
--- a/src/Common/typeid_cast.h
+++ b/src/Common/typeid_cast.h
@@ -18,6 +18,9 @@ namespace DB
}
}
+template
+concept is_any_of = (std::same_as || ...);
+
/** Checks type by comparing typeid.
* The exact match of the type is checked. That is, cast to the ancestor will be unsuccessful.
diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp
index 71a6d27e5b4..c1842d7e493 100644
--- a/src/Core/Field.cpp
+++ b/src/Core/Field.cpp
@@ -51,6 +51,18 @@ inline Field getBinaryValue(UInt8 type, ReadBuffer & buf)
readBinary(value, buf);
return value;
}
+ case Field::Types::IPv4:
+ {
+ IPv4 value;
+ readBinary(value, buf);
+ return value;
+ }
+ case Field::Types::IPv6:
+ {
+ IPv6 value;
+ readBinary(value.toUnderType(), buf);
+ return value;
+ }
case Field::Types::Int64:
{
Int64 value;
@@ -583,6 +595,8 @@ String fieldTypeToString(Field::Types::Which type)
case Field::Types::Which::UInt128: return "UInt128";
case Field::Types::Which::UInt256: return "UInt256";
case Field::Types::Which::UUID: return "UUID";
+ case Field::Types::Which::IPv4: return "IPv4";
+ case Field::Types::Which::IPv6: return "IPv6";
}
}
diff --git a/src/Core/Field.h b/src/Core/Field.h
index c3516b705a6..3c787389ef6 100644
--- a/src/Core/Field.h
+++ b/src/Core/Field.h
@@ -13,6 +13,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -192,6 +193,8 @@ template <> struct NearestFieldTypeImpl { using Type = UInt64; };
template <> struct NearestFieldTypeImpl { using Type = UInt64; };
template <> struct NearestFieldTypeImpl { using Type = UUID; };
+template <> struct NearestFieldTypeImpl { using Type = IPv4; };
+template <> struct NearestFieldTypeImpl { using Type = IPv6; };
template <> struct NearestFieldTypeImpl { using Type = Int64; };
template <> struct NearestFieldTypeImpl { using Type = Int64; };
@@ -292,6 +295,8 @@ public:
UUID = 27,
Bool = 28,
Object = 29,
+ IPv4 = 30,
+ IPv6 = 31,
};
};
@@ -468,6 +473,8 @@ public:
case Types::Int128: return get() < rhs.get();
case Types::Int256: return get() < rhs.get();
case Types::UUID: return get() < rhs.get();
+ case Types::IPv4: return get() < rhs.get();
+ case Types::IPv6: return get() < rhs.get();
case Types::Float64: return get() < rhs.get();
case Types::String: return get() < rhs.get();
case Types::Array: return get() < rhs.get();
@@ -507,6 +514,8 @@ public:
case Types::Int128: return get() <= rhs.get();
case Types::Int256: return get() <= rhs.get();
case Types::UUID: return get().toUnderType() <= rhs.get().toUnderType();
+ case Types::IPv4: return get() <= rhs.get();
+ case Types::IPv6: return get() <= rhs.get();
case Types::Float64: return get() <= rhs.get();
case Types::String: return get() <= rhs.get();
case Types::Array: return get() <= rhs.get();
@@ -547,6 +556,8 @@ public:
return std::bit_cast(get()) == std::bit_cast(rhs.get());
}
case Types::UUID: return get() == rhs.get();
+ case Types::IPv4: return get() == rhs.get();
+ case Types::IPv6: return get() == rhs.get();
case Types::String: return get() == rhs.get();
case Types::Array: return get() == rhs.get();
case Types::Tuple: return get() == rhs.get();
@@ -586,6 +597,8 @@ public:
case Types::Int128: return f(field.template get());
case Types::Int256: return f(field.template get());
case Types::UUID: return f(field.template get());
+ case Types::IPv4: return f(field.template get());
+ case Types::IPv6: return f(field.template get());
case Types::Float64: return f(field.template get());
case Types::String: return f(field.template get());
case Types::Array: return f(field.template get());
@@ -612,7 +625,7 @@ public:
private:
std::aligned_union_t, DecimalField, DecimalField, DecimalField,
AggregateFunctionStateData
> storage;
@@ -747,6 +760,8 @@ template <> struct Field::TypeToEnum { static constexpr Types::Which va
template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Int128; };
template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Int256; };
template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::UUID; };
+template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::IPv4; };
+template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::IPv6; };
template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Float64; };
template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::String; };
template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Array; };
@@ -769,6 +784,8 @@ template <> struct Field::EnumToType { using Type = Int64
template <> struct Field::EnumToType { using Type = Int128; };
template <> struct Field::EnumToType { using Type = Int256; };
template <> struct Field::EnumToType { using Type = UUID; };
+template <> struct Field::EnumToType { using Type = IPv4; };
+template <> struct Field::EnumToType { using Type = IPv6; };
template <> struct Field::EnumToType { using Type = Float64; };
template <> struct Field::EnumToType { using Type = String; };
template <> struct Field::EnumToType { using Type = Array; };
diff --git a/src/Core/IResolvedFunction.h b/src/Core/IResolvedFunction.h
index 64c69f597c7..d472d2ce734 100644
--- a/src/Core/IResolvedFunction.h
+++ b/src/Core/IResolvedFunction.h
@@ -12,6 +12,9 @@ using DataTypes = std::vector;
struct Array;
+/* Generic class for all functions.
+ * Represents interface for function signature.
+ */
class IResolvedFunction
{
public:
diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h
index abd3e3c85f8..3c412fa1f17 100644
--- a/src/Core/SortCursor.h
+++ b/src/Core/SortCursor.h
@@ -20,6 +20,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -696,6 +697,8 @@ private:
SortingQueueImpl>, strategy>,
SortingQueueImpl>, strategy>,
+ SortingQueueImpl>, strategy>,
+ SortingQueueImpl>, strategy>,
SortingQueueImpl, strategy>,
SortingQueueImpl, strategy>,
diff --git a/src/Core/TypeId.h b/src/Core/TypeId.h
index 39058773184..d2ae56b4280 100644
--- a/src/Core/TypeId.h
+++ b/src/Core/TypeId.h
@@ -47,6 +47,8 @@ TYPEID_MAP(Int256)
TYPEID_MAP(Float32)
TYPEID_MAP(Float64)
TYPEID_MAP(UUID)
+TYPEID_MAP(IPv4)
+TYPEID_MAP(IPv6)
TYPEID_MAP(Decimal32)
TYPEID_MAP(Decimal64)
diff --git a/src/Core/Types.h b/src/Core/Types.h
index 1eddd431c86..cd559661a96 100644
--- a/src/Core/Types.h
+++ b/src/Core/Types.h
@@ -8,6 +8,7 @@
#include