From 77e5a63648ef23cb182385a1059aa58d238536ae Mon Sep 17 00:00:00 2001 From: Kevin Chiang Date: Mon, 12 Oct 2020 12:58:38 -0700 Subject: [PATCH 001/264] Fix a markdown issue that causes a section of the note to appear outside of the Note box --- docs/en/sql-reference/statements/alter/index/index.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/index/index.md b/docs/en/sql-reference/statements/alter/index/index.md index 4660478551f..6b00696e07f 100644 --- a/docs/en/sql-reference/statements/alter/index/index.md +++ b/docs/en/sql-reference/statements/alter/index/index.md @@ -19,5 +19,4 @@ The first two commands areare lightweight in a sense that they only change metad Also, they are replicated, syncing indices metadata via ZooKeeper. !!! note "Note" - Index manipulation is supported only for tables with [`*MergeTree`](../../../../engines/table-engines/mergetree-family/mergetree.md) engine (including -[replicated](../../../../engines/table-engines/mergetree-family/replication.md) variants). + Index manipulation is supported only for tables with [`*MergeTree`](../../../../engines/table-engines/mergetree-family/mergetree.md) engine (including [replicated](../../../../engines/table-engines/mergetree-family/replication.md) variants). From 74bdc7e645df799d35fdb32fbe77c481c89a390a Mon Sep 17 00:00:00 2001 From: Kevin Chiang Date: Mon, 12 Oct 2020 12:59:18 -0700 Subject: [PATCH 002/264] MergeTrees can have data skipping indexes, removing section from architecture to avoid confusion. --- docs/en/development/architecture.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md index da5b4470704..c86818c7a1a 100644 --- a/docs/en/development/architecture.md +++ b/docs/en/development/architecture.md @@ -177,8 +177,6 @@ When you `INSERT` a bunch of data into `MergeTree`, that bunch is sorted by prim `MergeTree` is not an LSM tree because it doesn’t contain “memtable” and “log”: inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity’s sake, and because we are already inserting data in batches in our applications. -> MergeTree tables can only have one (primary) index: there aren’t any secondary indices. It would be nice to allow multiple physical representations under one logical table, for example, to store data in more than one physical order or even to allow representations with pre-aggregated data along with original data. - There are MergeTree engines that are doing additional work during background merges. Examples are `CollapsingMergeTree` and `AggregatingMergeTree`. This could be treated as special support for updates. Keep in mind that these are not real updates because users usually have no control over the time when background merges are executed, and data in a `MergeTree` table is almost always stored in more than one part, not in completely merged form. ## Replication {#replication} From ee890e7689305c24118666a4f829ea3b4e6fe1fe Mon Sep 17 00:00:00 2001 From: annvsh Date: Sat, 17 Oct 2020 23:39:01 +0700 Subject: [PATCH 003/264] Added RawBLOB format description --- docs/en/interfaces/formats.md | 38 +++++++++++++++++++++++++++++++++++ docs/ru/interfaces/formats.md | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index d96d48bdca3..7b5e887e04b 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -57,6 +57,7 @@ The supported formats are: | [XML](#xml) | ✗ | ✔ | | [CapnProto](#capnproto) | ✔ | ✗ | | [LineAsString](#lineasstring) | ✔ | ✗ | +| [RawBLOB](#rawblob) | ✔ | ✔ | You can control some format processing parameters with the ClickHouse settings. For more information read the [Settings](../operations/settings/settings.md) section. @@ -1338,4 +1339,41 @@ Result: └───────────────────────────────────────────────────┘ ``` +## RawBLOB {#rawblob} + +This format slurps all input data into a single value. This format can only parse a table with a single field of type [String](../sql-reference/data-types/string.md) or similar. +When an empty value is passed to the input, ClickHouse generates an exception: + + ``` text +Code: 108. DB::Exception: No data to insert +``` + +The result is output in binary format without delimiters and escaping. If more than one value is output, the format is ambiguous, and it will be impossible to read the data back. + +**Example** + +``` bash +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -n --query " +DROP TABLE IF EXISTS t; +CREATE TABLE t (a LowCardinality(Nullable(String))) ENGINE = Memory; + +${CLICKHOUSE_CLIENT} --query "INSERT INTO t FORMAT RawBLOB" < ${BASH_SOURCE[0]} + +cat ${BASH_SOURCE[0]} | md5sum + +${CLICKHOUSE_CLIENT} -n --query "SELECT * FROM t FORMAT RawBLOB" | md5sum + +${CLICKHOUSE_CLIENT} --query " +DROP TABLE t; +``` + +Result: + +``` text +f9725a22f9191e064120d718e26862a9 - +``` + [Original article](https://clickhouse.tech/docs/en/interfaces/formats/) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 2745139998f..b0901e70f20 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -39,6 +39,7 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT | [XML](#xml) | ✗ | ✔ | | [CapnProto](#capnproto) | ✔ | ✗ | | [LineAsString](#lineasstring) | ✔ | ✗ | +| [RawBLOB](#rawblob) | ✔ | ✔ | Вы можете регулировать некоторые параметры работы с форматами с помощью настроек ClickHouse. За дополнительной информацией обращайтесь к разделу [Настройки](../operations/settings/settings.md). @@ -1143,4 +1144,41 @@ SELECT * FROM line_as_string; └───────────────────────────────────────────────────┘ ``` +## RawBLOB {#rawblob} + +Этот формат объединяет все входные данные в одно значение. Этот формат может парсить только таблицу с одним полем типа [String](../sql-reference/data-types/string.md) или подобным ему. +При передаче на вход пустого значения ClickHouse сгенерирует исключение: + + ``` text +Code: 108. DB::Exception: No data to insert +``` + +Результат выводится в двоичном формате без разделителей и экранирования. При выводе более одного значения формат неоднозначен и будет невозможно прочитать данные снова. + +**Пример** + +``` bash +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -n --query " +DROP TABLE IF EXISTS t; +CREATE TABLE t (a LowCardinality(Nullable(String))) ENGINE = Memory; + +${CLICKHOUSE_CLIENT} --query "INSERT INTO t FORMAT RawBLOB" < ${BASH_SOURCE[0]} + +cat ${BASH_SOURCE[0]} | md5sum + +${CLICKHOUSE_CLIENT} -n --query "SELECT * FROM t FORMAT RawBLOB" | md5sum + +${CLICKHOUSE_CLIENT} --query " +DROP TABLE t; +``` + +Результат: + +``` text +f9725a22f9191e064120d718e26862a9 - +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/formats/) From a9532d611b34b4f427fd44dda719f86fc235a428 Mon Sep 17 00:00:00 2001 From: annvsh Date: Sat, 17 Oct 2020 23:50:19 +0700 Subject: [PATCH 004/264] Fixed --- docs/en/interfaces/formats.md | 1 + docs/ru/interfaces/formats.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 7b5e887e04b..1a96a2e4643 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1348,6 +1348,7 @@ When an empty value is passed to the input, ClickHouse generates an exception: Code: 108. DB::Exception: No data to insert ``` + The result is output in binary format without delimiters and escaping. If more than one value is output, the format is ambiguous, and it will be impossible to read the data back. **Example** diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index b0901e70f20..844c1a7fd59 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1153,6 +1153,7 @@ SELECT * FROM line_as_string; Code: 108. DB::Exception: No data to insert ``` + Результат выводится в двоичном формате без разделителей и экранирования. При выводе более одного значения формат неоднозначен и будет невозможно прочитать данные снова. **Пример** @@ -1181,4 +1182,5 @@ DROP TABLE t; f9725a22f9191e064120d718e26862a9 - ``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/formats/) From 42047068b7c1473632cb9efd508e394c63d7a7ca Mon Sep 17 00:00:00 2001 From: annvsh Date: Sat, 17 Oct 2020 23:51:26 +0700 Subject: [PATCH 005/264] Fixed --- docs/ru/interfaces/formats.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 844c1a7fd59..ad326e2cfef 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1182,5 +1182,4 @@ DROP TABLE t; f9725a22f9191e064120d718e26862a9 - ``` - [Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/formats/) From a6402336375563bbaa454bfc71316dedf231ea8d Mon Sep 17 00:00:00 2001 From: annvsh Date: Sat, 17 Oct 2020 23:58:53 +0700 Subject: [PATCH 006/264] Fixed --- docs/en/interfaces/formats.md | 2 +- docs/ru/interfaces/formats.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 1a96a2e4643..cccef0725b7 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1344,7 +1344,7 @@ Result: This format slurps all input data into a single value. This format can only parse a table with a single field of type [String](../sql-reference/data-types/string.md) or similar. When an empty value is passed to the input, ClickHouse generates an exception: - ``` text +``` text Code: 108. DB::Exception: No data to insert ``` diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index ad326e2cfef..11fb65fd0c0 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1149,7 +1149,7 @@ SELECT * FROM line_as_string; Этот формат объединяет все входные данные в одно значение. Этот формат может парсить только таблицу с одним полем типа [String](../sql-reference/data-types/string.md) или подобным ему. При передаче на вход пустого значения ClickHouse сгенерирует исключение: - ``` text +``` text Code: 108. DB::Exception: No data to insert ``` From ada6f7eb2469fc32c12b65153f2b6789a6b25d89 Mon Sep 17 00:00:00 2001 From: annvsh Date: Sun, 18 Oct 2020 00:05:24 +0700 Subject: [PATCH 007/264] Fixed --- docs/en/interfaces/formats.md | 1 + docs/ru/interfaces/formats.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index cccef0725b7..46b29478fc8 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1369,6 +1369,7 @@ ${CLICKHOUSE_CLIENT} -n --query "SELECT * FROM t FORMAT RawBLOB" | md5sum ${CLICKHOUSE_CLIENT} --query " DROP TABLE t; +" ``` Result: diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 11fb65fd0c0..654ad3de9f4 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1174,6 +1174,7 @@ ${CLICKHOUSE_CLIENT} -n --query "SELECT * FROM t FORMAT RawBLOB" | md5sum ${CLICKHOUSE_CLIENT} --query " DROP TABLE t; +" ``` Результат: From 8519f66555ea3bfb3060ea7c2bdcfe576702184e Mon Sep 17 00:00:00 2001 From: annvsh Date: Wed, 21 Oct 2020 19:46:31 +0700 Subject: [PATCH 008/264] x --- docs/ru/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 654ad3de9f4..3519b29c2d4 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1146,7 +1146,7 @@ SELECT * FROM line_as_string; ## RawBLOB {#rawblob} -Этот формат объединяет все входные данные в одно значение. Этот формат может парсить только таблицу с одним полем типа [String](../sql-reference/data-types/string.md) или подобным ему. +Этот формат считывает все входные данные в одно значение. Этот формат может парсить только таблицу с одним полем типа [String](../sql-reference/data-types/string.md) или подобным ему. При передаче на вход пустого значения ClickHouse сгенерирует исключение: ``` text From 1399f8889e9c4ae703e0da7833106ac9a1b21414 Mon Sep 17 00:00:00 2001 From: annvsh Date: Fri, 23 Oct 2020 22:51:08 +0700 Subject: [PATCH 009/264] Fixed --- docs/en/interfaces/formats.md | 35 ++++++++++++++------------------- docs/ru/interfaces/formats.md | 37 +++++++++++++++-------------------- 2 files changed, 31 insertions(+), 41 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 46b29478fc8..bb96f6c62a9 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1342,34 +1342,29 @@ Result: ## RawBLOB {#rawblob} This format slurps all input data into a single value. This format can only parse a table with a single field of type [String](../sql-reference/data-types/string.md) or similar. -When an empty value is passed to the input, ClickHouse generates an exception: +The result is output in binary format without delimiters and escaping. If more than one value is output, the format is ambiguous, and it will be impossible to read the data back. + +The difference between `RawBLOB` and `TSVRaw`: +- data is output in binary format, no escaping; +- no delimiters between values; +- no newline at the end of each value. + +In `Raw BLOB` unlike `Raw Binary` strings are output without their length. + +When an empty value is passed to the `RawBLOB` input, ClickHouse generates an exception: ``` text Code: 108. DB::Exception: No data to insert ``` - -The result is output in binary format without delimiters and escaping. If more than one value is output, the format is ambiguous, and it will be impossible to read the data back. - **Example** ``` bash -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. "$CURDIR"/../shell_config.sh - -${CLICKHOUSE_CLIENT} -n --query " -DROP TABLE IF EXISTS t; -CREATE TABLE t (a LowCardinality(Nullable(String))) ENGINE = Memory; - -${CLICKHOUSE_CLIENT} --query "INSERT INTO t FORMAT RawBLOB" < ${BASH_SOURCE[0]} - -cat ${BASH_SOURCE[0]} | md5sum - -${CLICKHOUSE_CLIENT} -n --query "SELECT * FROM t FORMAT RawBLOB" | md5sum - -${CLICKHOUSE_CLIENT} --query " -DROP TABLE t; -" +$ clickhouse-client --query "DROP TABLE IF EXISTS {some_table};" +$ clickhouse-client --query "CREATE TABLE {some_table} (a String) ENGINE = Memory;" +$ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT RawBLOB" +$ clickhouse-client --query "SELECT * FROM {some_table} FORMAT RawBLOB" | md5sum +$ clickhouse-client --query "DROP TABLE {some_table};" ``` Result: diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 3519b29c2d4..e65de3d74e0 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1146,35 +1146,30 @@ SELECT * FROM line_as_string; ## RawBLOB {#rawblob} -Этот формат считывает все входные данные в одно значение. Этот формат может парсить только таблицу с одним полем типа [String](../sql-reference/data-types/string.md) или подобным ему. -При передаче на вход пустого значения ClickHouse сгенерирует исключение: +Этот формат считывает все входные данные в одно значение. Формат может парсить только таблицу с одним полем типа [String](../sql-reference/data-types/string.md) или подобным ему. +Результат выводится в бинарном виде без разделителей и экранирования. При выводе более одного значения формат неоднозначен и будет невозможно прочитать данные снова. + +Отличия между `RawBLOB` и `TabSeparatedRaw`: +- данные выводятся в бинарном виде, без экранирования; +- нет разделителей между значениями; +- нет новой строки в конце каждого значения. + +В `RawBLOB`, в отличие от `RowBinary`, строки выводятся без их длины. + +При передаче на вход `RawBLOB` пустого значения ClickHouse сгенерирует исключение: ``` text Code: 108. DB::Exception: No data to insert ``` - -Результат выводится в двоичном формате без разделителей и экранирования. При выводе более одного значения формат неоднозначен и будет невозможно прочитать данные снова. - **Пример** ``` bash -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. "$CURDIR"/../shell_config.sh - -${CLICKHOUSE_CLIENT} -n --query " -DROP TABLE IF EXISTS t; -CREATE TABLE t (a LowCardinality(Nullable(String))) ENGINE = Memory; - -${CLICKHOUSE_CLIENT} --query "INSERT INTO t FORMAT RawBLOB" < ${BASH_SOURCE[0]} - -cat ${BASH_SOURCE[0]} | md5sum - -${CLICKHOUSE_CLIENT} -n --query "SELECT * FROM t FORMAT RawBLOB" | md5sum - -${CLICKHOUSE_CLIENT} --query " -DROP TABLE t; -" +$ clickhouse-client --query "DROP TABLE IF EXISTS {some_table};" +$ clickhouse-client --query "CREATE TABLE {some_table} (a String) ENGINE = Memory;" +$ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT RawBLOB" +$ clickhouse-client --query "SELECT * FROM {some_table} FORMAT RawBLOB" | md5sum +$ clickhouse-client --query "DROP TABLE {some_table};" ``` Результат: From 5e2a3d12d762ee5222b2a70990e06c4255771454 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 29 Oct 2020 17:25:44 +0300 Subject: [PATCH 010/264] Split requests and responses part from zookeeper --- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 623 +++++++++++++++++++++++ src/Common/ZooKeeper/ZooKeeperCommon.h | 326 ++++++++++++ src/Common/ZooKeeper/ZooKeeperImpl.cpp | 455 ----------------- src/Common/ZooKeeper/ZooKeeperImpl.h | 33 +- 4 files changed, 951 insertions(+), 486 deletions(-) create mode 100644 src/Common/ZooKeeper/ZooKeeperCommon.cpp create mode 100644 src/Common/ZooKeeper/ZooKeeperCommon.h diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp new file mode 100644 index 00000000000..eb04536ae00 --- /dev/null +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -0,0 +1,623 @@ +#include +#include +#include +#include +#include +#include + + +namespace Coordination +{ + +using namespace DB; + +/// ZooKeeper has 1 MB node size and serialization limit by default, +/// but it can be raised up, so we have a slightly larger limit on our side. +#define MAX_STRING_OR_ARRAY_SIZE (1 << 28) /// 256 MiB + +/// Assuming we are at little endian. + +static void write(int64_t x, WriteBuffer & out) +{ + x = __builtin_bswap64(x); + writeBinary(x, out); +} + +static void write(int32_t x, WriteBuffer & out) +{ + x = __builtin_bswap32(x); + writeBinary(x, out); +} + +static void write(bool x, WriteBuffer & out) +{ + writeBinary(x, out); +} + +static void write(const String & s, WriteBuffer & out) +{ + write(int32_t(s.size()), out); + out.write(s.data(), s.size()); +} + +template void write(std::array s, WriteBuffer & out) +{ + write(int32_t(N), out); + out.write(s.data(), N); +} + +template void write(const std::vector & arr, WriteBuffer & out) +{ + write(int32_t(arr.size()), out); + for (const auto & elem : arr) + write(elem, out); +} + +static void write(const ACL & acl, WriteBuffer & out) +{ + write(acl.permissions, out); + write(acl.scheme, out); + write(acl.id, out); +} + +static void write(const Stat & stat, WriteBuffer & out) +{ + write(stat.czxid, out); + write(stat.mzxid, out); + write(stat.ctime, out); + write(stat.mtime, out); + write(stat.version, out); + write(stat.cversion, out); + write(stat.aversion, out); + write(stat.ephemeralOwner, out); + write(stat.dataLength, out); + write(stat.numChildren, out); + write(stat.pzxid, out); +} + +static void write(const Error & x, WriteBuffer & out) +{ + write(static_cast(x), out); +} + +static void read(int64_t & x, ReadBuffer & in) +{ + readBinary(x, in); + x = __builtin_bswap64(x); +} + +static void read(int32_t & x, ReadBuffer & in) +{ + readBinary(x, in); + x = __builtin_bswap32(x); +} + +static void read(Error & x, ReadBuffer & in) +{ + int32_t code; + read(code, in); + x = Error(code); +} + +static void read(bool & x, ReadBuffer & in) +{ + readBinary(x, in); +} + +static void read(String & s, ReadBuffer & in) +{ + int32_t size = 0; + read(size, in); + + if (size == -1) + { + /// It means that zookeeper node has NULL value. We will treat it like empty string. + s.clear(); + return; + } + + if (size < 0) + throw Exception("Negative size while reading string from ZooKeeper", Error::ZMARSHALLINGERROR); + + if (size > MAX_STRING_OR_ARRAY_SIZE) + throw Exception("Too large string size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); + + s.resize(size); + in.read(s.data(), size); +} + +template void read(std::array & s, ReadBuffer & in) +{ + int32_t size = 0; + read(size, in); + if (size != N) + throw Exception("Unexpected array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); + in.read(s.data(), N); +} + +static void read(Stat & stat, ReadBuffer & in) +{ + read(stat.czxid, in); + read(stat.mzxid, in); + read(stat.ctime, in); + read(stat.mtime, in); + read(stat.version, in); + read(stat.cversion, in); + read(stat.aversion, in); + read(stat.ephemeralOwner, in); + read(stat.dataLength, in); + read(stat.numChildren, in); + read(stat.pzxid, in); +} + +template void read(std::vector & arr, ReadBuffer & in) +{ + int32_t size = 0; + read(size, in); + if (size < 0) + throw Exception("Negative size while reading array from ZooKeeper", Error::ZMARSHALLINGERROR); + if (size > MAX_STRING_OR_ARRAY_SIZE) + throw Exception("Too large array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); + arr.resize(size); + for (auto & elem : arr) + read(elem, in); +} + +static void read(ACL & acl, ReadBuffer & in) +{ + read(acl.permissions, in); + read(acl.scheme, in); + read(acl.id, in); +} + +void ZooKeeperRequest::write(WriteBuffer & out) const +{ + /// Excessive copy to calculate length. + WriteBufferFromOwnString buf; + Coordination::write(xid, buf); + Coordination::write(getOpNum(), buf); + writeImpl(buf); + Coordination::write(buf.str(), out); + out.next(); +} + +void ZooKeeperWatchResponse::readImpl(ReadBuffer & in) +{ + Coordination::read(type, in); + Coordination::read(state, in); + Coordination::read(path, in); +} + +void ZooKeeperWatchResponse::writeImpl(WriteBuffer & out) const +{ + Coordination::write(type, out); + Coordination::write(state, out); + Coordination::write(path, out); +} + +void ZooKeeperAuthRequest::writeImpl(WriteBuffer & out) const +{ + Coordination::write(type, out); + Coordination::write(scheme, out); + Coordination::write(data, out); +} + +void ZooKeeperAuthRequest::readImpl(ReadBuffer & in) +{ + Coordination::read(type, in); + Coordination::read(scheme, in); + Coordination::read(data, in); +} + +void ZooKeeperCreateRequest::writeImpl(WriteBuffer & out) const +{ + Coordination::write(path, out); + Coordination::write(data, out); + Coordination::write(acls, out); + + int32_t flags = 0; + + if (is_ephemeral) + flags |= 1; + if (is_sequential) + flags |= 2; + + Coordination::write(flags, out); +} + +void ZooKeeperCreateRequest::readImpl(ReadBuffer & in) +{ + Coordination::read(path, in); + Coordination::read(data, in); + Coordination::read(acls, in); + + int32_t flags = 0; + Coordination::read(flags, in); + + if (flags & 1) + is_ephemeral = true; + if (flags & 2) + is_sequential = true; +} + +void ZooKeeperCreateResponse::readImpl(ReadBuffer & in) +{ + Coordination::read(path_created, in); +} + +void ZooKeeperCreateResponse::writeImpl(WriteBuffer & out) const +{ + Coordination::write(path_created, out); +} + +void ZooKeeperRemoveRequest::writeImpl(WriteBuffer & out) const +{ + Coordination::write(path, out); + Coordination::write(version, out); +} + +void ZooKeeperRemoveRequest::readImpl(ReadBuffer & in) +{ + Coordination::read(path, in); + Coordination::read(version, in); +} + +void ZooKeeperExistsRequest::writeImpl(WriteBuffer & out) const +{ + Coordination::write(path, out); + Coordination::write(has_watch, out); +} + +void ZooKeeperExistsRequest::readImpl(ReadBuffer & in) +{ + Coordination::read(path, in); + Coordination::read(has_watch, in); +} + +void ZooKeeperExistsResponse::readImpl(ReadBuffer & in) +{ + Coordination::read(stat, in); +} + +void ZooKeeperExistsResponse::writeImpl(WriteBuffer & out) const +{ + Coordination::write(stat, out); +} + +void ZooKeeperGetRequest::writeImpl(WriteBuffer & out) const +{ + Coordination::write(path, out); + Coordination::write(has_watch, out); +} + +void ZooKeeperGetRequest::readImpl(ReadBuffer & in) +{ + Coordination::read(path, in); + Coordination::read(has_watch, in); +} + +void ZooKeeperGetResponse::readImpl(ReadBuffer & in) +{ + Coordination::read(data, in); + Coordination::read(stat, in); +} + +void ZooKeeperGetResponse::writeImpl(WriteBuffer & out) const +{ + Coordination::write(data, out); + Coordination::write(stat, out); +} + +void ZooKeeperSetRequest::writeImpl(WriteBuffer & out) const +{ + Coordination::write(path, out); + Coordination::write(data, out); + Coordination::write(version, out); +} + +void ZooKeeperSetRequest::readImpl(ReadBuffer & in) +{ + Coordination::read(path, in); + Coordination::read(data, in); + Coordination::read(version, in); +} + +void ZooKeeperSetResponse::readImpl(ReadBuffer & in) +{ + Coordination::read(stat, in); +} + +void ZooKeeperSetResponse::writeImpl(WriteBuffer & out) const +{ + Coordination::write(stat, out); +} + +void ZooKeeperListRequest::writeImpl(WriteBuffer & out) const +{ + Coordination::write(path, out); + Coordination::write(has_watch, out); +} + +void ZooKeeperListRequest::readImpl(ReadBuffer & in) +{ + Coordination::read(path, in); + Coordination::read(has_watch, in); +} + +void ZooKeeperListResponse::readImpl(ReadBuffer & in) +{ + Coordination::read(names, in); + Coordination::read(stat, in); +} + +void ZooKeeperListResponse::writeImpl(WriteBuffer & out) const +{ + Coordination::write(names, out); + Coordination::write(stat, out); +} + +void ZooKeeperCheckRequest::writeImpl(WriteBuffer & out) const +{ + Coordination::write(path, out); + Coordination::write(version, out); +} + +void ZooKeeperCheckRequest::readImpl(ReadBuffer & in) +{ + Coordination::read(path, in); + Coordination::read(version, in); +} + +void ZooKeeperErrorResponse::readImpl(ReadBuffer & in) +{ + Coordination::Error read_error; + Coordination::read(read_error, in); + + if (read_error != error) + throw Exception(fmt::format("Error code in ErrorResponse ({}) doesn't match error code in header ({})", read_error, error), + Error::ZMARSHALLINGERROR); +} + +void ZooKeeperErrorResponse::writeImpl(WriteBuffer & out) const +{ + Coordination::write(error, out); +} + +ZooKeeperMultiRequest::ZooKeeperMultiRequest(const Requests & generic_requests, const ACLs & default_acls) +{ + /// Convert nested Requests to ZooKeeperRequests. + /// Note that deep copy is required to avoid modifying path in presence of chroot prefix. + requests.reserve(generic_requests.size()); + + for (const auto & generic_request : generic_requests) + { + if (const auto * concrete_request_create = dynamic_cast(generic_request.get())) + { + auto create = std::make_shared(*concrete_request_create); + if (create->acls.empty()) + create->acls = default_acls; + requests.push_back(create); + } + else if (const auto * concrete_request_remove = dynamic_cast(generic_request.get())) + { + requests.push_back(std::make_shared(*concrete_request_remove)); + } + else if (const auto * concrete_request_set = dynamic_cast(generic_request.get())) + { + requests.push_back(std::make_shared(*concrete_request_set)); + } + else if (const auto * concrete_request_check = dynamic_cast(generic_request.get())) + { + requests.push_back(std::make_shared(*concrete_request_check)); + } + else + throw Exception("Illegal command as part of multi ZooKeeper request", Error::ZBADARGUMENTS); + } +} + +void ZooKeeperMultiRequest::writeImpl(WriteBuffer & out) const +{ + for (const auto & request : requests) + { + const auto & zk_request = dynamic_cast(*request); + + bool done = false; + int32_t error = -1; + + Coordination::write(zk_request.getOpNum(), out); + Coordination::write(done, out); + Coordination::write(error, out); + + zk_request.writeImpl(out); + } + + OpNum op_num = -1; + bool done = true; + int32_t error = -1; + + Coordination::write(op_num, out); + Coordination::write(done, out); + Coordination::write(error, out); +} + +void ZooKeeperMultiRequest::readImpl(ReadBuffer & in) +{ + + while (true) + { + OpNum op_num; + bool done; + int32_t error; + Coordination::read(op_num, in); + Coordination::read(done, in); + Coordination::read(error, in); + + if (done) + { + if (op_num != -1) + throw Exception("Unexpected op_num received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); + if (error != -1) + throw Exception("Unexpected error value received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); + break; + } + + ZooKeeperRequestPtr request = ZooKeeperRequestFactory::instance().get(op_num); + request->readImpl(in); + requests.push_back(request); + + if (in.eof()) + throw Exception("Not enough results received for multi transaction", Error::ZMARSHALLINGERROR); + } +} + +void ZooKeeperMultiResponse::readImpl(ReadBuffer & in) +{ + for (auto & response : responses) + { + OpNum op_num; + bool done; + Error op_error; + + Coordination::read(op_num, in); + Coordination::read(done, in); + Coordination::read(op_error, in); + + if (done) + throw Exception("Not enough results received for multi transaction", Error::ZMARSHALLINGERROR); + + /// op_num == -1 is special for multi transaction. + /// For unknown reason, error code is duplicated in header and in response body. + + if (op_num == -1) + response = std::make_shared(); + + if (op_error != Error::ZOK) + { + response->error = op_error; + + /// Set error for whole transaction. + /// If some operations fail, ZK send global error as zero and then send details about each operation. + /// It will set error code for first failed operation and it will set special "runtime inconsistency" code for other operations. + if (error == Error::ZOK && op_error != Error::ZRUNTIMEINCONSISTENCY) + error = op_error; + } + + if (op_error == Error::ZOK || op_num == -1) + dynamic_cast(*response).readImpl(in); + } + + /// Footer. + { + OpNum op_num; + bool done; + int32_t error_read; + + Coordination::read(op_num, in); + Coordination::read(done, in); + Coordination::read(error_read, in); + + if (!done) + throw Exception("Too many results received for multi transaction", Error::ZMARSHALLINGERROR); + if (op_num != -1) + throw Exception("Unexpected op_num received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); + if (error_read != -1) + throw Exception("Unexpected error value received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); + } +} + +void ZooKeeperMultiResponse::writeImpl(WriteBuffer & out) const +{ + for (auto & response : responses) + { + const ZooKeeperResponse & zk_response = dynamic_cast(*response); + OpNum op_num = zk_response.getOpNum(); + bool done = false; + Error op_error = zk_response.error; + + Coordination::write(op_num, out); + Coordination::write(done, out); + Coordination::write(op_error, out); + zk_response.writeImpl(out); + } + + /// Footer. + { + OpNum op_num = -1; + bool done = true; + int32_t error_read = - 1; + + Coordination::write(op_num, out); + Coordination::write(done, out); + Coordination::write(error_read, out); + } +} + +ZooKeeperResponsePtr ZooKeeperHeartbeatRequest::makeResponse() const { return std::make_shared(); } +ZooKeeperResponsePtr ZooKeeperAuthRequest::makeResponse() const { return std::make_shared(); } +ZooKeeperResponsePtr ZooKeeperCreateRequest::makeResponse() const { return std::make_shared(); } +ZooKeeperResponsePtr ZooKeeperRemoveRequest::makeResponse() const { return std::make_shared(); } +ZooKeeperResponsePtr ZooKeeperExistsRequest::makeResponse() const { return std::make_shared(); } +ZooKeeperResponsePtr ZooKeeperGetRequest::makeResponse() const { return std::make_shared(); } +ZooKeeperResponsePtr ZooKeeperSetRequest::makeResponse() const { return std::make_shared(); } +ZooKeeperResponsePtr ZooKeeperListRequest::makeResponse() const { return std::make_shared(); } +ZooKeeperResponsePtr ZooKeeperCheckRequest::makeResponse() const { return std::make_shared(); } +ZooKeeperResponsePtr ZooKeeperMultiRequest::makeResponse() const { return std::make_shared(requests); } +ZooKeeperResponsePtr ZooKeeperCloseRequest::makeResponse() const { return std::make_shared(); } + +void ZooKeeperRequestFactory::registerRequest(OpNum op_num, Creator creator) +{ + if (!op_num_to_request.try_emplace(op_num, creator).second) + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Request with op num {} already registered", op_num); +} + +std::shared_ptr ZooKeeperRequest::read(ReadBuffer & in) +{ + XID xid; + OpNum op_num; + + Coordination::read(xid, in); + Coordination::read(op_num, in); + + auto request = ZooKeeperRequestFactory::instance().get(op_num); + request->xid = xid; + request->readImpl(in); + return request; +} + +ZooKeeperRequestPtr ZooKeeperRequestFactory::get(OpNum op_num) const +{ + auto it = op_num_to_request.find(op_num); + if (it == op_num_to_request.end()) + throw Exception("Unknown operation type " + std::to_string(op_num), Error::ZBADARGUMENTS); + + return it->second(); +} + +ZooKeeperRequestFactory & ZooKeeperRequestFactory::instance() +{ + static ZooKeeperRequestFactory factory; + return factory; +} + +template +void registerZooKeeperRequest(ZooKeeperRequestFactory & factory) +{ + factory.registerRequest(num, [] { return std::make_shared(); }); +} + +ZooKeeperRequestFactory::ZooKeeperRequestFactory() +{ + registerZooKeeperRequest<11, ZooKeeperHeartbeatRequest>(*this); + registerZooKeeperRequest<100, ZooKeeperAuthRequest>(*this); + registerZooKeeperRequest<-11, ZooKeeperCloseRequest>(*this); + registerZooKeeperRequest<1, ZooKeeperCreateRequest>(*this); + registerZooKeeperRequest<2, ZooKeeperRemoveRequest>(*this); + registerZooKeeperRequest<3, ZooKeeperExistsRequest>(*this); + registerZooKeeperRequest<4, ZooKeeperGetRequest>(*this); + registerZooKeeperRequest<5, ZooKeeperSetRequest>(*this); + registerZooKeeperRequest<12, ZooKeeperListRequest>(*this); + registerZooKeeperRequest<13, ZooKeeperCheckRequest>(*this); + registerZooKeeperRequest<14, ZooKeeperMultiRequest>(*this); +} + +} diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h new file mode 100644 index 00000000000..0b19869dd5a --- /dev/null +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -0,0 +1,326 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + + +namespace Coordination +{ + +using XID = int32_t; +using OpNum = int32_t; + + +struct ZooKeeperResponse : virtual Response +{ + virtual ~ZooKeeperResponse() override = default; + virtual void readImpl(ReadBuffer &) = 0; + virtual void writeImpl(WriteBuffer &) const = 0; + virtual OpNum getOpNum() const = 0; +}; + +using ZooKeeperResponsePtr = std::shared_ptr; + +/// Exposed in header file for Yandex.Metrica code. +struct ZooKeeperRequest : virtual Request +{ + XID xid = 0; + bool has_watch = false; + /// If the request was not send and the error happens, we definitely sure, that it has not been processed by the server. + /// If the request was sent and we didn't get the response and the error happens, then we cannot be sure was it processed or not. + bool probably_sent = false; + + ZooKeeperRequest() = default; + ZooKeeperRequest(const ZooKeeperRequest &) = default; + virtual ~ZooKeeperRequest() override = default; + + virtual OpNum getOpNum() const = 0; + + /// Writes length, xid, op_num, then the rest. + void write(WriteBuffer & out) const; + + virtual void writeImpl(WriteBuffer &) const = 0; + virtual void readImpl(ReadBuffer &) = 0; + + static std::shared_ptr read(ReadBuffer & in); + + virtual ZooKeeperResponsePtr makeResponse() const = 0; +}; + +using ZooKeeperRequestPtr = std::shared_ptr; + +struct ZooKeeperHeartbeatRequest final : ZooKeeperRequest +{ + String getPath() const override { return {}; } + OpNum getOpNum() const override { return 11; } + void writeImpl(WriteBuffer &) const override {} + void readImpl(ReadBuffer &) override {} + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperHeartbeatResponse final : ZooKeeperResponse +{ + void readImpl(ReadBuffer &) override {} + void writeImpl(WriteBuffer &) const override {} + OpNum getOpNum() const override { return 11; } +}; + +struct ZooKeeperWatchResponse final : WatchResponse, ZooKeeperResponse +{ + void readImpl(ReadBuffer & in) override; + + void writeImpl(WriteBuffer & out) const override; + + /// TODO FIXME alesap + OpNum getOpNum() const override { return 0; } +}; + +struct ZooKeeperAuthRequest final : ZooKeeperRequest +{ + int32_t type = 0; /// ignored by the server + String scheme; + String data; + + String getPath() const override { return {}; } + OpNum getOpNum() const override { return 100; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperAuthResponse final : ZooKeeperResponse +{ + void readImpl(ReadBuffer &) override {} + void writeImpl(WriteBuffer &) const override {} + + OpNum getOpNum() const override { return 100; } +}; + +struct ZooKeeperCloseRequest final : ZooKeeperRequest +{ + String getPath() const override { return {}; } + OpNum getOpNum() const override { return -11; } + void writeImpl(WriteBuffer &) const override {} + void readImpl(ReadBuffer &) override {} + + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperCloseResponse final : ZooKeeperResponse +{ + void readImpl(ReadBuffer &) override + { + throw Exception("Received response for close request", Error::ZRUNTIMEINCONSISTENCY); + } + + void writeImpl(WriteBuffer &) const override {} + + OpNum getOpNum() const override { return -11; } +}; + +struct ZooKeeperCreateRequest final : CreateRequest, ZooKeeperRequest +{ + ZooKeeperCreateRequest() = default; + explicit ZooKeeperCreateRequest(const CreateRequest & base) : CreateRequest(base) {} + + OpNum getOpNum() const override { return 1; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperCreateResponse final : CreateResponse, ZooKeeperResponse +{ + void readImpl(ReadBuffer & in) override; + + void writeImpl(WriteBuffer & out) const override; + + OpNum getOpNum() const override { return 1; } +}; + +struct ZooKeeperRemoveRequest final : RemoveRequest, ZooKeeperRequest +{ + ZooKeeperRemoveRequest() = default; + explicit ZooKeeperRemoveRequest(const RemoveRequest & base) : RemoveRequest(base) {} + + OpNum getOpNum() const override { return 2; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperRemoveResponse final : RemoveResponse, ZooKeeperResponse +{ + void readImpl(ReadBuffer &) override {} + void writeImpl(WriteBuffer &) const override {} + OpNum getOpNum() const override { return 2; } +}; + +struct ZooKeeperExistsRequest final : ExistsRequest, ZooKeeperRequest +{ + OpNum getOpNum() const override { return 3; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperExistsResponse final : ExistsResponse, ZooKeeperResponse +{ + void readImpl(ReadBuffer & in) override; + void writeImpl(WriteBuffer & out) const override; + OpNum getOpNum() const override { return 3; } +}; + +struct ZooKeeperGetRequest final : GetRequest, ZooKeeperRequest +{ + OpNum getOpNum() const override { return 4; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperGetResponse final : GetResponse, ZooKeeperResponse +{ + void readImpl(ReadBuffer & in) override; + void writeImpl(WriteBuffer & out) const override; + OpNum getOpNum() const override { return 4; } +}; + +struct ZooKeeperSetRequest final : SetRequest, ZooKeeperRequest +{ + ZooKeeperSetRequest() = default; + explicit ZooKeeperSetRequest(const SetRequest & base) : SetRequest(base) {} + + OpNum getOpNum() const override { return 5; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperSetResponse final : SetResponse, ZooKeeperResponse +{ + void readImpl(ReadBuffer & in) override; + void writeImpl(WriteBuffer & out) const override; + OpNum getOpNum() const override { return 5; } +}; + +struct ZooKeeperListRequest final : ListRequest, ZooKeeperRequest +{ + OpNum getOpNum() const override { return 12; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperListResponse final : ListResponse, ZooKeeperResponse +{ + void readImpl(ReadBuffer & in) override; + void writeImpl(WriteBuffer & out) const override; + OpNum getOpNum() const override { return 12; } +}; + +struct ZooKeeperCheckRequest final : CheckRequest, ZooKeeperRequest +{ + ZooKeeperCheckRequest() = default; + explicit ZooKeeperCheckRequest(const CheckRequest & base) : CheckRequest(base) {} + + OpNum getOpNum() const override { return 13; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperCheckResponse final : CheckResponse, ZooKeeperResponse +{ + void readImpl(ReadBuffer &) override {} + void writeImpl(WriteBuffer &) const override {} + OpNum getOpNum() const override { return 13; } +}; + +/// This response may be received only as an element of responses in MultiResponse. +struct ZooKeeperErrorResponse final : ErrorResponse, ZooKeeperResponse +{ + void readImpl(ReadBuffer & in) override; + void writeImpl(WriteBuffer & out) const override; + + OpNum getOpNum() const override { return -1; } +}; + +struct ZooKeeperMultiRequest final : MultiRequest, ZooKeeperRequest +{ + OpNum getOpNum() const override { return 14; } + ZooKeeperMultiRequest() = default; + + ZooKeeperMultiRequest(const Requests & generic_requests, const ACLs & default_acls); + + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperMultiResponse final : MultiResponse, ZooKeeperResponse +{ + OpNum getOpNum() const override { return 14; } + + explicit ZooKeeperMultiResponse(const Requests & requests) + { + responses.reserve(requests.size()); + + for (const auto & request : requests) + responses.emplace_back(dynamic_cast(*request).makeResponse()); + } + + explicit ZooKeeperMultiResponse(const Responses & responses_) + { + responses = responses_; + } + + void readImpl(ReadBuffer & in) override; + + void writeImpl(WriteBuffer & out) const override; + +}; + +class ZooKeeperRequestFactory final : private boost::noncopyable +{ + +public: + using Creator = std::function; + using OpNumToRequest = std::unordered_map; + + static ZooKeeperRequestFactory & instance(); + + ZooKeeperRequestPtr get(OpNum op_num) const; + + void registerRequest(OpNum op_num, Creator creator); + +private: + OpNumToRequest op_num_to_request; + +private: + ZooKeeperRequestFactory(); +}; + +} diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index f5c57781eef..f3c8b537cf2 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -281,17 +281,6 @@ static void write(int32_t x, WriteBuffer & out) writeBinary(x, out); } -static void write(bool x, WriteBuffer & out) -{ - writeBinary(x, out); -} - -static void write(const String & s, WriteBuffer & out) -{ - write(int32_t(s.size()), out); - out.write(s.data(), s.size()); -} - template void write(std::array s, WriteBuffer & out) { write(int32_t(N), out); @@ -305,14 +294,6 @@ template void write(const std::vector & arr, WriteBuffer & out) write(elem, out); } -static void write(const ACL & acl, WriteBuffer & out) -{ - write(acl.permissions, out); - write(acl.scheme, out); - write(acl.id, out); -} - - static void read(int64_t & x, ReadBuffer & in) { readBinary(x, in); @@ -332,33 +313,6 @@ static void read(Error & x, ReadBuffer & in) x = Error(code); } -static void read(bool & x, ReadBuffer & in) -{ - readBinary(x, in); -} - -static void read(String & s, ReadBuffer & in) -{ - int32_t size = 0; - read(size, in); - - if (size == -1) - { - /// It means that zookeeper node has NULL value. We will treat it like empty string. - s.clear(); - return; - } - - if (size < 0) - throw Exception("Negative size while reading string from ZooKeeper", Error::ZMARSHALLINGERROR); - - if (size > MAX_STRING_OR_ARRAY_SIZE) - throw Exception("Too large string size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); - - s.resize(size); - in.read(s.data(), size); -} - template void read(std::array & s, ReadBuffer & in) { int32_t size = 0; @@ -368,21 +322,6 @@ template void read(std::array & s, ReadBuffer & in) in.read(s.data(), N); } -static void read(Stat & stat, ReadBuffer & in) -{ - read(stat.czxid, in); - read(stat.mzxid, in); - read(stat.ctime, in); - read(stat.mtime, in); - read(stat.version, in); - read(stat.cversion, in); - read(stat.aversion, in); - read(stat.ephemeralOwner, in); - read(stat.dataLength, in); - read(stat.numChildren, in); - read(stat.pzxid, in); -} - template void read(std::vector & arr, ReadBuffer & in) { int32_t size = 0; @@ -396,7 +335,6 @@ template void read(std::vector & arr, ReadBuffer & in) read(elem, in); } - template void ZooKeeper::write(const T & x) { @@ -409,19 +347,6 @@ void ZooKeeper::read(T & x) Coordination::read(x, *in); } - -void ZooKeeperRequest::write(WriteBuffer & out) const -{ - /// Excessive copy to calculate length. - WriteBufferFromOwnString buf; - Coordination::write(xid, buf); - Coordination::write(getOpNum(), buf); - writeImpl(buf); - Coordination::write(buf.str(), out); - out.next(); -} - - static void removeRootPath(String & path, const String & root_path) { if (root_path.empty()) @@ -433,385 +358,6 @@ static void removeRootPath(String & path, const String & root_path) path = path.substr(root_path.size()); } - -struct ZooKeeperResponse : virtual Response -{ - virtual ~ZooKeeperResponse() override = default; - virtual void readImpl(ReadBuffer &) = 0; -}; - - -struct ZooKeeperHeartbeatRequest final : ZooKeeperRequest -{ - String getPath() const override { return {}; } - ZooKeeper::OpNum getOpNum() const override { return 11; } - void writeImpl(WriteBuffer &) const override {} - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperHeartbeatResponse final : ZooKeeperResponse -{ - void readImpl(ReadBuffer &) override {} -}; - -struct ZooKeeperWatchResponse final : WatchResponse, ZooKeeperResponse -{ - void readImpl(ReadBuffer & in) override - { - Coordination::read(type, in); - Coordination::read(state, in); - Coordination::read(path, in); - } -}; - -struct ZooKeeperAuthRequest final : ZooKeeperRequest -{ - int32_t type = 0; /// ignored by the server - String scheme; - String data; - - String getPath() const override { return {}; } - ZooKeeper::OpNum getOpNum() const override { return 100; } - void writeImpl(WriteBuffer & out) const override - { - Coordination::write(type, out); - Coordination::write(scheme, out); - Coordination::write(data, out); - } - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperAuthResponse final : ZooKeeperResponse -{ - void readImpl(ReadBuffer &) override {} -}; - -struct ZooKeeperCloseRequest final : ZooKeeperRequest -{ - String getPath() const override { return {}; } - ZooKeeper::OpNum getOpNum() const override { return -11; } - void writeImpl(WriteBuffer &) const override {} - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperCloseResponse final : ZooKeeperResponse -{ - void readImpl(ReadBuffer &) override - { - throw Exception("Received response for close request", Error::ZRUNTIMEINCONSISTENCY); - } -}; - -struct ZooKeeperCreateRequest final : CreateRequest, ZooKeeperRequest -{ - ZooKeeperCreateRequest() = default; - explicit ZooKeeperCreateRequest(const CreateRequest & base) : CreateRequest(base) {} - - ZooKeeper::OpNum getOpNum() const override { return 1; } - void writeImpl(WriteBuffer & out) const override - { - Coordination::write(path, out); - Coordination::write(data, out); - Coordination::write(acls, out); - - int32_t flags = 0; - - if (is_ephemeral) - flags |= 1; - if (is_sequential) - flags |= 2; - - Coordination::write(flags, out); - } - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperCreateResponse final : CreateResponse, ZooKeeperResponse -{ - void readImpl(ReadBuffer & in) override - { - Coordination::read(path_created, in); - } -}; - -struct ZooKeeperRemoveRequest final : RemoveRequest, ZooKeeperRequest -{ - ZooKeeperRemoveRequest() = default; - explicit ZooKeeperRemoveRequest(const RemoveRequest & base) : RemoveRequest(base) {} - - ZooKeeper::OpNum getOpNum() const override { return 2; } - void writeImpl(WriteBuffer & out) const override - { - Coordination::write(path, out); - Coordination::write(version, out); - } - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperRemoveResponse final : RemoveResponse, ZooKeeperResponse -{ - void readImpl(ReadBuffer &) override {} -}; - -struct ZooKeeperExistsRequest final : ExistsRequest, ZooKeeperRequest -{ - ZooKeeper::OpNum getOpNum() const override { return 3; } - void writeImpl(WriteBuffer & out) const override - { - Coordination::write(path, out); - Coordination::write(has_watch, out); - } - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperExistsResponse final : ExistsResponse, ZooKeeperResponse -{ - void readImpl(ReadBuffer & in) override - { - Coordination::read(stat, in); - } -}; - -struct ZooKeeperGetRequest final : GetRequest, ZooKeeperRequest -{ - ZooKeeper::OpNum getOpNum() const override { return 4; } - void writeImpl(WriteBuffer & out) const override - { - Coordination::write(path, out); - Coordination::write(has_watch, out); - } - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperGetResponse final : GetResponse, ZooKeeperResponse -{ - void readImpl(ReadBuffer & in) override - { - Coordination::read(data, in); - Coordination::read(stat, in); - } -}; - -struct ZooKeeperSetRequest final : SetRequest, ZooKeeperRequest -{ - ZooKeeperSetRequest() = default; - explicit ZooKeeperSetRequest(const SetRequest & base) : SetRequest(base) {} - - ZooKeeper::OpNum getOpNum() const override { return 5; } - void writeImpl(WriteBuffer & out) const override - { - Coordination::write(path, out); - Coordination::write(data, out); - Coordination::write(version, out); - } - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperSetResponse final : SetResponse, ZooKeeperResponse -{ - void readImpl(ReadBuffer & in) override - { - Coordination::read(stat, in); - } -}; - -struct ZooKeeperListRequest final : ListRequest, ZooKeeperRequest -{ - ZooKeeper::OpNum getOpNum() const override { return 12; } - void writeImpl(WriteBuffer & out) const override - { - Coordination::write(path, out); - Coordination::write(has_watch, out); - } - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperListResponse final : ListResponse, ZooKeeperResponse -{ - void readImpl(ReadBuffer & in) override - { - Coordination::read(names, in); - Coordination::read(stat, in); - } -}; - -struct ZooKeeperCheckRequest final : CheckRequest, ZooKeeperRequest -{ - ZooKeeperCheckRequest() = default; - explicit ZooKeeperCheckRequest(const CheckRequest & base) : CheckRequest(base) {} - - ZooKeeper::OpNum getOpNum() const override { return 13; } - void writeImpl(WriteBuffer & out) const override - { - Coordination::write(path, out); - Coordination::write(version, out); - } - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperCheckResponse final : CheckResponse, ZooKeeperResponse -{ - void readImpl(ReadBuffer &) override {} -}; - -/// This response may be received only as an element of responses in MultiResponse. -struct ZooKeeperErrorResponse final : ErrorResponse, ZooKeeperResponse -{ - void readImpl(ReadBuffer & in) override - { - Coordination::Error read_error; - Coordination::read(read_error, in); - - if (read_error != error) - throw Exception(fmt::format("Error code in ErrorResponse ({}) doesn't match error code in header ({})", read_error, error), - Error::ZMARSHALLINGERROR); - } -}; - -struct ZooKeeperMultiRequest final : MultiRequest, ZooKeeperRequest -{ - ZooKeeper::OpNum getOpNum() const override { return 14; } - - ZooKeeperMultiRequest(const Requests & generic_requests, const ACLs & default_acls) - { - /// Convert nested Requests to ZooKeeperRequests. - /// Note that deep copy is required to avoid modifying path in presence of chroot prefix. - requests.reserve(generic_requests.size()); - - for (const auto & generic_request : generic_requests) - { - if (const auto * concrete_request_create = dynamic_cast(generic_request.get())) - { - auto create = std::make_shared(*concrete_request_create); - if (create->acls.empty()) - create->acls = default_acls; - requests.push_back(create); - } - else if (const auto * concrete_request_remove = dynamic_cast(generic_request.get())) - { - requests.push_back(std::make_shared(*concrete_request_remove)); - } - else if (const auto * concrete_request_set = dynamic_cast(generic_request.get())) - { - requests.push_back(std::make_shared(*concrete_request_set)); - } - else if (const auto * concrete_request_check = dynamic_cast(generic_request.get())) - { - requests.push_back(std::make_shared(*concrete_request_check)); - } - else - throw Exception("Illegal command as part of multi ZooKeeper request", Error::ZBADARGUMENTS); - } - } - - void writeImpl(WriteBuffer & out) const override - { - for (const auto & request : requests) - { - const auto & zk_request = dynamic_cast(*request); - - bool done = false; - int32_t error = -1; - - Coordination::write(zk_request.getOpNum(), out); - Coordination::write(done, out); - Coordination::write(error, out); - - zk_request.writeImpl(out); - } - - ZooKeeper::OpNum op_num = -1; - bool done = true; - int32_t error = -1; - - Coordination::write(op_num, out); - Coordination::write(done, out); - Coordination::write(error, out); - } - - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperMultiResponse final : MultiResponse, ZooKeeperResponse -{ - explicit ZooKeeperMultiResponse(const Requests & requests) - { - responses.reserve(requests.size()); - - for (const auto & request : requests) - responses.emplace_back(dynamic_cast(*request).makeResponse()); - } - - void readImpl(ReadBuffer & in) override - { - for (auto & response : responses) - { - ZooKeeper::OpNum op_num; - bool done; - Error op_error; - - Coordination::read(op_num, in); - Coordination::read(done, in); - Coordination::read(op_error, in); - - if (done) - throw Exception("Not enough results received for multi transaction", Error::ZMARSHALLINGERROR); - - /// op_num == -1 is special for multi transaction. - /// For unknown reason, error code is duplicated in header and in response body. - - if (op_num == -1) - response = std::make_shared(); - - if (op_error != Error::ZOK) - { - response->error = op_error; - - /// Set error for whole transaction. - /// If some operations fail, ZK send global error as zero and then send details about each operation. - /// It will set error code for first failed operation and it will set special "runtime inconsistency" code for other operations. - if (error == Error::ZOK && op_error != Error::ZRUNTIMEINCONSISTENCY) - error = op_error; - } - - if (op_error == Error::ZOK || op_num == -1) - dynamic_cast(*response).readImpl(in); - } - - /// Footer. - { - ZooKeeper::OpNum op_num; - bool done; - int32_t error_read; - - Coordination::read(op_num, in); - Coordination::read(done, in); - Coordination::read(error_read, in); - - if (!done) - throw Exception("Too many results received for multi transaction", Error::ZMARSHALLINGERROR); - if (op_num != -1) - throw Exception("Unexpected op_num received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); - if (error_read != -1) - throw Exception("Unexpected error value received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); - } - } -}; - - -ZooKeeperResponsePtr ZooKeeperHeartbeatRequest::makeResponse() const { return std::make_shared(); } -ZooKeeperResponsePtr ZooKeeperAuthRequest::makeResponse() const { return std::make_shared(); } -ZooKeeperResponsePtr ZooKeeperCreateRequest::makeResponse() const { return std::make_shared(); } -ZooKeeperResponsePtr ZooKeeperRemoveRequest::makeResponse() const { return std::make_shared(); } -ZooKeeperResponsePtr ZooKeeperExistsRequest::makeResponse() const { return std::make_shared(); } -ZooKeeperResponsePtr ZooKeeperGetRequest::makeResponse() const { return std::make_shared(); } -ZooKeeperResponsePtr ZooKeeperSetRequest::makeResponse() const { return std::make_shared(); } -ZooKeeperResponsePtr ZooKeeperListRequest::makeResponse() const { return std::make_shared(); } -ZooKeeperResponsePtr ZooKeeperCheckRequest::makeResponse() const { return std::make_shared(); } -ZooKeeperResponsePtr ZooKeeperMultiRequest::makeResponse() const { return std::make_shared(requests); } -ZooKeeperResponsePtr ZooKeeperCloseRequest::makeResponse() const { return std::make_shared(); } - - static constexpr int32_t protocol_version = 0; static constexpr ZooKeeper::XID watch_xid = -1; @@ -1688,5 +1234,4 @@ void ZooKeeper::close() ProfileEvents::increment(ProfileEvents::ZooKeeperClose); } - } diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index 085b0e9856a..c96d7d2f0cb 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -85,9 +86,6 @@ namespace Coordination using namespace DB; -struct ZooKeeperRequest; - - /** Usage scenario: look at the documentation for IKeeper class. */ class ZooKeeper : public IKeeper @@ -193,7 +191,7 @@ private: struct RequestInfo { - std::shared_ptr request; + ZooKeeperRequestPtr request; ResponseCallback callback; WatchCallback watch; clock::time_point time; @@ -246,31 +244,4 @@ private: CurrentMetrics::Increment active_session_metric_increment{CurrentMetrics::ZooKeeperSession}; }; -struct ZooKeeperResponse; -using ZooKeeperResponsePtr = std::shared_ptr; - -/// Exposed in header file for Yandex.Metrica code. -struct ZooKeeperRequest : virtual Request -{ - ZooKeeper::XID xid = 0; - bool has_watch = false; - /// If the request was not send and the error happens, we definitely sure, that is has not been processed by the server. - /// If the request was sent and we didn't get the response and the error happens, then we cannot be sure was it processed or not. - bool probably_sent = false; - - ZooKeeperRequest() = default; - ZooKeeperRequest(const ZooKeeperRequest &) = default; - virtual ~ZooKeeperRequest() override = default; - - virtual ZooKeeper::OpNum getOpNum() const = 0; - - /// Writes length, xid, op_num, then the rest. - void write(WriteBuffer & out) const; - - virtual void writeImpl(WriteBuffer &) const = 0; - - virtual ZooKeeperResponsePtr makeResponse() const = 0; -}; - - } From c2525ef211a5cc1b93914cc67a7a6a36a2f2df5e Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 30 Oct 2020 17:16:47 +0300 Subject: [PATCH 011/264] Server and client pinging each other --- programs/server/Server.cpp | 16 + src/Common/ZooKeeper/TestKeeperStorage.cpp | 6 + src/Common/ZooKeeper/TestKeeperStorage.h | 77 +++++ src/Common/ZooKeeper/ZooKeeperCommon.h | 1 - src/Interpreters/Context.cpp | 12 + src/Interpreters/Context.h | 3 + src/Server/TCPHandlerFactory.h | 11 +- src/Server/TestKeeperTCPHandler.cpp | 339 +++++++++++++++++++++ src/Server/TestKeeperTCPHandler.h | 48 +++ 9 files changed, 510 insertions(+), 3 deletions(-) create mode 100644 src/Common/ZooKeeper/TestKeeperStorage.cpp create mode 100644 src/Common/ZooKeeper/TestKeeperStorage.h create mode 100644 src/Server/TestKeeperTCPHandler.cpp create mode 100644 src/Server/TestKeeperTCPHandler.h diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index b85cb5e75f2..f855a76e362 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -949,6 +949,22 @@ int Server::main(const std::vector & /*args*/) LOG_INFO(log, "Listening for connections with native protocol (tcp): {}", address.toString()); }); + /// TCP TestKeeper + create_server("test_keeper_tcp_port", [&](UInt16 port) + { + Poco::Net::ServerSocket socket; + auto address = socket_bind_listen(socket, listen_host, port); + socket.setReceiveTimeout(settings.receive_timeout); + socket.setSendTimeout(settings.send_timeout); + servers.emplace_back(std::make_unique( + new TCPHandlerFactory(*this, false, true), + server_pool, + socket, + new Poco::Net::TCPServerParams)); + + LOG_INFO(log, "Listening for connections to fake zookeeper (tcp): {}", address.toString()); + }); + /// TCP with SSL create_server("tcp_port_secure", [&](UInt16 port) { diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp new file mode 100644 index 00000000000..c238fa2620f --- /dev/null +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -0,0 +1,6 @@ +#include + +namespace DB +{ + +} diff --git a/src/Common/ZooKeeper/TestKeeperStorage.h b/src/Common/ZooKeeper/TestKeeperStorage.h new file mode 100644 index 00000000000..f0c8a942dff --- /dev/null +++ b/src/Common/ZooKeeper/TestKeeperStorage.h @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include +#include + +namespace zkutil +{ + +using namespace DB; + +class TestKeeperStorage +{ + struct TestKeeperRequest; + using TestKeeperRequestPtr = std::shared_ptr; + + std::atomic session_id_counter{0}; + + struct Node + { + String data; + Coordination::ACLs acls; + bool is_ephemeral = false; + bool is_sequental = false; + Coordination::Stat stat{}; + int32_t seq_num = 0; + }; + + using Container = std::map; + + using WatchCallbacks = std::vector; + using Watches = std::map; + + Container container; + + String root_path; + + std::atomic zxid{0}; + + Watches watches; + Watches list_watches; /// Watches for 'list' request (watches on children). + + using clock = std::chrono::steady_clock; + + struct RequestInfo + { + TestKeeperRequestPtr request; + Coordination::ResponseCallback callback; + Coordination::WatchCallback watch; + clock::time_point time; + }; + using RequestsQueue = ConcurrentBoundedQueue; + RequestsQueue requests_queue{1}; + + void pushRequest(RequestInfo && request); + + void finalize(); + + ThreadFromGlobalPool processing_thread; + + void processingThread(); + +public: + void putRequest(const Coordination::ZooKeeperRequestPtr & request, std::shared_ptr response_out); + + int64_t getSessionID() + { + return session_id_counter.fetch_add(1); + } + int64_t getZXID() + { + return zxid.fetch_add(1); + } +}; + +} diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index 0b19869dd5a..6293cbb09fe 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -25,7 +25,6 @@ namespace Coordination using XID = int32_t; using OpNum = int32_t; - struct ZooKeeperResponse : virtual Response { virtual ~ZooKeeperResponse() override = default; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index b75e9ae9d58..863b179df90 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -300,6 +301,8 @@ struct ContextShared mutable zkutil::ZooKeeperPtr zookeeper; /// Client for ZooKeeper. + mutable std::mutex test_keeper_storage_mutex; + mutable std::shared_ptr test_keeper_storage; mutable std::mutex auxiliary_zookeepers_mutex; mutable std::map auxiliary_zookeepers; /// Map for auxiliary ZooKeeper clients. @@ -1492,6 +1495,15 @@ zkutil::ZooKeeperPtr Context::getZooKeeper() const return shared->zookeeper; } +std::shared_ptr & Context::getTestKeeperStorage() const +{ + std::lock_guard lock(shared->test_keeper_storage_mutex); + if (!shared->test_keeper_storage) + shared->test_keeper_storage = std::make_shared(); + + return shared->test_keeper_storage; +} + zkutil::ZooKeeperPtr Context::getAuxiliaryZooKeeper(const String & name) const { std::lock_guard lock(shared->auxiliary_zookeepers_mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index fbf578494ed..22e90d8eb5a 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -40,6 +40,7 @@ namespace Poco namespace zkutil { class ZooKeeper; + class TestKeeperStorage; } @@ -483,6 +484,8 @@ public: std::shared_ptr getZooKeeper() const; /// Same as above but return a zookeeper connection from auxiliary_zookeepers configuration entry. std::shared_ptr getAuxiliaryZooKeeper(const String & name) const; + + std::shared_ptr & getTestKeeperStorage() const; /// Has ready or expired ZooKeeper bool hasZooKeeper() const; /// Reset current zookeeper session. Do not create a new one. diff --git a/src/Server/TCPHandlerFactory.h b/src/Server/TCPHandlerFactory.h index 5ecd427bf8b..945a2350508 100644 --- a/src/Server/TCPHandlerFactory.h +++ b/src/Server/TCPHandlerFactory.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace Poco { class Logger; } @@ -16,6 +17,7 @@ class TCPHandlerFactory : public Poco::Net::TCPServerConnectionFactory private: IServer & server; Poco::Logger * log; + bool test_keeper; class DummyTCPHandler : public Poco::Net::TCPServerConnection { @@ -25,9 +27,10 @@ private: }; public: - explicit TCPHandlerFactory(IServer & server_, bool secure_ = false) + explicit TCPHandlerFactory(IServer & server_, bool secure_ = false, bool test_keeper_ = false) : server(server_) , log(&Poco::Logger::get(std::string("TCP") + (secure_ ? "S" : "") + "HandlerFactory")) + , test_keeper(test_keeper_) { } @@ -36,7 +39,11 @@ public: try { LOG_TRACE(log, "TCP Request. Address: {}", socket.peerAddress().toString()); - return new TCPHandler(server, socket); + + if (test_keeper) + return new TestKeeperTCPHandler(server, socket); + else + return new TCPHandler(server, socket); } catch (const Poco::Net::NetException &) { diff --git a/src/Server/TestKeeperTCPHandler.cpp b/src/Server/TestKeeperTCPHandler.cpp new file mode 100644 index 00000000000..a0679554f64 --- /dev/null +++ b/src/Server/TestKeeperTCPHandler.cpp @@ -0,0 +1,339 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + +} + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-function" +#endif + +/// ZooKeeper has 1 MB node size and serialization limit by default, +/// but it can be raised up, so we have a slightly larger limit on our side. +#define MAX_STRING_OR_ARRAY_SIZE (1 << 28) /// 256 MiB + +/// Assuming we are at little endian. + +static void write(int64_t x, WriteBuffer & out) +{ + x = __builtin_bswap64(x); + writeBinary(x, out); +} + +static void write(int32_t x, WriteBuffer & out) +{ + x = __builtin_bswap32(x); + writeBinary(x, out); +} + +static void write(bool x, WriteBuffer & out) +{ + writeBinary(x, out); +} + +static void write(const String & s, WriteBuffer & out) +{ + write(int32_t(s.size()), out); + out.write(s.data(), s.size()); +} + +template void write(std::array s, WriteBuffer & out) +{ + write(int32_t(N), out); + out.write(s.data(), N); +} + +template void write(const std::vector & arr, WriteBuffer & out) +{ + write(int32_t(arr.size()), out); + for (const auto & elem : arr) + write(elem, out); +} + +static void write(const Coordination::ACL & acl, WriteBuffer & out) +{ + write(acl.permissions, out); + write(acl.scheme, out); + write(acl.id, out); +} + +static void write(const Coordination::Stat & stat, WriteBuffer & out) +{ + write(stat.czxid, out); + write(stat.mzxid, out); + write(stat.ctime, out); + write(stat.mtime, out); + write(stat.version, out); + write(stat.cversion, out); + write(stat.aversion, out); + write(stat.ephemeralOwner, out); + write(stat.dataLength, out); + write(stat.numChildren, out); + write(stat.pzxid, out); +} + +static void write(const Coordination::Error & x, WriteBuffer & out) +{ + write(static_cast(x), out); +} + +static void read(int64_t & x, ReadBuffer & in) +{ + readBinary(x, in); + x = __builtin_bswap64(x); +} + +static void read(int32_t & x, ReadBuffer & in) +{ + readBinary(x, in); + x = __builtin_bswap32(x); +} + +static void read(Coordination::Error & x, ReadBuffer & in) +{ + int32_t code; + read(code, in); + x = Coordination::Error(code); +} + +static void read(bool & x, ReadBuffer & in) +{ + readBinary(x, in); +} + +static void read(String & s, ReadBuffer & in) +{ + int32_t size = 0; + read(size, in); + + if (size == -1) + { + /// It means that zookeeper node has NULL value. We will treat it like empty string. + s.clear(); + return; + } + + if (size < 0) + throw Exception("Negative size while reading string from ZooKeeper", ErrorCodes::LOGICAL_ERROR); + + if (size > MAX_STRING_OR_ARRAY_SIZE) + throw Exception("Too large string size while reading from ZooKeeper", ErrorCodes::LOGICAL_ERROR); + + s.resize(size); + in.read(s.data(), size); +} + +template void read(std::array & s, ReadBuffer & in) +{ + int32_t size = 0; + read(size, in); + if (size != N) + throw Exception("Unexpected array size while reading from ZooKeeper", ErrorCodes::LOGICAL_ERROR); + in.read(s.data(), N); +} + +static void read(Coordination::Stat & stat, ReadBuffer & in) +{ + read(stat.czxid, in); + read(stat.mzxid, in); + read(stat.ctime, in); + read(stat.mtime, in); + read(stat.version, in); + read(stat.cversion, in); + read(stat.aversion, in); + read(stat.ephemeralOwner, in); + read(stat.dataLength, in); + read(stat.numChildren, in); + read(stat.pzxid, in); +} + +template void read(std::vector & arr, ReadBuffer & in) +{ + int32_t size = 0; + read(size, in); + if (size < 0) + throw Exception("Negative size while reading array from ZooKeeper", ErrorCodes::LOGICAL_ERROR); + if (size > MAX_STRING_OR_ARRAY_SIZE) + throw Exception("Too large array size while reading from ZooKeeper", ErrorCodes::LOGICAL_ERROR); + arr.resize(size); + for (auto & elem : arr) + read(elem, in); +} + +static void read(Coordination::ACL & acl, ReadBuffer & in) +{ + read(acl.permissions, in); + read(acl.scheme, in); + read(acl.id, in); +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +void TestKeeperTCPHandler::sendHandshake() +{ + static constexpr int32_t handshake_length = 36; + static constexpr int32_t protocol_version = 0; + static constexpr int32_t DEFAULT_SESSION_TIMEOUT = 30000; + + write(handshake_length, *out); + write(protocol_version, *out); + write(DEFAULT_SESSION_TIMEOUT, *out); + write(test_keeper_storage->getSessionID(), *out); + constexpr int32_t passwd_len = 16; + std::array passwd{}; + write(passwd, *out); + out->next(); +} + +void TestKeeperTCPHandler::run() +{ + runImpl(); +} + +void TestKeeperTCPHandler::receiveHandshake() +{ + int32_t handshake_length; + int32_t protocol_version; + int64_t last_zxid_seen; + int32_t timeout; + int64_t previous_session_id = 0; /// We don't support session restore. So previous session_id is always zero. + constexpr int32_t passwd_len = 16; + std::array passwd {}; + + read(handshake_length, *in); + if (handshake_length != 44) + throw Exception("Unexpected handshake length received: " + toString(handshake_length), ErrorCodes::LOGICAL_ERROR); + + read(protocol_version, *in); + + if (protocol_version != 0) + throw Exception("Unexpected protocol version: " + toString(protocol_version), ErrorCodes::LOGICAL_ERROR); + + read(last_zxid_seen, *in); + + if (last_zxid_seen != 0) + throw Exception("Non zero last_zxid_seen is not supported", ErrorCodes::LOGICAL_ERROR); + + read(timeout, *in); + read(previous_session_id, *in); + + if (previous_session_id != 0) + throw Exception("Non zero previous session id is not supported", ErrorCodes::LOGICAL_ERROR); + + read(passwd, *in); +} + + +void TestKeeperTCPHandler::runImpl() +{ + setThreadName("TstKprHandler"); + ThreadStatus thread_status; + auto global_receive_timeout = global_context.getSettingsRef().receive_timeout; + auto global_send_timeout = global_context.getSettingsRef().send_timeout; + + socket().setReceiveTimeout(global_receive_timeout); + socket().setSendTimeout(global_send_timeout); + socket().setNoDelay(true); + + in = std::make_shared(socket()); + out = std::make_shared(socket()); + + if (in->eof()) + { + LOG_WARNING(log, "Client has not sent any data."); + return; + } + + try + { + receiveHandshake(); + } + catch (const Exception & e) /// Typical for an incorrect username, password, or address. + { + LOG_DEBUG(log, "Cannot receive handshake {}", e.displayText()); + } + + sendHandshake(); + + while (true) + { + UInt64 max_wait = operation_timeout.totalMicroseconds(); + if (in->poll(max_wait)) + { + receiveHeartbeatRequest(); + sendHeartbeatResponse(); + } + } +} + + +void TestKeeperTCPHandler::receiveHeartbeatRequest() +{ + LOG_DEBUG(log, "Receiving heartbeat event"); + int32_t length; + read(length, *in); + int32_t total_count = in->count(); + LOG_DEBUG(log, "RECEIVED LENGTH {}", length); + int32_t xid; + LOG_DEBUG(log, "READING XID"); + read(xid, *in); + + LOG_DEBUG(log, "Received xid {}", xid); + + if (xid == -2) + { + int32_t opnum; + read(opnum, *in); + LOG_DEBUG(log, "RRECEIVED OP NUM {}", opnum); + auto request = std::make_shared(); + request->readImpl(*in); + int32_t readed = in->count() - total_count; + if (readed != length) + LOG_DEBUG(log, "EXPECTED TO READ {}, BUT GOT {}", length, readed); + } + else + { + LOG_INFO(log, "UNKNOWN EVENT xid:{}", xid); + } + + LOG_DEBUG(log, "Event received"); +} + + +void TestKeeperTCPHandler::sendHeartbeatResponse() +{ + LOG_DEBUG(log, "Sending heartbeat event"); + int32_t length = sizeof(int32_t) + sizeof(int64_t) + sizeof(Coordination::Error); + write(length, *out); + int64_t zxid = test_keeper_storage->getZXID(); + int32_t xid = -2; + write(xid, *out); + write(zxid, *out); + write(Coordination::Error::ZOK, *out); + auto response = std::make_shared(); + response->writeImpl(*out); + out->next(); +} + +} diff --git a/src/Server/TestKeeperTCPHandler.h b/src/Server/TestKeeperTCPHandler.h new file mode 100644 index 00000000000..967ea1d29e7 --- /dev/null +++ b/src/Server/TestKeeperTCPHandler.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include "IServer.h" +#include +#include +#include +#include +#include + +namespace DB +{ + +class TestKeeperTCPHandler : public Poco::Net::TCPServerConnection +{ +public: + TestKeeperTCPHandler(IServer & server_, const Poco::Net::StreamSocket & socket_) + : Poco::Net::TCPServerConnection(socket_) + , server(server_) + , log(&Poco::Logger::get("TestKeeperTCPHandler")) + , global_context(server.context()) + , test_keeper_storage(global_context.getTestKeeperStorage()) + , operation_timeout(10000) + { + } + + void run() override; +private: + IServer & server; + Poco::Logger * log; + Context global_context; + std::shared_ptr test_keeper_storage; + Poco::Timespan operation_timeout; + + /// Streams for reading/writing from/to client connection socket. + std::shared_ptr in; + std::shared_ptr out; + + void runImpl(); + + void sendHandshake(); + void receiveHandshake(); + + void receiveHeartbeatRequest(); + void sendHeartbeatResponse(); +}; + +} From 1fc763328965d7cab2890f15b073b3824ca5c068 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 30 Oct 2020 22:57:30 +0300 Subject: [PATCH 012/264] Heartbeats working --- src/Common/ZooKeeper/TestKeeperStorage.cpp | 107 ++++++++++++++++++++- src/Common/ZooKeeper/TestKeeperStorage.h | 4 + src/Common/ZooKeeper/ZooKeeperCommon.h | 2 +- src/Server/TestKeeperTCPHandler.h | 2 + 4 files changed, 112 insertions(+), 3 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index c238fa2620f..96593b291d6 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -1,6 +1,109 @@ #include +#include -namespace DB +namespace zkutil { - +using namespace DB; + +static String parentPath(const String & path) +{ + auto rslash_pos = path.rfind('/'); + if (rslash_pos > 0) + return path.substr(0, rslash_pos); + return "/"; +} + +using Undo = std::function; + +struct TestKeeperStorageRequest +{ + Coordination::ZooKeeperRequestPtr zk_request; + + TestKeeperStorageRequest(const Coordination::ZooKeeperRequestPtr & zk_request_) + : zk_request(zk_request_) + {} + virtual bool isMutable() const { return false; } + virtual std::pair process(TestKeeperStorage::Container & container, int64_t zxid) const = 0; + virtual ~TestKeeperStorageRequest() {} +}; + + +struct TestKeeperStorageCreateRequest final : public TestKeeperStorageRequest +{ + std::pair process(TestKeeperStorage::Container & container, int64_t zxid) const override + { + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); + Undo undo; + Coordination::ZooKeeperCreateResponse & response = dynamic_cast(*response_ptr); + Coordination::ZooKeeperCreateRequest & request = dynamic_cast(*zk_request); + + if (container.count(request.path)) + { + response.error = Coordination::Error::ZNODEEXISTS; + } + else + { + auto it = container.find(parentPath(request.path)); + + if (it == container.end()) + { + response.error = Coordination::Error::ZNONODE; + } + else if (it->second.is_ephemeral) + { + response.error = Coordination::Error::ZNOCHILDRENFOREPHEMERALS; + } + else + { + TestKeeperStorage::Node created_node; + created_node.seq_num = 0; + created_node.stat.czxid = zxid; + created_node.stat.mzxid = zxid; + created_node.stat.ctime = std::chrono::system_clock::now().time_since_epoch() / std::chrono::milliseconds(1); + created_node.stat.mtime = created_node.stat.ctime; + created_node.stat.numChildren = 0; + created_node.stat.dataLength = request.data.length(); + created_node.data = request.data; + created_node.is_ephemeral = request.is_ephemeral; + created_node.is_sequental = request.is_sequential; + std::string path_created = request.path; + + if (request.is_sequential) + { + auto seq_num = it->second.seq_num; + ++it->second.seq_num; + + std::stringstream seq_num_str; + seq_num_str << std::setw(10) << std::setfill('0') << seq_num; + + path_created += seq_num_str.str(); + } + + response.path_created = path_created; + container.emplace(path_created, std::move(created_node)); + + undo = [&container, path_created, is_sequential = request.is_sequential, parent_path = it->first] + { + container.erase(path_created); + auto & undo_parent = container.at(parent_path); + --undo_parent.stat.cversion; + --undo_parent.stat.numChildren; + + if (is_sequential) + --undo_parent.seq_num; + }; + + ++it->second.stat.cversion; + ++it->second.stat.numChildren; + + response.error = Coordination::Error::ZOK; + } + } + + return { response_ptr, undo }; + } +}; + + + } diff --git a/src/Common/ZooKeeper/TestKeeperStorage.h b/src/Common/ZooKeeper/TestKeeperStorage.h index f0c8a942dff..893f9f2842c 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.h +++ b/src/Common/ZooKeeper/TestKeeperStorage.h @@ -12,6 +12,8 @@ using namespace DB; class TestKeeperStorage { + +public: struct TestKeeperRequest; using TestKeeperRequestPtr = std::shared_ptr; @@ -61,6 +63,8 @@ class TestKeeperStorage void processingThread(); + void writeResponse(const Coordination::ZooKeeperResponsePtr & response); + public: void putRequest(const Coordination::ZooKeeperRequestPtr & request, std::shared_ptr response_out); diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index 6293cbb09fe..05886fc0468 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -133,7 +133,7 @@ struct ZooKeeperCloseResponse final : ZooKeeperResponse OpNum getOpNum() const override { return -11; } }; -struct ZooKeeperCreateRequest final : CreateRequest, ZooKeeperRequest +struct ZooKeeperCreateRequest final : public CreateRequest, ZooKeeperRequest { ZooKeeperCreateRequest() = default; explicit ZooKeeperCreateRequest(const CreateRequest & base) : CreateRequest(base) {} diff --git a/src/Server/TestKeeperTCPHandler.h b/src/Server/TestKeeperTCPHandler.h index 967ea1d29e7..695f5f04382 100644 --- a/src/Server/TestKeeperTCPHandler.h +++ b/src/Server/TestKeeperTCPHandler.h @@ -43,6 +43,8 @@ private: void receiveHeartbeatRequest(); void sendHeartbeatResponse(); + + void receiveCreateRequest(); }; } From 93c2ad603426e7de3deea8c5226bbf4884c4e2a1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 3 Nov 2020 17:49:30 +0300 Subject: [PATCH 013/264] Something work --- src/Common/ZooKeeper/TestKeeperStorage.cpp | 180 ++++++++++++++++++++- src/Common/ZooKeeper/TestKeeperStorage.h | 22 +-- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 18 +++ src/Common/ZooKeeper/ZooKeeperCommon.h | 4 + src/Common/ZooKeeper/ZooKeeperImpl.cpp | 6 +- src/Server/TestKeeperTCPHandler.cpp | 33 +++- src/Server/TestKeeperTCPHandler.h | 6 +- 7 files changed, 252 insertions(+), 17 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index 96593b291d6..2024daab7ee 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -1,5 +1,9 @@ #include #include +#include +#include +#include +#include namespace zkutil { @@ -13,6 +17,14 @@ static String parentPath(const String & path) return "/"; } + +TestKeeperStorage::TestKeeperStorage() +{ + container.emplace("/", Node()); + + processing_thread = ThreadFromGlobalPool([this] { processingThread(); }); +} + using Undo = std::function; struct TestKeeperStorageRequest @@ -27,11 +39,22 @@ struct TestKeeperStorageRequest virtual ~TestKeeperStorageRequest() {} }; +struct TestKeeperStorageHeartbeatRequest final : public TestKeeperStorageRequest +{ + using TestKeeperStorageRequest::TestKeeperStorageRequest; + std::pair process(TestKeeperStorage::Container & /* container */, int64_t /* zxid */) const override + { + return {zk_request->makeResponse(), {}}; + } +}; + struct TestKeeperStorageCreateRequest final : public TestKeeperStorageRequest { + using TestKeeperStorageRequest::TestKeeperStorageRequest; std::pair process(TestKeeperStorage::Container & container, int64_t zxid) const override { + LOG_DEBUG(&Poco::Logger::get("STORAGE"), "EXECUTING CREATE REQUEST"); Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Undo undo; Coordination::ZooKeeperCreateResponse & response = dynamic_cast(*response_ptr); @@ -104,6 +127,161 @@ struct TestKeeperStorageCreateRequest final : public TestKeeperStorageRequest } }; +struct TestKeeperStorageGetRequest final : public TestKeeperStorageRequest +{ + using TestKeeperStorageRequest::TestKeeperStorageRequest; + std::pair process(TestKeeperStorage::Container & container, int64_t /* zxid */) const override + { + LOG_DEBUG(&Poco::Logger::get("STORAGE"), "EXECUTING GET REQUEST"); + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); + Coordination::ZooKeeperGetResponse & response = dynamic_cast(*response_ptr); + Coordination::ZooKeeperGetRequest & request = dynamic_cast(*zk_request); + + auto it = container.find(request.path); + if (it == container.end()) + { + response.error = Coordination::Error::ZNONODE; + } + else + { + response.stat = it->second.stat; + response.data = it->second.data; + response.error = Coordination::Error::ZOK; + } + + return { response_ptr, {} }; + } +}; + +void TestKeeperStorage::processingThread() +{ + setThreadName("TestKeeperSProc"); + + LOG_DEBUG(&Poco::Logger::get("STORAGE"), "LOOPING IN THREAD"); + try + { + while (!shutdown) + { + RequestInfo info; + + UInt64 max_wait = UInt64(operation_timeout.totalMilliseconds()); + + if (requests_queue.tryPop(info, max_wait)) + { + if (shutdown) + break; + + ++zxid; + + auto zk_request = info.request->zk_request; + LOG_DEBUG(&Poco::Logger::get("STORAGE"), "GOT REQUEST {}", zk_request->getOpNum()); + Coordination::ZooKeeperResponsePtr response; + if (zk_request->xid == -2) + { + response = std::make_shared(); + response->xid = zk_request->xid; + response->zxid = zxid; + } + else + { + zk_request->addRootPath(root_path); + LOG_DEBUG(&Poco::Logger::get("STORAGE"), "PROCESSING REQUEST"); + std::tie(response, std::ignore) = info.request->process(container, zxid); + response->xid = zk_request->xid; + LOG_DEBUG(&Poco::Logger::get("STORAGE"), "SENDING XID {}", response->xid); + response->zxid = zxid; + + response->removeRootPath(root_path); + } + + LOG_DEBUG(&Poco::Logger::get("STORAGE"), "SENDING RESPONSE"); + info.response_callback(response); + LOG_DEBUG(&Poco::Logger::get("STORAGE"), "DONE"); + } + } + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + finalize(); + } +} + + +void TestKeeperStorage::finalize() +{ + { + std::lock_guard lock(push_request_mutex); + + if (shutdown) + return; + shutdown = true; + } + try + { + RequestInfo info; + while (requests_queue.tryPop(info)) + { + auto response = info.request->zk_request->makeResponse(); + response->error = Coordination::Error::ZSESSIONEXPIRED; + try + { + info.response_callback(response); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + +} + +TestKeeperStorage::AsyncResponse TestKeeperStorage::putRequest(const Coordination::ZooKeeperRequestPtr & request) +{ + auto promise = std::make_shared>(); + auto future = promise->get_future(); + TestKeeperStorageRequestPtr storage_request; + if (request->xid == -2) + storage_request = std::make_shared(request); + else if (request->getOpNum() == 1) + storage_request = std::make_shared(request); + else if (request->getOpNum() == 4) + storage_request = std::make_shared(request); + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "UNKNOWN EVENT WITH OPNUM {}", request->getOpNum()) +; + RequestInfo request_info; + request_info.time = clock::now(); + request_info.request = storage_request; + request_info.response_callback = [promise] (const Coordination::ZooKeeperResponsePtr & response) { promise->set_value(response); }; + + std::lock_guard lock(push_request_mutex); + if (!requests_queue.tryPush(std::move(request_info), operation_timeout.totalMilliseconds())) + throw Exception("Cannot push request to queue within operation timeout", ErrorCodes::LOGICAL_ERROR); + LOG_DEBUG(&Poco::Logger::get("STORAGE"), "PUSHED"); + return future; +} + + +TestKeeperStorage::~TestKeeperStorage() +{ + try + { + finalize(); + if (processing_thread.joinable()) + processing_thread.join(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } +} + + - } diff --git a/src/Common/ZooKeeper/TestKeeperStorage.h b/src/Common/ZooKeeper/TestKeeperStorage.h index 893f9f2842c..86be2e0eeaf 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.h +++ b/src/Common/ZooKeeper/TestKeeperStorage.h @@ -4,19 +4,21 @@ #include #include #include +#include namespace zkutil { using namespace DB; +struct TestKeeperStorageRequest; +using TestKeeperStorageRequestPtr = std::shared_ptr; class TestKeeperStorage { public: - struct TestKeeperRequest; - using TestKeeperRequestPtr = std::shared_ptr; + Poco::Timespan operation_timeout{10000}; std::atomic session_id_counter{0}; struct Node @@ -39,6 +41,7 @@ public: String root_path; std::atomic zxid{0}; + std::atomic shutdown{false}; Watches watches; Watches list_watches; /// Watches for 'list' request (watches on children). @@ -47,26 +50,25 @@ public: struct RequestInfo { - TestKeeperRequestPtr request; - Coordination::ResponseCallback callback; - Coordination::WatchCallback watch; + TestKeeperStorageRequestPtr request; + std::function response_callback; clock::time_point time; }; + std::mutex push_request_mutex; using RequestsQueue = ConcurrentBoundedQueue; RequestsQueue requests_queue{1}; - void pushRequest(RequestInfo && request); - void finalize(); ThreadFromGlobalPool processing_thread; void processingThread(); - void writeResponse(const Coordination::ZooKeeperResponsePtr & response); - public: - void putRequest(const Coordination::ZooKeeperRequestPtr & request, std::shared_ptr response_out); + using AsyncResponse = std::future; + TestKeeperStorage(); + ~TestKeeperStorage(); + AsyncResponse putRequest(const Coordination::ZooKeeperRequestPtr & request); int64_t getSessionID() { diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index eb04536ae00..7d7bbbd8dc3 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -36,6 +37,8 @@ static void write(bool x, WriteBuffer & out) static void write(const String & s, WriteBuffer & out) { + + LOG_DEBUG(&Poco::Logger::get("LOG"), "S SIZE {}", s.size()); write(int32_t(s.size()), out); out.write(s.data(), s.size()); } @@ -170,6 +173,20 @@ static void read(ACL & acl, ReadBuffer & in) read(acl.id, in); } +void ZooKeeperResponse::write(WriteBuffer & out) const +{ + /// Excessive copy to calculate length. + WriteBufferFromOwnString buf; + LOG_DEBUG(&Poco::Logger::get("LOG"), "WRITING {}", xid); + Coordination::write(xid, buf); + Coordination::write(zxid, buf); + Coordination::write(error, buf); + writeImpl(buf); + LOG_DEBUG(&Poco::Logger::get("LOG"), "BUFFER LENGTH {}", buf.str().length()); + Coordination::write(buf.str(), out); + out.next(); +} + void ZooKeeperRequest::write(WriteBuffer & out) const { /// Excessive copy to calculate length. @@ -247,6 +264,7 @@ void ZooKeeperCreateResponse::readImpl(ReadBuffer & in) void ZooKeeperCreateResponse::writeImpl(WriteBuffer & out) const { + LOG_DEBUG(&Poco::Logger::get("LOG"), "WRITE IMPL ON: {}", path_created); Coordination::write(path_created, out); } diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index 05886fc0468..f40de116da2 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -27,9 +27,13 @@ using OpNum = int32_t; struct ZooKeeperResponse : virtual Response { + XID xid = 0; + int64_t zxid; + virtual ~ZooKeeperResponse() override = default; virtual void readImpl(ReadBuffer &) = 0; virtual void writeImpl(WriteBuffer &) const = 0; + void write(WriteBuffer & out) const; virtual OpNum getOpNum() const = 0; }; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index f3c8b537cf2..30b30695eea 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) # include @@ -750,9 +751,12 @@ void ZooKeeper::receiveEvent() Error err; read(length); + std::cerr << "RECEIVED LENGTH " << length << std::endl; size_t count_before_event = in->count(); read(xid); + std::cerr << "RECEIVED XID " << xid << std::endl; read(zxid); + std::cerr << "RECEIVED ZXID " << zxid << std::endl; read(err); RequestInfo request_info; @@ -806,7 +810,7 @@ void ZooKeeper::receiveEvent() auto it = operations.find(xid); if (it == operations.end()) - throw Exception("Received response for unknown xid", Error::ZRUNTIMEINCONSISTENCY); + throw Exception("Received response for unknown xid " + toString(xid), Error::ZRUNTIMEINCONSISTENCY); /// After this point, we must invoke callback, that we've grabbed from 'operations'. /// Invariant: all callbacks are invoked either in case of success or in case of error. diff --git a/src/Server/TestKeeperTCPHandler.cpp b/src/Server/TestKeeperTCPHandler.cpp index a0679554f64..baf78e1fc59 100644 --- a/src/Server/TestKeeperTCPHandler.cpp +++ b/src/Server/TestKeeperTCPHandler.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace DB { @@ -279,16 +280,23 @@ void TestKeeperTCPHandler::runImpl() while (true) { UInt64 max_wait = operation_timeout.totalMicroseconds(); + using namespace std::chrono_literals; + if (!responses.empty() && responses.front().wait_for(100ms) == std::future_status::ready) + { + auto response = responses.front().get(); + response->write(*out); + responses.pop(); + } + if (in->poll(max_wait)) { receiveHeartbeatRequest(); - sendHeartbeatResponse(); } } } -void TestKeeperTCPHandler::receiveHeartbeatRequest() +bool TestKeeperTCPHandler::receiveHeartbeatRequest() { LOG_DEBUG(log, "Receiving heartbeat event"); int32_t length; @@ -301,12 +309,14 @@ void TestKeeperTCPHandler::receiveHeartbeatRequest() LOG_DEBUG(log, "Received xid {}", xid); + Coordination::ZooKeeperRequestPtr request; if (xid == -2) { int32_t opnum; read(opnum, *in); LOG_DEBUG(log, "RRECEIVED OP NUM {}", opnum); - auto request = std::make_shared(); + request = std::make_shared(); + request->xid = xid; request->readImpl(*in); int32_t readed = in->count() - total_count; if (readed != length) @@ -314,10 +324,25 @@ void TestKeeperTCPHandler::receiveHeartbeatRequest() } else { - LOG_INFO(log, "UNKNOWN EVENT xid:{}", xid); + int32_t opnum; + read(opnum, *in); + LOG_DEBUG(log, "RRECEIVED OP NUM {}", opnum); + if (opnum == 1) + request = std::make_shared(); + else if (opnum == 4) + request = std::make_shared(); + request->readImpl(*in); + request->xid = xid; + int32_t readed = in->count() - total_count; + if (readed != length) + LOG_DEBUG(log, "EXPECTED TO READ {}, BUT GOT {}", length, readed); + LOG_DEBUG(log, "REQUEST PUTTED TO STORAGE"); } + responses.push(test_keeper_storage->putRequest(request)); + LOG_DEBUG(log, "Event received"); + return false; } diff --git a/src/Server/TestKeeperTCPHandler.h b/src/Server/TestKeeperTCPHandler.h index 695f5f04382..fd2c6227e73 100644 --- a/src/Server/TestKeeperTCPHandler.h +++ b/src/Server/TestKeeperTCPHandler.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB { @@ -32,6 +33,9 @@ private: std::shared_ptr test_keeper_storage; Poco::Timespan operation_timeout; + std::queue responses; + + /// Streams for reading/writing from/to client connection socket. std::shared_ptr in; std::shared_ptr out; @@ -41,7 +45,7 @@ private: void sendHandshake(); void receiveHandshake(); - void receiveHeartbeatRequest(); + bool receiveHeartbeatRequest(); void sendHeartbeatResponse(); void receiveCreateRequest(); From 5700e5e46d3c495342149a6484798e4157766fa7 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 3 Nov 2020 17:57:40 +0300 Subject: [PATCH 014/264] Fix response serialization --- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index 7d7bbbd8dc3..5ab22474d72 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -181,7 +181,8 @@ void ZooKeeperResponse::write(WriteBuffer & out) const Coordination::write(xid, buf); Coordination::write(zxid, buf); Coordination::write(error, buf); - writeImpl(buf); + if (error == Error::ZOK) + writeImpl(buf); LOG_DEBUG(&Poco::Logger::get("LOG"), "BUFFER LENGTH {}", buf.str().length()); Coordination::write(buf.str(), out); out.next(); From 598532b58216d1ab1cf9f3ee766778a55476801f Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 3 Nov 2020 18:01:56 +0300 Subject: [PATCH 015/264] Remove strange method --- src/Common/ZooKeeper/TestKeeper.cpp | 3 --- src/Common/ZooKeeper/TestKeeperStorage.cpp | 1 - 2 files changed, 4 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index 4f7beadef5f..b25e20afeda 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -31,7 +31,6 @@ using Undo = std::function; struct TestKeeperRequest : virtual Request { - virtual bool isMutable() const { return false; } virtual ResponsePtr createResponse() const = 0; virtual std::pair process(TestKeeper::Container & container, int64_t zxid) const = 0; virtual void processWatches(TestKeeper::Watches & /*watches*/, TestKeeper::Watches & /*list_watches*/) const {} @@ -85,7 +84,6 @@ struct TestKeeperRemoveRequest final : RemoveRequest, TestKeeperRequest { TestKeeperRemoveRequest() = default; explicit TestKeeperRemoveRequest(const RemoveRequest & base) : RemoveRequest(base) {} - bool isMutable() const override { return true; } ResponsePtr createResponse() const override; std::pair process(TestKeeper::Container & container, int64_t zxid) const override; @@ -112,7 +110,6 @@ struct TestKeeperSetRequest final : SetRequest, TestKeeperRequest { TestKeeperSetRequest() = default; explicit TestKeeperSetRequest(const SetRequest & base) : SetRequest(base) {} - bool isMutable() const override { return true; } ResponsePtr createResponse() const override; std::pair process(TestKeeper::Container & container, int64_t zxid) const override; diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index 2024daab7ee..50564e6ae54 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -34,7 +34,6 @@ struct TestKeeperStorageRequest TestKeeperStorageRequest(const Coordination::ZooKeeperRequestPtr & zk_request_) : zk_request(zk_request_) {} - virtual bool isMutable() const { return false; } virtual std::pair process(TestKeeperStorage::Container & container, int64_t zxid) const = 0; virtual ~TestKeeperStorageRequest() {} }; From a9529e8d6558f54f171906c7f906c7cea91b49fb Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 4 Nov 2020 21:54:55 +0300 Subject: [PATCH 016/264] Bad code but all events --- src/Common/ZooKeeper/TestKeeperStorage.cpp | 347 +++++++++++++++++++-- src/Server/TestKeeperTCPHandler.cpp | 58 +--- src/Server/TestKeeperTCPHandler.h | 5 +- 3 files changed, 328 insertions(+), 82 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index 50564e6ae54..a3e637a4ce8 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace zkutil { @@ -17,6 +18,11 @@ static String parentPath(const String & path) return "/"; } +static String baseName(const String & path) +{ + auto rslash_pos = path.rfind('/'); + return path.substr(rslash_pos + 1); +} TestKeeperStorage::TestKeeperStorage() { @@ -152,6 +158,255 @@ struct TestKeeperStorageGetRequest final : public TestKeeperStorageRequest } }; +struct TestKeeperStorageRemoveRequest final : public TestKeeperStorageRequest +{ + using TestKeeperStorageRequest::TestKeeperStorageRequest; + std::pair process(TestKeeperStorage::Container & container, int64_t /*zxid*/) const override + { + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); + Coordination::ZooKeeperRemoveResponse & response = dynamic_cast(*response_ptr); + Coordination::ZooKeeperRemoveRequest & request = dynamic_cast(*zk_request); + Undo undo; + + auto it = container.find(request.path); + if (it == container.end()) + { + response.error = Coordination::Error::ZNONODE; + } + else if (request.version != -1 && request.version != it->second.stat.version) + { + response.error = Coordination::Error::ZBADVERSION; + } + else if (it->second.stat.numChildren) + { + response.error = Coordination::Error::ZNOTEMPTY; + } + else + { + auto prev_node = it->second; + container.erase(it); + auto & parent = container.at(parentPath(request.path)); + --parent.stat.numChildren; + ++parent.stat.cversion; + response.error = Coordination::Error::ZOK; + + undo = [prev_node, &container, path = request.path] + { + container.emplace(path, prev_node); + auto & undo_parent = container.at(parentPath(path)); + ++undo_parent.stat.numChildren; + --undo_parent.stat.cversion; + }; + } + + return { response_ptr, undo }; + } +}; + +struct TestKeeperStorageExistsRequest final : public TestKeeperStorageRequest +{ + using TestKeeperStorageRequest::TestKeeperStorageRequest; + std::pair process(TestKeeperStorage::Container & container, int64_t /*zxid*/) const override + { + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); + Coordination::ZooKeeperExistsResponse & response = dynamic_cast(*response_ptr); + Coordination::ZooKeeperExistsRequest & request = dynamic_cast(*zk_request); + + auto it = container.find(request.path); + if (it != container.end()) + { + response.stat = it->second.stat; + response.error = Coordination::Error::ZOK; + } + else + { + response.error = Coordination::Error::ZNONODE; + } + + return { response_ptr, {} }; + } +}; + +struct TestKeeperStorageSetRequest final : public TestKeeperStorageRequest +{ + using TestKeeperStorageRequest::TestKeeperStorageRequest; + std::pair process(TestKeeperStorage::Container & container, int64_t zxid) const override + { + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); + Coordination::ZooKeeperSetResponse & response = dynamic_cast(*response_ptr); + Coordination::ZooKeeperSetRequest & request = dynamic_cast(*zk_request); + Undo undo; + + auto it = container.find(request.path); + if (it == container.end()) + { + response.error = Coordination::Error::ZNONODE; + } + else if (request.version == -1 || request.version == it->second.stat.version) + { + auto prev_node = it->second; + + it->second.data = request.data; + ++it->second.stat.version; + it->second.stat.mzxid = zxid; + it->second.stat.mtime = std::chrono::system_clock::now().time_since_epoch() / std::chrono::milliseconds(1); + it->second.data = request.data; + ++container.at(parentPath(request.path)).stat.cversion; + response.stat = it->second.stat; + response.error = Coordination::Error::ZOK; + + undo = [prev_node, &container, path = request.path] + { + container.at(path) = prev_node; + --container.at(parentPath(path)).stat.cversion; + }; + } + else + { + response.error = Coordination::Error::ZBADVERSION; + } + + return { response_ptr, {} }; + } +}; + +struct TestKeeperStorageListRequest final : public TestKeeperStorageRequest +{ + using TestKeeperStorageRequest::TestKeeperStorageRequest; + std::pair process(TestKeeperStorage::Container & container, int64_t /*zxid*/) const override + { + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); + Coordination::ZooKeeperListResponse & response = dynamic_cast(*response_ptr); + Coordination::ZooKeeperListRequest & request = dynamic_cast(*zk_request); + Undo undo; + auto it = container.find(request.path); + if (it == container.end()) + { + response.error = Coordination::Error::ZNONODE; + } + else + { + auto path_prefix = request.path; + if (path_prefix.empty()) + throw Coordination::Exception("Logical error: path cannot be empty", Coordination::Error::ZSESSIONEXPIRED); + + if (path_prefix.back() != '/') + path_prefix += '/'; + + /// Fairly inefficient. + for (auto child_it = container.upper_bound(path_prefix); + child_it != container.end() && startsWith(child_it->first, path_prefix); + ++child_it) + { + if (parentPath(child_it->first) == request.path) + response.names.emplace_back(baseName(child_it->first)); + } + + response.stat = it->second.stat; + response.error = Coordination::Error::ZOK; + } + + return { response_ptr, {} }; + } +}; + +struct TestKeeperStorageCheckRequest final : public TestKeeperStorageRequest +{ + using TestKeeperStorageRequest::TestKeeperStorageRequest; + std::pair process(TestKeeperStorage::Container & container, int64_t /*zxid*/) const override + { + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); + Coordination::ZooKeeperCheckResponse & response = dynamic_cast(*response_ptr); + Coordination::ZooKeeperCheckRequest & request = dynamic_cast(*zk_request); + auto it = container.find(request.path); + if (it == container.end()) + { + response.error = Coordination::Error::ZNONODE; + } + else if (request.version != -1 && request.version != it->second.stat.version) + { + response.error = Coordination::Error::ZBADVERSION; + } + else + { + response.error = Coordination::Error::ZOK; + } + + return { response_ptr, {} }; + } +}; + +struct TestKeeperStorageMultiRequest final : public TestKeeperStorageRequest +{ + std::vector concrete_requests; + TestKeeperStorageMultiRequest(const Coordination::ZooKeeperRequestPtr & zk_request_) + : TestKeeperStorageRequest(zk_request_) + { + Coordination::ZooKeeperMultiRequest & request = dynamic_cast(*zk_request); + concrete_requests.reserve(request.requests.size()); + + for (const auto & zk_request : request.requests) + { + if (const auto * concrete_request_create = dynamic_cast(zk_request.get())) + { + concrete_requests.push_back(std::make_shared(dynamic_pointer_cast(zk_request))); + } + else if (const auto * concrete_request_remove = dynamic_cast(zk_request.get())) + { + concrete_requests.push_back(std::make_shared(dynamic_pointer_cast(zk_request))); + } + else if (const auto * concrete_request_set = dynamic_cast(zk_request.get())) + { + concrete_requests.push_back(std::make_shared(dynamic_pointer_cast(zk_request))); + } + else if (const auto * concrete_request_check = dynamic_cast(zk_request.get())) + { + concrete_requests.push_back(std::make_shared(dynamic_pointer_cast(zk_request))); + } + else + throw Coordination::Exception("Illegal command as part of multi ZooKeeper request", Coordination::Error::ZBADARGUMENTS); + } + } + + std::pair process(TestKeeperStorage::Container & container, int64_t zxid) const override + { + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); + Coordination::ZooKeeperMultiResponse & response = dynamic_cast(*response_ptr); + std::vector undo_actions; + + try + { + for (const auto & concrete_request : concrete_requests) + { + auto [ cur_response, undo_action ] = concrete_request->process(container, zxid); + response.responses.emplace_back(cur_response); + if (cur_response->error != Coordination::Error::ZOK) + { + response.error = cur_response->error; + + for (auto it = undo_actions.rbegin(); it != undo_actions.rend(); ++it) + if (*it) + (*it)(); + + return { response_ptr, {} }; + } + else + undo_actions.emplace_back(std::move(undo_action)); + } + + response.error = Coordination::Error::ZOK; + return { response_ptr, {} }; + } + catch (...) + { + for (auto it = undo_actions.rbegin(); it != undo_actions.rend(); ++it) + if (*it) + (*it)(); + throw; + } + } +}; + void TestKeeperStorage::processingThread() { setThreadName("TestKeeperSProc"); @@ -174,24 +429,11 @@ void TestKeeperStorage::processingThread() auto zk_request = info.request->zk_request; LOG_DEBUG(&Poco::Logger::get("STORAGE"), "GOT REQUEST {}", zk_request->getOpNum()); - Coordination::ZooKeeperResponsePtr response; - if (zk_request->xid == -2) - { - response = std::make_shared(); - response->xid = zk_request->xid; - response->zxid = zxid; - } - else - { - zk_request->addRootPath(root_path); - LOG_DEBUG(&Poco::Logger::get("STORAGE"), "PROCESSING REQUEST"); - std::tie(response, std::ignore) = info.request->process(container, zxid); - response->xid = zk_request->xid; - LOG_DEBUG(&Poco::Logger::get("STORAGE"), "SENDING XID {}", response->xid); - response->zxid = zxid; - response->removeRootPath(root_path); - } + auto [response, _] = info.request->process(container, zxid); + response->xid = zk_request->xid; + response->zxid = zxid; + response->removeRootPath(root_path); LOG_DEBUG(&Poco::Logger::get("STORAGE"), "SENDING RESPONSE"); info.response_callback(response); @@ -237,23 +479,74 @@ void TestKeeperStorage::finalize() { tryLogCurrentException(__PRETTY_FUNCTION__); } +} + + + + +class TestKeeperWrapperFactory final : private boost::noncopyable +{ + +public: + using Creator = std::function; + using OpNumToRequest = std::unordered_map; + + static TestKeeperWrapperFactory & instance() + { + static TestKeeperWrapperFactory factory; + return factory; + } + + TestKeeperStorageRequestPtr get(const Coordination::ZooKeeperRequestPtr & zk_request) const + { + auto it = op_num_to_request.find(zk_request->getOpNum()); + if (it == op_num_to_request.end()) + throw Coordination::Exception("Unknown operation type " + std::to_string(zk_request->getOpNum()), Coordination::Error::ZBADARGUMENTS); + + return it->second(zk_request); + } + + void registerRequest(int32_t op_num, Creator creator) + { + if (!op_num_to_request.try_emplace(op_num, creator).second) + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Request with op num {} already registered", op_num); + } + +private: + OpNumToRequest op_num_to_request; + +private: + TestKeeperWrapperFactory(); +}; + +template +void registerTestKeeperRequestWrapper(TestKeeperWrapperFactory & factory) +{ + factory.registerRequest(num, [] (const Coordination::ZooKeeperRequestPtr & zk_request) { return std::make_shared(zk_request); }); +} + + +TestKeeperWrapperFactory::TestKeeperWrapperFactory() +{ + registerTestKeeperRequestWrapper<11, TestKeeperStorageHeartbeatRequest>(*this); + //registerTestKeeperRequestWrapper<100, TestKeeperStorageAuthRequest>(*this); + //registerTestKeeperRequestWrapper<-11, TestKeeperStorageCloseRequest>(*this); + registerTestKeeperRequestWrapper<1, TestKeeperStorageCreateRequest>(*this); + registerTestKeeperRequestWrapper<2, TestKeeperStorageRemoveRequest>(*this); + registerTestKeeperRequestWrapper<3, TestKeeperStorageExistsRequest>(*this); + registerTestKeeperRequestWrapper<4, TestKeeperStorageGetRequest>(*this); + registerTestKeeperRequestWrapper<5, TestKeeperStorageSetRequest>(*this); + registerTestKeeperRequestWrapper<12, TestKeeperStorageListRequest>(*this); + registerTestKeeperRequestWrapper<13, TestKeeperStorageCheckRequest>(*this); + registerTestKeeperRequestWrapper<14, TestKeeperStorageMultiRequest>(*this); } TestKeeperStorage::AsyncResponse TestKeeperStorage::putRequest(const Coordination::ZooKeeperRequestPtr & request) { auto promise = std::make_shared>(); auto future = promise->get_future(); - TestKeeperStorageRequestPtr storage_request; - if (request->xid == -2) - storage_request = std::make_shared(request); - else if (request->getOpNum() == 1) - storage_request = std::make_shared(request); - else if (request->getOpNum() == 4) - storage_request = std::make_shared(request); - else - throw Exception(ErrorCodes::LOGICAL_ERROR, "UNKNOWN EVENT WITH OPNUM {}", request->getOpNum()) -; + TestKeeperStorageRequestPtr storage_request = TestKeeperWrapperFactory::instance().get(request); RequestInfo request_info; request_info.time = clock::now(); request_info.request = storage_request; diff --git a/src/Server/TestKeeperTCPHandler.cpp b/src/Server/TestKeeperTCPHandler.cpp index baf78e1fc59..0ca106d0e32 100644 --- a/src/Server/TestKeeperTCPHandler.cpp +++ b/src/Server/TestKeeperTCPHandler.cpp @@ -289,19 +289,16 @@ void TestKeeperTCPHandler::runImpl() } if (in->poll(max_wait)) - { - receiveHeartbeatRequest(); - } + receiveRequest(); } } -bool TestKeeperTCPHandler::receiveHeartbeatRequest() +void TestKeeperTCPHandler::receiveRequest() { LOG_DEBUG(log, "Receiving heartbeat event"); int32_t length; read(length, *in); - int32_t total_count = in->count(); LOG_DEBUG(log, "RECEIVED LENGTH {}", length); int32_t xid; LOG_DEBUG(log, "READING XID"); @@ -309,56 +306,15 @@ bool TestKeeperTCPHandler::receiveHeartbeatRequest() LOG_DEBUG(log, "Received xid {}", xid); - Coordination::ZooKeeperRequestPtr request; - if (xid == -2) - { - int32_t opnum; - read(opnum, *in); - LOG_DEBUG(log, "RRECEIVED OP NUM {}", opnum); - request = std::make_shared(); - request->xid = xid; - request->readImpl(*in); - int32_t readed = in->count() - total_count; - if (readed != length) - LOG_DEBUG(log, "EXPECTED TO READ {}, BUT GOT {}", length, readed); - } - else - { - int32_t opnum; - read(opnum, *in); - LOG_DEBUG(log, "RRECEIVED OP NUM {}", opnum); - if (opnum == 1) - request = std::make_shared(); - else if (opnum == 4) - request = std::make_shared(); - request->readImpl(*in); - request->xid = xid; - int32_t readed = in->count() - total_count; - if (readed != length) - LOG_DEBUG(log, "EXPECTED TO READ {}, BUT GOT {}", length, readed); - LOG_DEBUG(log, "REQUEST PUTTED TO STORAGE"); - } - + int32_t opnum; + read(opnum, *in); + Coordination::ZooKeeperRequestPtr request = Coordination::ZooKeeperRequestFactory::instance().get(opnum); + request->xid = xid; + request->readImpl(*in); responses.push(test_keeper_storage->putRequest(request)); LOG_DEBUG(log, "Event received"); - return false; } -void TestKeeperTCPHandler::sendHeartbeatResponse() -{ - LOG_DEBUG(log, "Sending heartbeat event"); - int32_t length = sizeof(int32_t) + sizeof(int64_t) + sizeof(Coordination::Error); - write(length, *out); - int64_t zxid = test_keeper_storage->getZXID(); - int32_t xid = -2; - write(xid, *out); - write(zxid, *out); - write(Coordination::Error::ZOK, *out); - auto response = std::make_shared(); - response->writeImpl(*out); - out->next(); -} - } diff --git a/src/Server/TestKeeperTCPHandler.h b/src/Server/TestKeeperTCPHandler.h index fd2c6227e73..2a796daa4e9 100644 --- a/src/Server/TestKeeperTCPHandler.h +++ b/src/Server/TestKeeperTCPHandler.h @@ -45,10 +45,7 @@ private: void sendHandshake(); void receiveHandshake(); - bool receiveHeartbeatRequest(); - void sendHeartbeatResponse(); - - void receiveCreateRequest(); + void receiveRequest(); }; } From 456b0b94c1661a37d52b2261e6725710c693f393 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 9 Nov 2020 09:54:35 +0300 Subject: [PATCH 017/264] Remove some debug --- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 2 -- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 3 --- 2 files changed, 5 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index 5ab22474d72..b1a11b2d473 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -37,8 +37,6 @@ static void write(bool x, WriteBuffer & out) static void write(const String & s, WriteBuffer & out) { - - LOG_DEBUG(&Poco::Logger::get("LOG"), "S SIZE {}", s.size()); write(int32_t(s.size()), out); out.write(s.data(), s.size()); } diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 30b30695eea..33204686dce 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -751,12 +751,9 @@ void ZooKeeper::receiveEvent() Error err; read(length); - std::cerr << "RECEIVED LENGTH " << length << std::endl; size_t count_before_event = in->count(); read(xid); - std::cerr << "RECEIVED XID " << xid << std::endl; read(zxid); - std::cerr << "RECEIVED ZXID " << zxid << std::endl; read(err); RequestInfo request_info; From 45b089425250edcafc4ef3ce982134c0b448e7db Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 9 Nov 2020 20:50:39 +0300 Subject: [PATCH 018/264] Some fixes --- src/Common/ZooKeeper/TestKeeperStorage.cpp | 8 ++++---- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 2 ++ src/Common/ZooKeeper/ZooKeeperImpl.cpp | 4 ++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index a3e637a4ce8..22a06f67988 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -376,12 +376,14 @@ struct TestKeeperStorageMultiRequest final : public TestKeeperStorageRequest try { + size_t i = 0; for (const auto & concrete_request : concrete_requests) { auto [ cur_response, undo_action ] = concrete_request->process(container, zxid); - response.responses.emplace_back(cur_response); + response.responses[i] = cur_response; if (cur_response->error != Coordination::Error::ZOK) { + std::cerr << "GOT ERROR ON: " << i << " error" << static_cast(cur_response->error) << std::endl; response.error = cur_response->error; for (auto it = undo_actions.rbegin(); it != undo_actions.rend(); ++it) @@ -392,6 +394,7 @@ struct TestKeeperStorageMultiRequest final : public TestKeeperStorageRequest } else undo_actions.emplace_back(std::move(undo_action)); + ++i; } response.error = Coordination::Error::ZOK; @@ -482,9 +485,6 @@ void TestKeeperStorage::finalize() } - - - class TestKeeperWrapperFactory final : private boost::noncopyable { diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index b1a11b2d473..f26a72a3d55 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -544,12 +544,14 @@ void ZooKeeperMultiResponse::readImpl(ReadBuffer & in) void ZooKeeperMultiResponse::writeImpl(WriteBuffer & out) const { + std::cerr << "WRITING MULTIRESPONSE " << responses.size() << std::endl; for (auto & response : responses) { const ZooKeeperResponse & zk_response = dynamic_cast(*response); OpNum op_num = zk_response.getOpNum(); bool done = false; Error op_error = zk_response.error; + std::cerr << "WRITING OP ERROR:" << static_cast(op_error) << std::endl; Coordination::write(op_num, out); Coordination::write(done, out); diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 33204686dce..4d81748ca3d 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -824,11 +824,15 @@ void ZooKeeper::receiveEvent() try { + std::cerr << "READING RESPONSE FOR REQUEST ID:" << request_info.request->getOpNum() << std::endl; if (!response) response = request_info.request->makeResponse(); if (err != Error::ZOK) + { + std::cerr << "GOT ERROR:" << static_cast(err) << std::endl; response->error = err; + } else { response->readImpl(*in); From ba449d7c929a8be63d09290e8f790b5733738f00 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 9 Nov 2020 20:51:22 +0300 Subject: [PATCH 019/264] Add small test --- utils/zookeeper-test/CMakeLists.txt | 3 + utils/zookeeper-test/main.cpp | 129 ++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 utils/zookeeper-test/CMakeLists.txt create mode 100644 utils/zookeeper-test/main.cpp diff --git a/utils/zookeeper-test/CMakeLists.txt b/utils/zookeeper-test/CMakeLists.txt new file mode 100644 index 00000000000..aa26c840ba3 --- /dev/null +++ b/utils/zookeeper-test/CMakeLists.txt @@ -0,0 +1,3 @@ +add_executable(zk-test main.cpp) +target_link_libraries(zk-test PRIVATE clickhouse_common_zookeeper) +INSTALL(TARGETS zk-test RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse-utils) diff --git a/utils/zookeeper-test/main.cpp b/utils/zookeeper-test/main.cpp new file mode 100644 index 00000000000..694a6205adb --- /dev/null +++ b/utils/zookeeper-test/main.cpp @@ -0,0 +1,129 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace std; + +void checkEq(zkutil::ZooKeeper & zk, const std::string & path, const std::string & expected) +{ + auto result = zk.get(path); + if (result != expected) + throw std::runtime_error(fmt::format("Data on path '{}' = '{}' doesn't match expected '{}'", + path, result, expected)); +} + +void checkExists(zkutil::ZooKeeper & zk, const std::string & path) +{ + if (!zk.exists(path)) + throw std::runtime_error(fmt::format("Path '{}' doesn't exists", path)); +} + +void testCreateGetExistsNode(zkutil::ZooKeeper & zk) +{ + zk.create("/data", "test_string", zkutil::CreateMode::Persistent); + zk.create("/data/seq-", "another_string", zkutil::CreateMode::PersistentSequential); + checkEq(zk, "/data", "test_string"); + checkExists(zk, "/data/seq-0000000000"); + checkEq(zk, "/data/seq-0000000000", "another_string"); +} + +void testCreateSetNode(zkutil::ZooKeeper & zk) +{ + zk.create("/data/set", "sssss", zkutil::CreateMode::Persistent); + checkEq(zk, "/data/set", "sssss"); + zk.set("/data/set", "qqqqq"); + checkEq(zk, "/data/set", "qqqqq"); +} + +void testCreateList(zkutil::ZooKeeper & zk) +{ + zk.create("/data/lst", "", zkutil::CreateMode::Persistent); + zk.create("/data/lst/d1", "", zkutil::CreateMode::Persistent); + zk.create("/data/lst/d2", "", zkutil::CreateMode::Persistent); + zk.create("/data/lst/d3", "", zkutil::CreateMode::Persistent); + auto children = zk.getChildren("/data/lst"); + if (children.size() != 3) + throw std::runtime_error("Children of /data/lst doesn't equal to three"); + for (size_t i = 0; i < children.size(); ++i) + { + if (children[i] != "d" + std::to_string(i + 1)) + throw std::runtime_error(fmt::format("Incorrect children #{} got {}, expected {}", i, children[i], "d" + std::to_string(i + 1))); + } +} + +void testCreateSetVersionRequest(zkutil::ZooKeeper & zk) +{ + zk.create("/data/check_data", "d", zkutil::CreateMode::Persistent); + Coordination::Stat stat; + std::string result = zk.get("/data/check_data", &stat); + try + { + zk.set("/data/check_data", "e", stat.version + 2); + std::terminate(); + } + catch (...) + { + std::cerr << "Got exception on incorrect version (it's ok)\n"; + } + + checkEq(zk, "/data/check_data", "d"); + zk.set("/data/check_data", "e", stat.version); + + checkEq(zk, "/data/check_data", "e"); +} + +void testMultiRequest(zkutil::ZooKeeper & zk) +{ + Coordination::Requests requests; + requests.push_back(zkutil::makeCreateRequest("/data/multirequest", "aaa", zkutil::CreateMode::Persistent)); + requests.push_back(zkutil::makeSetRequest("/data/multirequest", "bbb", -1)); + zk.multi(requests); + + try + { + requests.clear(); + requests.push_back(zkutil::makeCreateRequest("/data/multirequest", "qweqwe", zkutil::CreateMode::Persistent)); + requests.push_back(zkutil::makeSetRequest("/data/multirequest", "bbb", -1)); + requests.push_back(zkutil::makeSetRequest("/data/multirequest", "ccc", -1)); + zk.multi(requests); + std::terminate(); + } + catch(...) + { + std::cerr << "Got exception on multy request (it's ok)\n"; + } + + checkEq(zk, "/data/multirequest", "bbb"); +} + +int main(int argc, char *argv[]) { + if (argc != 2) + { + std::cerr << "usage: " << argv[0] << " hosts" << std::endl; + return 2; + } + Poco::AutoPtr channel = new Poco::ConsoleChannel(std::cerr); + Poco::Logger::root().setChannel(channel); + Poco::Logger::root().setLevel("trace"); + + zkutil::ZooKeeper zk(argv[1]); + + testCreateGetExistsNode(zk); + testCreateSetNode(zk); + testCreateList(zk); + testCreateSetVersionRequest(zk); + testMultiRequest(zk); + + //zk.removeRecursive("/data"); + return 0; +} From 09fac0da9d37b3365bd9a574535816cb4123277b Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 9 Nov 2020 21:16:02 +0300 Subject: [PATCH 020/264] Better --- src/Common/ZooKeeper/TestKeeperStorage.cpp | 1 - src/Common/ZooKeeper/ZooKeeperCommon.cpp | 3 ++- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 1 + utils/CMakeLists.txt | 1 + utils/zookeeper-test/main.cpp | 22 +++++++++++++++------- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index 22a06f67988..5ac8daf2e09 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -384,7 +384,6 @@ struct TestKeeperStorageMultiRequest final : public TestKeeperStorageRequest if (cur_response->error != Coordination::Error::ZOK) { std::cerr << "GOT ERROR ON: " << i << " error" << static_cast(cur_response->error) << std::endl; - response.error = cur_response->error; for (auto it = undo_actions.rbegin(); it != undo_actions.rend(); ++it) if (*it) diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index f26a72a3d55..604bb2137c4 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -556,7 +556,8 @@ void ZooKeeperMultiResponse::writeImpl(WriteBuffer & out) const Coordination::write(op_num, out); Coordination::write(done, out); Coordination::write(op_error, out); - zk_response.writeImpl(out); + if (op_error == Error::ZOK) + zk_response.writeImpl(out); } /// Footer. diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 4d81748ca3d..fef0b7063f5 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -835,6 +835,7 @@ void ZooKeeper::receiveEvent() } else { + std::cerr << "NO ERROR RECEIVED\n"; response->readImpl(*in); response->removeRootPath(root_path); } diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 0987d64abed..9e1872ded7b 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -19,6 +19,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (iotest) add_subdirectory (corrector_utf8) add_subdirectory (zookeeper-cli) + add_subdirectory (zookeeper-test) add_subdirectory (zookeeper-dump-tree) add_subdirectory (zookeeper-remove-by-list) add_subdirectory (zookeeper-create-entry-to-download-part) diff --git a/utils/zookeeper-test/main.cpp b/utils/zookeeper-test/main.cpp index 694a6205adb..802fbc17708 100644 --- a/utils/zookeeper-test/main.cpp +++ b/utils/zookeeper-test/main.cpp @@ -107,6 +107,7 @@ void testMultiRequest(zkutil::ZooKeeper & zk) } int main(int argc, char *argv[]) { + if (argc != 2) { std::cerr << "usage: " << argv[0] << " hosts" << std::endl; @@ -118,12 +119,19 @@ int main(int argc, char *argv[]) { zkutil::ZooKeeper zk(argv[1]); - testCreateGetExistsNode(zk); - testCreateSetNode(zk); - testCreateList(zk); - testCreateSetVersionRequest(zk); - testMultiRequest(zk); - - //zk.removeRecursive("/data"); + try + { + zk.tryRemoveRecursive("/data"); + testCreateGetExistsNode(zk); + testCreateSetNode(zk); + testCreateList(zk); + testCreateSetVersionRequest(zk); + testMultiRequest(zk); + } + catch(...) + { + zk.tryRemoveRecursive("/data"); + throw; + } return 0; } From d8f515c4f31026fb4089ce147eed95fde4f42053 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 10 Nov 2020 16:43:10 +0300 Subject: [PATCH 021/264] Almost watches --- src/Common/ZooKeeper/TestKeeperStorage.cpp | 85 +++++++++++++++++++++- src/Common/ZooKeeper/TestKeeperStorage.h | 13 +++- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 9 ++- src/Server/TestKeeperTCPHandler.cpp | 30 ++++++-- src/Server/TestKeeperTCPHandler.h | 3 +- 5 files changed, 122 insertions(+), 18 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index 5ac8daf2e09..9ddb7a7e82f 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -24,6 +24,37 @@ static String baseName(const String & path) return path.substr(rslash_pos + 1); } +static void processWatchesImpl(const String & path, TestKeeperStorage::Watches & watches, TestKeeperStorage::Watches & list_watches) +{ + Coordination::ZooKeeperWatchResponse watch_response; + watch_response.path = path; + watch_response.xid = -1; + + auto it = watches.find(watch_response.path); + if (it != watches.end()) + { + for (auto & callback : it->second) + if (callback) + callback(std::make_shared(watch_response)); + + watches.erase(it); + } + + Coordination::ZooKeeperWatchResponse watch_list_response; + watch_list_response.path = parentPath(path); + watch_list_response.xid = -1; + + it = list_watches.find(watch_list_response.path); + if (it != list_watches.end()) + { + for (auto & callback : it->second) + if (callback) + callback(std::make_shared(watch_list_response)); + + list_watches.erase(it); + } +} + TestKeeperStorage::TestKeeperStorage() { container.emplace("/", Node()); @@ -41,6 +72,8 @@ struct TestKeeperStorageRequest : zk_request(zk_request_) {} virtual std::pair process(TestKeeperStorage::Container & container, int64_t zxid) const = 0; + virtual void processWatches(TestKeeperStorage::Watches & /*watches*/, TestKeeperStorage::Watches & /*list_watches*/) const {} + virtual ~TestKeeperStorageRequest() {} }; @@ -57,6 +90,12 @@ struct TestKeeperStorageHeartbeatRequest final : public TestKeeperStorageRequest struct TestKeeperStorageCreateRequest final : public TestKeeperStorageRequest { using TestKeeperStorageRequest::TestKeeperStorageRequest; + + void processWatches(TestKeeperStorage::Watches & watches, TestKeeperStorage::Watches & list_watches) const override + { + processWatchesImpl(zk_request->getPath(), watches, list_watches); + } + std::pair process(TestKeeperStorage::Container & container, int64_t zxid) const override { LOG_DEBUG(&Poco::Logger::get("STORAGE"), "EXECUTING CREATE REQUEST"); @@ -201,6 +240,11 @@ struct TestKeeperStorageRemoveRequest final : public TestKeeperStorageRequest return { response_ptr, undo }; } + + void processWatches(TestKeeperStorage::Watches & watches, TestKeeperStorage::Watches & list_watches) const override + { + processWatchesImpl(zk_request->getPath(), watches, list_watches); + } }; struct TestKeeperStorageExistsRequest final : public TestKeeperStorageRequest @@ -268,6 +312,12 @@ struct TestKeeperStorageSetRequest final : public TestKeeperStorageRequest return { response_ptr, {} }; } + + void processWatches(TestKeeperStorage::Watches & watches, TestKeeperStorage::Watches & list_watches) const override + { + processWatchesImpl(zk_request->getPath(), watches, list_watches); + } + }; struct TestKeeperStorageListRequest final : public TestKeeperStorageRequest @@ -407,6 +457,12 @@ struct TestKeeperStorageMultiRequest final : public TestKeeperStorageRequest throw; } } + + void processWatches(TestKeeperStorage::Watches & watches, TestKeeperStorage::Watches & list_watches) const override + { + for (const auto & generic_request : concrete_requests) + generic_request->processWatches(watches, list_watches); + } }; void TestKeeperStorage::processingThread() @@ -427,12 +483,26 @@ void TestKeeperStorage::processingThread() if (shutdown) break; - ++zxid; + if (info.watch_callback) + { + auto & watches_type = dynamic_cast(info.request->zk_request.get()) + ? list_watches + : watches; + + watches_type[info.request->zk_request->getPath()].emplace_back(std::move(info.watch_callback)); + } auto zk_request = info.request->zk_request; LOG_DEBUG(&Poco::Logger::get("STORAGE"), "GOT REQUEST {}", zk_request->getOpNum()); + info.request->zk_request->addRootPath(root_path); auto [response, _] = info.request->process(container, zxid); + if (response->error == Coordination::Error::ZOK) + { + info.request->processWatches(watches, list_watches); + } + + ++zxid; response->xid = zk_request->xid; response->zxid = zxid; response->removeRootPath(root_path); @@ -541,7 +611,7 @@ TestKeeperWrapperFactory::TestKeeperWrapperFactory() registerTestKeeperRequestWrapper<14, TestKeeperStorageMultiRequest>(*this); } -TestKeeperStorage::AsyncResponse TestKeeperStorage::putRequest(const Coordination::ZooKeeperRequestPtr & request) +TestKeeperStorage::ResponsePair TestKeeperStorage::putRequest(const Coordination::ZooKeeperRequestPtr & request) { auto promise = std::make_shared>(); auto future = promise->get_future(); @@ -550,12 +620,19 @@ TestKeeperStorage::AsyncResponse TestKeeperStorage::putRequest(const Coordinatio request_info.time = clock::now(); request_info.request = storage_request; request_info.response_callback = [promise] (const Coordination::ZooKeeperResponsePtr & response) { promise->set_value(response); }; + std::optional watch_future; + if (request->has_watch) + { + auto watch_promise = std::make_shared>(); + watch_future.emplace(watch_promise->get_future()); + request_info.watch_callback = [watch_promise] (const Coordination::ZooKeeperResponsePtr & response) { watch_promise->set_value(response); }; + } std::lock_guard lock(push_request_mutex); if (!requests_queue.tryPush(std::move(request_info), operation_timeout.totalMilliseconds())) throw Exception("Cannot push request to queue within operation timeout", ErrorCodes::LOGICAL_ERROR); - LOG_DEBUG(&Poco::Logger::get("STORAGE"), "PUSHED"); - return future; + //LOG_DEBUG(&Poco::Logger::get("STORAGE"), "PUSHED"); + return ResponsePair{std::move(future), std::move(watch_future)}; } diff --git a/src/Common/ZooKeeper/TestKeeperStorage.h b/src/Common/ZooKeeper/TestKeeperStorage.h index 86be2e0eeaf..edcae28af85 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.h +++ b/src/Common/ZooKeeper/TestKeeperStorage.h @@ -12,6 +12,7 @@ namespace zkutil using namespace DB; struct TestKeeperStorageRequest; using TestKeeperStorageRequestPtr = std::shared_ptr; +using ResponseCallback = std::function; class TestKeeperStorage { @@ -33,7 +34,7 @@ public: using Container = std::map; - using WatchCallbacks = std::vector; + using WatchCallbacks = std::vector; using Watches = std::map; Container container; @@ -51,7 +52,8 @@ public: struct RequestInfo { TestKeeperStorageRequestPtr request; - std::function response_callback; + ResponseCallback response_callback; + ResponseCallback watch_callback; clock::time_point time; }; std::mutex push_request_mutex; @@ -68,7 +70,12 @@ public: using AsyncResponse = std::future; TestKeeperStorage(); ~TestKeeperStorage(); - AsyncResponse putRequest(const Coordination::ZooKeeperRequestPtr & request); + struct ResponsePair + { + AsyncResponse response; + std::optional watch_response; + }; + ResponsePair putRequest(const Coordination::ZooKeeperRequestPtr & request); int64_t getSessionID() { diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index fef0b7063f5..093b2bec7df 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -654,11 +654,14 @@ void ZooKeeper::sendThread() } if (expired) + { break; + } info.request->addRootPath(root_path); info.request->probably_sent = true; + std::cerr << "SENDING GENERAL REQUEST\n"; info.request->write(*out); if (info.request->xid == close_xid) @@ -899,7 +902,6 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) if (expired) return; - expired = true; } active_session_metric_increment.destroy(); @@ -922,6 +924,11 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) send_thread.join(); } + { + std::lock_guard lock(push_request_mutex); + expired = true; + } + try { /// This will also wakeup the receiving thread. diff --git a/src/Server/TestKeeperTCPHandler.cpp b/src/Server/TestKeeperTCPHandler.cpp index 0ca106d0e32..38d9f7e54c5 100644 --- a/src/Server/TestKeeperTCPHandler.cpp +++ b/src/Server/TestKeeperTCPHandler.cpp @@ -289,31 +289,45 @@ void TestKeeperTCPHandler::runImpl() } if (in->poll(max_wait)) - receiveRequest(); + { + bool close_received = receiveRequest(); + if (close_received) + { + LOG_DEBUG(log, "Received close request"); + break; + } + } } } -void TestKeeperTCPHandler::receiveRequest() +bool TestKeeperTCPHandler::receiveRequest() { - LOG_DEBUG(log, "Receiving heartbeat event"); + LOG_DEBUG(log, "Receiving event"); int32_t length; read(length, *in); - LOG_DEBUG(log, "RECEIVED LENGTH {}", length); + //LOG_DEBUG(log, "RECEIVED LENGTH {}", length); int32_t xid; - LOG_DEBUG(log, "READING XID"); + //LOG_DEBUG(log, "READING XID"); read(xid, *in); - LOG_DEBUG(log, "Received xid {}", xid); + //LOG_DEBUG(log, "Received xid {}", xid); int32_t opnum; read(opnum, *in); + if (opnum == -11) + return true; + Coordination::ZooKeeperRequestPtr request = Coordination::ZooKeeperRequestFactory::instance().get(opnum); request->xid = xid; request->readImpl(*in); - responses.push(test_keeper_storage->putRequest(request)); + auto request_future_responses = test_keeper_storage->putRequest(request); + responses.push(std::move(request_future_responses.response)); + if (request_future_responses.watch_response) + responses.push(std::move(*request_future_responses.watch_response)); - LOG_DEBUG(log, "Event received"); + return false; + //LOG_DEBUG(log, "Event received"); } diff --git a/src/Server/TestKeeperTCPHandler.h b/src/Server/TestKeeperTCPHandler.h index 2a796daa4e9..b50af3363c1 100644 --- a/src/Server/TestKeeperTCPHandler.h +++ b/src/Server/TestKeeperTCPHandler.h @@ -35,7 +35,6 @@ private: std::queue responses; - /// Streams for reading/writing from/to client connection socket. std::shared_ptr in; std::shared_ptr out; @@ -45,7 +44,7 @@ private: void sendHandshake(); void receiveHandshake(); - void receiveRequest(); + bool receiveRequest(); }; } From bb3b420057609620fa39d3f7468b929f0da3e65b Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 11 Nov 2020 11:51:54 +0300 Subject: [PATCH 022/264] Ugly working code --- src/Common/ZooKeeper/TestKeeperStorage.cpp | 12 +++++++ src/Common/ZooKeeper/ZooKeeperImpl.cpp | 16 ++++++--- src/Server/TestKeeperTCPHandler.cpp | 29 +++++++++++++--- src/Server/TestKeeperTCPHandler.h | 1 + utils/zookeeper-test/main.cpp | 39 ++++++++++++++++++++++ 5 files changed, 88 insertions(+), 9 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index 9ddb7a7e82f..8f80fa1a0c5 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -30,12 +30,18 @@ static void processWatchesImpl(const String & path, TestKeeperStorage::Watches & watch_response.path = path; watch_response.xid = -1; + std::cerr << "WATCHES SIZE:" << watches.size() << " path:" << path << std::endl; auto it = watches.find(watch_response.path); if (it != watches.end()) { for (auto & callback : it->second) + { if (callback) + { + std::cerr << "CALLING WATCH CALLBACK\n"; callback(std::make_shared(watch_response)); + } + } watches.erase(it); } @@ -44,12 +50,18 @@ static void processWatchesImpl(const String & path, TestKeeperStorage::Watches & watch_list_response.path = parentPath(path); watch_list_response.xid = -1; + std::cerr << "LIST WATCHES SIZE:" << list_watches.size() << " path:" << path << std::endl; it = list_watches.find(watch_list_response.path); if (it != list_watches.end()) { for (auto & callback : it->second) + { if (callback) + { + std::cerr << "Calling list watch callback\n" << std::endl; callback(std::make_shared(watch_list_response)); + } + } list_watches.erase(it); } diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 17999044a77..8e10adb4500 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -650,6 +650,7 @@ void ZooKeeper::sendThread() if (info.watch) { info.request->has_watch = true; + std::cerr << "REQUEST" << info.request->getOpNum() << " HAS WATCH" << std::endl; CurrentMetrics::add(CurrentMetrics::ZooKeeperWatch); } @@ -661,7 +662,7 @@ void ZooKeeper::sendThread() info.request->addRootPath(root_path); info.request->probably_sent = true; - std::cerr << "SENDING GENERAL REQUEST\n"; + std::cerr << "SENDING GENERAL REQUEST:" << info.request->getOpNum() << std::endl; info.request->write(*out); /// We sent close request, exit @@ -729,7 +730,9 @@ void ZooKeeper::receiveThread() else { if (earliest_operation) - throw Exception("Operation timeout (no response) for path: " + earliest_operation->request->getPath(), Error::ZOPERATIONTIMEOUT); + { + throw Exception("Operation timeout (no response) for request " + std::to_string(earliest_operation->request->getOpNum()) + " for path: " + earliest_operation->request->getPath(), Error::ZOPERATIONTIMEOUT); + } waited += max_wait; if (waited >= session_timeout.totalMicroseconds()) throw Exception("Nothing is received in session timeout", Error::ZOPERATIONTIMEOUT); @@ -772,6 +775,7 @@ void ZooKeeper::receiveEvent() } else if (xid == watch_xid) { + std::cerr << "Receiving watch\n"; ProfileEvents::increment(ProfileEvents::ZooKeeperWatchResponse); response = std::make_shared(); @@ -828,21 +832,23 @@ void ZooKeeper::receiveEvent() try { - std::cerr << "READING RESPONSE FOR REQUEST ID:" << request_info.request->getOpNum() << std::endl; + std::cerr << "READING RESPONSE FOR REQUEST\n"; if (!response) response = request_info.request->makeResponse(); if (err != Error::ZOK) { - std::cerr << "GOT ERROR:" << static_cast(err) << std::endl; + //std::cerr << "GOT ERROR:" << static_cast(err) << std::endl; response->error = err; } else { - std::cerr << "NO ERROR RECEIVED\n"; + //std::cerr << "NO ERROR RECEIVED\n"; response->readImpl(*in); response->removeRootPath(root_path); } + if (request_info.request) + std::cerr << "Response Request ID" << request_info.request->getOpNum() << std::endl; /// Instead of setting the watch in sendEvent, set it in receiveEvent because need to check the response. /// The watch shouldn't be set if the node does not exist and it will never exist like sequential ephemeral nodes. diff --git a/src/Server/TestKeeperTCPHandler.cpp b/src/Server/TestKeeperTCPHandler.cpp index 38d9f7e54c5..82949dfc547 100644 --- a/src/Server/TestKeeperTCPHandler.cpp +++ b/src/Server/TestKeeperTCPHandler.cpp @@ -279,16 +279,33 @@ void TestKeeperTCPHandler::runImpl() while (true) { - UInt64 max_wait = operation_timeout.totalMicroseconds(); + //UInt64 max_wait = operation_timeout.totalMicroseconds(); using namespace std::chrono_literals; + LOG_DEBUG(log, "TRYING TO GET RESPONSE (size {})", responses.size()); if (!responses.empty() && responses.front().wait_for(100ms) == std::future_status::ready) { auto response = responses.front().get(); + + LOG_DEBUG(log, "Writing response bytes to socket {}", response->getOpNum()); response->write(*out); responses.pop(); + LOG_DEBUG(log, "Responses size {}", responses.size()); + } + for (auto it = watch_responses.begin(); it != watch_responses.end();) + { + if (it->wait_for(0s) == std::future_status::ready) + { + it->get()->write(*out); + it = watch_responses.erase(it); + } + else + { + ++it; + } } - if (in->poll(max_wait)) + LOG_DEBUG(log, "WAITING ON POLL"); + if (in->poll(100 * 1000)) { bool close_received = receiveRequest(); if (close_received) @@ -297,6 +314,10 @@ void TestKeeperTCPHandler::runImpl() break; } } + else + { + //LOG_DEBUG(log, "NOTHING POLLED"); + } } } @@ -324,10 +345,10 @@ bool TestKeeperTCPHandler::receiveRequest() auto request_future_responses = test_keeper_storage->putRequest(request); responses.push(std::move(request_future_responses.response)); if (request_future_responses.watch_response) - responses.push(std::move(*request_future_responses.watch_response)); + watch_responses.emplace_back(std::move(*request_future_responses.watch_response)); + LOG_DEBUG(log, "Responses size {}", responses.size()); return false; - //LOG_DEBUG(log, "Event received"); } diff --git a/src/Server/TestKeeperTCPHandler.h b/src/Server/TestKeeperTCPHandler.h index b50af3363c1..7eea5419006 100644 --- a/src/Server/TestKeeperTCPHandler.h +++ b/src/Server/TestKeeperTCPHandler.h @@ -34,6 +34,7 @@ private: Poco::Timespan operation_timeout; std::queue responses; + std::vector watch_responses; /// Streams for reading/writing from/to client connection socket. std::shared_ptr in; diff --git a/utils/zookeeper-test/main.cpp b/utils/zookeeper-test/main.cpp index 802fbc17708..a924465d02a 100644 --- a/utils/zookeeper-test/main.cpp +++ b/utils/zookeeper-test/main.cpp @@ -82,6 +82,43 @@ void testCreateSetVersionRequest(zkutil::ZooKeeper & zk) checkEq(zk, "/data/check_data", "e"); } +void testCreateSetWatchEvent(zkutil::ZooKeeper & zk) +{ + + std::shared_ptr event = std::make_shared(); + zk.create("/data/nodeforwatch", "", zkutil::CreateMode::Persistent); + Coordination::Stat stat; + zk.get("/data/nodeforwatch", &stat, event); + + if (event->tryWait(300)) + throw std::runtime_error(fmt::format("Event for path {} was set without any actions", "/data/nodeforwatch")); + + zk.set("/data/nodeforwatch", "x"); + if (!event->tryWait(300)) + throw std::runtime_error(fmt::format("Event for path {} was not set after set", "/data/nodeforwatch")); + else + std::cerr << "Event was set well\n"; +} + +void testCreateListWatchEvent(zkutil::ZooKeeper & zk) +{ + std::shared_ptr event = std::make_shared(); + std::string path = "/data/pathforwatch"; + zk.create(path, "", zkutil::CreateMode::Persistent); + zk.create(path + "/n1", "", zkutil::CreateMode::Persistent); + zk.create(path + "/n2", "", zkutil::CreateMode::Persistent); + zk.getChildren(path, nullptr, event); + + if (event->tryWait(300)) + throw std::runtime_error(fmt::format("ListEvent for path {} was set without any actions", path)); + + zk.create(path + "/n3", "", zkutil::CreateMode::Persistent); + if (!event->tryWait(300)) + throw std::runtime_error(fmt::format("ListEvent for path {} was not set after create", path)); + else + std::cerr << "ListEvent was set well\n"; +} + void testMultiRequest(zkutil::ZooKeeper & zk) { Coordination::Requests requests; @@ -127,6 +164,8 @@ int main(int argc, char *argv[]) { testCreateList(zk); testCreateSetVersionRequest(zk); testMultiRequest(zk); + testCreateSetWatchEvent(zk); + testCreateListWatchEvent(zk); } catch(...) { From 0388006ed672a570925993ca43c5a60860ab63de Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 11 Nov 2020 12:05:45 +0300 Subject: [PATCH 023/264] Less garbage --- src/Common/ZooKeeper/TestKeeperStorage.cpp | 21 +-------------------- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 6 ++---- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 9 ++------- src/Server/TestKeeperTCPHandler.cpp | 13 +++++++------ 4 files changed, 12 insertions(+), 37 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index 8f80fa1a0c5..b60144f4a87 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -30,18 +30,12 @@ static void processWatchesImpl(const String & path, TestKeeperStorage::Watches & watch_response.path = path; watch_response.xid = -1; - std::cerr << "WATCHES SIZE:" << watches.size() << " path:" << path << std::endl; auto it = watches.find(watch_response.path); if (it != watches.end()) { for (auto & callback : it->second) - { if (callback) - { - std::cerr << "CALLING WATCH CALLBACK\n"; callback(std::make_shared(watch_response)); - } - } watches.erase(it); } @@ -50,18 +44,12 @@ static void processWatchesImpl(const String & path, TestKeeperStorage::Watches & watch_list_response.path = parentPath(path); watch_list_response.xid = -1; - std::cerr << "LIST WATCHES SIZE:" << list_watches.size() << " path:" << path << std::endl; it = list_watches.find(watch_list_response.path); if (it != list_watches.end()) { for (auto & callback : it->second) - { if (callback) - { - std::cerr << "Calling list watch callback\n" << std::endl; callback(std::make_shared(watch_list_response)); - } - } list_watches.erase(it); } @@ -110,7 +98,6 @@ struct TestKeeperStorageCreateRequest final : public TestKeeperStorageRequest std::pair process(TestKeeperStorage::Container & container, int64_t zxid) const override { - LOG_DEBUG(&Poco::Logger::get("STORAGE"), "EXECUTING CREATE REQUEST"); Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Undo undo; Coordination::ZooKeeperCreateResponse & response = dynamic_cast(*response_ptr); @@ -188,7 +175,7 @@ struct TestKeeperStorageGetRequest final : public TestKeeperStorageRequest using TestKeeperStorageRequest::TestKeeperStorageRequest; std::pair process(TestKeeperStorage::Container & container, int64_t /* zxid */) const override { - LOG_DEBUG(&Poco::Logger::get("STORAGE"), "EXECUTING GET REQUEST"); + //LOG_DEBUG(&Poco::Logger::get("STORAGE"), "EXECUTING GET REQUEST"); Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Coordination::ZooKeeperGetResponse & response = dynamic_cast(*response_ptr); Coordination::ZooKeeperGetRequest & request = dynamic_cast(*zk_request); @@ -445,8 +432,6 @@ struct TestKeeperStorageMultiRequest final : public TestKeeperStorageRequest response.responses[i] = cur_response; if (cur_response->error != Coordination::Error::ZOK) { - std::cerr << "GOT ERROR ON: " << i << " error" << static_cast(cur_response->error) << std::endl; - for (auto it = undo_actions.rbegin(); it != undo_actions.rend(); ++it) if (*it) (*it)(); @@ -481,7 +466,6 @@ void TestKeeperStorage::processingThread() { setThreadName("TestKeeperSProc"); - LOG_DEBUG(&Poco::Logger::get("STORAGE"), "LOOPING IN THREAD"); try { while (!shutdown) @@ -505,7 +489,6 @@ void TestKeeperStorage::processingThread() } auto zk_request = info.request->zk_request; - LOG_DEBUG(&Poco::Logger::get("STORAGE"), "GOT REQUEST {}", zk_request->getOpNum()); info.request->zk_request->addRootPath(root_path); auto [response, _] = info.request->process(container, zxid); @@ -519,9 +502,7 @@ void TestKeeperStorage::processingThread() response->zxid = zxid; response->removeRootPath(root_path); - LOG_DEBUG(&Poco::Logger::get("STORAGE"), "SENDING RESPONSE"); info.response_callback(response); - LOG_DEBUG(&Poco::Logger::get("STORAGE"), "DONE"); } } } diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index 604bb2137c4..cb61f22fb3c 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -175,13 +175,13 @@ void ZooKeeperResponse::write(WriteBuffer & out) const { /// Excessive copy to calculate length. WriteBufferFromOwnString buf; - LOG_DEBUG(&Poco::Logger::get("LOG"), "WRITING {}", xid); + //LOG_DEBUG(&Poco::Logger::get("LOG"), "WRITING {}", xid); Coordination::write(xid, buf); Coordination::write(zxid, buf); Coordination::write(error, buf); if (error == Error::ZOK) writeImpl(buf); - LOG_DEBUG(&Poco::Logger::get("LOG"), "BUFFER LENGTH {}", buf.str().length()); + //LOG_DEBUG(&Poco::Logger::get("LOG"), "BUFFER LENGTH {}", buf.str().length()); Coordination::write(buf.str(), out); out.next(); } @@ -544,14 +544,12 @@ void ZooKeeperMultiResponse::readImpl(ReadBuffer & in) void ZooKeeperMultiResponse::writeImpl(WriteBuffer & out) const { - std::cerr << "WRITING MULTIRESPONSE " << responses.size() << std::endl; for (auto & response : responses) { const ZooKeeperResponse & zk_response = dynamic_cast(*response); OpNum op_num = zk_response.getOpNum(); bool done = false; Error op_error = zk_response.error; - std::cerr << "WRITING OP ERROR:" << static_cast(op_error) << std::endl; Coordination::write(op_num, out); Coordination::write(done, out); diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 8e10adb4500..23a5c6c4301 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -650,7 +650,7 @@ void ZooKeeper::sendThread() if (info.watch) { info.request->has_watch = true; - std::cerr << "REQUEST" << info.request->getOpNum() << " HAS WATCH" << std::endl; + //std::cerr << "REQUEST" << info.request->getOpNum() << " HAS WATCH" << std::endl; CurrentMetrics::add(CurrentMetrics::ZooKeeperWatch); } @@ -662,7 +662,7 @@ void ZooKeeper::sendThread() info.request->addRootPath(root_path); info.request->probably_sent = true; - std::cerr << "SENDING GENERAL REQUEST:" << info.request->getOpNum() << std::endl; + //std::cerr << "SENDING GENERAL REQUEST:" << info.request->getOpNum() << std::endl; info.request->write(*out); /// We sent close request, exit @@ -775,7 +775,6 @@ void ZooKeeper::receiveEvent() } else if (xid == watch_xid) { - std::cerr << "Receiving watch\n"; ProfileEvents::increment(ProfileEvents::ZooKeeperWatchResponse); response = std::make_shared(); @@ -832,7 +831,6 @@ void ZooKeeper::receiveEvent() try { - std::cerr << "READING RESPONSE FOR REQUEST\n"; if (!response) response = request_info.request->makeResponse(); @@ -847,9 +845,6 @@ void ZooKeeper::receiveEvent() response->readImpl(*in); response->removeRootPath(root_path); } - if (request_info.request) - std::cerr << "Response Request ID" << request_info.request->getOpNum() << std::endl; - /// Instead of setting the watch in sendEvent, set it in receiveEvent because need to check the response. /// The watch shouldn't be set if the node does not exist and it will never exist like sequential ephemeral nodes. /// By using getData() instead of exists(), a watch won't be set if the node doesn't exist. diff --git a/src/Server/TestKeeperTCPHandler.cpp b/src/Server/TestKeeperTCPHandler.cpp index 82949dfc547..1e7f69dc14f 100644 --- a/src/Server/TestKeeperTCPHandler.cpp +++ b/src/Server/TestKeeperTCPHandler.cpp @@ -273,6 +273,7 @@ void TestKeeperTCPHandler::runImpl() catch (const Exception & e) /// Typical for an incorrect username, password, or address. { LOG_DEBUG(log, "Cannot receive handshake {}", e.displayText()); + return; } sendHandshake(); @@ -281,15 +282,15 @@ void TestKeeperTCPHandler::runImpl() { //UInt64 max_wait = operation_timeout.totalMicroseconds(); using namespace std::chrono_literals; - LOG_DEBUG(log, "TRYING TO GET RESPONSE (size {})", responses.size()); + //LOG_DEBUG(log, "TRYING TO GET RESPONSE (size {})", responses.size()); if (!responses.empty() && responses.front().wait_for(100ms) == std::future_status::ready) { auto response = responses.front().get(); - LOG_DEBUG(log, "Writing response bytes to socket {}", response->getOpNum()); + //LOG_DEBUG(log, "Writing response bytes to socket {}", response->getOpNum()); response->write(*out); responses.pop(); - LOG_DEBUG(log, "Responses size {}", responses.size()); + //LOG_DEBUG(log, "Responses size {}", responses.size()); } for (auto it = watch_responses.begin(); it != watch_responses.end();) { @@ -304,7 +305,7 @@ void TestKeeperTCPHandler::runImpl() } } - LOG_DEBUG(log, "WAITING ON POLL"); + //LOG_DEBUG(log, "WAITING ON POLL"); if (in->poll(100 * 1000)) { bool close_received = receiveRequest(); @@ -324,7 +325,7 @@ void TestKeeperTCPHandler::runImpl() bool TestKeeperTCPHandler::receiveRequest() { - LOG_DEBUG(log, "Receiving event"); + //LOG_DEBUG(log, "Receiving event"); int32_t length; read(length, *in); //LOG_DEBUG(log, "RECEIVED LENGTH {}", length); @@ -347,7 +348,7 @@ bool TestKeeperTCPHandler::receiveRequest() if (request_future_responses.watch_response) watch_responses.emplace_back(std::move(*request_future_responses.watch_response)); - LOG_DEBUG(log, "Responses size {}", responses.size()); + //LOG_DEBUG(log, "Responses size {}", responses.size()); return false; } From 66236d6ebb8af22eed790ddea1aa2f83d5ef404d Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 11 Nov 2020 16:07:06 +0300 Subject: [PATCH 024/264] Refactored a little --- programs/server/Server.cpp | 274 ++++++++++---------- programs/server/Server.h | 14 + src/Common/ZooKeeper/TestKeeperStorage.cpp | 70 +++-- src/Common/ZooKeeper/TestKeeperStorage.h | 3 +- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 197 ++------------ src/Common/ZooKeeper/ZooKeeperCommon.h | 56 ++-- src/Common/ZooKeeper/ZooKeeperConstants.cpp | 64 +++++ src/Common/ZooKeeper/ZooKeeperConstants.h | 47 ++++ src/Common/ZooKeeper/ZooKeeperIO.cpp | 135 ++++++++++ src/Common/ZooKeeper/ZooKeeperIO.h | 73 ++++++ src/Common/ZooKeeper/ZooKeeperImpl.cpp | 134 ++-------- src/Common/ZooKeeper/ZooKeeperImpl.h | 3 - src/Server/TestKeeperTCPHandler.cpp | 246 +++--------------- src/Server/TestKeeperTCPHandler.h | 5 +- 14 files changed, 625 insertions(+), 696 deletions(-) create mode 100644 src/Common/ZooKeeper/ZooKeeperConstants.cpp create mode 100644 src/Common/ZooKeeper/ZooKeeperConstants.h create mode 100644 src/Common/ZooKeeper/ZooKeeperIO.cpp create mode 100644 src/Common/ZooKeeper/ZooKeeperIO.h diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 344082086e9..597b6c43fe5 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -180,6 +180,85 @@ static std::string getUserName(uid_t user_id) return toString(user_id); } +Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port, Poco::Logger * log) +{ + Poco::Net::SocketAddress socket_address; + try + { + socket_address = Poco::Net::SocketAddress(host, port); + } + catch (const Poco::Net::DNSException & e) + { + const auto code = e.code(); + if (code == EAI_FAMILY +#if defined(EAI_ADDRFAMILY) + || code == EAI_ADDRFAMILY +#endif + ) + { + LOG_ERROR(log, "Cannot resolve listen_host ({}), error {}: {}. " + "If it is an IPv6 address and your host has disabled IPv6, then consider to " + "specify IPv4 address to listen in element of configuration " + "file. Example: 0.0.0.0", + host, e.code(), e.message()); + } + + throw; + } + return socket_address; +} + +Poco::Net::SocketAddress Server::socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure) +{ + auto address = makeSocketAddress(host, port, &logger()); +#if !defined(POCO_CLICKHOUSE_PATCH) || POCO_VERSION < 0x01090100 + if (secure) + /// Bug in old (<1.9.1) poco, listen() after bind() with reusePort param will fail because have no implementation in SecureServerSocketImpl + /// https://github.com/pocoproject/poco/pull/2257 + socket.bind(address, /* reuseAddress = */ true); + else +#endif +#if POCO_VERSION < 0x01080000 + socket.bind(address, /* reuseAddress = */ true); +#else + socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ config().getBool("listen_reuse_port", false)); +#endif + + socket.listen(/* backlog = */ config().getUInt("listen_backlog", 64)); + + return address; +} + +void Server::createServer(const std::string & listen_host, const char * port_name, bool listen_try, CreateServerFunc && func) +{ + /// For testing purposes, user may omit tcp_port or http_port or https_port in configuration file. + if (!config().has(port_name)) + return; + + auto port = config().getInt(port_name); + try + { + func(port); + } + catch (const Poco::Exception &) + { + std::string message = "Listen [" + listen_host + "]:" + std::to_string(port) + " failed: " + getCurrentExceptionMessage(false); + + if (listen_try) + { + LOG_WARNING(&logger(), "{}. If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " + "specify not disabled IPv4 or IPv6 address to listen in element of configuration " + "file. Example for disabled IPv6: 0.0.0.0 ." + " Example for disabled IPv4: ::", + message); + } + else + { + throw Exception{message, ErrorCodes::NETWORK_ERROR}; + } + } +} + void Server::uninitialize() { logger().information("shutting down"); @@ -667,6 +746,47 @@ int Server::main(const std::vector & /*args*/) total_memory_tracker.setDescription("(total)"); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); + Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0); + + Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024)); + Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams; + http_params->setTimeout(settings.http_receive_timeout); + http_params->setKeepAliveTimeout(keep_alive_timeout); + + std::vector> servers; + + std::vector listen_hosts = DB::getMultipleValuesFromConfig(config(), "", "listen_host"); + + bool listen_try = config().getBool("listen_try", false); + if (listen_hosts.empty()) + { + listen_hosts.emplace_back("::1"); + listen_hosts.emplace_back("127.0.0.1"); + listen_try = true; + } + + for (const auto & listen_host : listen_hosts) + { + /// TCP TestKeeper + createServer(listen_host, "test_keeper_tcp_port", listen_try, [&](UInt16 port) + { + Poco::Net::ServerSocket socket; + auto address = socketBindListen(socket, listen_host, port); + socket.setReceiveTimeout(settings.receive_timeout); + socket.setSendTimeout(settings.send_timeout); + servers.emplace_back(std::make_unique( + new TCPHandlerFactory(*this, false, true), + server_pool, + socket, + new Poco::Net::TCPServerParams)); + + LOG_INFO(log, "Listening for connections to fake zookeeper (tcp): {}", address.toString()); + }); + } + + for (auto & server : servers) + server->start(); + /// Set current database name before loading tables and databases because /// system logs may copy global context. global_context->setCurrentDatabaseNameInGlobalContext(default_database); @@ -797,74 +917,6 @@ int Server::main(const std::vector & /*args*/) #endif { - Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0); - - Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024)); - Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams; - http_params->setTimeout(settings.http_receive_timeout); - http_params->setKeepAliveTimeout(keep_alive_timeout); - - std::vector> servers; - - std::vector listen_hosts = DB::getMultipleValuesFromConfig(config(), "", "listen_host"); - - bool listen_try = config().getBool("listen_try", false); - if (listen_hosts.empty()) - { - listen_hosts.emplace_back("::1"); - listen_hosts.emplace_back("127.0.0.1"); - listen_try = true; - } - - auto make_socket_address = [&](const std::string & host, UInt16 port) - { - Poco::Net::SocketAddress socket_address; - try - { - socket_address = Poco::Net::SocketAddress(host, port); - } - catch (const Poco::Net::DNSException & e) - { - const auto code = e.code(); - if (code == EAI_FAMILY -#if defined(EAI_ADDRFAMILY) - || code == EAI_ADDRFAMILY -#endif - ) - { - LOG_ERROR(log, "Cannot resolve listen_host ({}), error {}: {}. " - "If it is an IPv6 address and your host has disabled IPv6, then consider to " - "specify IPv4 address to listen in element of configuration " - "file. Example: 0.0.0.0", - host, e.code(), e.message()); - } - - throw; - } - return socket_address; - }; - - auto socket_bind_listen = [&](auto & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = false) - { - auto address = make_socket_address(host, port); -#if !defined(POCO_CLICKHOUSE_PATCH) || POCO_VERSION < 0x01090100 - if (secure) - /// Bug in old (<1.9.1) poco, listen() after bind() with reusePort param will fail because have no implementation in SecureServerSocketImpl - /// https://github.com/pocoproject/poco/pull/2257 - socket.bind(address, /* reuseAddress = */ true); - else -#endif -#if POCO_VERSION < 0x01080000 - socket.bind(address, /* reuseAddress = */ true); -#else - socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ config().getBool("listen_reuse_port", false)); -#endif - - socket.listen(/* backlog = */ config().getUInt("listen_backlog", 64)); - - return address; - }; - /// This object will periodically calculate some metrics. AsynchronousMetrics async_metrics(*global_context, config().getUInt("asynchronous_metrics_update_period_s", 60)); @@ -872,41 +924,11 @@ int Server::main(const std::vector & /*args*/) for (const auto & listen_host : listen_hosts) { - auto create_server = [&](const char * port_name, auto && func) - { - /// For testing purposes, user may omit tcp_port or http_port or https_port in configuration file. - if (!config().has(port_name)) - return; - - auto port = config().getInt(port_name); - try - { - func(port); - } - catch (const Poco::Exception &) - { - std::string message = "Listen [" + listen_host + "]:" + std::to_string(port) + " failed: " + getCurrentExceptionMessage(false); - - if (listen_try) - { - LOG_WARNING(log, "{}. If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " - "specify not disabled IPv4 or IPv6 address to listen in element of configuration " - "file. Example for disabled IPv6: 0.0.0.0 ." - " Example for disabled IPv4: ::", - message); - } - else - { - throw Exception{message, ErrorCodes::NETWORK_ERROR}; - } - } - }; - /// HTTP - create_server("http_port", [&](UInt16 port) + createServer(listen_host, "http_port", listen_try, [&](UInt16 port) { Poco::Net::ServerSocket socket; - auto address = socket_bind_listen(socket, listen_host, port); + auto address = socketBindListen(socket, listen_host, port); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); @@ -917,11 +939,11 @@ int Server::main(const std::vector & /*args*/) }); /// HTTPS - create_server("https_port", [&](UInt16 port) + createServer(listen_host, "https_port", listen_try, [&](UInt16 port) { #if USE_SSL Poco::Net::SecureServerSocket socket; - auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true); + auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(std::make_unique( @@ -936,10 +958,10 @@ int Server::main(const std::vector & /*args*/) }); /// TCP - create_server("tcp_port", [&](UInt16 port) + createServer(listen_host, "tcp_port", listen_try, [&](UInt16 port) { Poco::Net::ServerSocket socket; - auto address = socket_bind_listen(socket, listen_host, port); + auto address = socketBindListen(socket, listen_host, port); socket.setReceiveTimeout(settings.receive_timeout); socket.setSendTimeout(settings.send_timeout); servers.emplace_back(std::make_unique( @@ -951,28 +973,12 @@ int Server::main(const std::vector & /*args*/) LOG_INFO(log, "Listening for connections with native protocol (tcp): {}", address.toString()); }); - /// TCP TestKeeper - create_server("test_keeper_tcp_port", [&](UInt16 port) - { - Poco::Net::ServerSocket socket; - auto address = socket_bind_listen(socket, listen_host, port); - socket.setReceiveTimeout(settings.receive_timeout); - socket.setSendTimeout(settings.send_timeout); - servers.emplace_back(std::make_unique( - new TCPHandlerFactory(*this, false, true), - server_pool, - socket, - new Poco::Net::TCPServerParams)); - - LOG_INFO(log, "Listening for connections to fake zookeeper (tcp): {}", address.toString()); - }); - /// TCP with SSL - create_server("tcp_port_secure", [&](UInt16 port) + createServer(listen_host, "tcp_port_secure", listen_try, [&](UInt16 port) { #if USE_SSL Poco::Net::SecureServerSocket socket; - auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true); + auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(settings.receive_timeout); socket.setSendTimeout(settings.send_timeout); servers.emplace_back(std::make_unique( @@ -989,10 +995,10 @@ int Server::main(const std::vector & /*args*/) }); /// Interserver IO HTTP - create_server("interserver_http_port", [&](UInt16 port) + createServer(listen_host, "interserver_http_port", listen_try, [&](UInt16 port) { Poco::Net::ServerSocket socket; - auto address = socket_bind_listen(socket, listen_host, port); + auto address = socketBindListen(socket, listen_host, port); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(std::make_unique( @@ -1001,11 +1007,11 @@ int Server::main(const std::vector & /*args*/) LOG_INFO(log, "Listening for replica communication (interserver): http://{}", address.toString()); }); - create_server("interserver_https_port", [&](UInt16 port) + createServer(listen_host, "interserver_https_port", listen_try, [&](UInt16 port) { #if USE_SSL Poco::Net::SecureServerSocket socket; - auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true); + auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(std::make_unique( @@ -1019,10 +1025,10 @@ int Server::main(const std::vector & /*args*/) #endif }); - create_server("mysql_port", [&](UInt16 port) + createServer(listen_host, "mysql_port", listen_try, [&](UInt16 port) { Poco::Net::ServerSocket socket; - auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true); + auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(Poco::Timespan()); socket.setSendTimeout(settings.send_timeout); servers.emplace_back(std::make_unique( @@ -1034,10 +1040,10 @@ int Server::main(const std::vector & /*args*/) LOG_INFO(log, "Listening for MySQL compatibility protocol: {}", address.toString()); }); - create_server("postgresql_port", [&](UInt16 port) + createServer(listen_host, "postgresql_port", listen_try, [&](UInt16 port) { Poco::Net::ServerSocket socket; - auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true); + auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(Poco::Timespan()); socket.setSendTimeout(settings.send_timeout); servers.emplace_back(std::make_unique( @@ -1050,10 +1056,10 @@ int Server::main(const std::vector & /*args*/) }); /// Prometheus (if defined and not setup yet with http_port) - create_server("prometheus.port", [&](UInt16 port) + createServer(listen_host, "prometheus.port", listen_try, [&](UInt16 port) { Poco::Net::ServerSocket socket; - auto address = socket_bind_listen(socket, listen_host, port); + auto address = socketBindListen(socket, listen_host, port); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(std::make_unique( @@ -1069,8 +1075,8 @@ int Server::main(const std::vector & /*args*/) global_context->enableNamedSessions(); - for (auto & server : servers) - server->start(); + for (size_t i = 1; i < servers.size(); ++i) + servers[i]->start(); { String level_str = config().getString("text_log.level", ""); diff --git a/programs/server/Server.h b/programs/server/Server.h index ad9e51c881c..57fcea3cee8 100644 --- a/programs/server/Server.h +++ b/programs/server/Server.h @@ -14,6 +14,13 @@ * 3. Interserver HTTP - for replication. */ +namespace Poco +{ + namespace Net + { + class ServerSocket; + } +} namespace DB { @@ -57,6 +64,13 @@ protected: private: Context * global_context_ptr = nullptr; + +private: + + Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = false); + + using CreateServerFunc = std::function; + void createServer(const std::string & listen_host, const char * port_name, bool listen_try, CreateServerFunc && func); }; } diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index b60144f4a87..4e291679b78 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -175,7 +175,6 @@ struct TestKeeperStorageGetRequest final : public TestKeeperStorageRequest using TestKeeperStorageRequest::TestKeeperStorageRequest; std::pair process(TestKeeperStorage::Container & container, int64_t /* zxid */) const override { - //LOG_DEBUG(&Poco::Logger::get("STORAGE"), "EXECUTING GET REQUEST"); Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Coordination::ZooKeeperGetResponse & response = dynamic_cast(*response_ptr); Coordination::ZooKeeperGetRequest & request = dynamic_cast(*zk_request); @@ -309,7 +308,7 @@ struct TestKeeperStorageSetRequest final : public TestKeeperStorageRequest response.error = Coordination::Error::ZBADVERSION; } - return { response_ptr, {} }; + return { response_ptr, undo }; } void processWatches(TestKeeperStorage::Watches & watches, TestKeeperStorage::Watches & list_watches) const override @@ -327,7 +326,6 @@ struct TestKeeperStorageListRequest final : public TestKeeperStorageRequest Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Coordination::ZooKeeperListResponse & response = dynamic_cast(*response_ptr); Coordination::ZooKeeperListRequest & request = dynamic_cast(*zk_request); - Undo undo; auto it = container.find(request.path); if (it == container.end()) { @@ -490,17 +488,14 @@ void TestKeeperStorage::processingThread() auto zk_request = info.request->zk_request; - info.request->zk_request->addRootPath(root_path); auto [response, _] = info.request->process(container, zxid); if (response->error == Coordination::Error::ZOK) { info.request->processWatches(watches, list_watches); } - ++zxid; response->xid = zk_request->xid; - response->zxid = zxid; - response->removeRootPath(root_path); + response->zxid = getZXID(); info.response_callback(response); } @@ -525,6 +520,36 @@ void TestKeeperStorage::finalize() } try { + { + auto finish_watch = [] (const auto & watch_pair) + { + Coordination::ZooKeeperWatchResponse response; + response.type = Coordination::SESSION; + response.state = Coordination::EXPIRED_SESSION; + response.error = Coordination::Error::ZSESSIONEXPIRED; + + for (auto & callback : watch_pair.second) + { + if (callback) + { + try + { + callback(std::make_shared(response)); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + } + }; + for (auto & path_watch : watches) + finish_watch(path_watch); + watches.clear(); + for (auto & path_watch : list_watches) + finish_watch(path_watch); + list_watches.clear(); + } RequestInfo info; while (requests_queue.tryPop(info)) { @@ -552,7 +577,7 @@ class TestKeeperWrapperFactory final : private boost::noncopyable public: using Creator = std::function; - using OpNumToRequest = std::unordered_map; + using OpNumToRequest = std::unordered_map; static TestKeeperWrapperFactory & instance() { @@ -564,12 +589,12 @@ public: { auto it = op_num_to_request.find(zk_request->getOpNum()); if (it == op_num_to_request.end()) - throw Coordination::Exception("Unknown operation type " + std::to_string(zk_request->getOpNum()), Coordination::Error::ZBADARGUMENTS); + throw Coordination::Exception("Unknown operation type " + toString(zk_request->getOpNum()), Coordination::Error::ZBADARGUMENTS); return it->second(zk_request); } - void registerRequest(int32_t op_num, Creator creator) + void registerRequest(Coordination::OpNum op_num, Creator creator) { if (!op_num_to_request.try_emplace(op_num, creator).second) throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Request with op num {} already registered", op_num); @@ -582,7 +607,7 @@ private: TestKeeperWrapperFactory(); }; -template +template void registerTestKeeperRequestWrapper(TestKeeperWrapperFactory & factory) { factory.registerRequest(num, [] (const Coordination::ZooKeeperRequestPtr & zk_request) { return std::make_shared(zk_request); }); @@ -591,17 +616,17 @@ void registerTestKeeperRequestWrapper(TestKeeperWrapperFactory & factory) TestKeeperWrapperFactory::TestKeeperWrapperFactory() { - registerTestKeeperRequestWrapper<11, TestKeeperStorageHeartbeatRequest>(*this); - //registerTestKeeperRequestWrapper<100, TestKeeperStorageAuthRequest>(*this); - //registerTestKeeperRequestWrapper<-11, TestKeeperStorageCloseRequest>(*this); - registerTestKeeperRequestWrapper<1, TestKeeperStorageCreateRequest>(*this); - registerTestKeeperRequestWrapper<2, TestKeeperStorageRemoveRequest>(*this); - registerTestKeeperRequestWrapper<3, TestKeeperStorageExistsRequest>(*this); - registerTestKeeperRequestWrapper<4, TestKeeperStorageGetRequest>(*this); - registerTestKeeperRequestWrapper<5, TestKeeperStorageSetRequest>(*this); - registerTestKeeperRequestWrapper<12, TestKeeperStorageListRequest>(*this); - registerTestKeeperRequestWrapper<13, TestKeeperStorageCheckRequest>(*this); - registerTestKeeperRequestWrapper<14, TestKeeperStorageMultiRequest>(*this); + registerTestKeeperRequestWrapper(*this); + //registerTestKeeperRequestWrapper(*this); + //registerTestKeeperRequestWrapper(*this); + registerTestKeeperRequestWrapper(*this); + registerTestKeeperRequestWrapper(*this); + registerTestKeeperRequestWrapper(*this); + registerTestKeeperRequestWrapper(*this); + registerTestKeeperRequestWrapper(*this); + registerTestKeeperRequestWrapper(*this); + registerTestKeeperRequestWrapper(*this); + registerTestKeeperRequestWrapper(*this); } TestKeeperStorage::ResponsePair TestKeeperStorage::putRequest(const Coordination::ZooKeeperRequestPtr & request) @@ -624,7 +649,6 @@ TestKeeperStorage::ResponsePair TestKeeperStorage::putRequest(const Coordination std::lock_guard lock(push_request_mutex); if (!requests_queue.tryPush(std::move(request_info), operation_timeout.totalMilliseconds())) throw Exception("Cannot push request to queue within operation timeout", ErrorCodes::LOGICAL_ERROR); - //LOG_DEBUG(&Poco::Logger::get("STORAGE"), "PUSHED"); return ResponsePair{std::move(future), std::move(watch_future)}; } diff --git a/src/Common/ZooKeeper/TestKeeperStorage.h b/src/Common/ZooKeeper/TestKeeperStorage.h index edcae28af85..875cb9f0253 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.h +++ b/src/Common/ZooKeeper/TestKeeperStorage.h @@ -39,8 +39,6 @@ public: Container container; - String root_path; - std::atomic zxid{0}; std::atomic shutdown{false}; @@ -56,6 +54,7 @@ public: ResponseCallback watch_callback; clock::time_point time; }; + std::mutex push_request_mutex; using RequestsQueue = ConcurrentBoundedQueue; RequestsQueue requests_queue{1}; diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index cb61f22fb3c..7b162492d1a 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -12,165 +13,6 @@ namespace Coordination using namespace DB; -/// ZooKeeper has 1 MB node size and serialization limit by default, -/// but it can be raised up, so we have a slightly larger limit on our side. -#define MAX_STRING_OR_ARRAY_SIZE (1 << 28) /// 256 MiB - -/// Assuming we are at little endian. - -static void write(int64_t x, WriteBuffer & out) -{ - x = __builtin_bswap64(x); - writeBinary(x, out); -} - -static void write(int32_t x, WriteBuffer & out) -{ - x = __builtin_bswap32(x); - writeBinary(x, out); -} - -static void write(bool x, WriteBuffer & out) -{ - writeBinary(x, out); -} - -static void write(const String & s, WriteBuffer & out) -{ - write(int32_t(s.size()), out); - out.write(s.data(), s.size()); -} - -template void write(std::array s, WriteBuffer & out) -{ - write(int32_t(N), out); - out.write(s.data(), N); -} - -template void write(const std::vector & arr, WriteBuffer & out) -{ - write(int32_t(arr.size()), out); - for (const auto & elem : arr) - write(elem, out); -} - -static void write(const ACL & acl, WriteBuffer & out) -{ - write(acl.permissions, out); - write(acl.scheme, out); - write(acl.id, out); -} - -static void write(const Stat & stat, WriteBuffer & out) -{ - write(stat.czxid, out); - write(stat.mzxid, out); - write(stat.ctime, out); - write(stat.mtime, out); - write(stat.version, out); - write(stat.cversion, out); - write(stat.aversion, out); - write(stat.ephemeralOwner, out); - write(stat.dataLength, out); - write(stat.numChildren, out); - write(stat.pzxid, out); -} - -static void write(const Error & x, WriteBuffer & out) -{ - write(static_cast(x), out); -} - -static void read(int64_t & x, ReadBuffer & in) -{ - readBinary(x, in); - x = __builtin_bswap64(x); -} - -static void read(int32_t & x, ReadBuffer & in) -{ - readBinary(x, in); - x = __builtin_bswap32(x); -} - -static void read(Error & x, ReadBuffer & in) -{ - int32_t code; - read(code, in); - x = Error(code); -} - -static void read(bool & x, ReadBuffer & in) -{ - readBinary(x, in); -} - -static void read(String & s, ReadBuffer & in) -{ - int32_t size = 0; - read(size, in); - - if (size == -1) - { - /// It means that zookeeper node has NULL value. We will treat it like empty string. - s.clear(); - return; - } - - if (size < 0) - throw Exception("Negative size while reading string from ZooKeeper", Error::ZMARSHALLINGERROR); - - if (size > MAX_STRING_OR_ARRAY_SIZE) - throw Exception("Too large string size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); - - s.resize(size); - in.read(s.data(), size); -} - -template void read(std::array & s, ReadBuffer & in) -{ - int32_t size = 0; - read(size, in); - if (size != N) - throw Exception("Unexpected array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); - in.read(s.data(), N); -} - -static void read(Stat & stat, ReadBuffer & in) -{ - read(stat.czxid, in); - read(stat.mzxid, in); - read(stat.ctime, in); - read(stat.mtime, in); - read(stat.version, in); - read(stat.cversion, in); - read(stat.aversion, in); - read(stat.ephemeralOwner, in); - read(stat.dataLength, in); - read(stat.numChildren, in); - read(stat.pzxid, in); -} - -template void read(std::vector & arr, ReadBuffer & in) -{ - int32_t size = 0; - read(size, in); - if (size < 0) - throw Exception("Negative size while reading array from ZooKeeper", Error::ZMARSHALLINGERROR); - if (size > MAX_STRING_OR_ARRAY_SIZE) - throw Exception("Too large array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); - arr.resize(size); - for (auto & elem : arr) - read(elem, in); -} - -static void read(ACL & acl, ReadBuffer & in) -{ - read(acl.permissions, in); - read(acl.scheme, in); - read(acl.id, in); -} - void ZooKeeperResponse::write(WriteBuffer & out) const { /// Excessive copy to calculate length. @@ -263,7 +105,6 @@ void ZooKeeperCreateResponse::readImpl(ReadBuffer & in) void ZooKeeperCreateResponse::writeImpl(WriteBuffer & out) const { - LOG_DEBUG(&Poco::Logger::get("LOG"), "WRITE IMPL ON: {}", path_created); Coordination::write(path_created, out); } @@ -448,7 +289,7 @@ void ZooKeeperMultiRequest::writeImpl(WriteBuffer & out) const zk_request.writeImpl(out); } - OpNum op_num = -1; + OpNum op_num = OpNum::Error; bool done = true; int32_t error = -1; @@ -471,7 +312,7 @@ void ZooKeeperMultiRequest::readImpl(ReadBuffer & in) if (done) { - if (op_num != -1) + if (op_num != OpNum::Error) throw Exception("Unexpected op_num received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); if (error != -1) throw Exception("Unexpected error value received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); @@ -505,7 +346,7 @@ void ZooKeeperMultiResponse::readImpl(ReadBuffer & in) /// op_num == -1 is special for multi transaction. /// For unknown reason, error code is duplicated in header and in response body. - if (op_num == -1) + if (op_num == OpNum::Error) response = std::make_shared(); if (op_error != Error::ZOK) @@ -519,7 +360,7 @@ void ZooKeeperMultiResponse::readImpl(ReadBuffer & in) error = op_error; } - if (op_error == Error::ZOK || op_num == -1) + if (op_error == Error::ZOK || op_num == OpNum::Error) dynamic_cast(*response).readImpl(in); } @@ -535,7 +376,7 @@ void ZooKeeperMultiResponse::readImpl(ReadBuffer & in) if (!done) throw Exception("Too many results received for multi transaction", Error::ZMARSHALLINGERROR); - if (op_num != -1) + if (op_num != OpNum::Error) throw Exception("Unexpected op_num received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); if (error_read != -1) throw Exception("Unexpected error value received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); @@ -560,7 +401,7 @@ void ZooKeeperMultiResponse::writeImpl(WriteBuffer & out) const /// Footer. { - OpNum op_num = -1; + OpNum op_num = OpNum::Error; bool done = true; int32_t error_read = - 1; @@ -606,7 +447,7 @@ ZooKeeperRequestPtr ZooKeeperRequestFactory::get(OpNum op_num) const { auto it = op_num_to_request.find(op_num); if (it == op_num_to_request.end()) - throw Exception("Unknown operation type " + std::to_string(op_num), Error::ZBADARGUMENTS); + throw Exception("Unknown operation type " + toString(op_num), Error::ZBADARGUMENTS); return it->second(); } @@ -625,17 +466,17 @@ void registerZooKeeperRequest(ZooKeeperRequestFactory & factory) ZooKeeperRequestFactory::ZooKeeperRequestFactory() { - registerZooKeeperRequest<11, ZooKeeperHeartbeatRequest>(*this); - registerZooKeeperRequest<100, ZooKeeperAuthRequest>(*this); - registerZooKeeperRequest<-11, ZooKeeperCloseRequest>(*this); - registerZooKeeperRequest<1, ZooKeeperCreateRequest>(*this); - registerZooKeeperRequest<2, ZooKeeperRemoveRequest>(*this); - registerZooKeeperRequest<3, ZooKeeperExistsRequest>(*this); - registerZooKeeperRequest<4, ZooKeeperGetRequest>(*this); - registerZooKeeperRequest<5, ZooKeeperSetRequest>(*this); - registerZooKeeperRequest<12, ZooKeeperListRequest>(*this); - registerZooKeeperRequest<13, ZooKeeperCheckRequest>(*this); - registerZooKeeperRequest<14, ZooKeeperMultiRequest>(*this); + registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); } } diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index f40de116da2..286330bd769 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -22,9 +23,6 @@ namespace Coordination { -using XID = int32_t; -using OpNum = int32_t; - struct ZooKeeperResponse : virtual Response { XID xid = 0; @@ -70,7 +68,7 @@ using ZooKeeperRequestPtr = std::shared_ptr; struct ZooKeeperHeartbeatRequest final : ZooKeeperRequest { String getPath() const override { return {}; } - OpNum getOpNum() const override { return 11; } + OpNum getOpNum() const override { return OpNum::Heartbeat; } void writeImpl(WriteBuffer &) const override {} void readImpl(ReadBuffer &) override {} ZooKeeperResponsePtr makeResponse() const override; @@ -80,7 +78,7 @@ struct ZooKeeperHeartbeatResponse final : ZooKeeperResponse { void readImpl(ReadBuffer &) override {} void writeImpl(WriteBuffer &) const override {} - OpNum getOpNum() const override { return 11; } + OpNum getOpNum() const override { return OpNum::Heartbeat; } }; struct ZooKeeperWatchResponse final : WatchResponse, ZooKeeperResponse @@ -89,8 +87,10 @@ struct ZooKeeperWatchResponse final : WatchResponse, ZooKeeperResponse void writeImpl(WriteBuffer & out) const override; - /// TODO FIXME alesap - OpNum getOpNum() const override { return 0; } + OpNum getOpNum() const override + { + throw Exception("OpNum for watch response doesn't exist", Error::ZRUNTIMEINCONSISTENCY); + } }; struct ZooKeeperAuthRequest final : ZooKeeperRequest @@ -100,7 +100,7 @@ struct ZooKeeperAuthRequest final : ZooKeeperRequest String data; String getPath() const override { return {}; } - OpNum getOpNum() const override { return 100; } + OpNum getOpNum() const override { return OpNum::Auth; } void writeImpl(WriteBuffer & out) const override; void readImpl(ReadBuffer & in) override; @@ -112,13 +112,13 @@ struct ZooKeeperAuthResponse final : ZooKeeperResponse void readImpl(ReadBuffer &) override {} void writeImpl(WriteBuffer &) const override {} - OpNum getOpNum() const override { return 100; } + OpNum getOpNum() const override { return OpNum::Auth; } }; struct ZooKeeperCloseRequest final : ZooKeeperRequest { String getPath() const override { return {}; } - OpNum getOpNum() const override { return -11; } + OpNum getOpNum() const override { return OpNum::Close; } void writeImpl(WriteBuffer &) const override {} void readImpl(ReadBuffer &) override {} @@ -134,7 +134,7 @@ struct ZooKeeperCloseResponse final : ZooKeeperResponse void writeImpl(WriteBuffer &) const override {} - OpNum getOpNum() const override { return -11; } + OpNum getOpNum() const override { return OpNum::Close; } }; struct ZooKeeperCreateRequest final : public CreateRequest, ZooKeeperRequest @@ -142,7 +142,7 @@ struct ZooKeeperCreateRequest final : public CreateRequest, ZooKeeperRequest ZooKeeperCreateRequest() = default; explicit ZooKeeperCreateRequest(const CreateRequest & base) : CreateRequest(base) {} - OpNum getOpNum() const override { return 1; } + OpNum getOpNum() const override { return OpNum::Create; } void writeImpl(WriteBuffer & out) const override; void readImpl(ReadBuffer & in) override; @@ -155,7 +155,7 @@ struct ZooKeeperCreateResponse final : CreateResponse, ZooKeeperResponse void writeImpl(WriteBuffer & out) const override; - OpNum getOpNum() const override { return 1; } + OpNum getOpNum() const override { return OpNum::Create; } }; struct ZooKeeperRemoveRequest final : RemoveRequest, ZooKeeperRequest @@ -163,7 +163,7 @@ struct ZooKeeperRemoveRequest final : RemoveRequest, ZooKeeperRequest ZooKeeperRemoveRequest() = default; explicit ZooKeeperRemoveRequest(const RemoveRequest & base) : RemoveRequest(base) {} - OpNum getOpNum() const override { return 2; } + OpNum getOpNum() const override { return OpNum::Remove; } void writeImpl(WriteBuffer & out) const override; void readImpl(ReadBuffer & in) override; @@ -174,12 +174,12 @@ struct ZooKeeperRemoveResponse final : RemoveResponse, ZooKeeperResponse { void readImpl(ReadBuffer &) override {} void writeImpl(WriteBuffer &) const override {} - OpNum getOpNum() const override { return 2; } + OpNum getOpNum() const override { return OpNum::Remove; } }; struct ZooKeeperExistsRequest final : ExistsRequest, ZooKeeperRequest { - OpNum getOpNum() const override { return 3; } + OpNum getOpNum() const override { return OpNum::Exists; } void writeImpl(WriteBuffer & out) const override; void readImpl(ReadBuffer & in) override; @@ -190,12 +190,12 @@ struct ZooKeeperExistsResponse final : ExistsResponse, ZooKeeperResponse { void readImpl(ReadBuffer & in) override; void writeImpl(WriteBuffer & out) const override; - OpNum getOpNum() const override { return 3; } + OpNum getOpNum() const override { return OpNum::Exists; } }; struct ZooKeeperGetRequest final : GetRequest, ZooKeeperRequest { - OpNum getOpNum() const override { return 4; } + OpNum getOpNum() const override { return OpNum::Get; } void writeImpl(WriteBuffer & out) const override; void readImpl(ReadBuffer & in) override; @@ -206,7 +206,7 @@ struct ZooKeeperGetResponse final : GetResponse, ZooKeeperResponse { void readImpl(ReadBuffer & in) override; void writeImpl(WriteBuffer & out) const override; - OpNum getOpNum() const override { return 4; } + OpNum getOpNum() const override { return OpNum::Get; } }; struct ZooKeeperSetRequest final : SetRequest, ZooKeeperRequest @@ -214,7 +214,7 @@ struct ZooKeeperSetRequest final : SetRequest, ZooKeeperRequest ZooKeeperSetRequest() = default; explicit ZooKeeperSetRequest(const SetRequest & base) : SetRequest(base) {} - OpNum getOpNum() const override { return 5; } + OpNum getOpNum() const override { return OpNum::Set; } void writeImpl(WriteBuffer & out) const override; void readImpl(ReadBuffer & in) override; ZooKeeperResponsePtr makeResponse() const override; @@ -224,12 +224,12 @@ struct ZooKeeperSetResponse final : SetResponse, ZooKeeperResponse { void readImpl(ReadBuffer & in) override; void writeImpl(WriteBuffer & out) const override; - OpNum getOpNum() const override { return 5; } + OpNum getOpNum() const override { return OpNum::Set; } }; struct ZooKeeperListRequest final : ListRequest, ZooKeeperRequest { - OpNum getOpNum() const override { return 12; } + OpNum getOpNum() const override { return OpNum::List; } void writeImpl(WriteBuffer & out) const override; void readImpl(ReadBuffer & in) override; ZooKeeperResponsePtr makeResponse() const override; @@ -239,7 +239,7 @@ struct ZooKeeperListResponse final : ListResponse, ZooKeeperResponse { void readImpl(ReadBuffer & in) override; void writeImpl(WriteBuffer & out) const override; - OpNum getOpNum() const override { return 12; } + OpNum getOpNum() const override { return OpNum::List; } }; struct ZooKeeperCheckRequest final : CheckRequest, ZooKeeperRequest @@ -247,7 +247,7 @@ struct ZooKeeperCheckRequest final : CheckRequest, ZooKeeperRequest ZooKeeperCheckRequest() = default; explicit ZooKeeperCheckRequest(const CheckRequest & base) : CheckRequest(base) {} - OpNum getOpNum() const override { return 13; } + OpNum getOpNum() const override { return OpNum::Check; } void writeImpl(WriteBuffer & out) const override; void readImpl(ReadBuffer & in) override; @@ -258,7 +258,7 @@ struct ZooKeeperCheckResponse final : CheckResponse, ZooKeeperResponse { void readImpl(ReadBuffer &) override {} void writeImpl(WriteBuffer &) const override {} - OpNum getOpNum() const override { return 13; } + OpNum getOpNum() const override { return OpNum::Check; } }; /// This response may be received only as an element of responses in MultiResponse. @@ -267,12 +267,12 @@ struct ZooKeeperErrorResponse final : ErrorResponse, ZooKeeperResponse void readImpl(ReadBuffer & in) override; void writeImpl(WriteBuffer & out) const override; - OpNum getOpNum() const override { return -1; } + OpNum getOpNum() const override { return OpNum::Error; } }; struct ZooKeeperMultiRequest final : MultiRequest, ZooKeeperRequest { - OpNum getOpNum() const override { return 14; } + OpNum getOpNum() const override { return OpNum::Multi; } ZooKeeperMultiRequest() = default; ZooKeeperMultiRequest(const Requests & generic_requests, const ACLs & default_acls); @@ -285,7 +285,7 @@ struct ZooKeeperMultiRequest final : MultiRequest, ZooKeeperRequest struct ZooKeeperMultiResponse final : MultiResponse, ZooKeeperResponse { - OpNum getOpNum() const override { return 14; } + OpNum getOpNum() const override { return OpNum::Multi; } explicit ZooKeeperMultiResponse(const Requests & requests) { diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.cpp b/src/Common/ZooKeeper/ZooKeeperConstants.cpp new file mode 100644 index 00000000000..9a1a43d43a7 --- /dev/null +++ b/src/Common/ZooKeeper/ZooKeeperConstants.cpp @@ -0,0 +1,64 @@ +#include +#include +#include + +namespace Coordination +{ + +static const std::unordered_set VALID_OPERATIONS = +{ + static_cast(OpNum::Close), + static_cast(OpNum::Error), + static_cast(OpNum::Create), + static_cast(OpNum::Remove), + static_cast(OpNum::Exists), + static_cast(OpNum::Get), + static_cast(OpNum::Set), + static_cast(OpNum::Heartbeat), + static_cast(OpNum::List), + static_cast(OpNum::Check), + static_cast(OpNum::Multi), + static_cast(OpNum::Auth), +}; + +std::string toString(OpNum op_num) +{ + switch (op_num) + { + case OpNum::Close: + return "Close"; + case OpNum::Error: + return "Error"; + case OpNum::Create: + return "Create"; + case OpNum::Remove: + return "Remove"; + case OpNum::Exists: + return "Exists"; + case OpNum::Get: + return "Get"; + case OpNum::Set: + return "Set"; + case OpNum::List: + return "List"; + case OpNum::Check: + return "Check"; + case OpNum::Multi: + return "Multi"; + case OpNum::Heartbeat: + return "Heartbeat"; + case OpNum::Auth: + return "Auth"; + } + int32_t raw_op = static_cast(op_num); + throw Exception("Operation " + std::to_string(raw_op) + " is unknown", Error::ZUNIMPLEMENTED); +} + +OpNum getOpNum(int32_t raw_op_num) +{ + if (!VALID_OPERATIONS.count(raw_op_num)) + throw Exception("Operation " + std::to_string(raw_op_num) + " is unknown", Error::ZUNIMPLEMENTED); + return static_cast(raw_op_num); +} + +} diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.h b/src/Common/ZooKeeper/ZooKeeperConstants.h new file mode 100644 index 00000000000..47b597f1c9d --- /dev/null +++ b/src/Common/ZooKeeper/ZooKeeperConstants.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + + +namespace Coordination +{ + +using XID = int32_t; + +static constexpr XID WATCH_XID = -1; +static constexpr XID PING_XID = -2; +static constexpr XID AUTH_XID = -4; +static constexpr XID CLOSE_XID = 0x7FFFFFFF; + +enum class OpNum : int32_t +{ + Close = -11, + Error = -1, + Create = 1, + Remove = 2, + Exists = 3, + Get = 4, + Set = 5, + Heartbeat = 11, + List = 12, + Check = 13, + Multi = 14, + Auth = 100, +}; + +std::string toString(OpNum op_num); +OpNum getOpNum(int32_t raw_op_num); + +static constexpr int32_t ZOOKEEPER_PROTOCOL_VERSION = 0; +static constexpr int32_t CLIENT_HANDSHAKE_LENGTH = 44; +static constexpr int32_t SERVER_HANDSHAKE_LENGTH = 36; +static constexpr int32_t PASSWORD_LENGTH = 16; + +/// ZooKeeper has 1 MB node size and serialization limit by default, +/// but it can be raised up, so we have a slightly larger limit on our side. +static constexpr int32_t MAX_STRING_OR_ARRAY_SIZE = 1 << 28; /// 256 MiB +static constexpr int32_t DEFAULT_SESSION_TIMEOUT = 30000; +static constexpr int32_t DEFAULT_OPERATION_TIMEOUT = 10000; + +} diff --git a/src/Common/ZooKeeper/ZooKeeperIO.cpp b/src/Common/ZooKeeper/ZooKeeperIO.cpp new file mode 100644 index 00000000000..157325cb8e6 --- /dev/null +++ b/src/Common/ZooKeeper/ZooKeeperIO.cpp @@ -0,0 +1,135 @@ +#include + +namespace Coordination +{ + +void write(int64_t x, WriteBuffer & out) +{ + x = __builtin_bswap64(x); + writeBinary(x, out); +} +void write(int32_t x, WriteBuffer & out) +{ + x = __builtin_bswap32(x); + writeBinary(x, out); +} + +void write(OpNum x, WriteBuffer & out) +{ + write(static_cast(x), out); +} + +void write(bool x, WriteBuffer & out) +{ + writeBinary(x, out); +} + +void write(const std::string & s, WriteBuffer & out) +{ + write(int32_t(s.size()), out); + out.write(s.data(), s.size()); +} + +void write(const ACL & acl, WriteBuffer & out) +{ + write(acl.permissions, out); + write(acl.scheme, out); + write(acl.id, out); +} + +void write(const Stat & stat, WriteBuffer & out) +{ + write(stat.czxid, out); + write(stat.mzxid, out); + write(stat.ctime, out); + write(stat.mtime, out); + write(stat.version, out); + write(stat.cversion, out); + write(stat.aversion, out); + write(stat.ephemeralOwner, out); + write(stat.dataLength, out); + write(stat.numChildren, out); + write(stat.pzxid, out); +} + +void write(const Error & x, WriteBuffer & out) +{ + write(static_cast(x), out); +} + +void read(int64_t & x, ReadBuffer & in) +{ + readBinary(x, in); + x = __builtin_bswap64(x); +} + +void read(int32_t & x, ReadBuffer & in) +{ + readBinary(x, in); + x = __builtin_bswap32(x); +} + +void read(OpNum & x, ReadBuffer & in) +{ + int32_t raw_op_num; + read(raw_op_num, in); + x = getOpNum(raw_op_num); +} + +void read(bool & x, ReadBuffer & in) +{ + readBinary(x, in); +} + +void read(std::string & s, ReadBuffer & in) +{ + int32_t size = 0; + read(size, in); + + if (size == -1) + { + /// It means that zookeeper node has NULL value. We will treat it like empty string. + s.clear(); + return; + } + + if (size < 0) + throw Exception("Negative size while reading string from ZooKeeper", Error::ZMARSHALLINGERROR); + + if (size > MAX_STRING_OR_ARRAY_SIZE) + throw Exception("Too large string size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); + + s.resize(size); + in.read(s.data(), size); +} + +void read(ACL & acl, ReadBuffer & in) +{ + read(acl.permissions, in); + read(acl.scheme, in); + read(acl.id, in); +} + +void read(Stat & stat, ReadBuffer & in) +{ + read(stat.czxid, in); + read(stat.mzxid, in); + read(stat.ctime, in); + read(stat.mtime, in); + read(stat.version, in); + read(stat.cversion, in); + read(stat.aversion, in); + read(stat.ephemeralOwner, in); + read(stat.dataLength, in); + read(stat.numChildren, in); + read(stat.pzxid, in); +} + +void read(Error & x, ReadBuffer & in) +{ + int32_t code; + read(code, in); + x = Coordination::Error(code); +} + +} diff --git a/src/Common/ZooKeeper/ZooKeeperIO.h b/src/Common/ZooKeeper/ZooKeeperIO.h new file mode 100644 index 00000000000..bd6d6a6f849 --- /dev/null +++ b/src/Common/ZooKeeper/ZooKeeperIO.h @@ -0,0 +1,73 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Coordination +{ + +using namespace DB; + +void write(int64_t x, WriteBuffer & out); +void write(int32_t x, WriteBuffer & out); +void write(OpNum x, WriteBuffer & out); +void write(bool x, WriteBuffer & out); +void write(const std::string & s, WriteBuffer & out); +void write(const ACL & acl, WriteBuffer & out); +void write(const Stat & stat, WriteBuffer & out); +void write(const Error & x, WriteBuffer & out); + +template +void write(const std::array s, WriteBuffer & out) +{ + write(int32_t(N), out); + out.write(s.data(), N); +} + +template +void write(const std::vector & arr, WriteBuffer & out) +{ + write(int32_t(arr.size()), out); + for (const auto & elem : arr) + write(elem, out); +} + +void read(int64_t & x, ReadBuffer & in); +void read(int32_t & x, ReadBuffer & in); +void read(OpNum & x, ReadBuffer & in); +void read(bool & x, ReadBuffer & in); +void read(std::string & s, ReadBuffer & in); +void read(ACL & acl, ReadBuffer & in); +void read(Stat & stat, ReadBuffer & in); +void read(Error & x, ReadBuffer & in); + +template +void read(std::array & s, ReadBuffer & in) +{ + int32_t size = 0; + read(size, in); + if (size != N) + throw Exception("Unexpected array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); + in.read(s.data(), N); +} + +template +void read(std::vector & arr, ReadBuffer & in) +{ + int32_t size = 0; + read(size, in); + if (size < 0) + throw Exception("Negative size while reading array from ZooKeeper", Error::ZMARSHALLINGERROR); + if (size > MAX_STRING_OR_ARRAY_SIZE) + throw Exception("Too large array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); + arr.resize(size); + for (auto & elem : arr) + read(elem, in); +} + +} diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 23a5c6c4301..c9ef2a26edd 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -2,7 +2,7 @@ #include #include #include - +#include #include #include #include @@ -20,11 +20,6 @@ #include -/// ZooKeeper has 1 MB node size and serialization limit by default, -/// but it can be raised up, so we have a slightly larger limit on our side. -#define MAX_STRING_OR_ARRAY_SIZE (1 << 28) /// 256 MiB - - namespace ProfileEvents { extern const Event ZooKeeperInit; @@ -267,75 +262,6 @@ namespace Coordination using namespace DB; - -/// Assuming we are at little endian. - -static void write(int64_t x, WriteBuffer & out) -{ - x = __builtin_bswap64(x); - writeBinary(x, out); -} - -static void write(int32_t x, WriteBuffer & out) -{ - x = __builtin_bswap32(x); - writeBinary(x, out); -} - -template void write(std::array s, WriteBuffer & out) -{ - write(int32_t(N), out); - out.write(s.data(), N); -} - -template void write(const std::vector & arr, WriteBuffer & out) -{ - write(int32_t(arr.size()), out); - for (const auto & elem : arr) - write(elem, out); -} - -static void read(int64_t & x, ReadBuffer & in) -{ - readBinary(x, in); - x = __builtin_bswap64(x); -} - -static void read(int32_t & x, ReadBuffer & in) -{ - readBinary(x, in); - x = __builtin_bswap32(x); -} - -static void read(Error & x, ReadBuffer & in) -{ - int32_t code; - read(code, in); - x = Error(code); -} - -template void read(std::array & s, ReadBuffer & in) -{ - int32_t size = 0; - read(size, in); - if (size != N) - throw Exception("Unexpected array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); - in.read(s.data(), N); -} - -template void read(std::vector & arr, ReadBuffer & in) -{ - int32_t size = 0; - read(size, in); - if (size < 0) - throw Exception("Negative size while reading array from ZooKeeper", Error::ZMARSHALLINGERROR); - if (size > MAX_STRING_OR_ARRAY_SIZE) - throw Exception("Too large array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); - arr.resize(size); - for (auto & elem : arr) - read(elem, in); -} - template void ZooKeeper::write(const T & x) { @@ -359,15 +285,6 @@ static void removeRootPath(String & path, const String & root_path) path = path.substr(root_path.size()); } -static constexpr int32_t protocol_version = 0; - -static constexpr ZooKeeper::XID watch_xid = -1; -static constexpr ZooKeeper::XID ping_xid = -2; -static constexpr ZooKeeper::XID auth_xid = -4; - -static constexpr ZooKeeper::XID close_xid = 0x7FFFFFFF; - - ZooKeeper::~ZooKeeper() { try @@ -542,7 +459,7 @@ void ZooKeeper::sendHandshake() std::array passwd {}; write(handshake_length); - write(protocol_version); + write(ZOOKEEPER_PROTOCOL_VERSION); write(last_zxid_seen); write(timeout); write(previous_session_id); @@ -557,16 +474,15 @@ void ZooKeeper::receiveHandshake() int32_t handshake_length; int32_t protocol_version_read; int32_t timeout; - constexpr int32_t passwd_len = 16; - std::array passwd; + std::array passwd; read(handshake_length); - if (handshake_length != 36) - throw Exception("Unexpected handshake length received: " + toString(handshake_length), Error::ZMARSHALLINGERROR); + if (handshake_length != SERVER_HANDSHAKE_LENGTH) + throw Exception("Unexpected handshake length received: " + DB::toString(handshake_length), Error::ZMARSHALLINGERROR); read(protocol_version_read); - if (protocol_version_read != protocol_version) - throw Exception("Unexpected protocol version: " + toString(protocol_version_read), Error::ZMARSHALLINGERROR); + if (protocol_version_read != ZOOKEEPER_PROTOCOL_VERSION) + throw Exception("Unexpected protocol version: " + DB::toString(protocol_version_read), Error::ZMARSHALLINGERROR); read(timeout); if (timeout != session_timeout.totalMilliseconds()) @@ -583,7 +499,7 @@ void ZooKeeper::sendAuth(const String & scheme, const String & data) ZooKeeperAuthRequest request; request.scheme = scheme; request.data = data; - request.xid = auth_xid; + request.xid = AUTH_XID; request.write(*out); int32_t length; @@ -597,17 +513,17 @@ void ZooKeeper::sendAuth(const String & scheme, const String & data) read(zxid); read(err); - if (read_xid != auth_xid) - throw Exception("Unexpected event received in reply to auth request: " + toString(read_xid), + if (read_xid != AUTH_XID) + throw Exception("Unexpected event received in reply to auth request: " + DB::toString(read_xid), Error::ZMARSHALLINGERROR); int32_t actual_length = in->count() - count_before_event; if (length != actual_length) - throw Exception("Response length doesn't match. Expected: " + toString(length) + ", actual: " + toString(actual_length), + throw Exception("Response length doesn't match. Expected: " + DB::toString(length) + ", actual: " + DB::toString(actual_length), Error::ZMARSHALLINGERROR); if (err != Error::ZOK) - throw Exception("Error received in reply to auth request. Code: " + toString(int32_t(err)) + ". Message: " + String(errorMessage(err)), + throw Exception("Error received in reply to auth request. Code: " + DB::toString(int32_t(err)) + ". Message: " + String(errorMessage(err)), Error::ZMARSHALLINGERROR); } @@ -640,7 +556,7 @@ void ZooKeeper::sendThread() /// After we popped element from the queue, we must register callbacks (even in the case when expired == true right now), /// because they must not be lost (callbacks must be called because the user will wait for them). - if (info.request->xid != close_xid) + if (info.request->xid != CLOSE_XID) { CurrentMetrics::add(CurrentMetrics::ZooKeeperRequest); std::lock_guard lock(operations_mutex); @@ -650,7 +566,6 @@ void ZooKeeper::sendThread() if (info.watch) { info.request->has_watch = true; - //std::cerr << "REQUEST" << info.request->getOpNum() << " HAS WATCH" << std::endl; CurrentMetrics::add(CurrentMetrics::ZooKeeperWatch); } @@ -662,11 +577,10 @@ void ZooKeeper::sendThread() info.request->addRootPath(root_path); info.request->probably_sent = true; - //std::cerr << "SENDING GENERAL REQUEST:" << info.request->getOpNum() << std::endl; info.request->write(*out); /// We sent close request, exit - if (info.request->xid == close_xid) + if (info.request->xid == CLOSE_XID) break; } } @@ -676,7 +590,7 @@ void ZooKeeper::sendThread() prev_heartbeat_time = clock::now(); ZooKeeperHeartbeatRequest request; - request.xid = ping_xid; + request.xid = PING_XID; request.write(*out); } @@ -731,7 +645,7 @@ void ZooKeeper::receiveThread() { if (earliest_operation) { - throw Exception("Operation timeout (no response) for request " + std::to_string(earliest_operation->request->getOpNum()) + " for path: " + earliest_operation->request->getPath(), Error::ZOPERATIONTIMEOUT); + throw Exception("Operation timeout (no response) for request " + toString(earliest_operation->request->getOpNum()) + " for path: " + earliest_operation->request->getPath(), Error::ZOPERATIONTIMEOUT); } waited += max_wait; if (waited >= session_timeout.totalMicroseconds()) @@ -766,14 +680,14 @@ void ZooKeeper::receiveEvent() RequestInfo request_info; ZooKeeperResponsePtr response; - if (xid == ping_xid) + if (xid == PING_XID) { if (err != Error::ZOK) throw Exception("Received error in heartbeat response: " + String(errorMessage(err)), Error::ZRUNTIMEINCONSISTENCY); response = std::make_shared(); } - else if (xid == watch_xid) + else if (xid == WATCH_XID) { ProfileEvents::increment(ProfileEvents::ZooKeeperWatchResponse); response = std::make_shared(); @@ -814,7 +728,7 @@ void ZooKeeper::receiveEvent() auto it = operations.find(xid); if (it == operations.end()) - throw Exception("Received response for unknown xid " + toString(xid), Error::ZRUNTIMEINCONSISTENCY); + throw Exception("Received response for unknown xid " + DB::toString(xid), Error::ZRUNTIMEINCONSISTENCY); /// After this point, we must invoke callback, that we've grabbed from 'operations'. /// Invariant: all callbacks are invoked either in case of success or in case of error. @@ -836,12 +750,10 @@ void ZooKeeper::receiveEvent() if (err != Error::ZOK) { - //std::cerr << "GOT ERROR:" << static_cast(err) << std::endl; response->error = err; } else { - //std::cerr << "NO ERROR RECEIVED\n"; response->readImpl(*in); response->removeRootPath(root_path); } @@ -854,7 +766,7 @@ void ZooKeeper::receiveEvent() /// 3 indicates the ZooKeeperExistsRequest. // For exists, we set the watch on both node exist and nonexist case. // For other case like getData, we only set the watch when node exists. - if (request_info.request->getOpNum() == 3) + if (request_info.request->getOpNum() == OpNum::Exists) add_watch = (response->error == Error::ZOK || response->error == Error::ZNONODE); else add_watch = response->error == Error::ZOK; @@ -871,7 +783,7 @@ void ZooKeeper::receiveEvent() int32_t actual_length = in->count() - count_before_event; if (length != actual_length) - throw Exception("Response length doesn't match. Expected: " + toString(length) + ", actual: " + toString(actual_length), Error::ZMARSHALLINGERROR); + throw Exception("Response length doesn't match. Expected: " + DB::toString(length) + ", actual: " + DB::toString(actual_length), Error::ZMARSHALLINGERROR); } catch (...) { @@ -1058,7 +970,7 @@ void ZooKeeper::pushRequest(RequestInfo && info) if (!info.request->xid) { info.request->xid = next_xid.fetch_add(1); - if (info.request->xid == close_xid) + if (info.request->xid == CLOSE_XID) throw Exception("xid equal to close_xid", Error::ZSESSIONEXPIRED); if (info.request->xid < 0) throw Exception("XID overflow", Error::ZSESSIONEXPIRED); @@ -1238,7 +1150,7 @@ void ZooKeeper::multi( void ZooKeeper::close() { ZooKeeperCloseRequest request; - request.xid = close_xid; + request.xid = CLOSE_XID; RequestInfo request_info; request_info.request = std::make_shared(std::move(request)); diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index c96d7d2f0cb..65f8fe91793 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -99,9 +99,6 @@ public: using Nodes = std::vector; - using XID = int32_t; - using OpNum = int32_t; - /** Connection to nodes is performed in order. If you want, shuffle them manually. * Operation timeout couldn't be greater than session timeout. * Operation timeout applies independently for network read, network write, waiting for events and synchronization. diff --git a/src/Server/TestKeeperTCPHandler.cpp b/src/Server/TestKeeperTCPHandler.cpp index 1e7f69dc14f..7afd1af1aa0 100644 --- a/src/Server/TestKeeperTCPHandler.cpp +++ b/src/Server/TestKeeperTCPHandler.cpp @@ -1,9 +1,6 @@ #include +#include #include -#include -#include -#include -#include #include #include #include @@ -23,187 +20,15 @@ namespace ErrorCodes } -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-function" -#endif - -/// ZooKeeper has 1 MB node size and serialization limit by default, -/// but it can be raised up, so we have a slightly larger limit on our side. -#define MAX_STRING_OR_ARRAY_SIZE (1 << 28) /// 256 MiB - -/// Assuming we are at little endian. - -static void write(int64_t x, WriteBuffer & out) -{ - x = __builtin_bswap64(x); - writeBinary(x, out); -} - -static void write(int32_t x, WriteBuffer & out) -{ - x = __builtin_bswap32(x); - writeBinary(x, out); -} - -static void write(bool x, WriteBuffer & out) -{ - writeBinary(x, out); -} - -static void write(const String & s, WriteBuffer & out) -{ - write(int32_t(s.size()), out); - out.write(s.data(), s.size()); -} - -template void write(std::array s, WriteBuffer & out) -{ - write(int32_t(N), out); - out.write(s.data(), N); -} - -template void write(const std::vector & arr, WriteBuffer & out) -{ - write(int32_t(arr.size()), out); - for (const auto & elem : arr) - write(elem, out); -} - -static void write(const Coordination::ACL & acl, WriteBuffer & out) -{ - write(acl.permissions, out); - write(acl.scheme, out); - write(acl.id, out); -} - -static void write(const Coordination::Stat & stat, WriteBuffer & out) -{ - write(stat.czxid, out); - write(stat.mzxid, out); - write(stat.ctime, out); - write(stat.mtime, out); - write(stat.version, out); - write(stat.cversion, out); - write(stat.aversion, out); - write(stat.ephemeralOwner, out); - write(stat.dataLength, out); - write(stat.numChildren, out); - write(stat.pzxid, out); -} - -static void write(const Coordination::Error & x, WriteBuffer & out) -{ - write(static_cast(x), out); -} - -static void read(int64_t & x, ReadBuffer & in) -{ - readBinary(x, in); - x = __builtin_bswap64(x); -} - -static void read(int32_t & x, ReadBuffer & in) -{ - readBinary(x, in); - x = __builtin_bswap32(x); -} - -static void read(Coordination::Error & x, ReadBuffer & in) -{ - int32_t code; - read(code, in); - x = Coordination::Error(code); -} - -static void read(bool & x, ReadBuffer & in) -{ - readBinary(x, in); -} - -static void read(String & s, ReadBuffer & in) -{ - int32_t size = 0; - read(size, in); - - if (size == -1) - { - /// It means that zookeeper node has NULL value. We will treat it like empty string. - s.clear(); - return; - } - - if (size < 0) - throw Exception("Negative size while reading string from ZooKeeper", ErrorCodes::LOGICAL_ERROR); - - if (size > MAX_STRING_OR_ARRAY_SIZE) - throw Exception("Too large string size while reading from ZooKeeper", ErrorCodes::LOGICAL_ERROR); - - s.resize(size); - in.read(s.data(), size); -} - -template void read(std::array & s, ReadBuffer & in) -{ - int32_t size = 0; - read(size, in); - if (size != N) - throw Exception("Unexpected array size while reading from ZooKeeper", ErrorCodes::LOGICAL_ERROR); - in.read(s.data(), N); -} - -static void read(Coordination::Stat & stat, ReadBuffer & in) -{ - read(stat.czxid, in); - read(stat.mzxid, in); - read(stat.ctime, in); - read(stat.mtime, in); - read(stat.version, in); - read(stat.cversion, in); - read(stat.aversion, in); - read(stat.ephemeralOwner, in); - read(stat.dataLength, in); - read(stat.numChildren, in); - read(stat.pzxid, in); -} - -template void read(std::vector & arr, ReadBuffer & in) -{ - int32_t size = 0; - read(size, in); - if (size < 0) - throw Exception("Negative size while reading array from ZooKeeper", ErrorCodes::LOGICAL_ERROR); - if (size > MAX_STRING_OR_ARRAY_SIZE) - throw Exception("Too large array size while reading from ZooKeeper", ErrorCodes::LOGICAL_ERROR); - arr.resize(size); - for (auto & elem : arr) - read(elem, in); -} - -static void read(Coordination::ACL & acl, ReadBuffer & in) -{ - read(acl.permissions, in); - read(acl.scheme, in); - read(acl.id, in); -} - -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - void TestKeeperTCPHandler::sendHandshake() { - static constexpr int32_t handshake_length = 36; - static constexpr int32_t protocol_version = 0; - static constexpr int32_t DEFAULT_SESSION_TIMEOUT = 30000; - write(handshake_length, *out); - write(protocol_version, *out); - write(DEFAULT_SESSION_TIMEOUT, *out); - write(test_keeper_storage->getSessionID(), *out); - constexpr int32_t passwd_len = 16; - std::array passwd{}; - write(passwd, *out); + Coordination::write(Coordination::SERVER_HANDSHAKE_LENGTH, *out); + Coordination::write(Coordination::ZOOKEEPER_PROTOCOL_VERSION, *out); + Coordination::write(Coordination::DEFAULT_SESSION_TIMEOUT, *out); + Coordination::write(test_keeper_storage->getSessionID(), *out); + std::array passwd{}; + Coordination::write(passwd, *out); out->next(); } @@ -219,30 +44,29 @@ void TestKeeperTCPHandler::receiveHandshake() int64_t last_zxid_seen; int32_t timeout; int64_t previous_session_id = 0; /// We don't support session restore. So previous session_id is always zero. - constexpr int32_t passwd_len = 16; - std::array passwd {}; + std::array passwd {}; - read(handshake_length, *in); - if (handshake_length != 44) + Coordination::read(handshake_length, *in); + if (handshake_length != Coordination::CLIENT_HANDSHAKE_LENGTH) throw Exception("Unexpected handshake length received: " + toString(handshake_length), ErrorCodes::LOGICAL_ERROR); - read(protocol_version, *in); + Coordination::read(protocol_version, *in); - if (protocol_version != 0) + if (protocol_version != Coordination::ZOOKEEPER_PROTOCOL_VERSION) throw Exception("Unexpected protocol version: " + toString(protocol_version), ErrorCodes::LOGICAL_ERROR); - read(last_zxid_seen, *in); + Coordination::read(last_zxid_seen, *in); if (last_zxid_seen != 0) throw Exception("Non zero last_zxid_seen is not supported", ErrorCodes::LOGICAL_ERROR); - read(timeout, *in); - read(previous_session_id, *in); + Coordination::read(timeout, *in); + Coordination::read(previous_session_id, *in); if (previous_session_id != 0) throw Exception("Non zero previous session id is not supported", ErrorCodes::LOGICAL_ERROR); - read(passwd, *in); + Coordination::read(passwd, *in); } @@ -272,7 +96,7 @@ void TestKeeperTCPHandler::runImpl() } catch (const Exception & e) /// Typical for an incorrect username, password, or address. { - LOG_DEBUG(log, "Cannot receive handshake {}", e.displayText()); + LOG_WARNING(log, "Cannot receive handshake {}", e.displayText()); return; } @@ -280,18 +104,17 @@ void TestKeeperTCPHandler::runImpl() while (true) { - //UInt64 max_wait = operation_timeout.totalMicroseconds(); using namespace std::chrono_literals; - //LOG_DEBUG(log, "TRYING TO GET RESPONSE (size {})", responses.size()); - if (!responses.empty() && responses.front().wait_for(100ms) == std::future_status::ready) + while(!responses.empty()) { - auto response = responses.front().get(); + if (responses.front().wait_for(10ms) != std::future_status::ready) + break; - //LOG_DEBUG(log, "Writing response bytes to socket {}", response->getOpNum()); + auto response = responses.front().get(); response->write(*out); responses.pop(); - //LOG_DEBUG(log, "Responses size {}", responses.size()); } + for (auto it = watch_responses.begin(); it != watch_responses.end();) { if (it->wait_for(0s) == std::future_status::ready) @@ -305,8 +128,9 @@ void TestKeeperTCPHandler::runImpl() } } - //LOG_DEBUG(log, "WAITING ON POLL"); - if (in->poll(100 * 1000)) + long poll_wait = responses.empty() ? session_timeout.totalMicroseconds() : 10000; + + if (in->poll(poll_wait)) { bool close_received = receiveRequest(); if (close_received) @@ -315,29 +139,20 @@ void TestKeeperTCPHandler::runImpl() break; } } - else - { - //LOG_DEBUG(log, "NOTHING POLLED"); - } } } bool TestKeeperTCPHandler::receiveRequest() { - //LOG_DEBUG(log, "Receiving event"); int32_t length; - read(length, *in); - //LOG_DEBUG(log, "RECEIVED LENGTH {}", length); + Coordination::read(length, *in); int32_t xid; - //LOG_DEBUG(log, "READING XID"); - read(xid, *in); + Coordination::read(xid, *in); - //LOG_DEBUG(log, "Received xid {}", xid); - - int32_t opnum; - read(opnum, *in); - if (opnum == -11) + Coordination::OpNum opnum; + Coordination::read(opnum, *in); + if (opnum == Coordination::OpNum::Close) return true; Coordination::ZooKeeperRequestPtr request = Coordination::ZooKeeperRequestFactory::instance().get(opnum); @@ -348,7 +163,6 @@ bool TestKeeperTCPHandler::receiveRequest() if (request_future_responses.watch_response) watch_responses.emplace_back(std::move(*request_future_responses.watch_response)); - //LOG_DEBUG(log, "Responses size {}", responses.size()); return false; } diff --git a/src/Server/TestKeeperTCPHandler.h b/src/Server/TestKeeperTCPHandler.h index 7eea5419006..06b3c102df5 100644 --- a/src/Server/TestKeeperTCPHandler.h +++ b/src/Server/TestKeeperTCPHandler.h @@ -4,6 +4,7 @@ #include "IServer.h" #include #include +#include #include #include #include @@ -21,7 +22,8 @@ public: , log(&Poco::Logger::get("TestKeeperTCPHandler")) , global_context(server.context()) , test_keeper_storage(global_context.getTestKeeperStorage()) - , operation_timeout(10000) + , operation_timeout(Coordination::DEFAULT_OPERATION_TIMEOUT) + , session_timeout(Coordination::DEFAULT_SESSION_TIMEOUT) { } @@ -32,6 +34,7 @@ private: Context global_context; std::shared_ptr test_keeper_storage; Poco::Timespan operation_timeout; + Poco::Timespan session_timeout; std::queue responses; std::vector watch_responses; From d83c68fca806af5c40060c857ba47951905b0330 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 11 Nov 2020 16:55:28 +0300 Subject: [PATCH 025/264] Fix timeouts --- programs/server/Server.cpp | 20 ++++++++++---------- src/Common/ZooKeeper/TestKeeperStorage.cpp | 12 ++++++++++++ src/Common/ZooKeeper/TestKeeperStorage.h | 4 ++-- src/Common/ZooKeeper/ZooKeeper.cpp | 4 ++-- src/Common/ZooKeeper/ZooKeeper.h | 8 +++----- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 6 ++---- src/Common/ZooKeeper/ZooKeeperConstants.h | 4 ++-- src/Common/ZooKeeper/ZooKeeperIO.cpp | 2 +- src/Common/ya.make | 4 ++++ src/Server/TestKeeperTCPHandler.cpp | 4 ++-- src/Server/TestKeeperTCPHandler.h | 4 ++-- src/Server/ya.make | 1 + utils/zookeeper-test/main.cpp | 7 ++++--- 13 files changed, 47 insertions(+), 33 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 597b6c43fe5..064fa391813 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -195,17 +195,17 @@ Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port || code == EAI_ADDRFAMILY #endif ) - { - LOG_ERROR(log, "Cannot resolve listen_host ({}), error {}: {}. " - "If it is an IPv6 address and your host has disabled IPv6, then consider to " - "specify IPv4 address to listen in element of configuration " - "file. Example: 0.0.0.0", - host, e.code(), e.message()); - } + { + LOG_ERROR(log, "Cannot resolve listen_host ({}), error {}: {}. " + "If it is an IPv6 address and your host has disabled IPv6, then consider to " + "specify IPv4 address to listen in element of configuration " + "file. Example: 0.0.0.0", + host, e.code(), e.message()); + } - throw; - } - return socket_address; + throw; + } + return socket_address; } Poco::Net::SocketAddress Server::socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure) diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index 4e291679b78..19d5fe6ad22 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -6,10 +6,22 @@ #include #include +namespace DB +{ + namespace ErrorCodes + { + extern const int LOGICAL_ERROR; + } +} + namespace zkutil { + + using namespace DB; + + static String parentPath(const String & path) { auto rslash_pos = path.rfind('/'); diff --git a/src/Common/ZooKeeper/TestKeeperStorage.h b/src/Common/ZooKeeper/TestKeeperStorage.h index 875cb9f0253..02fdc21cc58 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.h +++ b/src/Common/ZooKeeper/TestKeeperStorage.h @@ -19,7 +19,7 @@ class TestKeeperStorage public: - Poco::Timespan operation_timeout{10000}; + Poco::Timespan operation_timeout{0, Coordination::DEFAULT_OPERATION_TIMEOUT_MS * 1000}; std::atomic session_id_counter{0}; struct Node @@ -85,5 +85,5 @@ public: return zxid.fetch_add(1); } }; - + } diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index bee875d1c74..b3a0a082a9f 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -129,8 +129,8 @@ struct ZooKeeperArgs std::vector hosts_strings; - session_timeout_ms = DEFAULT_SESSION_TIMEOUT; - operation_timeout_ms = DEFAULT_OPERATION_TIMEOUT; + session_timeout_ms = Coordination::DEFAULT_SESSION_TIMEOUT_MS; + operation_timeout_ms = Coordination::DEFAULT_OPERATION_TIMEOUT_MS; implementation = "zookeeper"; for (const auto & key : keys) { diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index b1a69646db5..0d9dc104c48 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -28,9 +29,6 @@ namespace CurrentMetrics namespace zkutil { -const UInt32 DEFAULT_SESSION_TIMEOUT = 30000; -const UInt32 DEFAULT_OPERATION_TIMEOUT = 10000; - /// Preferred size of multi() command (in number of ops) constexpr size_t MULTI_BATCH_SIZE = 100; @@ -53,8 +51,8 @@ public: using Ptr = std::shared_ptr; ZooKeeper(const std::string & hosts_, const std::string & identity_ = "", - int32_t session_timeout_ms_ = DEFAULT_SESSION_TIMEOUT, - int32_t operation_timeout_ms_ = DEFAULT_OPERATION_TIMEOUT, + int32_t session_timeout_ms_ = Coordination::DEFAULT_SESSION_TIMEOUT_MS, + int32_t operation_timeout_ms_ = Coordination::DEFAULT_OPERATION_TIMEOUT_MS, const std::string & chroot_ = "", const std::string & implementation_ = "zookeeper"); diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index 7b162492d1a..d1c6b1ee80b 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -17,13 +17,11 @@ void ZooKeeperResponse::write(WriteBuffer & out) const { /// Excessive copy to calculate length. WriteBufferFromOwnString buf; - //LOG_DEBUG(&Poco::Logger::get("LOG"), "WRITING {}", xid); Coordination::write(xid, buf); Coordination::write(zxid, buf); Coordination::write(error, buf); if (error == Error::ZOK) writeImpl(buf); - //LOG_DEBUG(&Poco::Logger::get("LOG"), "BUFFER LENGTH {}", buf.str().length()); Coordination::write(buf.str(), out); out.next(); } @@ -426,7 +424,7 @@ ZooKeeperResponsePtr ZooKeeperCloseRequest::makeResponse() const { return std::m void ZooKeeperRequestFactory::registerRequest(OpNum op_num, Creator creator) { if (!op_num_to_request.try_emplace(op_num, creator).second) - throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Request with op num {} already registered", op_num); + throw Coordination::Exception("Request type " + toString(op_num) + " already registered", Coordination::Error::ZRUNTIMEINCONSISTENCY); } std::shared_ptr ZooKeeperRequest::read(ReadBuffer & in) @@ -478,5 +476,5 @@ ZooKeeperRequestFactory::ZooKeeperRequestFactory() registerZooKeeperRequest(*this); registerZooKeeperRequest(*this); } - + } diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.h b/src/Common/ZooKeeper/ZooKeeperConstants.h index 47b597f1c9d..292bccd86b4 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.h +++ b/src/Common/ZooKeeper/ZooKeeperConstants.h @@ -41,7 +41,7 @@ static constexpr int32_t PASSWORD_LENGTH = 16; /// ZooKeeper has 1 MB node size and serialization limit by default, /// but it can be raised up, so we have a slightly larger limit on our side. static constexpr int32_t MAX_STRING_OR_ARRAY_SIZE = 1 << 28; /// 256 MiB -static constexpr int32_t DEFAULT_SESSION_TIMEOUT = 30000; -static constexpr int32_t DEFAULT_OPERATION_TIMEOUT = 10000; +static constexpr int32_t DEFAULT_SESSION_TIMEOUT_MS = 30000; +static constexpr int32_t DEFAULT_OPERATION_TIMEOUT_MS = 10000; } diff --git a/src/Common/ZooKeeper/ZooKeeperIO.cpp b/src/Common/ZooKeeper/ZooKeeperIO.cpp index 157325cb8e6..07227669366 100644 --- a/src/Common/ZooKeeper/ZooKeeperIO.cpp +++ b/src/Common/ZooKeeper/ZooKeeperIO.cpp @@ -16,7 +16,7 @@ void write(int32_t x, WriteBuffer & out) void write(OpNum x, WriteBuffer & out) { - write(static_cast(x), out); + write(static_cast(x), out); } void write(bool x, WriteBuffer & out) diff --git a/src/Common/ya.make b/src/Common/ya.make index 0d6caa22f3a..2d8110b13d0 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -78,7 +78,11 @@ SRCS( WeakHash.cpp ZooKeeper/IKeeper.cpp ZooKeeper/TestKeeper.cpp + ZooKeeper/TestKeeperStorage.cpp ZooKeeper/ZooKeeper.cpp + ZooKeeper/ZooKeeperCommon.cpp + ZooKeeper/ZooKeeperConstants.cpp + ZooKeeper/ZooKeeperIO.cpp ZooKeeper/ZooKeeperImpl.cpp ZooKeeper/ZooKeeperNodeCache.cpp checkStackSize.cpp diff --git a/src/Server/TestKeeperTCPHandler.cpp b/src/Server/TestKeeperTCPHandler.cpp index 7afd1af1aa0..cdac4fdd9b2 100644 --- a/src/Server/TestKeeperTCPHandler.cpp +++ b/src/Server/TestKeeperTCPHandler.cpp @@ -25,7 +25,7 @@ void TestKeeperTCPHandler::sendHandshake() Coordination::write(Coordination::SERVER_HANDSHAKE_LENGTH, *out); Coordination::write(Coordination::ZOOKEEPER_PROTOCOL_VERSION, *out); - Coordination::write(Coordination::DEFAULT_SESSION_TIMEOUT, *out); + Coordination::write(Coordination::DEFAULT_SESSION_TIMEOUT_MS, *out); Coordination::write(test_keeper_storage->getSessionID(), *out); std::array passwd{}; Coordination::write(passwd, *out); @@ -105,7 +105,7 @@ void TestKeeperTCPHandler::runImpl() while (true) { using namespace std::chrono_literals; - while(!responses.empty()) + while (!responses.empty()) { if (responses.front().wait_for(10ms) != std::future_status::ready) break; diff --git a/src/Server/TestKeeperTCPHandler.h b/src/Server/TestKeeperTCPHandler.h index 06b3c102df5..ec44a481522 100644 --- a/src/Server/TestKeeperTCPHandler.h +++ b/src/Server/TestKeeperTCPHandler.h @@ -22,8 +22,8 @@ public: , log(&Poco::Logger::get("TestKeeperTCPHandler")) , global_context(server.context()) , test_keeper_storage(global_context.getTestKeeperStorage()) - , operation_timeout(Coordination::DEFAULT_OPERATION_TIMEOUT) - , session_timeout(Coordination::DEFAULT_SESSION_TIMEOUT) + , operation_timeout(0, Coordination::DEFAULT_OPERATION_TIMEOUT_MS * 1000) + , session_timeout(0, Coordination::DEFAULT_SESSION_TIMEOUT_MS * 1000) { } diff --git a/src/Server/ya.make b/src/Server/ya.make index 8a9bbd3bbc2..a83b642dc8b 100644 --- a/src/Server/ya.make +++ b/src/Server/ya.make @@ -21,6 +21,7 @@ SRCS( ReplicasStatusHandler.cpp StaticRequestHandler.cpp TCPHandler.cpp + TestKeeperTCPHandler.cpp WebUIRequestHandler.cpp ) diff --git a/utils/zookeeper-test/main.cpp b/utils/zookeeper-test/main.cpp index a924465d02a..1bf17a035bf 100644 --- a/utils/zookeeper-test/main.cpp +++ b/utils/zookeeper-test/main.cpp @@ -135,7 +135,7 @@ void testMultiRequest(zkutil::ZooKeeper & zk) zk.multi(requests); std::terminate(); } - catch(...) + catch (...) { std::cerr << "Got exception on multy request (it's ok)\n"; } @@ -143,7 +143,8 @@ void testMultiRequest(zkutil::ZooKeeper & zk) checkEq(zk, "/data/multirequest", "bbb"); } -int main(int argc, char *argv[]) { +int main(int argc, char *argv[]) +{ if (argc != 2) { @@ -167,7 +168,7 @@ int main(int argc, char *argv[]) { testCreateSetWatchEvent(zk); testCreateListWatchEvent(zk); } - catch(...) + catch (...) { zk.tryRemoveRecursive("/data"); throw; From df02bb23151af67364dc6a3695cc703b698b7a6a Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 11 Nov 2020 17:00:28 +0300 Subject: [PATCH 026/264] Change test config --- tests/config/config.d/zookeeper.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/config/config.d/zookeeper.xml b/tests/config/config.d/zookeeper.xml index 68c85788c98..06ed7fcd39f 100644 --- a/tests/config/config.d/zookeeper.xml +++ b/tests/config/config.d/zookeeper.xml @@ -1,5 +1,8 @@ - testkeeper + + localhost + 9181 + From d57b57156ecda30fcdb01aa7cf2f0ff07311bbbf Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 11 Nov 2020 17:22:11 +0300 Subject: [PATCH 027/264] Fix style check --- src/Common/ZooKeeper/TestKeeperStorage.cpp | 5 ----- src/Common/ZooKeeper/ZooKeeperCommon.h | 1 - 2 files changed, 6 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeperStorage.cpp b/src/Common/ZooKeeper/TestKeeperStorage.cpp index 19d5fe6ad22..70702dbecc9 100644 --- a/src/Common/ZooKeeper/TestKeeperStorage.cpp +++ b/src/Common/ZooKeeper/TestKeeperStorage.cpp @@ -17,11 +17,8 @@ namespace DB namespace zkutil { - using namespace DB; - - static String parentPath(const String & path) { auto rslash_pos = path.rfind('/'); @@ -679,6 +676,4 @@ TestKeeperStorage::~TestKeeperStorage() } } - - } diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index 286330bd769..fbf3e13fbfe 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -19,7 +19,6 @@ #include - namespace Coordination { From ade14da63484a8c44c034f8c4bdb8a48f4a471e4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 11 Nov 2020 18:45:10 +0300 Subject: [PATCH 028/264] Remove some redundant files --- src/Common/ZooKeeper/TestKeeper.h | 2 -- utils/zookeeper-test/main.cpp | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h index 01c92c98778..ca9f584304f 100644 --- a/src/Common/ZooKeeper/TestKeeper.h +++ b/src/Common/ZooKeeper/TestKeeper.h @@ -125,8 +125,6 @@ private: Watches watches; Watches list_watches; /// Watches for 'list' request (watches on children). - void createWatchCallBack(const String & path); - using RequestsQueue = ConcurrentBoundedQueue; RequestsQueue requests_queue{1}; diff --git a/utils/zookeeper-test/main.cpp b/utils/zookeeper-test/main.cpp index 1bf17a035bf..fe7bf93fd15 100644 --- a/utils/zookeeper-test/main.cpp +++ b/utils/zookeeper-test/main.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -14,6 +13,8 @@ using namespace std; +/// TODO: Remove ME + void checkEq(zkutil::ZooKeeper & zk, const std::string & path, const std::string & expected) { auto result = zk.get(path); From 7d32873cb9a75ed3ad730bba85317d8cbe50e761 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 11 Nov 2020 19:59:13 +0300 Subject: [PATCH 029/264] Add missed file --- tests/config/config.d/test_keeper_port.xml | 3 +++ tests/config/install.sh | 1 + 2 files changed, 4 insertions(+) create mode 100644 tests/config/config.d/test_keeper_port.xml diff --git a/tests/config/config.d/test_keeper_port.xml b/tests/config/config.d/test_keeper_port.xml new file mode 100644 index 00000000000..23b744630f2 --- /dev/null +++ b/tests/config/config.d/test_keeper_port.xml @@ -0,0 +1,3 @@ + + 9181 + diff --git a/tests/config/install.sh b/tests/config/install.sh index f6fae181ac8..4eb665b01ba 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -28,6 +28,7 @@ ln -sf $SRC_PATH/config.d/clusters.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/graphite.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/database_atomic.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/test_cluster_with_incorrect_pw.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/test_keeper_port.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/users.d/log_queries.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/readonly.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/access_management.xml $DEST_SERVER_PATH/users.d/ From eb9604ecd005d8c076e001a04efae822d5567e03 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 12 Nov 2020 23:23:26 +0300 Subject: [PATCH 030/264] Fixed comparison of DateTime4 with different scales Made DateTime64 a stand-alone type rather than typedef to allow function resolution/deduction based on type. Added tests. --- src/Columns/ColumnDecimal.cpp | 1 + src/Core/DecimalComparison.h | 14 ++--- src/Core/DecimalFunctions.h | 60 ++++++++++++++++++- src/Core/Field.cpp | 4 ++ src/Core/Field.h | 3 + src/Core/Types.h | 17 +++++- src/Core/callOnTypeIndex.h | 2 +- src/DataTypes/DataTypeDateTime64.h | 1 + src/DataTypes/DataTypeDecimalBase.cpp | 1 + src/DataTypes/DataTypeDecimalBase.h | 42 +++++++------ src/DataTypes/IDataType.h | 8 +++ src/IO/ReadHelpers.h | 14 +++-- .../01553_datetime64_comparison.reference | 3 + .../01553_datetime64_comparison.sql | 20 +++++++ 14 files changed, 156 insertions(+), 34 deletions(-) create mode 100644 tests/queries/0_stateless/01553_datetime64_comparison.reference create mode 100644 tests/queries/0_stateless/01553_datetime64_comparison.sql diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 7c3af5fe095..196f77d0db8 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -421,4 +421,5 @@ template class ColumnDecimal; template class ColumnDecimal; template class ColumnDecimal; template class ColumnDecimal; +template class ColumnDecimal; } diff --git a/src/Core/DecimalComparison.h b/src/Core/DecimalComparison.h index 674ed31683b..600e876f326 100644 --- a/src/Core/DecimalComparison.h +++ b/src/Core/DecimalComparison.h @@ -113,15 +113,15 @@ private: static std::enable_if_t && IsDecimalNumber, Shift> getScales(const DataTypePtr & left_type, const DataTypePtr & right_type) { - const DataTypeDecimal * decimal0 = checkDecimal(*left_type); - const DataTypeDecimal * decimal1 = checkDecimal(*right_type); + const DataTypeDecimalBase * decimal0 = checkDecimalBase(*left_type); + const DataTypeDecimalBase * decimal1 = checkDecimalBase(*right_type); Shift shift; if (decimal0 && decimal1) { - auto result_type = decimalResultType(*decimal0, *decimal1); - shift.a = static_cast(result_type.scaleFactorFor(*decimal0, false).value); - shift.b = static_cast(result_type.scaleFactorFor(*decimal1, false).value); + auto result_type = DecimalUtils::binaryOpResult(*decimal0, *decimal1); + shift.a = static_cast(result_type.scaleFactorFor(decimal0->getTrait(), false).value); + shift.b = static_cast(result_type.scaleFactorFor(decimal1->getTrait(), false).value); } else if (decimal0) shift.b = static_cast(decimal0->getScaleMultiplier().value); @@ -136,7 +136,7 @@ private: getScales(const DataTypePtr & left_type, const DataTypePtr &) { Shift shift; - const DataTypeDecimal * decimal0 = checkDecimal(*left_type); + const DataTypeDecimalBase * decimal0 = checkDecimalBase(*left_type); if (decimal0) shift.b = static_cast(decimal0->getScaleMultiplier().value); return shift; @@ -147,7 +147,7 @@ private: getScales(const DataTypePtr &, const DataTypePtr & right_type) { Shift shift; - const DataTypeDecimal * decimal1 = checkDecimal(*right_type); + const DataTypeDecimalBase * decimal1 = checkDecimalBase(*right_type); if (decimal1) shift.a = static_cast(decimal1->getScaleMultiplier().value); return shift; diff --git a/src/Core/DecimalFunctions.h b/src/Core/DecimalFunctions.h index 52e8ebae322..b82cfd88e98 100644 --- a/src/Core/DecimalFunctions.h +++ b/src/Core/DecimalFunctions.h @@ -11,9 +11,13 @@ namespace DB { +template +class DataTypeNumber; + namespace ErrorCodes { extern const int DECIMAL_OVERFLOW; + extern const int ARGUMENT_OUT_OF_BOUND; } namespace DecimalUtils @@ -23,6 +27,7 @@ static constexpr size_t minPrecision() { return 1; } template static constexpr size_t maxPrecision() { return 0; } template <> constexpr size_t maxPrecision() { return 9; } template <> constexpr size_t maxPrecision() { return 18; } +template <> constexpr size_t maxPrecision() { return 18; } template <> constexpr size_t maxPrecision() { return 38; } template <> constexpr size_t maxPrecision() { return 76; } @@ -31,7 +36,7 @@ inline auto scaleMultiplier(UInt32 scale) { if constexpr (std::is_same_v || std::is_same_v) return common::exp10_i32(scale); - else if constexpr (std::is_same_v || std::is_same_v) + else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) return common::exp10_i64(scale); else if constexpr (std::is_same_v || std::is_same_v) return common::exp10_i128(scale); @@ -51,6 +56,30 @@ struct DecimalComponents T fractional; }; +/// Traits used for determining final Type/Precision/Scale for certain math operations on decimals. +template +struct DataTypeDecimalTrait +{ + using FieldType = T; + const UInt32 precision; + const UInt32 scale; + + DataTypeDecimalTrait(UInt32 precision_, UInt32 scale_) + : precision(precision_), + scale(scale_) + {} + + /// @returns multiplier for U to become T with correct scale + template + T scaleFactorFor(const DataTypeDecimalTrait & x, bool) const + { + if (scale < x.scale) + throw Exception("Decimal result's scale is less than argument's one", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + const UInt32 scale_delta = scale - x.scale; /// scale_delta >= 0 + return DecimalUtils::scaleMultiplier(scale_delta); + } +}; + /** Make a decimal value from whole and fractional components with given scale multiplier. * where scale_multiplier = scaleMultiplier(scale) * this is to reduce number of calls to scaleMultiplier when scale is known. @@ -211,6 +240,35 @@ To convertTo(const DecimalType & decimal, size_t scale) } } +template typename DecimalType> +inline auto binaryOpResult(const DecimalType & tx, const DecimalType & ty) +{ + UInt32 scale{}; + if constexpr (is_multiply) + scale = tx.getScale() + ty.getScale(); + else if constexpr (is_division) + scale = tx.getScale(); + else + scale = (tx.getScale() > ty.getScale() ? tx.getScale() : ty.getScale()); + + if constexpr (sizeof(T) < sizeof(U)) + return DataTypeDecimalTrait(DecimalUtils::maxPrecision(), scale); + else + return DataTypeDecimalTrait(DecimalUtils::maxPrecision(), scale); +} + +template typename DecimalType> +inline const DataTypeDecimalTrait binaryOpResult(const DecimalType & tx, const DataTypeNumber &) +{ + return DataTypeDecimalTrait(DecimalUtils::maxPrecision(), tx.getScale()); +} + +template typename DecimalType> +inline const DataTypeDecimalTrait binaryOpResult(const DataTypeNumber &, const DecimalType & ty) +{ + return DataTypeDecimalTrait(DecimalUtils::maxPrecision(), ty.getScale()); +} + } } diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 2309a36dc40..6fda3b39b3b 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -592,6 +592,10 @@ template <> bool decimalEqual(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 template <> bool decimalLess(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 y_scale) { return decLess(x, y, x_scale, y_scale); } template <> bool decimalLessOrEqual(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 y_scale) { return decLessOrEqual(x, y, x_scale, y_scale); } +template <> bool decimalEqual(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale) { return decEqual(x, y, x_scale, y_scale); } +template <> bool decimalLess(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale) { return decLess(x, y, x_scale, y_scale); } +template <> bool decimalLessOrEqual(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale) { return decLessOrEqual(x, y, x_scale, y_scale); } + inline void writeText(const Null &, WriteBuffer & buf) { writeText(std::string("Null"), buf); diff --git a/src/Core/Field.h b/src/Core/Field.h index 66e4f0ac8db..433a1eaa5be 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -193,10 +193,12 @@ template <> struct NearestFieldTypeImpl { using Type = DecimalField struct NearestFieldTypeImpl { using Type = DecimalField; }; template <> struct NearestFieldTypeImpl { using Type = DecimalField; }; template <> struct NearestFieldTypeImpl { using Type = DecimalField; }; +template <> struct NearestFieldTypeImpl { using Type = DecimalField; }; template <> struct NearestFieldTypeImpl> { using Type = DecimalField; }; template <> struct NearestFieldTypeImpl> { using Type = DecimalField; }; template <> struct NearestFieldTypeImpl> { using Type = DecimalField; }; template <> struct NearestFieldTypeImpl> { using Type = DecimalField; }; +template <> struct NearestFieldTypeImpl> { using Type = DecimalField; }; template <> struct NearestFieldTypeImpl { using Type = Float64; }; template <> struct NearestFieldTypeImpl { using Type = Float64; }; template <> struct NearestFieldTypeImpl { using Type = String; }; @@ -731,6 +733,7 @@ template <> struct Field::TypeToEnum>{ static const Type template <> struct Field::TypeToEnum>{ static const Types::Which value = Types::Decimal64; }; template <> struct Field::TypeToEnum>{ static const Types::Which value = Types::Decimal128; }; template <> struct Field::TypeToEnum>{ static const Types::Which value = Types::Decimal256; }; +template <> struct Field::TypeToEnum>{ static const Types::Which value = Types::Decimal64; }; template <> struct Field::TypeToEnum{ static const Types::Which value = Types::AggregateFunctionState; }; template <> struct Field::TypeToEnum { static const Types::Which value = Types::UInt256; }; template <> struct Field::TypeToEnum { static const Types::Which value = Types::Int256; }; diff --git a/src/Core/Types.h b/src/Core/Types.h index 3157598adc0..aafe87e2f2a 100644 --- a/src/Core/Types.h +++ b/src/Core/Types.h @@ -184,29 +184,44 @@ using Decimal64 = Decimal; using Decimal128 = Decimal; using Decimal256 = Decimal; -using DateTime64 = Decimal64; +// Distinguishable type to allow function resultion/deduction based on value type, +// but also relatively easy to convert to/from Decimal64. +class DateTime64 : public Decimal64 +{ +public: + using Base = Decimal64; + using Base::Base; + + DateTime64(const Base & v) + : Base(v) + {} +}; template <> struct TypeName { static constexpr const char * get() { return "Decimal32"; } }; template <> struct TypeName { static constexpr const char * get() { return "Decimal64"; } }; template <> struct TypeName { static constexpr const char * get() { return "Decimal128"; } }; template <> struct TypeName { static constexpr const char * get() { return "Decimal256"; } }; +template <> struct TypeName { static constexpr const char * get() { return "DateTime64"; } }; template <> struct TypeId { static constexpr const TypeIndex value = TypeIndex::Decimal32; }; template <> struct TypeId { static constexpr const TypeIndex value = TypeIndex::Decimal64; }; template <> struct TypeId { static constexpr const TypeIndex value = TypeIndex::Decimal128; }; template <> struct TypeId { static constexpr const TypeIndex value = TypeIndex::Decimal256; }; +template <> struct TypeId { static constexpr const TypeIndex value = TypeIndex::DateTime64; }; template constexpr bool IsDecimalNumber = false; template <> inline constexpr bool IsDecimalNumber = true; template <> inline constexpr bool IsDecimalNumber = true; template <> inline constexpr bool IsDecimalNumber = true; template <> inline constexpr bool IsDecimalNumber = true; +template <> inline constexpr bool IsDecimalNumber = true; template struct NativeType { using Type = T; }; template <> struct NativeType { using Type = Int32; }; template <> struct NativeType { using Type = Int64; }; template <> struct NativeType { using Type = Int128; }; template <> struct NativeType { using Type = Int256; }; +template <> struct NativeType { using Type = Int64; }; template constexpr bool OverBigInt = false; template <> inline constexpr bool OverBigInt = true; diff --git a/src/Core/callOnTypeIndex.h b/src/Core/callOnTypeIndex.h index 62eec37c0cb..2ef049762f1 100644 --- a/src/Core/callOnTypeIndex.h +++ b/src/Core/callOnTypeIndex.h @@ -141,7 +141,7 @@ inline bool callOnBasicTypes(TypeIndex type_num1, TypeIndex type_num2, F && f) { case TypeIndex::Date: return callOnBasicType(type_num2, std::forward(f)); case TypeIndex::DateTime: return callOnBasicType(type_num2, std::forward(f)); - case TypeIndex::DateTime64: return callOnBasicType(type_num2, std::forward(f)); + case TypeIndex::DateTime64: return callOnBasicType(type_num2, std::forward(f)); default: break; } diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/DataTypeDateTime64.h index 8cbeedfb48c..003e83b7195 100644 --- a/src/DataTypes/DataTypeDateTime64.h +++ b/src/DataTypes/DataTypeDateTime64.h @@ -16,6 +16,7 @@ namespace DB class DataTypeDateTime64 final : public DataTypeDecimalBase, public TimezoneMixin { public: + using Base = DataTypeDecimalBase; static constexpr UInt8 default_scale = 3; static constexpr auto family_name = "DateTime64"; diff --git a/src/DataTypes/DataTypeDecimalBase.cpp b/src/DataTypes/DataTypeDecimalBase.cpp index 123f370cb91..9fb445ab00d 100644 --- a/src/DataTypes/DataTypeDecimalBase.cpp +++ b/src/DataTypes/DataTypeDecimalBase.cpp @@ -102,5 +102,6 @@ template class DataTypeDecimalBase; template class DataTypeDecimalBase; template class DataTypeDecimalBase; template class DataTypeDecimalBase; +template class DataTypeDecimalBase; } diff --git a/src/DataTypes/DataTypeDecimalBase.h b/src/DataTypes/DataTypeDecimalBase.h index c5669ab735a..4be98e4d22d 100644 --- a/src/DataTypes/DataTypeDecimalBase.h +++ b/src/DataTypes/DataTypeDecimalBase.h @@ -150,39 +150,45 @@ public: static T getScaleMultiplier(UInt32 scale); + inline DecimalUtils::DataTypeDecimalTrait getTrait() const + { + return {precision, scale}; + } + protected: const UInt32 precision; const UInt32 scale; }; +template +inline const DataTypeDecimalBase * checkDecimalBase(const IDataType & data_type) +{ + if (isColumnedAsDecimalT(data_type)) + return static_cast *>(&data_type); + + return nullptr; +} + template typename DecimalType> inline auto decimalResultType(const DecimalType & tx, const DecimalType & ty) { - UInt32 scale{}; - if constexpr (is_multiply) - scale = tx.getScale() + ty.getScale(); - else if constexpr (is_division) - scale = tx.getScale(); - else - scale = (tx.getScale() > ty.getScale() ? tx.getScale() : ty.getScale()); - - if constexpr (sizeof(T) < sizeof(U)) - return DecimalType(DecimalUtils::maxPrecision(), scale); - else - return DecimalType(DecimalUtils::maxPrecision(), scale); + const auto result_trait = DecimalUtils::binaryOpResult(tx, ty); + return DecimalType(result_trait.precision, result_trait.scale); } -template typename DecimalType> -inline const DecimalType decimalResultType(const DecimalType & tx, const DataTypeNumber &) +template typename DecimalType> +inline const DecimalType decimalResultType(const DecimalType & tx, const DataTypeNumber & ty) { - return DecimalType(DecimalUtils::maxPrecision(), tx.getScale()); + const auto result_trait = DecimalUtils::binaryOpResult(tx, ty); + return DecimalType(result_trait.precision, result_trait.scale); } -template typename DecimalType> -inline const DecimalType decimalResultType(const DataTypeNumber &, const DecimalType & ty) +template typename DecimalType> +inline const DecimalType decimalResultType(const DataTypeNumber & tx, const DecimalType & ty) { - return DecimalType(DecimalUtils::maxPrecision(), ty.getScale()); + const auto result_trait = DecimalUtils::binaryOpResult(tx, ty); + return DecimalType(result_trait.precision, result_trait.scale); } template